Adaptation de la base de données pour sql server
This commit is contained in:
@@ -0,0 +1,124 @@
|
|||||||
|
/*
|
||||||
|
Script de création des tables SQL Server pour la synchronisation Meta Lead Gen.
|
||||||
|
|
||||||
|
IMPORTANT :
|
||||||
|
Avant de lancer ce script, la base de données doit déjà exister.
|
||||||
|
|
||||||
|
Étapes :
|
||||||
|
1. Se connecter à SQL Server avec un outil comme Adminer, SSMS, Azure Data Studio ou DBeaver.
|
||||||
|
2. Créer la base de données si elle n'existe pas encore :
|
||||||
|
|
||||||
|
CREATE DATABASE MetaLeadDb;
|
||||||
|
|
||||||
|
3. Se connecter ensuite à la base MetaLeadDb.
|
||||||
|
4. Lancer ce script pour créer les tables :
|
||||||
|
- leadgen_forms
|
||||||
|
- lead_data
|
||||||
|
- sync_logs
|
||||||
|
|
||||||
|
Ce script ne crée pas la base de données.
|
||||||
|
Il crée uniquement les tables nécessaires au projet.*/
|
||||||
|
|
||||||
|
|
||||||
|
IF OBJECT_ID(N'dbo.sync_logs', N'U') IS NOT NULL
|
||||||
|
DROP TABLE dbo.sync_logs;
|
||||||
|
|
||||||
|
IF OBJECT_ID(N'dbo.lead_data', N'U') IS NOT NULL
|
||||||
|
DROP TABLE dbo.lead_data;
|
||||||
|
|
||||||
|
IF OBJECT_ID(N'dbo.leadgen_forms', N'U') IS NOT NULL
|
||||||
|
DROP TABLE dbo.leadgen_forms;
|
||||||
|
|
||||||
|
CREATE TABLE dbo.leadgen_forms (
|
||||||
|
id BIGINT IDENTITY(1,1) NOT NULL,
|
||||||
|
meta_form_id NVARCHAR(64) NOT NULL,
|
||||||
|
page_id NVARCHAR(64) NOT NULL,
|
||||||
|
name NVARCHAR(255) NOT NULL,
|
||||||
|
locale NVARCHAR(20) NULL,
|
||||||
|
status NVARCHAR(50) NOT NULL,
|
||||||
|
created_at DATETIME2 NOT NULL CONSTRAINT DF_leadgen_forms_created_at DEFAULT SYSDATETIME(),
|
||||||
|
updated_at DATETIME2 NULL,
|
||||||
|
|
||||||
|
CONSTRAINT PK_leadgen_forms PRIMARY KEY (id),
|
||||||
|
CONSTRAINT UQ_leadgen_forms_meta_form_id UNIQUE (meta_form_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE dbo.lead_data (
|
||||||
|
id BIGINT IDENTITY(1,1) NOT NULL,
|
||||||
|
meta_lead_id NVARCHAR(64) NOT NULL,
|
||||||
|
meta_form_id NVARCHAR(64) NOT NULL,
|
||||||
|
created_time DATETIME2 NULL,
|
||||||
|
je_suis NVARCHAR(255) NULL,
|
||||||
|
nom NVARCHAR(255) NULL,
|
||||||
|
first_name NVARCHAR(255) NULL,
|
||||||
|
adresse_email NVARCHAR(255) NULL,
|
||||||
|
phone_number NVARCHAR(50) NULL,
|
||||||
|
city NVARCHAR(255) NULL,
|
||||||
|
campus NVARCHAR(100) NULL,
|
||||||
|
status NVARCHAR(50) NOT NULL CONSTRAINT DF_lead_data_status DEFAULT N'RECEIVED',
|
||||||
|
error_message NVARCHAR(MAX) NULL,
|
||||||
|
synced_at DATETIME2 NULL,
|
||||||
|
created_at DATETIME2 NOT NULL CONSTRAINT DF_lead_data_created_at DEFAULT SYSDATETIME(),
|
||||||
|
updated_at DATETIME2 NULL,
|
||||||
|
|
||||||
|
CONSTRAINT PK_lead_data PRIMARY KEY (id),
|
||||||
|
CONSTRAINT UQ_lead_data_meta_lead_id UNIQUE (meta_lead_id),
|
||||||
|
|
||||||
|
CONSTRAINT FK_lead_data_leadgen_forms
|
||||||
|
FOREIGN KEY (meta_form_id)
|
||||||
|
REFERENCES dbo.leadgen_forms(meta_form_id)
|
||||||
|
ON UPDATE CASCADE
|
||||||
|
ON DELETE NO ACTION,
|
||||||
|
|
||||||
|
CONSTRAINT CK_lead_data_status
|
||||||
|
CHECK (status IN (N'RECEIVED', N'READY_TO_SEND', N'SENT', N'ERROR', N'INVALID'))
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE dbo.sync_logs (
|
||||||
|
id BIGINT IDENTITY(1,1) NOT NULL,
|
||||||
|
lead_id BIGINT NOT NULL,
|
||||||
|
oscar_payload NVARCHAR(MAX) NULL,
|
||||||
|
oscar_response NVARCHAR(MAX) NULL,
|
||||||
|
status NVARCHAR(50) NOT NULL,
|
||||||
|
error_message NVARCHAR(MAX) NULL,
|
||||||
|
attempted_at DATETIME2 NOT NULL CONSTRAINT DF_sync_logs_attempted_at DEFAULT SYSDATETIME(),
|
||||||
|
|
||||||
|
CONSTRAINT PK_sync_logs PRIMARY KEY (id),
|
||||||
|
|
||||||
|
CONSTRAINT FK_sync_logs_lead_data
|
||||||
|
FOREIGN KEY (lead_id)
|
||||||
|
REFERENCES dbo.lead_data(id)
|
||||||
|
ON UPDATE CASCADE
|
||||||
|
ON DELETE CASCADE,
|
||||||
|
|
||||||
|
CONSTRAINT CK_sync_logs_status
|
||||||
|
CHECK (status IN (N'SENT', N'ERROR', N'INVALID'))
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IX_leadgen_forms_status
|
||||||
|
ON dbo.leadgen_forms(status);
|
||||||
|
|
||||||
|
CREATE INDEX IX_lead_data_meta_form_id
|
||||||
|
ON dbo.lead_data(meta_form_id);
|
||||||
|
|
||||||
|
CREATE INDEX IX_lead_data_status
|
||||||
|
ON dbo.lead_data(status);
|
||||||
|
|
||||||
|
CREATE INDEX IX_lead_data_created_time
|
||||||
|
ON dbo.lead_data(created_time);
|
||||||
|
|
||||||
|
CREATE INDEX IX_lead_data_adresse_email
|
||||||
|
ON dbo.lead_data(adresse_email);
|
||||||
|
|
||||||
|
CREATE INDEX IX_lead_data_campus
|
||||||
|
ON dbo.lead_data(campus);
|
||||||
|
|
||||||
|
CREATE INDEX IX_sync_logs_lead_id
|
||||||
|
ON dbo.sync_logs(lead_id);
|
||||||
|
|
||||||
|
CREATE INDEX IX_sync_logs_status
|
||||||
|
ON dbo.sync_logs(status);
|
||||||
|
|
||||||
|
CREATE INDEX IX_sync_logs_attempted_at
|
||||||
|
ON dbo.sync_logs(attempted_at);
|
||||||
|
|
||||||
@@ -1,52 +1,34 @@
|
|||||||
"""
|
"""
|
||||||
Synchronisation Meta Lead Gen vers MySQL local (V1).
|
Synchronisation Meta Lead Gen vers SQL Server local.
|
||||||
|
|
||||||
|
Récupère les formulaires actifs et leurs leads via l'API Graph Meta,
|
||||||
|
puis les stocke dans SQL Server sans créer de doublons.
|
||||||
|
|
||||||
Récupère les formulaires actifs, leurs questions et leurs leads via l'API
|
|
||||||
Graph Meta, puis les stocke dans la base locale sans créer de doublons.
|
|
||||||
L'injection vers OscarCRM n'est pas gérée à ce stade.
|
L'injection vers OscarCRM n'est pas gérée à ce stade.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
|
||||||
import os
|
import os
|
||||||
from datetime import datetime
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
import mysql.connector
|
import pyodbc
|
||||||
import requests
|
import requests
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# Chargement de la configuration
|
|
||||||
# ============================================================
|
|
||||||
# Les informations sensibles sont lues depuis le fichier .env.
|
|
||||||
# Cela évite de stocker le token Meta ou le mot de passe MySQL
|
|
||||||
# directement dans le code source.
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
load_dotenv()
|
load_dotenv()
|
||||||
|
|
||||||
META_ACCESS_TOKEN = os.getenv("META_ACCESS_TOKEN")
|
META_ACCESS_TOKEN = os.getenv("META_ACCESS_TOKEN")
|
||||||
META_API_VERSION = os.getenv("META_API_VERSION", "v25.0")
|
META_API_VERSION = os.getenv("META_API_VERSION", "v25.0")
|
||||||
META_PAGE_ID = os.getenv("META_PAGE_ID")
|
META_PAGE_ID = os.getenv("META_PAGE_ID")
|
||||||
|
|
||||||
DB_CONFIG = {
|
DB_DRIVER = os.getenv("DB_DRIVER", "SQL Server")
|
||||||
"host": os.getenv("DB_HOST", "localhost"),
|
DB_SERVER = os.getenv("DB_SERVER", "localhost")
|
||||||
"port": int(os.getenv("DB_PORT", "3306")),
|
DB_PORT = os.getenv("DB_PORT", "1433")
|
||||||
"database": os.getenv("DB_NAME", "meta_lead_db"),
|
DB_NAME = os.getenv("DB_NAME", "MetaLeadDb")
|
||||||
"user": os.getenv("DB_USER", "root"),
|
DB_USER = os.getenv("DB_USER", "sa")
|
||||||
"password": os.getenv("DB_PASSWORD", ""),
|
DB_PASSWORD = os.getenv("DB_PASSWORD")
|
||||||
"charset": "utf8mb4",
|
|
||||||
"use_unicode": True,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# Formulaires autorisés pour le test
|
|
||||||
# ============================================================
|
|
||||||
# On limite volontairement la synchronisation aux formulaires
|
|
||||||
# actifs connus afin d’éviter de traiter des formulaires archivés.
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
FORM_CAMPUSES = {
|
FORM_CAMPUSES = {
|
||||||
"1016111097572602": "Marseille",
|
"1016111097572602": "Marseille",
|
||||||
"1032481262797041": "Nantes",
|
"1032481262797041": "Nantes",
|
||||||
@@ -56,25 +38,22 @@ FORM_CAMPUSES = {
|
|||||||
|
|
||||||
|
|
||||||
def get_database_connection():
|
def get_database_connection():
|
||||||
"""Ouvre une connexion MySQL locale."""
|
"""Ouvre une connexion vers SQL Server."""
|
||||||
|
|
||||||
return mysql.connector.connect(**DB_CONFIG)
|
connection_string = (
|
||||||
|
f"DRIVER={{{DB_DRIVER}}};"
|
||||||
|
f"SERVER={DB_SERVER},{DB_PORT};"
|
||||||
|
f"DATABASE={DB_NAME};"
|
||||||
|
f"UID={DB_USER};"
|
||||||
|
f"PWD={DB_PASSWORD};"
|
||||||
|
"TrustServerCertificate=yes;"
|
||||||
|
)
|
||||||
|
|
||||||
|
return pyodbc.connect(connection_string)
|
||||||
|
|
||||||
|
|
||||||
def call_meta_api(path, params=None):
|
def call_meta_api(path, params=None):
|
||||||
"""
|
"""Exécute un appel GET vers l'API Graph Meta."""
|
||||||
Exécute un appel GET vers l’API Graph Meta.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
path: Chemin Graph API, par exemple 'PAGE_ID/leadgen_forms'.
|
|
||||||
params: Paramètres complémentaires de la requête.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Réponse JSON retournée par Meta.
|
|
||||||
|
|
||||||
Raises:
|
|
||||||
requests.HTTPError: si Meta retourne une erreur HTTP.
|
|
||||||
"""
|
|
||||||
|
|
||||||
url = f"https://graph.facebook.com/{META_API_VERSION}/{path}"
|
url = f"https://graph.facebook.com/{META_API_VERSION}/{path}"
|
||||||
|
|
||||||
@@ -88,39 +67,19 @@ def call_meta_api(path, params=None):
|
|||||||
|
|
||||||
|
|
||||||
def parse_meta_datetime(value):
|
def parse_meta_datetime(value):
|
||||||
"""
|
"""Convertit une date Meta en datetime compatible SQL Server."""
|
||||||
Convertit une date Meta en format compatible MySQL.
|
|
||||||
|
|
||||||
Exemple Meta :
|
|
||||||
2026-06-23T06:37:24+0000
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Date au format 'YYYY-MM-DD HH:MM:SS' ou None.
|
|
||||||
"""
|
|
||||||
|
|
||||||
if not value:
|
if not value:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
parsed_date = datetime.strptime(value, "%Y-%m-%dT%H:%M:%S%z")
|
parsed_date = datetime.strptime(value, "%Y-%m-%dT%H:%M:%S%z")
|
||||||
return parsed_date.strftime("%Y-%m-%d %H:%M:%S")
|
utc_date = parsed_date.astimezone(timezone.utc)
|
||||||
|
|
||||||
|
return utc_date.replace(tzinfo=None)
|
||||||
|
|
||||||
|
|
||||||
def normalize_field_data(field_data):
|
def normalize_field_data(field_data):
|
||||||
"""
|
"""Convertit field_data Meta en dictionnaire simple."""
|
||||||
Transforme field_data Meta en dictionnaire simple.
|
|
||||||
|
|
||||||
Exemple d’entrée Meta :
|
|
||||||
[
|
|
||||||
{"name": "nom", "values": ["Dupont"]},
|
|
||||||
{"name": "first_name", "values": ["Jean"]}
|
|
||||||
]
|
|
||||||
|
|
||||||
Exemple de sortie :
|
|
||||||
{
|
|
||||||
"nom": "Dupont",
|
|
||||||
"first_name": "Jean"
|
|
||||||
}
|
|
||||||
"""
|
|
||||||
|
|
||||||
normalized = {}
|
normalized = {}
|
||||||
|
|
||||||
@@ -134,8 +93,19 @@ def normalize_field_data(field_data):
|
|||||||
return normalized
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
|
def get_email_from_fields(fields):
|
||||||
|
"""Récupère l'email en gérant les variantes possibles du champ Meta."""
|
||||||
|
|
||||||
|
return (
|
||||||
|
fields.get("adresse_e-mail")
|
||||||
|
or fields.get("adresse-e-mail")
|
||||||
|
or fields.get("adresse_email")
|
||||||
|
or fields.get("email")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def fetch_active_forms():
|
def fetch_active_forms():
|
||||||
"""Récupère les formulaires Lead Gen actifs de la Page Meta."""
|
"""Récupère les formulaires Lead Gen actifs autorisés."""
|
||||||
|
|
||||||
result = call_meta_api(
|
result = call_meta_api(
|
||||||
f"{META_PAGE_ID}/leadgen_forms",
|
f"{META_PAGE_ID}/leadgen_forms",
|
||||||
@@ -149,30 +119,13 @@ def fetch_active_forms():
|
|||||||
|
|
||||||
return [
|
return [
|
||||||
form for form in forms
|
form for form in forms
|
||||||
if form.get("status") == "ACTIVE" and form.get("id") in FORM_CAMPUSES
|
if form.get("status") == "ACTIVE"
|
||||||
|
and form.get("id") in FORM_CAMPUSES
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def fetch_form_questions(form_id):
|
|
||||||
"""Récupère les questions/champs d’un formulaire Meta."""
|
|
||||||
|
|
||||||
result = call_meta_api(
|
|
||||||
form_id,
|
|
||||||
{
|
|
||||||
"fields": "id,name,status,locale,questions",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
return result.get("questions", []), result
|
|
||||||
|
|
||||||
|
|
||||||
def fetch_form_leads(form_id):
|
def fetch_form_leads(form_id):
|
||||||
"""
|
"""Récupère tous les leads d'un formulaire Meta avec pagination."""
|
||||||
Récupère les leads d’un formulaire Meta.
|
|
||||||
|
|
||||||
La pagination est prise en compte afin de récupérer toutes les
|
|
||||||
pages de résultats retournées par Meta.
|
|
||||||
"""
|
|
||||||
|
|
||||||
all_leads = []
|
all_leads = []
|
||||||
|
|
||||||
@@ -185,7 +138,6 @@ def fetch_form_leads(form_id):
|
|||||||
)
|
)
|
||||||
|
|
||||||
all_leads.extend(result.get("data", []))
|
all_leads.extend(result.get("data", []))
|
||||||
|
|
||||||
next_url = result.get("paging", {}).get("next")
|
next_url = result.get("paging", {}).get("next")
|
||||||
|
|
||||||
while next_url:
|
while next_url:
|
||||||
@@ -194,103 +146,89 @@ def fetch_form_leads(form_id):
|
|||||||
|
|
||||||
result = response.json()
|
result = response.json()
|
||||||
all_leads.extend(result.get("data", []))
|
all_leads.extend(result.get("data", []))
|
||||||
|
|
||||||
next_url = result.get("paging", {}).get("next")
|
next_url = result.get("paging", {}).get("next")
|
||||||
|
|
||||||
return all_leads
|
return all_leads
|
||||||
|
|
||||||
|
|
||||||
def upsert_form(connection, form, raw_payload):
|
def upsert_leadgen_form(connection, form):
|
||||||
"""Insère ou met à jour un formulaire Meta en base."""
|
"""Insère ou met à jour un formulaire Meta dans leadgen_forms."""
|
||||||
|
|
||||||
query = """
|
update_query = """
|
||||||
INSERT INTO meta_forms (
|
UPDATE dbo.leadgen_forms
|
||||||
|
SET
|
||||||
|
page_id = ?,
|
||||||
|
name = ?,
|
||||||
|
locale = ?,
|
||||||
|
status = ?,
|
||||||
|
updated_at = SYSDATETIME()
|
||||||
|
WHERE meta_form_id = ?
|
||||||
|
"""
|
||||||
|
|
||||||
|
insert_query = """
|
||||||
|
INSERT INTO dbo.leadgen_forms (
|
||||||
meta_form_id,
|
meta_form_id,
|
||||||
page_id,
|
page_id,
|
||||||
name,
|
name,
|
||||||
status,
|
|
||||||
locale,
|
locale,
|
||||||
campus,
|
status
|
||||||
raw_payload
|
|
||||||
)
|
)
|
||||||
VALUES (%s, %s, %s, %s, %s, %s, %s)
|
VALUES (?, ?, ?, ?, ?)
|
||||||
ON DUPLICATE KEY UPDATE
|
|
||||||
name = VALUES(name),
|
|
||||||
status = VALUES(status),
|
|
||||||
locale = VALUES(locale),
|
|
||||||
campus = VALUES(campus),
|
|
||||||
raw_payload = VALUES(raw_payload),
|
|
||||||
updated_at = CURRENT_TIMESTAMP
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
values = (
|
cursor = connection.cursor()
|
||||||
form.get("id"),
|
|
||||||
META_PAGE_ID,
|
cursor.execute(
|
||||||
form.get("name"),
|
update_query,
|
||||||
form.get("status"),
|
(
|
||||||
form.get("locale"),
|
META_PAGE_ID,
|
||||||
FORM_CAMPUSES.get(form.get("id")),
|
form.get("name"),
|
||||||
json.dumps(raw_payload, ensure_ascii=False),
|
form.get("locale"),
|
||||||
|
form.get("status"),
|
||||||
|
form.get("id"),
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
cursor = connection.cursor()
|
if cursor.rowcount == 0:
|
||||||
cursor.execute(query, values)
|
cursor.execute(
|
||||||
|
insert_query,
|
||||||
|
(
|
||||||
|
form.get("id"),
|
||||||
|
META_PAGE_ID,
|
||||||
|
form.get("name"),
|
||||||
|
form.get("locale"),
|
||||||
|
form.get("status"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
connection.commit()
|
connection.commit()
|
||||||
cursor.close()
|
cursor.close()
|
||||||
|
|
||||||
|
|
||||||
def upsert_question(connection, form_id, question):
|
def upsert_lead_data(connection, form, lead):
|
||||||
"""Insère ou met à jour une question de formulaire Meta."""
|
"""Insère ou met à jour un lead Meta dans lead_data."""
|
||||||
|
|
||||||
query = """
|
|
||||||
INSERT INTO meta_form_questions (
|
|
||||||
meta_form_id,
|
|
||||||
question_id,
|
|
||||||
field_key,
|
|
||||||
label,
|
|
||||||
question_type,
|
|
||||||
question_options,
|
|
||||||
raw_payload
|
|
||||||
)
|
|
||||||
VALUES (%s, %s, %s, %s, %s, %s, %s)
|
|
||||||
ON DUPLICATE KEY UPDATE
|
|
||||||
question_id = VALUES(question_id),
|
|
||||||
label = VALUES(label),
|
|
||||||
question_type = VALUES(question_type),
|
|
||||||
question_options = VALUES(question_options),
|
|
||||||
raw_payload = VALUES(raw_payload),
|
|
||||||
updated_at = CURRENT_TIMESTAMP
|
|
||||||
"""
|
|
||||||
|
|
||||||
values = (
|
|
||||||
form_id,
|
|
||||||
question.get("id"),
|
|
||||||
question.get("key"),
|
|
||||||
question.get("label"),
|
|
||||||
question.get("type"),
|
|
||||||
json.dumps(question.get("options"), ensure_ascii=False)
|
|
||||||
if question.get("options") is not None else None,
|
|
||||||
json.dumps(question, ensure_ascii=False),
|
|
||||||
)
|
|
||||||
|
|
||||||
cursor = connection.cursor()
|
|
||||||
cursor.execute(query, values)
|
|
||||||
connection.commit()
|
|
||||||
cursor.close()
|
|
||||||
|
|
||||||
|
|
||||||
def insert_lead(connection, form, lead):
|
|
||||||
"""
|
|
||||||
Insère un lead Meta en base.
|
|
||||||
|
|
||||||
Si le lead existe déjà, il n’est pas dupliqué grâce à la clé
|
|
||||||
unique meta_lead_id. Les données existantes sont conservées.
|
|
||||||
"""
|
|
||||||
|
|
||||||
fields = normalize_field_data(lead.get("field_data", []))
|
fields = normalize_field_data(lead.get("field_data", []))
|
||||||
|
campus = FORM_CAMPUSES.get(form.get("id"))
|
||||||
|
|
||||||
query = """
|
update_query = """
|
||||||
INSERT INTO meta_leads (
|
UPDATE dbo.lead_data
|
||||||
|
SET
|
||||||
|
meta_form_id = ?,
|
||||||
|
created_time = ?,
|
||||||
|
je_suis = ?,
|
||||||
|
nom = ?,
|
||||||
|
first_name = ?,
|
||||||
|
adresse_email = ?,
|
||||||
|
phone_number = ?,
|
||||||
|
city = ?,
|
||||||
|
campus = ?,
|
||||||
|
updated_at = SYSDATETIME()
|
||||||
|
WHERE meta_lead_id = ?
|
||||||
|
"""
|
||||||
|
|
||||||
|
insert_query = """
|
||||||
|
INSERT INTO dbo.lead_data (
|
||||||
meta_lead_id,
|
meta_lead_id,
|
||||||
meta_form_id,
|
meta_form_id,
|
||||||
created_time,
|
created_time,
|
||||||
@@ -301,75 +239,71 @@ def insert_lead(connection, form, lead):
|
|||||||
phone_number,
|
phone_number,
|
||||||
city,
|
city,
|
||||||
campus,
|
campus,
|
||||||
raw_payload,
|
|
||||||
status
|
status
|
||||||
)
|
)
|
||||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'RECEIVED')
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, N'RECEIVED')
|
||||||
ON DUPLICATE KEY UPDATE
|
|
||||||
updated_at = CURRENT_TIMESTAMP
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
values = (
|
values_for_update = (
|
||||||
|
form.get("id"),
|
||||||
|
parse_meta_datetime(lead.get("created_time")),
|
||||||
|
fields.get("je_suis"),
|
||||||
|
fields.get("nom"),
|
||||||
|
fields.get("first_name"),
|
||||||
|
get_email_from_fields(fields),
|
||||||
|
fields.get("phone_number"),
|
||||||
|
fields.get("city"),
|
||||||
|
campus,
|
||||||
|
lead.get("id"),
|
||||||
|
)
|
||||||
|
|
||||||
|
values_for_insert = (
|
||||||
lead.get("id"),
|
lead.get("id"),
|
||||||
form.get("id"),
|
form.get("id"),
|
||||||
parse_meta_datetime(lead.get("created_time")),
|
parse_meta_datetime(lead.get("created_time")),
|
||||||
fields.get("je_suis"),
|
fields.get("je_suis"),
|
||||||
fields.get("nom"),
|
fields.get("nom"),
|
||||||
fields.get("first_name"),
|
fields.get("first_name"),
|
||||||
fields.get("adresse_e-mail"),
|
get_email_from_fields(fields),
|
||||||
fields.get("phone_number"),
|
fields.get("phone_number"),
|
||||||
fields.get("city"),
|
fields.get("city"),
|
||||||
FORM_CAMPUSES.get(form.get("id")),
|
campus,
|
||||||
json.dumps(lead, ensure_ascii=False),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
cursor = connection.cursor()
|
cursor = connection.cursor()
|
||||||
cursor.execute(query, values)
|
|
||||||
|
cursor.execute(update_query, values_for_update)
|
||||||
|
|
||||||
|
if cursor.rowcount == 0:
|
||||||
|
cursor.execute(insert_query, values_for_insert)
|
||||||
|
|
||||||
connection.commit()
|
connection.commit()
|
||||||
cursor.close()
|
cursor.close()
|
||||||
|
|
||||||
|
|
||||||
def synchronize_meta_to_mysql():
|
def synchronize_meta_to_sql_server():
|
||||||
"""
|
"""Synchronise les formulaires et les leads Meta vers SQL Server."""
|
||||||
Lance la synchronisation Meta vers MySQL.
|
|
||||||
|
|
||||||
Étapes :
|
|
||||||
1. Récupération des formulaires actifs.
|
|
||||||
2. Stockage des formulaires.
|
|
||||||
3. Récupération et stockage des questions.
|
|
||||||
4. Récupération et stockage des leads.
|
|
||||||
"""
|
|
||||||
|
|
||||||
connection = get_database_connection()
|
connection = get_database_connection()
|
||||||
|
|
||||||
total_forms = 0
|
total_forms = 0
|
||||||
total_questions = 0
|
|
||||||
total_leads = 0
|
total_leads = 0
|
||||||
|
|
||||||
try:
|
try:
|
||||||
forms = fetch_active_forms()
|
forms = fetch_active_forms()
|
||||||
|
|
||||||
for form in forms:
|
for form in forms:
|
||||||
form_id = form.get("id")
|
upsert_leadgen_form(connection, form)
|
||||||
|
|
||||||
questions, form_details = fetch_form_questions(form_id)
|
|
||||||
|
|
||||||
upsert_form(connection, form, form_details)
|
|
||||||
total_forms += 1
|
total_forms += 1
|
||||||
|
|
||||||
for question in questions:
|
leads = fetch_form_leads(form.get("id"))
|
||||||
upsert_question(connection, form_id, question)
|
|
||||||
total_questions += 1
|
|
||||||
|
|
||||||
leads = fetch_form_leads(form_id)
|
|
||||||
|
|
||||||
for lead in leads:
|
for lead in leads:
|
||||||
insert_lead(connection, form, lead)
|
upsert_lead_data(connection, form, lead)
|
||||||
total_leads += 1
|
total_leads += 1
|
||||||
|
|
||||||
print("Synchronisation terminée.")
|
print("Synchronisation terminée.")
|
||||||
print(f"Formulaires traités : {total_forms}")
|
print(f"Formulaires traités : {total_forms}")
|
||||||
print(f"Questions traitées : {total_questions}")
|
|
||||||
print(f"Leads traités : {total_leads}")
|
print(f"Leads traités : {total_leads}")
|
||||||
|
|
||||||
finally:
|
finally:
|
||||||
@@ -383,4 +317,7 @@ if __name__ == "__main__":
|
|||||||
if not META_PAGE_ID:
|
if not META_PAGE_ID:
|
||||||
raise RuntimeError("META_PAGE_ID est manquant dans le fichier .env.")
|
raise RuntimeError("META_PAGE_ID est manquant dans le fichier .env.")
|
||||||
|
|
||||||
synchronize_meta_to_mysql()
|
if not DB_PASSWORD:
|
||||||
|
raise RuntimeError("DB_PASSWORD est manquant dans le fichier .env.")
|
||||||
|
|
||||||
|
synchronize_meta_to_sql_server()
|
||||||
@@ -1,222 +0,0 @@
|
|||||||
-- ============================================================
|
|
||||||
-- Script : meta_lead_db.sql
|
|
||||||
-- Base : meta_lead_db
|
|
||||||
--
|
|
||||||
-- Objectif :
|
|
||||||
-- Créer une base intermédiaire permettant de stocker les
|
|
||||||
-- formulaires et leads Meta Lead Gen avant leur injection
|
|
||||||
-- dans OscarCRM.
|
|
||||||
--
|
|
||||||
-- Remarques techniques :
|
|
||||||
-- - Les IDs Meta sont stockés en VARCHAR car ils sont fournis
|
|
||||||
-- sous forme de chaînes numériques.
|
|
||||||
-- - Les données brutes Meta sont conservées en JSON pour garder
|
|
||||||
-- une traçabilité complète.
|
|
||||||
-- - Les colonnes indexées sont limitées pour rester compatibles
|
|
||||||
-- avec les anciennes limites MySQL/InnoDB en utf8mb4.
|
|
||||||
-- ============================================================
|
|
||||||
|
|
||||||
DROP DATABASE IF EXISTS meta_lead_db;
|
|
||||||
|
|
||||||
CREATE DATABASE meta_lead_db
|
|
||||||
CHARACTER SET utf8mb4
|
|
||||||
COLLATE utf8mb4_unicode_ci;
|
|
||||||
|
|
||||||
USE meta_lead_db;
|
|
||||||
|
|
||||||
-- ============================================================
|
|
||||||
-- Table : meta_forms
|
|
||||||
--
|
|
||||||
-- Rôle :
|
|
||||||
-- Stocke les formulaires Lead Gen récupérés depuis Meta.
|
|
||||||
-- Chaque formulaire peut être lié à un campus ou à une campagne.
|
|
||||||
-- ============================================================
|
|
||||||
|
|
||||||
CREATE TABLE meta_forms (
|
|
||||||
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
|
|
||||||
|
|
||||||
meta_form_id VARCHAR(64) NOT NULL,
|
|
||||||
page_id VARCHAR(64) NOT NULL,
|
|
||||||
|
|
||||||
name VARCHAR(255) NOT NULL,
|
|
||||||
status VARCHAR(50) NOT NULL,
|
|
||||||
locale VARCHAR(20) NULL,
|
|
||||||
|
|
||||||
campus VARCHAR(100) NULL,
|
|
||||||
|
|
||||||
raw_payload JSON NULL,
|
|
||||||
|
|
||||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
updated_at DATETIME NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP,
|
|
||||||
|
|
||||||
CONSTRAINT uq_meta_forms_meta_form_id
|
|
||||||
UNIQUE (meta_form_id),
|
|
||||||
|
|
||||||
INDEX idx_meta_forms_status (status),
|
|
||||||
INDEX idx_meta_forms_campus (campus)
|
|
||||||
) ENGINE=InnoDB
|
|
||||||
DEFAULT CHARSET=utf8mb4
|
|
||||||
COLLATE=utf8mb4_unicode_ci
|
|
||||||
COMMENT='Formulaires Meta Lead Gen récupérés depuis l API Meta.';
|
|
||||||
|
|
||||||
|
|
||||||
-- ============================================================
|
|
||||||
-- Table : meta_form_questions
|
|
||||||
--
|
|
||||||
-- Rôle :
|
|
||||||
-- Stocke les questions/champs associés à chaque formulaire Meta.
|
|
||||||
-- Cette table permet de connaître la structure exacte des
|
|
||||||
-- formulaires utilisés pour générer les leads.
|
|
||||||
--
|
|
||||||
-- Note :
|
|
||||||
-- field_key est limité à 100 caractères afin d éviter les erreurs
|
|
||||||
-- d index avec utf8mb4 sur certaines versions MySQL/InnoDB.
|
|
||||||
-- ============================================================
|
|
||||||
|
|
||||||
CREATE TABLE meta_form_questions (
|
|
||||||
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
|
|
||||||
|
|
||||||
meta_form_id VARCHAR(64) NOT NULL,
|
|
||||||
|
|
||||||
question_id VARCHAR(64) NULL,
|
|
||||||
field_key VARCHAR(100) NOT NULL,
|
|
||||||
label VARCHAR(255) NULL,
|
|
||||||
question_type VARCHAR(100) NULL,
|
|
||||||
|
|
||||||
question_options JSON NULL,
|
|
||||||
raw_payload JSON NULL,
|
|
||||||
|
|
||||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
updated_at DATETIME NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP,
|
|
||||||
|
|
||||||
CONSTRAINT uq_meta_form_questions_form_field
|
|
||||||
UNIQUE (meta_form_id, field_key),
|
|
||||||
|
|
||||||
CONSTRAINT fk_meta_form_questions_form
|
|
||||||
FOREIGN KEY (meta_form_id)
|
|
||||||
REFERENCES meta_forms(meta_form_id)
|
|
||||||
ON UPDATE CASCADE
|
|
||||||
ON DELETE CASCADE,
|
|
||||||
|
|
||||||
INDEX idx_meta_form_questions_form (meta_form_id)
|
|
||||||
) ENGINE=InnoDB
|
|
||||||
DEFAULT CHARSET=utf8mb4
|
|
||||||
COLLATE=utf8mb4_unicode_ci
|
|
||||||
COMMENT='Questions et champs des formulaires Meta Lead Gen.';
|
|
||||||
|
|
||||||
|
|
||||||
-- ============================================================
|
|
||||||
-- Table : meta_leads
|
|
||||||
--
|
|
||||||
-- Rôle :
|
|
||||||
-- Stocke les leads récupérés depuis les formulaires Meta.
|
|
||||||
-- Les colonnes principales reprennent les champs du formulaire
|
|
||||||
-- Meta et correspondent au format attendu pour OscarCRM.
|
|
||||||
--
|
|
||||||
-- Statuts :
|
|
||||||
-- RECEIVED : lead récupéré depuis Meta
|
|
||||||
-- READY_TO_SEND : lead prêt à être envoyé vers OscarCRM
|
|
||||||
-- SENT : lead envoyé avec succès vers OscarCRM
|
|
||||||
-- ERROR : erreur pendant le traitement ou l envoi
|
|
||||||
-- INVALID : données obligatoires manquantes ou invalides
|
|
||||||
-- ============================================================
|
|
||||||
|
|
||||||
CREATE TABLE meta_leads (
|
|
||||||
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
|
|
||||||
|
|
||||||
meta_lead_id VARCHAR(64) NOT NULL,
|
|
||||||
meta_form_id VARCHAR(64) NOT NULL,
|
|
||||||
|
|
||||||
meta_ad_id VARCHAR(64) NULL,
|
|
||||||
meta_campaign_id VARCHAR(64) NULL,
|
|
||||||
|
|
||||||
created_time DATETIME NULL,
|
|
||||||
|
|
||||||
je_suis VARCHAR(255) NULL,
|
|
||||||
nom VARCHAR(255) NULL,
|
|
||||||
first_name VARCHAR(255) NULL,
|
|
||||||
adresse_email VARCHAR(191) NULL,
|
|
||||||
phone_number VARCHAR(50) NULL,
|
|
||||||
city VARCHAR(255) NULL,
|
|
||||||
|
|
||||||
source VARCHAR(100) NOT NULL DEFAULT 'META_LEAD_GEN',
|
|
||||||
campus VARCHAR(100) NULL,
|
|
||||||
|
|
||||||
raw_payload JSON NOT NULL,
|
|
||||||
oscar_payload JSON NULL,
|
|
||||||
oscar_response JSON NULL,
|
|
||||||
|
|
||||||
status VARCHAR(50) NOT NULL DEFAULT 'RECEIVED',
|
|
||||||
|
|
||||||
error_message TEXT NULL,
|
|
||||||
synced_at DATETIME NULL,
|
|
||||||
|
|
||||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
updated_at DATETIME NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP,
|
|
||||||
|
|
||||||
CONSTRAINT uq_meta_leads_meta_lead_id
|
|
||||||
UNIQUE (meta_lead_id),
|
|
||||||
|
|
||||||
CONSTRAINT fk_meta_leads_form
|
|
||||||
FOREIGN KEY (meta_form_id)
|
|
||||||
REFERENCES meta_forms(meta_form_id)
|
|
||||||
ON UPDATE CASCADE
|
|
||||||
ON DELETE RESTRICT,
|
|
||||||
|
|
||||||
CONSTRAINT chk_meta_leads_status
|
|
||||||
CHECK (status IN ('RECEIVED', 'READY_TO_SEND', 'SENT', 'ERROR', 'INVALID')),
|
|
||||||
|
|
||||||
INDEX idx_meta_leads_status (status),
|
|
||||||
INDEX idx_meta_leads_meta_form_id (meta_form_id),
|
|
||||||
INDEX idx_meta_leads_created_time (created_time),
|
|
||||||
INDEX idx_meta_leads_adresse_email (adresse_email),
|
|
||||||
INDEX idx_meta_leads_synced_at (synced_at)
|
|
||||||
) ENGINE=InnoDB
|
|
||||||
DEFAULT CHARSET=utf8mb4
|
|
||||||
COLLATE=utf8mb4_unicode_ci
|
|
||||||
COMMENT='Leads Meta Lead Gen stockés avant injection vers OscarCRM.';
|
|
||||||
|
|
||||||
|
|
||||||
-- ============================================================
|
|
||||||
-- Table : oscar_sync_logs
|
|
||||||
--
|
|
||||||
-- Rôle :
|
|
||||||
-- Trace chaque tentative d injection d un lead vers OscarCRM.
|
|
||||||
-- Cette table permet de conserver l historique des succès,
|
|
||||||
-- erreurs et réponses API sans écraser les données du lead.
|
|
||||||
--
|
|
||||||
-- Statuts :
|
|
||||||
-- SENT : injection réussie
|
|
||||||
-- ERROR : erreur lors de l injection
|
|
||||||
-- INVALID : payload non valide pour OscarCRM
|
|
||||||
-- ============================================================
|
|
||||||
|
|
||||||
CREATE TABLE oscar_sync_logs (
|
|
||||||
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
|
|
||||||
|
|
||||||
lead_id BIGINT UNSIGNED NOT NULL,
|
|
||||||
|
|
||||||
oscar_payload JSON NOT NULL,
|
|
||||||
oscar_response JSON NULL,
|
|
||||||
|
|
||||||
status VARCHAR(50) NOT NULL,
|
|
||||||
error_message TEXT NULL,
|
|
||||||
|
|
||||||
attempted_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
|
|
||||||
CONSTRAINT fk_oscar_sync_logs_lead
|
|
||||||
FOREIGN KEY (lead_id)
|
|
||||||
REFERENCES meta_leads(id)
|
|
||||||
ON UPDATE CASCADE
|
|
||||||
ON DELETE CASCADE,
|
|
||||||
|
|
||||||
CONSTRAINT chk_oscar_sync_logs_status
|
|
||||||
CHECK (status IN ('SENT', 'ERROR', 'INVALID')),
|
|
||||||
|
|
||||||
INDEX idx_oscar_sync_logs_lead_id (lead_id),
|
|
||||||
INDEX idx_oscar_sync_logs_status (status),
|
|
||||||
INDEX idx_oscar_sync_logs_attempted_at (attempted_at)
|
|
||||||
) ENGINE=InnoDB
|
|
||||||
DEFAULT CHARSET=utf8mb4
|
|
||||||
COLLATE=utf8mb4_unicode_ci
|
|
||||||
COMMENT='Historique des tentatives d injection des leads vers OscarCRM.';
|
|
||||||
+2
-2
@@ -1,3 +1,3 @@
|
|||||||
requests
|
requests
|
||||||
mysql-connector-python
|
python-dotenv
|
||||||
python-dotenv
|
pyodbc
|
||||||
Reference in New Issue
Block a user