V1 : synchronisation Meta Lead Gen vers MySQL local
- main.py : recuperation formulaires, questions et leads via Meta Graph API - meta_lead_db.sql : schema des tables MySQL - gestion anti-doublons et conservation des payloads bruts
This commit is contained in:
SaidSoighiri94
@@ -0,0 +1,386 @@
|
||||
"""
|
||||
Synchronisation Meta Lead Gen vers MySQL local (V1).
|
||||
|
||||
Récupère les formulaires actifs, leurs questions et leurs leads via l'API
|
||||
Graph Meta, puis les stocke dans la base locale sans créer de doublons.
|
||||
L'injection vers OscarCRM n'est pas gérée à ce stade.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
import mysql.connector
|
||||
import requests
|
||||
from dotenv import load_dotenv
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Chargement de la configuration
|
||||
# ============================================================
|
||||
# Les informations sensibles sont lues depuis le fichier .env.
|
||||
# Cela évite de stocker le token Meta ou le mot de passe MySQL
|
||||
# directement dans le code source.
|
||||
# ============================================================
|
||||
|
||||
load_dotenv()
|
||||
|
||||
META_ACCESS_TOKEN = os.getenv("META_ACCESS_TOKEN")
|
||||
META_API_VERSION = os.getenv("META_API_VERSION", "v25.0")
|
||||
META_PAGE_ID = os.getenv("META_PAGE_ID")
|
||||
|
||||
DB_CONFIG = {
|
||||
"host": os.getenv("DB_HOST", "localhost"),
|
||||
"port": int(os.getenv("DB_PORT", "3306")),
|
||||
"database": os.getenv("DB_NAME", "meta_lead_db"),
|
||||
"user": os.getenv("DB_USER", "root"),
|
||||
"password": os.getenv("DB_PASSWORD", ""),
|
||||
"charset": "utf8mb4",
|
||||
"use_unicode": True,
|
||||
}
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Formulaires autorisés pour le test
|
||||
# ============================================================
|
||||
# On limite volontairement la synchronisation aux formulaires
|
||||
# actifs connus afin d’éviter de traiter des formulaires archivés.
|
||||
# ============================================================
|
||||
|
||||
FORM_CAMPUSES = {
|
||||
"1016111097572602": "Marseille",
|
||||
"1032481262797041": "Nantes",
|
||||
"941574848893640": "Cergy",
|
||||
"1012012785055468": "Cergy",
|
||||
}
|
||||
|
||||
|
||||
def get_database_connection():
|
||||
"""Ouvre une connexion MySQL locale."""
|
||||
|
||||
return mysql.connector.connect(**DB_CONFIG)
|
||||
|
||||
|
||||
def call_meta_api(path, params=None):
|
||||
"""
|
||||
Exécute un appel GET vers l’API Graph Meta.
|
||||
|
||||
Args:
|
||||
path: Chemin Graph API, par exemple 'PAGE_ID/leadgen_forms'.
|
||||
params: Paramètres complémentaires de la requête.
|
||||
|
||||
Returns:
|
||||
Réponse JSON retournée par Meta.
|
||||
|
||||
Raises:
|
||||
requests.HTTPError: si Meta retourne une erreur HTTP.
|
||||
"""
|
||||
|
||||
url = f"https://graph.facebook.com/{META_API_VERSION}/{path}"
|
||||
|
||||
request_params = params.copy() if params else {}
|
||||
request_params["access_token"] = META_ACCESS_TOKEN
|
||||
|
||||
response = requests.get(url, params=request_params, timeout=30)
|
||||
response.raise_for_status()
|
||||
|
||||
return response.json()
|
||||
|
||||
|
||||
def parse_meta_datetime(value):
|
||||
"""
|
||||
Convertit une date Meta en format compatible MySQL.
|
||||
|
||||
Exemple Meta :
|
||||
2026-06-23T06:37:24+0000
|
||||
|
||||
Returns:
|
||||
Date au format 'YYYY-MM-DD HH:MM:SS' ou None.
|
||||
"""
|
||||
|
||||
if not value:
|
||||
return None
|
||||
|
||||
parsed_date = datetime.strptime(value, "%Y-%m-%dT%H:%M:%S%z")
|
||||
return parsed_date.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
def normalize_field_data(field_data):
|
||||
"""
|
||||
Transforme field_data Meta en dictionnaire simple.
|
||||
|
||||
Exemple d’entrée Meta :
|
||||
[
|
||||
{"name": "nom", "values": ["Dupont"]},
|
||||
{"name": "first_name", "values": ["Jean"]}
|
||||
]
|
||||
|
||||
Exemple de sortie :
|
||||
{
|
||||
"nom": "Dupont",
|
||||
"first_name": "Jean"
|
||||
}
|
||||
"""
|
||||
|
||||
normalized = {}
|
||||
|
||||
for field in field_data or []:
|
||||
name = field.get("name")
|
||||
values = field.get("values", [])
|
||||
|
||||
if name:
|
||||
normalized[name] = values[0] if values else None
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def fetch_active_forms():
|
||||
"""Récupère les formulaires Lead Gen actifs de la Page Meta."""
|
||||
|
||||
result = call_meta_api(
|
||||
f"{META_PAGE_ID}/leadgen_forms",
|
||||
{
|
||||
"fields": "id,name,status,locale",
|
||||
"limit": 100,
|
||||
},
|
||||
)
|
||||
|
||||
forms = result.get("data", [])
|
||||
|
||||
return [
|
||||
form for form in forms
|
||||
if form.get("status") == "ACTIVE" and form.get("id") in FORM_CAMPUSES
|
||||
]
|
||||
|
||||
|
||||
def fetch_form_questions(form_id):
|
||||
"""Récupère les questions/champs d’un formulaire Meta."""
|
||||
|
||||
result = call_meta_api(
|
||||
form_id,
|
||||
{
|
||||
"fields": "id,name,status,locale,questions",
|
||||
},
|
||||
)
|
||||
|
||||
return result.get("questions", []), result
|
||||
|
||||
|
||||
def fetch_form_leads(form_id):
|
||||
"""
|
||||
Récupère les leads d’un formulaire Meta.
|
||||
|
||||
La pagination est prise en compte afin de récupérer toutes les
|
||||
pages de résultats retournées par Meta.
|
||||
"""
|
||||
|
||||
all_leads = []
|
||||
|
||||
result = call_meta_api(
|
||||
f"{form_id}/leads",
|
||||
{
|
||||
"fields": "id,created_time,field_data",
|
||||
"limit": 100,
|
||||
},
|
||||
)
|
||||
|
||||
all_leads.extend(result.get("data", []))
|
||||
|
||||
next_url = result.get("paging", {}).get("next")
|
||||
|
||||
while next_url:
|
||||
response = requests.get(next_url, timeout=30)
|
||||
response.raise_for_status()
|
||||
|
||||
result = response.json()
|
||||
all_leads.extend(result.get("data", []))
|
||||
|
||||
next_url = result.get("paging", {}).get("next")
|
||||
|
||||
return all_leads
|
||||
|
||||
|
||||
def upsert_form(connection, form, raw_payload):
|
||||
"""Insère ou met à jour un formulaire Meta en base."""
|
||||
|
||||
query = """
|
||||
INSERT INTO meta_forms (
|
||||
meta_form_id,
|
||||
page_id,
|
||||
name,
|
||||
status,
|
||||
locale,
|
||||
campus,
|
||||
raw_payload
|
||||
)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
name = VALUES(name),
|
||||
status = VALUES(status),
|
||||
locale = VALUES(locale),
|
||||
campus = VALUES(campus),
|
||||
raw_payload = VALUES(raw_payload),
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
"""
|
||||
|
||||
values = (
|
||||
form.get("id"),
|
||||
META_PAGE_ID,
|
||||
form.get("name"),
|
||||
form.get("status"),
|
||||
form.get("locale"),
|
||||
FORM_CAMPUSES.get(form.get("id")),
|
||||
json.dumps(raw_payload, ensure_ascii=False),
|
||||
)
|
||||
|
||||
cursor = connection.cursor()
|
||||
cursor.execute(query, values)
|
||||
connection.commit()
|
||||
cursor.close()
|
||||
|
||||
|
||||
def upsert_question(connection, form_id, question):
|
||||
"""Insère ou met à jour une question de formulaire Meta."""
|
||||
|
||||
query = """
|
||||
INSERT INTO meta_form_questions (
|
||||
meta_form_id,
|
||||
question_id,
|
||||
field_key,
|
||||
label,
|
||||
question_type,
|
||||
question_options,
|
||||
raw_payload
|
||||
)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
question_id = VALUES(question_id),
|
||||
label = VALUES(label),
|
||||
question_type = VALUES(question_type),
|
||||
question_options = VALUES(question_options),
|
||||
raw_payload = VALUES(raw_payload),
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
"""
|
||||
|
||||
values = (
|
||||
form_id,
|
||||
question.get("id"),
|
||||
question.get("key"),
|
||||
question.get("label"),
|
||||
question.get("type"),
|
||||
json.dumps(question.get("options"), ensure_ascii=False)
|
||||
if question.get("options") is not None else None,
|
||||
json.dumps(question, ensure_ascii=False),
|
||||
)
|
||||
|
||||
cursor = connection.cursor()
|
||||
cursor.execute(query, values)
|
||||
connection.commit()
|
||||
cursor.close()
|
||||
|
||||
|
||||
def insert_lead(connection, form, lead):
|
||||
"""
|
||||
Insère un lead Meta en base.
|
||||
|
||||
Si le lead existe déjà, il n’est pas dupliqué grâce à la clé
|
||||
unique meta_lead_id. Les données existantes sont conservées.
|
||||
"""
|
||||
|
||||
fields = normalize_field_data(lead.get("field_data", []))
|
||||
|
||||
query = """
|
||||
INSERT INTO meta_leads (
|
||||
meta_lead_id,
|
||||
meta_form_id,
|
||||
created_time,
|
||||
je_suis,
|
||||
nom,
|
||||
first_name,
|
||||
adresse_email,
|
||||
phone_number,
|
||||
city,
|
||||
campus,
|
||||
raw_payload,
|
||||
status
|
||||
)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'RECEIVED')
|
||||
ON DUPLICATE KEY UPDATE
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
"""
|
||||
|
||||
values = (
|
||||
lead.get("id"),
|
||||
form.get("id"),
|
||||
parse_meta_datetime(lead.get("created_time")),
|
||||
fields.get("je_suis"),
|
||||
fields.get("nom"),
|
||||
fields.get("first_name"),
|
||||
fields.get("adresse_e-mail"),
|
||||
fields.get("phone_number"),
|
||||
fields.get("city"),
|
||||
FORM_CAMPUSES.get(form.get("id")),
|
||||
json.dumps(lead, ensure_ascii=False),
|
||||
)
|
||||
|
||||
cursor = connection.cursor()
|
||||
cursor.execute(query, values)
|
||||
connection.commit()
|
||||
cursor.close()
|
||||
|
||||
|
||||
def synchronize_meta_to_mysql():
|
||||
"""
|
||||
Lance la synchronisation Meta vers MySQL.
|
||||
|
||||
Étapes :
|
||||
1. Récupération des formulaires actifs.
|
||||
2. Stockage des formulaires.
|
||||
3. Récupération et stockage des questions.
|
||||
4. Récupération et stockage des leads.
|
||||
"""
|
||||
|
||||
connection = get_database_connection()
|
||||
|
||||
total_forms = 0
|
||||
total_questions = 0
|
||||
total_leads = 0
|
||||
|
||||
try:
|
||||
forms = fetch_active_forms()
|
||||
|
||||
for form in forms:
|
||||
form_id = form.get("id")
|
||||
|
||||
questions, form_details = fetch_form_questions(form_id)
|
||||
|
||||
upsert_form(connection, form, form_details)
|
||||
total_forms += 1
|
||||
|
||||
for question in questions:
|
||||
upsert_question(connection, form_id, question)
|
||||
total_questions += 1
|
||||
|
||||
leads = fetch_form_leads(form_id)
|
||||
|
||||
for lead in leads:
|
||||
insert_lead(connection, form, lead)
|
||||
total_leads += 1
|
||||
|
||||
print("Synchronisation terminée.")
|
||||
print(f"Formulaires traités : {total_forms}")
|
||||
print(f"Questions traitées : {total_questions}")
|
||||
print(f"Leads traités : {total_leads}")
|
||||
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if not META_ACCESS_TOKEN:
|
||||
raise RuntimeError("META_ACCESS_TOKEN est manquant dans le fichier .env.")
|
||||
|
||||
if not META_PAGE_ID:
|
||||
raise RuntimeError("META_PAGE_ID est manquant dans le fichier .env.")
|
||||
|
||||
synchronize_meta_to_mysql()
|
||||
Reference in New Issue
Block a user