SaidSoighiri94
b5fb954ab6
- main.py : recuperation formulaires, questions et leads via Meta Graph API - meta_lead_db.sql : schema des tables MySQL - gestion anti-doublons et conservation des payloads bruts
386 lines
9.8 KiB
Python
386 lines
9.8 KiB
Python
"""
|
||
Synchronisation Meta Lead Gen vers MySQL local (V1).
|
||
|
||
Récupère les formulaires actifs, leurs questions et leurs leads via l'API
|
||
Graph Meta, puis les stocke dans la base locale sans créer de doublons.
|
||
L'injection vers OscarCRM n'est pas gérée à ce stade.
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
from datetime import datetime
|
||
|
||
import mysql.connector
|
||
import requests
|
||
from dotenv import load_dotenv
|
||
|
||
|
||
# ============================================================
|
||
# Chargement de la configuration
|
||
# ============================================================
|
||
# Les informations sensibles sont lues depuis le fichier .env.
|
||
# Cela évite de stocker le token Meta ou le mot de passe MySQL
|
||
# directement dans le code source.
|
||
# ============================================================
|
||
|
||
load_dotenv()
|
||
|
||
META_ACCESS_TOKEN = os.getenv("META_ACCESS_TOKEN")
|
||
META_API_VERSION = os.getenv("META_API_VERSION", "v25.0")
|
||
META_PAGE_ID = os.getenv("META_PAGE_ID")
|
||
|
||
DB_CONFIG = {
|
||
"host": os.getenv("DB_HOST", "localhost"),
|
||
"port": int(os.getenv("DB_PORT", "3306")),
|
||
"database": os.getenv("DB_NAME", "meta_lead_db"),
|
||
"user": os.getenv("DB_USER", "root"),
|
||
"password": os.getenv("DB_PASSWORD", ""),
|
||
"charset": "utf8mb4",
|
||
"use_unicode": True,
|
||
}
|
||
|
||
|
||
# ============================================================
|
||
# Formulaires autorisés pour le test
|
||
# ============================================================
|
||
# On limite volontairement la synchronisation aux formulaires
|
||
# actifs connus afin d’éviter de traiter des formulaires archivés.
|
||
# ============================================================
|
||
|
||
FORM_CAMPUSES = {
|
||
"1016111097572602": "Marseille",
|
||
"1032481262797041": "Nantes",
|
||
"941574848893640": "Cergy",
|
||
"1012012785055468": "Cergy",
|
||
}
|
||
|
||
|
||
def get_database_connection():
|
||
"""Ouvre une connexion MySQL locale."""
|
||
|
||
return mysql.connector.connect(**DB_CONFIG)
|
||
|
||
|
||
def call_meta_api(path, params=None):
|
||
"""
|
||
Exécute un appel GET vers l’API Graph Meta.
|
||
|
||
Args:
|
||
path: Chemin Graph API, par exemple 'PAGE_ID/leadgen_forms'.
|
||
params: Paramètres complémentaires de la requête.
|
||
|
||
Returns:
|
||
Réponse JSON retournée par Meta.
|
||
|
||
Raises:
|
||
requests.HTTPError: si Meta retourne une erreur HTTP.
|
||
"""
|
||
|
||
url = f"https://graph.facebook.com/{META_API_VERSION}/{path}"
|
||
|
||
request_params = params.copy() if params else {}
|
||
request_params["access_token"] = META_ACCESS_TOKEN
|
||
|
||
response = requests.get(url, params=request_params, timeout=30)
|
||
response.raise_for_status()
|
||
|
||
return response.json()
|
||
|
||
|
||
def parse_meta_datetime(value):
|
||
"""
|
||
Convertit une date Meta en format compatible MySQL.
|
||
|
||
Exemple Meta :
|
||
2026-06-23T06:37:24+0000
|
||
|
||
Returns:
|
||
Date au format 'YYYY-MM-DD HH:MM:SS' ou None.
|
||
"""
|
||
|
||
if not value:
|
||
return None
|
||
|
||
parsed_date = datetime.strptime(value, "%Y-%m-%dT%H:%M:%S%z")
|
||
return parsed_date.strftime("%Y-%m-%d %H:%M:%S")
|
||
|
||
|
||
def normalize_field_data(field_data):
|
||
"""
|
||
Transforme field_data Meta en dictionnaire simple.
|
||
|
||
Exemple d’entrée Meta :
|
||
[
|
||
{"name": "nom", "values": ["Dupont"]},
|
||
{"name": "first_name", "values": ["Jean"]}
|
||
]
|
||
|
||
Exemple de sortie :
|
||
{
|
||
"nom": "Dupont",
|
||
"first_name": "Jean"
|
||
}
|
||
"""
|
||
|
||
normalized = {}
|
||
|
||
for field in field_data or []:
|
||
name = field.get("name")
|
||
values = field.get("values", [])
|
||
|
||
if name:
|
||
normalized[name] = values[0] if values else None
|
||
|
||
return normalized
|
||
|
||
|
||
def fetch_active_forms():
|
||
"""Récupère les formulaires Lead Gen actifs de la Page Meta."""
|
||
|
||
result = call_meta_api(
|
||
f"{META_PAGE_ID}/leadgen_forms",
|
||
{
|
||
"fields": "id,name,status,locale",
|
||
"limit": 100,
|
||
},
|
||
)
|
||
|
||
forms = result.get("data", [])
|
||
|
||
return [
|
||
form for form in forms
|
||
if form.get("status") == "ACTIVE" and form.get("id") in FORM_CAMPUSES
|
||
]
|
||
|
||
|
||
def fetch_form_questions(form_id):
|
||
"""Récupère les questions/champs d’un formulaire Meta."""
|
||
|
||
result = call_meta_api(
|
||
form_id,
|
||
{
|
||
"fields": "id,name,status,locale,questions",
|
||
},
|
||
)
|
||
|
||
return result.get("questions", []), result
|
||
|
||
|
||
def fetch_form_leads(form_id):
|
||
"""
|
||
Récupère les leads d’un formulaire Meta.
|
||
|
||
La pagination est prise en compte afin de récupérer toutes les
|
||
pages de résultats retournées par Meta.
|
||
"""
|
||
|
||
all_leads = []
|
||
|
||
result = call_meta_api(
|
||
f"{form_id}/leads",
|
||
{
|
||
"fields": "id,created_time,field_data",
|
||
"limit": 100,
|
||
},
|
||
)
|
||
|
||
all_leads.extend(result.get("data", []))
|
||
|
||
next_url = result.get("paging", {}).get("next")
|
||
|
||
while next_url:
|
||
response = requests.get(next_url, timeout=30)
|
||
response.raise_for_status()
|
||
|
||
result = response.json()
|
||
all_leads.extend(result.get("data", []))
|
||
|
||
next_url = result.get("paging", {}).get("next")
|
||
|
||
return all_leads
|
||
|
||
|
||
def upsert_form(connection, form, raw_payload):
|
||
"""Insère ou met à jour un formulaire Meta en base."""
|
||
|
||
query = """
|
||
INSERT INTO meta_forms (
|
||
meta_form_id,
|
||
page_id,
|
||
name,
|
||
status,
|
||
locale,
|
||
campus,
|
||
raw_payload
|
||
)
|
||
VALUES (%s, %s, %s, %s, %s, %s, %s)
|
||
ON DUPLICATE KEY UPDATE
|
||
name = VALUES(name),
|
||
status = VALUES(status),
|
||
locale = VALUES(locale),
|
||
campus = VALUES(campus),
|
||
raw_payload = VALUES(raw_payload),
|
||
updated_at = CURRENT_TIMESTAMP
|
||
"""
|
||
|
||
values = (
|
||
form.get("id"),
|
||
META_PAGE_ID,
|
||
form.get("name"),
|
||
form.get("status"),
|
||
form.get("locale"),
|
||
FORM_CAMPUSES.get(form.get("id")),
|
||
json.dumps(raw_payload, ensure_ascii=False),
|
||
)
|
||
|
||
cursor = connection.cursor()
|
||
cursor.execute(query, values)
|
||
connection.commit()
|
||
cursor.close()
|
||
|
||
|
||
def upsert_question(connection, form_id, question):
|
||
"""Insère ou met à jour une question de formulaire Meta."""
|
||
|
||
query = """
|
||
INSERT INTO meta_form_questions (
|
||
meta_form_id,
|
||
question_id,
|
||
field_key,
|
||
label,
|
||
question_type,
|
||
question_options,
|
||
raw_payload
|
||
)
|
||
VALUES (%s, %s, %s, %s, %s, %s, %s)
|
||
ON DUPLICATE KEY UPDATE
|
||
question_id = VALUES(question_id),
|
||
label = VALUES(label),
|
||
question_type = VALUES(question_type),
|
||
question_options = VALUES(question_options),
|
||
raw_payload = VALUES(raw_payload),
|
||
updated_at = CURRENT_TIMESTAMP
|
||
"""
|
||
|
||
values = (
|
||
form_id,
|
||
question.get("id"),
|
||
question.get("key"),
|
||
question.get("label"),
|
||
question.get("type"),
|
||
json.dumps(question.get("options"), ensure_ascii=False)
|
||
if question.get("options") is not None else None,
|
||
json.dumps(question, ensure_ascii=False),
|
||
)
|
||
|
||
cursor = connection.cursor()
|
||
cursor.execute(query, values)
|
||
connection.commit()
|
||
cursor.close()
|
||
|
||
|
||
def insert_lead(connection, form, lead):
|
||
"""
|
||
Insère un lead Meta en base.
|
||
|
||
Si le lead existe déjà, il n’est pas dupliqué grâce à la clé
|
||
unique meta_lead_id. Les données existantes sont conservées.
|
||
"""
|
||
|
||
fields = normalize_field_data(lead.get("field_data", []))
|
||
|
||
query = """
|
||
INSERT INTO meta_leads (
|
||
meta_lead_id,
|
||
meta_form_id,
|
||
created_time,
|
||
je_suis,
|
||
nom,
|
||
first_name,
|
||
adresse_email,
|
||
phone_number,
|
||
city,
|
||
campus,
|
||
raw_payload,
|
||
status
|
||
)
|
||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'RECEIVED')
|
||
ON DUPLICATE KEY UPDATE
|
||
updated_at = CURRENT_TIMESTAMP
|
||
"""
|
||
|
||
values = (
|
||
lead.get("id"),
|
||
form.get("id"),
|
||
parse_meta_datetime(lead.get("created_time")),
|
||
fields.get("je_suis"),
|
||
fields.get("nom"),
|
||
fields.get("first_name"),
|
||
fields.get("adresse_e-mail"),
|
||
fields.get("phone_number"),
|
||
fields.get("city"),
|
||
FORM_CAMPUSES.get(form.get("id")),
|
||
json.dumps(lead, ensure_ascii=False),
|
||
)
|
||
|
||
cursor = connection.cursor()
|
||
cursor.execute(query, values)
|
||
connection.commit()
|
||
cursor.close()
|
||
|
||
|
||
def synchronize_meta_to_mysql():
|
||
"""
|
||
Lance la synchronisation Meta vers MySQL.
|
||
|
||
Étapes :
|
||
1. Récupération des formulaires actifs.
|
||
2. Stockage des formulaires.
|
||
3. Récupération et stockage des questions.
|
||
4. Récupération et stockage des leads.
|
||
"""
|
||
|
||
connection = get_database_connection()
|
||
|
||
total_forms = 0
|
||
total_questions = 0
|
||
total_leads = 0
|
||
|
||
try:
|
||
forms = fetch_active_forms()
|
||
|
||
for form in forms:
|
||
form_id = form.get("id")
|
||
|
||
questions, form_details = fetch_form_questions(form_id)
|
||
|
||
upsert_form(connection, form, form_details)
|
||
total_forms += 1
|
||
|
||
for question in questions:
|
||
upsert_question(connection, form_id, question)
|
||
total_questions += 1
|
||
|
||
leads = fetch_form_leads(form_id)
|
||
|
||
for lead in leads:
|
||
insert_lead(connection, form, lead)
|
||
total_leads += 1
|
||
|
||
print("Synchronisation terminée.")
|
||
print(f"Formulaires traités : {total_forms}")
|
||
print(f"Questions traitées : {total_questions}")
|
||
print(f"Leads traités : {total_leads}")
|
||
|
||
finally:
|
||
connection.close()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
if not META_ACCESS_TOKEN:
|
||
raise RuntimeError("META_ACCESS_TOKEN est manquant dans le fichier .env.")
|
||
|
||
if not META_PAGE_ID:
|
||
raise RuntimeError("META_PAGE_ID est manquant dans le fichier .env.")
|
||
|
||
synchronize_meta_to_mysql() |