feat(campaigns): push leads and campaign figures into Maskan CRM
Server to server over the CRM's integration API, deliberately not through MCP. MCP is for an agent deciding to do something; leads have to reach the CRM on a schedule whether or not anyone is talking to the agent, and a lead that arrives only when someone asks for it arrives too late. Leads are re-sent for a trailing window on every run rather than tracked as new-since-last-time. Meta's lead id is the external id, so the CRM ignores one it already has — which makes a half-failed run heal itself on the next cycle with no bookkeeping. Meta returns form answers as a list under names the form's author chose, so mapping is best-effort against aliases: full_name or first/last, phone or phone_number or mobile. Unrecognised answers are kept in metadata rather than dropped, and a lead with a phone but no name still gets through — the CRM requires a first name, and losing a real enquiry to satisfy a validator would be the wrong trade. Campaign figures come from stored insights, not Meta: this runs every few minutes and re-reading Meta would spend the quota on numbers that change hourly. They go through an upsert rather than the idempotent create used for leads, because a campaign's figures change every time they are read. One care point, learnt the hard way and now commented and tested: stored metrics are already in minor units, so summing them with the raw-row normaliser reports spend a hundredfold. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
AFFAANh
@@ -323,6 +323,35 @@ async def lifespan(app: FastAPI): # pylint: disable=too-many-statements
|
||||
)
|
||||
app.state.insights_task = insights_task
|
||||
|
||||
# --- CRM sync ---
|
||||
# Server to server, not through MCP: leads have to reach the CRM on a
|
||||
# schedule whether or not anyone is talking to the agent.
|
||||
crm_task = None
|
||||
from ..campaigns.crm_sync import (
|
||||
configured_crm_client,
|
||||
crm_sync_interval_seconds,
|
||||
crm_sync_loop,
|
||||
)
|
||||
|
||||
if crm_sync_interval_seconds() > 0:
|
||||
from ..campaigns.insights_repo import InsightsRepository
|
||||
from ..campaigns.repo import CampaignRepository
|
||||
from ..meta.client import MetaClient, access_token_from_env
|
||||
|
||||
def _crm_meta_client():
|
||||
return MetaClient(access_token=access_token_from_env())
|
||||
|
||||
crm_task = asyncio.create_task(
|
||||
crm_sync_loop(
|
||||
CampaignRepository,
|
||||
InsightsRepository,
|
||||
_crm_meta_client,
|
||||
configured_crm_client,
|
||||
),
|
||||
name="campaign_crm_sync",
|
||||
)
|
||||
app.state.crm_task = crm_task
|
||||
|
||||
try:
|
||||
if mcp_initial_config is not None:
|
||||
mcp_init_task = _schedule_mcp_initialization(
|
||||
@@ -340,6 +369,8 @@ async def lifespan(app: FastAPI): # pylint: disable=too-many-statements
|
||||
guardrail_task.cancel()
|
||||
if insights_task is not None:
|
||||
insights_task.cancel()
|
||||
if crm_task is not None:
|
||||
crm_task.cancel()
|
||||
# stop order: watchers -> cron -> channels -> mcp -> runner
|
||||
try:
|
||||
await config_watcher.stop()
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Push leads and campaign figures into Maskan CRM.
|
||||
|
||||
Server to server, over the CRM's integration API — deliberately not through
|
||||
MCP. MCP is for an agent deciding to do something; this has to happen on a
|
||||
schedule whether or not anyone is talking to the agent, and a lead that
|
||||
reaches the CRM only when someone asks an agent to fetch it is a lead that
|
||||
arrives too late.
|
||||
|
||||
Two things carry the design:
|
||||
|
||||
**Idempotency.** Meta's own lead id is the external id, so re-sending a
|
||||
lead is harmless — the CRM returns the existing one. That is what lets this
|
||||
re-read a window of leads on every run without worrying about duplicates,
|
||||
which in turn is what makes a missed cycle self-healing.
|
||||
|
||||
**Campaigns upsert, leads dedupe.** A lead happened once; a campaign's
|
||||
figures change every time they are read. So leads go through the idempotent
|
||||
create and campaigns through an upsert keyed on the campaign id.
|
||||
|
||||
The CRM client is synchronous (urllib), so its calls are run in a worker
|
||||
thread rather than blocking the event loop.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
|
||||
from ..integrations.maskan_crm import (
|
||||
MaskanCRMConfigurationError,
|
||||
MaskanCRMError,
|
||||
configured_maskan_crm_client,
|
||||
)
|
||||
from ..meta.client import MetaNotConfiguredError
|
||||
from ..meta.leads import crm_payload
|
||||
from .analytics import summarise
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
INTERVAL_ENV = "MASKANX_CRM_SYNC_SECONDS"
|
||||
DEFAULT_INTERVAL_SECONDS = 300.0 # 5 minutes: leads should not wait long.
|
||||
|
||||
LEAD_WINDOW_DAYS_ENV = "MASKANX_CRM_LEAD_WINDOW_DAYS"
|
||||
# Re-read a week of leads each run. Long enough that a backend down for a
|
||||
# few days catches up by itself; short enough not to re-read a year of
|
||||
# history every five minutes.
|
||||
DEFAULT_LEAD_WINDOW_DAYS = 7
|
||||
|
||||
# Where the last successful push is recorded on the campaign, so the next
|
||||
# run can report what changed rather than what exists.
|
||||
CRM_STATE_KEY = "crm_sync"
|
||||
|
||||
|
||||
@dataclass
|
||||
class CRMSyncResult:
|
||||
leads_sent: int = 0
|
||||
campaigns_sent: int = 0
|
||||
failed: list[str] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def sent_anything(self) -> bool:
|
||||
return bool(self.leads_sent or self.campaigns_sent)
|
||||
|
||||
|
||||
def crm_sync_interval_seconds() -> float:
|
||||
"""How often to push; 0 or less disables the loop."""
|
||||
raw = os.environ.get(INTERVAL_ENV)
|
||||
if raw is None:
|
||||
return DEFAULT_INTERVAL_SECONDS
|
||||
try:
|
||||
return max(0.0, float(raw))
|
||||
except ValueError:
|
||||
logger.warning(
|
||||
"%s is not a number (%r); using %.0fs",
|
||||
INTERVAL_ENV,
|
||||
raw,
|
||||
DEFAULT_INTERVAL_SECONDS,
|
||||
)
|
||||
return DEFAULT_INTERVAL_SECONDS
|
||||
|
||||
|
||||
def lead_window_days() -> int:
|
||||
raw = os.environ.get(LEAD_WINDOW_DAYS_ENV)
|
||||
if raw is None:
|
||||
return DEFAULT_LEAD_WINDOW_DAYS
|
||||
try:
|
||||
return max(1, int(raw))
|
||||
except ValueError:
|
||||
return DEFAULT_LEAD_WINDOW_DAYS
|
||||
|
||||
|
||||
async def _in_thread(func, *args, **kwargs):
|
||||
"""Run a blocking CRM call without stalling the event loop."""
|
||||
loop = asyncio.get_running_loop()
|
||||
return await loop.run_in_executor(None, lambda: func(*args, **kwargs))
|
||||
|
||||
|
||||
async def push_campaign_leads(crm, meta, campaign) -> int:
|
||||
"""Send this campaign's recent leads to the CRM. Returns how many.
|
||||
|
||||
Every lead in the window is re-sent, not only new ones. The CRM
|
||||
deduplicates on Meta's lead id, so this costs a request and gains
|
||||
self-healing: a run that failed half-way is fixed by the next one
|
||||
without any bookkeeping of where it stopped.
|
||||
"""
|
||||
if not campaign.meta_campaign_id:
|
||||
return 0
|
||||
|
||||
since = datetime.now(timezone.utc) - timedelta(days=lead_window_days())
|
||||
leads = await meta.get_leads(
|
||||
campaign.meta_campaign_id,
|
||||
since=str(int(since.timestamp())),
|
||||
)
|
||||
|
||||
sent = 0
|
||||
for lead in leads:
|
||||
lead_id = str(lead.get("id") or "")
|
||||
if not lead_id:
|
||||
# Without Meta's id there is no stable external id, so sending
|
||||
# it would create a duplicate on every run.
|
||||
continue
|
||||
payload = crm_payload(lead, campaign_name=campaign.name)
|
||||
await _in_thread(crm.create_lead, payload, idempotency_key=f"meta-lead-{lead_id}")
|
||||
sent += 1
|
||||
return sent
|
||||
|
||||
|
||||
async def push_campaign_metrics(crm, insights_repo, campaign, *, days: int = 30) -> bool:
|
||||
"""Send this campaign's current figures to the CRM. Returns whether it did.
|
||||
|
||||
Read from stored insights rather than Meta: this runs every few
|
||||
minutes, and re-reading Meta each time would spend the API quota for
|
||||
numbers that only change hourly.
|
||||
|
||||
Note `totals_between`, not `meta.insights.total_metrics`. The two look
|
||||
interchangeable and are not: `total_metrics` normalises *raw* Meta rows
|
||||
on the way in, and stored rows are already normalised, so passing them
|
||||
through it converts spend to minor units a second time and reports it
|
||||
a hundredfold.
|
||||
"""
|
||||
until = date.today()
|
||||
since = until - timedelta(days=days)
|
||||
totals = summarise(
|
||||
await insights_repo.totals_between(since, until, campaign_id=campaign.id),
|
||||
)
|
||||
|
||||
payload = {
|
||||
"provider": "maskanx",
|
||||
"external_id": campaign.id,
|
||||
"name": campaign.name,
|
||||
"status": campaign.status,
|
||||
"objective": campaign.objective,
|
||||
"channel": (campaign.channels or ["facebook"])[0],
|
||||
"currency": campaign.budget.get("currency"),
|
||||
"daily_budget": campaign.budget.get("daily_budget"),
|
||||
"spend": totals["spend"],
|
||||
"impressions": totals["impressions"],
|
||||
"clicks": totals["clicks"],
|
||||
"leads": totals["leads"],
|
||||
"cost_per_lead": totals["cost_per_lead"],
|
||||
"metrics_from": since.isoformat(),
|
||||
"metrics_to": until.isoformat(),
|
||||
"metadata": {
|
||||
"meta_campaign_id": campaign.meta_campaign_id,
|
||||
"origin": campaign.origin,
|
||||
},
|
||||
}
|
||||
await _in_thread(crm.push_campaign, payload)
|
||||
return True
|
||||
|
||||
|
||||
async def sync_to_crm(repo, insights_repo, meta, crm) -> CRMSyncResult:
|
||||
"""Push every campaign that has reached Meta, and its recent leads."""
|
||||
result = CRMSyncResult()
|
||||
|
||||
for campaign in await repo.list_campaigns():
|
||||
if not campaign.meta_campaign_id:
|
||||
continue
|
||||
try:
|
||||
if await push_campaign_metrics(crm, insights_repo, campaign):
|
||||
result.campaigns_sent += 1
|
||||
result.leads_sent += await push_campaign_leads(crm, meta, campaign)
|
||||
except Exception as exc:
|
||||
# One campaign failing must not stop the rest reaching the CRM,
|
||||
# for the same reason it does not in guardrail enforcement.
|
||||
result.failed.append(campaign.id)
|
||||
logger.warning("CRM sync failed for campaign %s: %s", campaign.id, exc)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
async def crm_sync_loop(repo_factory, insights_factory, meta_factory, crm_factory) -> None:
|
||||
"""Push to the CRM on a schedule. Never raises."""
|
||||
interval = crm_sync_interval_seconds()
|
||||
if interval <= 0:
|
||||
logger.info("CRM sync disabled (%s=0)", INTERVAL_ENV)
|
||||
return
|
||||
|
||||
logger.info("CRM sync started, every %.0fs", interval)
|
||||
while True:
|
||||
try:
|
||||
await asyncio.sleep(interval)
|
||||
result = await sync_to_crm(
|
||||
repo_factory(), insights_factory(), meta_factory(), crm_factory(),
|
||||
)
|
||||
if result.sent_anything:
|
||||
logger.info(
|
||||
"CRM sync: %d campaigns, %d leads (%d failed)",
|
||||
result.campaigns_sent,
|
||||
result.leads_sent,
|
||||
len(result.failed),
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
logger.info("CRM sync stopped")
|
||||
raise
|
||||
except (MaskanCRMConfigurationError, MetaNotConfiguredError):
|
||||
# Not configured is a normal state for an install that does not
|
||||
# use the CRM, so this is quiet rather than a warning every
|
||||
# cycle — unlike guardrails, nothing unsafe follows from it.
|
||||
logger.debug("CRM sync skipped: CRM or Meta is not configured")
|
||||
except MaskanCRMError:
|
||||
logger.warning("CRM sync cycle failed to reach the CRM", exc_info=True)
|
||||
except Exception:
|
||||
logger.exception("CRM sync cycle failed")
|
||||
|
||||
|
||||
def configured_crm_client():
|
||||
"""Build a CRM client from the persisted environment."""
|
||||
return configured_maskan_crm_client()
|
||||
@@ -205,6 +205,17 @@ class MaskanCRMClient:
|
||||
idempotency_key=idempotency_key.strip(),
|
||||
)
|
||||
|
||||
def push_campaign(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Create or update one campaign's figures in the CRM.
|
||||
|
||||
No idempotency key, unlike `create_lead`. A lead is an event that
|
||||
happened once and must never be recorded twice; a campaign's
|
||||
figures change every time they are read, and this is called
|
||||
repeatedly as spend grows. An idempotency key here would pin the
|
||||
CRM to the first numbers it ever saw.
|
||||
"""
|
||||
return self._request("POST", "/integrations/campaigns", payload=payload)
|
||||
|
||||
|
||||
def configured_maskan_crm_client(
|
||||
envs: dict[str, str] | None = None,
|
||||
|
||||
@@ -263,6 +263,40 @@ class MetaClient:
|
||||
result = await self._get(f"/{object_id}/insights", params)
|
||||
return result.get("data") or []
|
||||
|
||||
async def get_leads(
|
||||
self,
|
||||
object_id: str,
|
||||
*,
|
||||
since: str | None = None,
|
||||
limit: int = 200,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return lead-ad submissions for an ad, ad set or campaign.
|
||||
|
||||
Each lead carries `campaign_id`, `adset_id` and `ad_id`, so a lead
|
||||
is attributable to the exact ad that produced it, and the answers
|
||||
themselves arrive in `field_data` as a list of
|
||||
`{"name": ..., "values": [...]}` — see `meta.leads` for turning
|
||||
that into a contact.
|
||||
|
||||
`since` is a Unix timestamp string, which is what Graph's
|
||||
`filtering` on `time_created` expects; it is how a caller avoids
|
||||
re-reading every lead a campaign has ever produced.
|
||||
"""
|
||||
params: dict[str, Any] = {
|
||||
"fields": "id,created_time,campaign_id,adset_id,ad_id,form_id,field_data",
|
||||
"limit": limit,
|
||||
}
|
||||
if since:
|
||||
params["filtering"] = json.dumps([
|
||||
{
|
||||
"field": "time_created",
|
||||
"operator": "GREATER_THAN",
|
||||
"value": since,
|
||||
},
|
||||
])
|
||||
result = await self._get(f"/{object_id}/leads", params)
|
||||
return result.get("data") or []
|
||||
|
||||
async def generate_previews(
|
||||
self,
|
||||
ad_account_id: str,
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Turn a Meta lead-ad submission into something a CRM can accept.
|
||||
|
||||
Meta returns answers as a list, not an object:
|
||||
|
||||
"field_data": [
|
||||
{"name": "full_name", "values": ["Asha Menon"]},
|
||||
{"name": "email", "values": ["asha@example.com"]}
|
||||
]
|
||||
|
||||
and the field names come from whatever the form's author typed. A form may
|
||||
ask for `full_name`, or `first_name` and `last_name`, or `name`; for phone
|
||||
it may be `phone_number`, `phone`, or `mobile_number`. So mapping is
|
||||
best-effort against a list of known aliases, and anything unrecognised is
|
||||
kept rather than dropped — an unmapped answer is still information the
|
||||
sales team wants, and silently discarding it would be worse than putting it
|
||||
in a metadata blob.
|
||||
|
||||
These functions do no I/O.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
# Ordered by preference: the first alias present wins.
|
||||
FULL_NAME_FIELDS = ("full_name", "name", "full name")
|
||||
FIRST_NAME_FIELDS = ("first_name", "firstname", "given_name", "first name")
|
||||
LAST_NAME_FIELDS = ("last_name", "lastname", "family_name", "surname", "last name")
|
||||
EMAIL_FIELDS = ("email", "email_address", "work_email")
|
||||
PHONE_FIELDS = ("phone_number", "phone", "mobile_number", "mobile", "contact_number")
|
||||
COMPANY_FIELDS = ("company_name", "company", "organisation", "organization")
|
||||
JOB_TITLE_FIELDS = ("job_title", "title", "role", "designation")
|
||||
|
||||
_MAPPED_FIELDS = frozenset(
|
||||
FULL_NAME_FIELDS
|
||||
+ FIRST_NAME_FIELDS
|
||||
+ LAST_NAME_FIELDS
|
||||
+ EMAIL_FIELDS
|
||||
+ PHONE_FIELDS
|
||||
+ COMPANY_FIELDS
|
||||
+ JOB_TITLE_FIELDS,
|
||||
)
|
||||
|
||||
|
||||
def field_map(lead: dict[str, Any]) -> dict[str, str]:
|
||||
"""Flatten `field_data` into `{name: first value}`, lowercased keys.
|
||||
|
||||
Meta gives every answer as a list because a checkbox question can have
|
||||
several. Only the first is taken for the mapped fields; the full list
|
||||
survives in `extra_fields`.
|
||||
"""
|
||||
flattened: dict[str, str] = {}
|
||||
for entry in lead.get("field_data") or []:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
name = str(entry.get("name") or "").strip().lower()
|
||||
values = entry.get("values")
|
||||
if not name or not isinstance(values, list) or not values:
|
||||
continue
|
||||
flattened[name] = str(values[0]).strip()
|
||||
return flattened
|
||||
|
||||
|
||||
def _first(fields: dict[str, str], names: tuple[str, ...]) -> str:
|
||||
for name in names:
|
||||
value = fields.get(name)
|
||||
if value:
|
||||
return value
|
||||
return ""
|
||||
|
||||
|
||||
def split_name(fields: dict[str, str]) -> tuple[str, str]:
|
||||
"""Return (first, last) from whichever name fields the form used.
|
||||
|
||||
A single `full_name` is split on the last space, so "Asha Menon" gives
|
||||
("Asha", "Menon") and a one-word name gives ("Asha", ""). Explicit
|
||||
first/last fields always win over splitting, because a split guesses
|
||||
and the form did not.
|
||||
"""
|
||||
first = _first(fields, FIRST_NAME_FIELDS)
|
||||
last = _first(fields, LAST_NAME_FIELDS)
|
||||
if first or last:
|
||||
return first, last
|
||||
|
||||
full = _first(fields, FULL_NAME_FIELDS)
|
||||
if not full:
|
||||
return "", ""
|
||||
parts = full.rsplit(" ", 1)
|
||||
return (parts[0], parts[1]) if len(parts) == 2 else (full, "")
|
||||
|
||||
|
||||
def crm_payload(
|
||||
lead: dict[str, Any],
|
||||
*,
|
||||
campaign_name: str | None = None,
|
||||
source: str = "Meta Lead Ads",
|
||||
) -> dict[str, Any]:
|
||||
"""Build the body for the CRM's lead ingest endpoint.
|
||||
|
||||
`external_id` is Meta's own lead id, which is what makes re-sending the
|
||||
same lead harmless: the CRM keys its deduplication on it.
|
||||
|
||||
`first_name` falls back to the email local part and then to a literal
|
||||
placeholder, because the CRM requires a non-empty first name and a lead
|
||||
with a phone number but no name is still a lead worth having — dropping
|
||||
it to satisfy a validator would lose a real enquiry.
|
||||
"""
|
||||
fields = field_map(lead)
|
||||
first, last = split_name(fields)
|
||||
email = _first(fields, EMAIL_FIELDS)
|
||||
|
||||
if not first:
|
||||
first = email.split("@")[0] if email else "Unknown"
|
||||
|
||||
extra = {
|
||||
name: value for name, value in fields.items() if name not in _MAPPED_FIELDS
|
||||
}
|
||||
|
||||
return {
|
||||
"provider": "maskanx",
|
||||
"external_id": str(lead.get("id") or ""),
|
||||
"first_name": first,
|
||||
"last_name": last,
|
||||
"email": email or None,
|
||||
"phone": _first(fields, PHONE_FIELDS) or None,
|
||||
"company_name": _first(fields, COMPANY_FIELDS) or None,
|
||||
"job_title": _first(fields, JOB_TITLE_FIELDS) or None,
|
||||
"lead_title": (
|
||||
f"{first} {last}".strip() + (f" - {campaign_name}" if campaign_name else "")
|
||||
),
|
||||
"source": source,
|
||||
"campaign": {
|
||||
"campaign_id": lead.get("campaign_id"),
|
||||
"adset_id": lead.get("adset_id"),
|
||||
"ad_id": lead.get("ad_id"),
|
||||
"form_id": lead.get("form_id"),
|
||||
"campaign_name": campaign_name,
|
||||
},
|
||||
"metadata": {
|
||||
"meta_created_time": lead.get("created_time"),
|
||||
# Answers the mapping did not recognise. Kept rather than
|
||||
# dropped: the form's author asked for them on purpose.
|
||||
"extra_fields": extra,
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,283 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Pushing leads and campaign figures into Maskan CRM.
|
||||
|
||||
The properties worth protecting: every lead in the window is re-sent so a
|
||||
half-failed run heals itself, a lead without Meta's id is never sent (it
|
||||
would duplicate on every run), campaign figures come from stored insights
|
||||
rather than Meta, and one campaign failing does not stop the rest.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from adclaw.campaigns.crm_sync import (
|
||||
DEFAULT_INTERVAL_SECONDS,
|
||||
crm_sync_interval_seconds,
|
||||
lead_window_days,
|
||||
push_campaign_leads,
|
||||
push_campaign_metrics,
|
||||
sync_to_crm,
|
||||
)
|
||||
from adclaw.campaigns.models import CampaignSpec
|
||||
|
||||
|
||||
class FakeCRM:
|
||||
def __init__(self):
|
||||
self.leads: list[tuple[dict, str]] = []
|
||||
self.campaigns: list[dict] = []
|
||||
self.fail_on_lead: str | None = None
|
||||
|
||||
def create_lead(self, payload, idempotency_key):
|
||||
if payload.get("external_id") == self.fail_on_lead:
|
||||
raise RuntimeError("CRM rejected the lead")
|
||||
self.leads.append((payload, idempotency_key))
|
||||
return {"lead_id": "crm_1", "created": True}
|
||||
|
||||
def push_campaign(self, payload):
|
||||
self.campaigns.append(payload)
|
||||
return {"campaign_id": "crm_camp_1", "created": True}
|
||||
|
||||
|
||||
class FakeMeta:
|
||||
def __init__(self, leads=None):
|
||||
self.leads = leads if leads is not None else []
|
||||
self.calls: list[dict] = []
|
||||
|
||||
async def get_leads(self, object_id, *, since=None, limit=200):
|
||||
self.calls.append({"object_id": object_id, "since": since})
|
||||
return self.leads
|
||||
|
||||
|
||||
class FakeInsightsRepo:
|
||||
"""Returns totals the way the real repository does: already normalised.
|
||||
|
||||
The real `totals_between` sums in SQL over stored rows, whose metrics
|
||||
were normalised on the way in. Returning raw Meta-shaped values here
|
||||
would let a double-conversion bug pass unnoticed.
|
||||
"""
|
||||
|
||||
def __init__(self, totals=None):
|
||||
self.totals = totals if totals is not None else {}
|
||||
|
||||
async def totals_between(self, since, until, campaign_id=None):
|
||||
return dict(self.totals)
|
||||
|
||||
|
||||
class FakeRepo:
|
||||
def __init__(self, campaigns=()):
|
||||
self.items = list(campaigns)
|
||||
|
||||
async def list_campaigns(self, company_id=None, status=None):
|
||||
return list(self.items)
|
||||
|
||||
|
||||
def _campaign(**overrides) -> CampaignSpec:
|
||||
data = {
|
||||
"id": "camp_1",
|
||||
"name": "Q3 lead gen",
|
||||
"status": "live",
|
||||
"objective": "OUTCOME_LEADS",
|
||||
"ad_account_id": "act_1",
|
||||
"meta_campaign_id": "meta_camp_1",
|
||||
"budget": {"daily_budget": 50000, "currency": "INR"},
|
||||
"channels": ["facebook"],
|
||||
}
|
||||
data.update(overrides)
|
||||
return CampaignSpec(**data)
|
||||
|
||||
|
||||
def _meta_lead(lead_id="lead_1", **overrides):
|
||||
lead = {
|
||||
"id": lead_id,
|
||||
"campaign_id": "meta_camp_1",
|
||||
"adset_id": "meta_set_1",
|
||||
"ad_id": "meta_ad_1",
|
||||
"field_data": [
|
||||
{"name": "full_name", "values": ["Asha Menon"]},
|
||||
{"name": "email", "values": ["asha@example.com"]},
|
||||
],
|
||||
}
|
||||
lead.update(overrides)
|
||||
return lead
|
||||
|
||||
|
||||
# --- configuration ---
|
||||
|
||||
|
||||
def test_the_interval_defaults_to_five_minutes(monkeypatch):
|
||||
monkeypatch.delenv("MASKANX_CRM_SYNC_SECONDS", raising=False)
|
||||
|
||||
assert crm_sync_interval_seconds() == DEFAULT_INTERVAL_SECONDS
|
||||
|
||||
|
||||
def test_a_nonsense_interval_falls_back_to_the_default(monkeypatch):
|
||||
monkeypatch.setenv("MASKANX_CRM_SYNC_SECONDS", "often")
|
||||
|
||||
assert crm_sync_interval_seconds() == DEFAULT_INTERVAL_SECONDS
|
||||
|
||||
|
||||
def test_the_lead_window_is_never_shorter_than_a_day(monkeypatch):
|
||||
monkeypatch.setenv("MASKANX_CRM_LEAD_WINDOW_DAYS", "0")
|
||||
|
||||
assert lead_window_days() == 1
|
||||
|
||||
|
||||
# --- leads ---
|
||||
|
||||
|
||||
async def test_each_lead_is_sent_keyed_on_metas_lead_id():
|
||||
"""That key is what lets the CRM ignore a lead it already has."""
|
||||
crm, meta = FakeCRM(), FakeMeta([_meta_lead("lead_1"), _meta_lead("lead_2")])
|
||||
|
||||
sent = await push_campaign_leads(crm, meta, _campaign())
|
||||
|
||||
assert sent == 2
|
||||
assert [key for _, key in crm.leads] == ["meta-lead-lead_1", "meta-lead-lead_2"]
|
||||
|
||||
|
||||
async def test_the_lead_carries_its_campaign_attribution():
|
||||
crm, meta = FakeCRM(), FakeMeta([_meta_lead()])
|
||||
|
||||
await push_campaign_leads(crm, meta, _campaign())
|
||||
|
||||
payload, _ = crm.leads[0]
|
||||
assert payload["campaign"]["ad_id"] == "meta_ad_1"
|
||||
assert payload["campaign"]["campaign_name"] == "Q3 lead gen"
|
||||
assert payload["first_name"] == "Asha"
|
||||
|
||||
|
||||
async def test_a_lead_without_an_id_is_never_sent():
|
||||
"""With no stable external id it would duplicate on every run."""
|
||||
crm, meta = FakeCRM(), FakeMeta([_meta_lead(lead_id="")])
|
||||
|
||||
assert await push_campaign_leads(crm, meta, _campaign()) == 0
|
||||
assert crm.leads == []
|
||||
|
||||
|
||||
async def test_leads_are_requested_from_a_trailing_window():
|
||||
crm, meta = FakeCRM(), FakeMeta()
|
||||
|
||||
await push_campaign_leads(crm, meta, _campaign())
|
||||
|
||||
assert meta.calls[0]["object_id"] == "meta_camp_1"
|
||||
assert meta.calls[0]["since"] is not None
|
||||
|
||||
|
||||
async def test_an_unsynced_campaign_asks_meta_for_nothing():
|
||||
crm, meta = FakeCRM(), FakeMeta()
|
||||
|
||||
assert await push_campaign_leads(crm, meta, _campaign(meta_campaign_id=None)) == 0
|
||||
assert meta.calls == []
|
||||
|
||||
|
||||
# --- campaign figures ---
|
||||
|
||||
|
||||
async def test_campaign_figures_come_from_stored_insights_not_meta():
|
||||
"""This runs every few minutes; re-reading Meta would burn the quota."""
|
||||
crm = FakeCRM()
|
||||
insights = FakeInsightsRepo(
|
||||
{"spend": 46000, "clicks": 30, "leads": 4, "impressions": 1800},
|
||||
)
|
||||
|
||||
await push_campaign_metrics(crm, insights, _campaign())
|
||||
|
||||
payload = crm.campaigns[0]
|
||||
assert payload["spend"] == 46000
|
||||
assert payload["leads"] == 4
|
||||
assert payload["cost_per_lead"] == 11500
|
||||
|
||||
|
||||
async def test_stored_spend_is_sent_as_is_not_converted_again():
|
||||
"""Stored metrics are already in minor units.
|
||||
|
||||
Passing them through the raw-row normaliser would multiply spend by a
|
||||
hundred, and the CRM would report a campaign that cost 460 rupees as
|
||||
having cost 46,000.
|
||||
"""
|
||||
crm = FakeCRM()
|
||||
|
||||
await push_campaign_metrics(
|
||||
crm, FakeInsightsRepo({"spend": 46000, "leads": 4}), _campaign(),
|
||||
)
|
||||
|
||||
assert crm.campaigns[0]["spend"] == 46000
|
||||
|
||||
|
||||
async def test_the_campaign_is_identified_by_its_maskanx_id():
|
||||
"""The CRM upserts on it, so it has to be stable across pushes."""
|
||||
crm = FakeCRM()
|
||||
|
||||
await push_campaign_metrics(crm, FakeInsightsRepo(), _campaign())
|
||||
|
||||
assert crm.campaigns[0]["external_id"] == "camp_1"
|
||||
assert crm.campaigns[0]["provider"] == "maskanx"
|
||||
|
||||
|
||||
async def test_a_campaign_with_no_insights_still_reports_zero():
|
||||
"""Absent numbers and zero numbers should look the same in the CRM."""
|
||||
crm = FakeCRM()
|
||||
|
||||
await push_campaign_metrics(crm, FakeInsightsRepo({}), _campaign())
|
||||
|
||||
assert crm.campaigns[0]["spend"] == 0
|
||||
assert crm.campaigns[0]["cost_per_lead"] is None
|
||||
|
||||
|
||||
async def test_the_budget_and_currency_are_carried_through():
|
||||
crm = FakeCRM()
|
||||
|
||||
await push_campaign_metrics(crm, FakeInsightsRepo(), _campaign())
|
||||
|
||||
assert crm.campaigns[0]["daily_budget"] == 50000
|
||||
assert crm.campaigns[0]["currency"] == "INR"
|
||||
|
||||
|
||||
# --- the sweep ---
|
||||
|
||||
|
||||
async def test_the_sweep_sends_both_figures_and_leads():
|
||||
repo = FakeRepo([_campaign()])
|
||||
crm, meta = FakeCRM(), FakeMeta([_meta_lead()])
|
||||
|
||||
result = await sync_to_crm(repo, FakeInsightsRepo(), meta, crm)
|
||||
|
||||
assert result.campaigns_sent == 1
|
||||
assert result.leads_sent == 1
|
||||
|
||||
|
||||
async def test_campaigns_that_never_reached_meta_are_skipped():
|
||||
repo = FakeRepo([_campaign(meta_campaign_id=None)])
|
||||
|
||||
result = await sync_to_crm(repo, FakeInsightsRepo(), FakeMeta(), FakeCRM())
|
||||
|
||||
assert result.campaigns_sent == 0
|
||||
|
||||
|
||||
async def test_one_campaign_failing_does_not_stop_the_rest():
|
||||
repo = FakeRepo([
|
||||
_campaign(id="c1", meta_campaign_id="meta_1"),
|
||||
_campaign(id="c2", meta_campaign_id="meta_2"),
|
||||
])
|
||||
crm = FakeCRM()
|
||||
crm.fail_on_lead = "lead_1"
|
||||
|
||||
class PerCampaignMeta(FakeMeta):
|
||||
async def get_leads(self, object_id, *, since=None, limit=200):
|
||||
# Only the first campaign has the lead the CRM will reject.
|
||||
return [_meta_lead("lead_1")] if object_id == "meta_1" else []
|
||||
|
||||
result = await sync_to_crm(repo, FakeInsightsRepo(), PerCampaignMeta(), crm)
|
||||
|
||||
assert result.failed == ["c1"]
|
||||
assert result.campaigns_sent == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", ["live", "paused", "stopped", "synced"])
|
||||
async def test_paused_and_stopped_campaigns_are_still_reported(status):
|
||||
"""They spent money; the CRM should keep showing what it bought."""
|
||||
repo = FakeRepo([_campaign(status=status)])
|
||||
crm = FakeCRM()
|
||||
|
||||
result = await sync_to_crm(repo, FakeInsightsRepo(), FakeMeta(), crm)
|
||||
|
||||
assert result.campaigns_sent == 1
|
||||
assert crm.campaigns[0]["status"] == status
|
||||
@@ -0,0 +1,159 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Mapping a Meta lead-ad submission onto a CRM lead.
|
||||
|
||||
Form field names come from whoever built the form, so the mapping is
|
||||
best-effort against aliases. The tests are mostly about the awkward cases:
|
||||
a name that arrives as one field, a lead with no name at all, and answers
|
||||
the mapping does not recognise.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from adclaw.meta.leads import crm_payload, field_map, split_name
|
||||
|
||||
|
||||
def _lead(fields, **overrides):
|
||||
lead = {
|
||||
"id": "lead_1",
|
||||
"created_time": "2026-08-01T10:00:00+0000",
|
||||
"campaign_id": "meta_camp_1",
|
||||
"adset_id": "meta_set_1",
|
||||
"ad_id": "meta_ad_1",
|
||||
"form_id": "form_1",
|
||||
"field_data": [
|
||||
{"name": name, "values": [value]} for name, value in fields.items()
|
||||
],
|
||||
}
|
||||
lead.update(overrides)
|
||||
return lead
|
||||
|
||||
|
||||
# --- flattening ---
|
||||
|
||||
|
||||
def test_field_data_is_flattened_to_a_dict():
|
||||
fields = field_map(_lead({"email": "a@example.com", "phone_number": "+91 99"}))
|
||||
|
||||
assert fields == {"email": "a@example.com", "phone_number": "+91 99"}
|
||||
|
||||
|
||||
def test_field_names_are_lowercased():
|
||||
"""Form authors capitalise inconsistently; the aliases are lowercase."""
|
||||
lead = {"field_data": [{"name": "Full_Name", "values": ["Asha"]}]}
|
||||
|
||||
assert field_map(lead) == {"full_name": "Asha"}
|
||||
|
||||
|
||||
def test_answers_with_no_values_are_skipped():
|
||||
lead = {
|
||||
"field_data": [
|
||||
{"name": "email", "values": []},
|
||||
{"name": "phone", "values": ["99"]},
|
||||
"not a dict",
|
||||
],
|
||||
}
|
||||
|
||||
assert field_map(lead) == {"phone": "99"}
|
||||
|
||||
|
||||
# --- names ---
|
||||
|
||||
|
||||
def test_explicit_first_and_last_names_win():
|
||||
first, last = split_name({"first_name": "Asha", "last_name": "Menon"})
|
||||
|
||||
assert (first, last) == ("Asha", "Menon")
|
||||
|
||||
|
||||
def test_a_full_name_is_split_on_the_last_space():
|
||||
assert split_name({"full_name": "Asha Priya Menon"}) == ("Asha Priya", "Menon")
|
||||
|
||||
|
||||
def test_a_one_word_name_has_no_surname():
|
||||
assert split_name({"full_name": "Asha"}) == ("Asha", "")
|
||||
|
||||
|
||||
def test_explicit_fields_beat_splitting_a_full_name():
|
||||
"""A split guesses where the surname starts; the form did not."""
|
||||
first, last = split_name({"full_name": "Asha Menon", "first_name": "Asha Priya"})
|
||||
|
||||
assert first == "Asha Priya"
|
||||
|
||||
|
||||
def test_no_name_fields_at_all():
|
||||
assert split_name({}) == ("", "")
|
||||
|
||||
|
||||
# --- the CRM payload ---
|
||||
|
||||
|
||||
def test_the_meta_lead_id_becomes_the_external_id():
|
||||
"""It is what makes re-sending the same lead harmless."""
|
||||
payload = crm_payload(_lead({"email": "a@example.com"}))
|
||||
|
||||
assert payload["external_id"] == "lead_1"
|
||||
assert payload["provider"] == "maskanx"
|
||||
|
||||
|
||||
def test_the_ad_that_produced_the_lead_is_carried_through():
|
||||
payload = crm_payload(_lead({"email": "a@example.com"}), campaign_name="Q3")
|
||||
|
||||
assert payload["campaign"] == {
|
||||
"campaign_id": "meta_camp_1",
|
||||
"adset_id": "meta_set_1",
|
||||
"ad_id": "meta_ad_1",
|
||||
"form_id": "form_1",
|
||||
"campaign_name": "Q3",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"alias", ["phone_number", "phone", "mobile_number", "mobile", "contact_number"],
|
||||
)
|
||||
def test_phone_is_found_under_any_known_alias(alias):
|
||||
payload = crm_payload(_lead({alias: "+91 99"}))
|
||||
|
||||
assert payload["phone"] == "+91 99"
|
||||
|
||||
|
||||
def test_a_lead_with_only_a_phone_number_still_gets_a_name():
|
||||
"""The CRM requires one, and a nameless enquiry is still an enquiry."""
|
||||
payload = crm_payload(_lead({"phone_number": "+91 99"}))
|
||||
|
||||
assert payload["first_name"] == "Unknown"
|
||||
assert payload["phone"] == "+91 99"
|
||||
|
||||
|
||||
def test_the_email_local_part_stands_in_for_a_missing_name():
|
||||
payload = crm_payload(_lead({"email": "asha.menon@example.com"}))
|
||||
|
||||
assert payload["first_name"] == "asha.menon"
|
||||
|
||||
|
||||
def test_unrecognised_answers_are_kept_rather_than_dropped():
|
||||
"""The form's author asked for them on purpose."""
|
||||
payload = crm_payload(
|
||||
_lead({"full_name": "Asha Menon", "budget_range": "50-70 lakh"}),
|
||||
)
|
||||
|
||||
assert payload["metadata"]["extra_fields"] == {"budget_range": "50-70 lakh"}
|
||||
|
||||
|
||||
def test_mapped_answers_are_not_repeated_in_extra_fields():
|
||||
payload = crm_payload(_lead({"email": "a@example.com", "full_name": "Asha Menon"}))
|
||||
|
||||
assert payload["metadata"]["extra_fields"] == {}
|
||||
|
||||
|
||||
def test_the_lead_title_names_the_campaign():
|
||||
payload = crm_payload(_lead({"full_name": "Asha Menon"}), campaign_name="Q3 leads")
|
||||
|
||||
assert payload["lead_title"] == "Asha Menon - Q3 leads"
|
||||
|
||||
|
||||
def test_absent_optional_fields_are_none_not_empty_strings():
|
||||
"""The CRM validates email; "" is not a valid address, None is absent."""
|
||||
payload = crm_payload(_lead({"full_name": "Asha Menon"}))
|
||||
|
||||
assert payload["email"] is None
|
||||
assert payload["phone"] is None
|
||||
assert payload["company_name"] is None
|
||||
Reference in New Issue
Block a user