commit 19e1e84fb772e2045fa6c44d6925dae688c5e5cd Author: AFFAANh Date: Sat Aug 1 10:28:22 2026 +0530 Initial commit: MaskanX backend Independent FastAPI backend for the MaskanX agentic growth platform. Includes the agent runtime, MCP client integrations (Meta Ads, LinkedIn, HubSpot, Tavily, Exa, xAI, Citedy, image generation), PostgreSQL storage for chats and cron jobs, provider and secret management, and the CLI. Co-Authored-By: Claude Opus 5 diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..91a9f31 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,52 @@ +# Git and IDE +.git +.gitignore +.idea +.vscode +*.md +!README.md + +# Python dev and cache +__pycache__ +*.py[cod] +*$py.class +.venv +venv +uv.lock +.pytest_cache +.coverage +htmlcov +.tox +.mypy_cache +.ruff_cache + +# Tests (not needed in runtime image) +tests +test.py +*_test.py +pytest.ini +.pre-commit-config.yaml +.flake8 +.eslintrc +.stylelintrc + +# Node command-runner dependencies +node_modules +**/node_modules +**/dist +**/.vite + +# Example and local config (mount at runtime instead) +example +config.json +jobs.json +sessions_mount_dir + +# Misc +.env +.env.* +!.env.example +.DS_Store +*.log +logs +cookbook diff --git a/.env.development.example b/.env.development.example new file mode 100644 index 0000000..ee70214 --- /dev/null +++ b/.env.development.example @@ -0,0 +1,11 @@ +MASKANX_ENV=development +MASKANX_STORAGE_BACKEND=postgres +MASKANX_PORT=8088 +MASKANX_CORS_ORIGINS=https://app-dev.example.com +DB_HOST=development-postgres.example.internal +DB_PORT=5432 +DB_NAME=campaign_dev +DB_USER=maskanx_dev +DB_PASSWORD=replace-me +DB_SSLMODE=require +LOG_LEVEL=INFO diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..1ab9f8d --- /dev/null +++ b/.env.example @@ -0,0 +1,65 @@ +# MaskanX Environment Variables +# Copy this file to .env and fill in your values: +# cp .env.example .env + +# Citedy API key — get a free key at https://www.citedy.com/developer +# This unlocks 52 SEO/marketing MCP tools and 6 pre-installed skills +CITEDY_API_KEY= + +# Telegram bot token — create a bot via https://t.me/BotFather +TELEGRAM_BOT_TOKEN= + +# Web UI port (default: 8088) +# MASKANX_PORT=8088 + +# Browser origins allowed to call this independent backend. +MASKANX_CORS_ORIGINS=http://127.0.0.1:5173,http://localhost:5173 + +# Enabled messaging channels (default: all) +# MASKANX_ENABLED_CHANNELS=discord,dingtalk,feishu,qq,console,telegram + +# Optional: Tavily search API key (https://tavily.com) +# TAVILY_API_KEY= + +# Optional: GitHub token to avoid rate limits when fetching skills +# GITHUB_TOKEN= + +# Optional: Exa search API key (https://exa.ai) +# EXA_API_KEY= + +# Optional: XAI (Grok) API key for xai_search MCP +# XAI_API_KEY= + +# Logging level: DEBUG, INFO, WARNING, ERROR +# LOG_LEVEL=INFO + +# PostgreSQL Database Configuration +# Local default: +# DB_HOST=localhost +# Docker Compose default: +# DB_HOST=postgres +# Production: +# set these to your managed Postgres credentials, or set DATABASE_URL. +MASKANX_STORAGE_BACKEND=postgres +DB_HOST=localhost +DB_PORT=5432 +DB_NAME=campaign +DB_USER=postgres +DB_PASSWORD=postgres +DB_SSLMODE=prefer +# Docker-only host port. Leave DB_PORT=5432 for the app container. +# POSTGRES_PUBLIC_PORT=5432 + +# Optional single-url alternative. If set, this overrides DB_* values. +# DATABASE_URL=postgresql://postgres:postgres@localhost:5432/campaign?sslmode=prefer + +# Optional first-party Maskan CRM connection. +# These can also be configured per company in Settings > Maskan CRM. +# MASKAN_CRM_API_URL=http://127.0.0.1:8091/api/v1 +# MASKAN_CRM_WEB_URL=http://127.0.0.1:5174 +# MASKAN_CRM_INTEGRATION_KEY= + +# --- For running live tests (tests/live_aom_test.py) --- +# QWEN_API_KEY= +# QWEN_API_URL=https://dashscope-intl.aliyuncs.com/compatible-mode/v1/chat/completions +# QWEN_MODEL=qwen-plus diff --git a/.env.local.example b/.env.local.example new file mode 100644 index 0000000..27952d6 --- /dev/null +++ b/.env.local.example @@ -0,0 +1,11 @@ +MASKANX_ENV=local +MASKANX_STORAGE_BACKEND=postgres +MASKANX_PORT=8088 +MASKANX_CORS_ORIGINS=http://127.0.0.1:5173,http://localhost:5173 +DB_HOST=127.0.0.1 +DB_PORT=5432 +DB_NAME=campaign +DB_USER=postgres +DB_PASSWORD=postgres +DB_SSLMODE=disable +LOG_LEVEL=INFO diff --git a/.env.production.example b/.env.production.example new file mode 100644 index 0000000..47179b8 --- /dev/null +++ b/.env.production.example @@ -0,0 +1,12 @@ +MASKANX_ENV=production +MASKANX_STORAGE_BACKEND=postgres +MASKANX_PORT=8088 +MASKANX_CORS_ORIGINS=https://app.example.com +DATABASE_URL=postgresql://maskanx:replace-me@postgres.example.internal:5432/campaign?sslmode=require +LOG_LEVEL=INFO + +# Store provider credentials in the deployment secret manager. +CITEDY_API_KEY= +TAVILY_API_KEY= +EXA_API_KEY= +XAI_API_KEY= diff --git a/.env.testing.example b/.env.testing.example new file mode 100644 index 0000000..63821c7 --- /dev/null +++ b/.env.testing.example @@ -0,0 +1,13 @@ +MASKANX_ENV=testing +MASKANX_STORAGE_BACKEND=postgres +MASKANX_PORT=8089 +MASKANX_CORS_ORIGINS=http://127.0.0.1:5173 +DB_HOST=127.0.0.1 +DB_PORT=5432 +DB_NAME=campaign_test +DB_USER=postgres +DB_PASSWORD=postgres +DB_SSLMODE=disable +ADCLAW_DISABLE_MEMORY_MANAGER=1 +MASKANX_DISABLE_MCP=1 +LOG_LEVEL=WARNING diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..b44183e --- /dev/null +++ b/.flake8 @@ -0,0 +1,12 @@ +[flake8] +exclude = + scripts/* + src/agentscope/rpc/* +max-line-length = 79 +inline-quotes = " +avoid-escape = no +ignore = + F401 + F403 + W503 + E731 \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2c0ef2b --- /dev/null +++ b/.gitignore @@ -0,0 +1,36 @@ +.venv/ +.venv311/ +venv/ +__pycache__/ +*.py[cod] +*.egg-info/ +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ +.coverage +htmlcov/ +dist/ +build/ + +node_modules/ + +.env +.env.* +!.env.example +!.env.local.example +!.env.development.example +!.env.testing.example +!.env.production.example + +working/ +working.secret/ +generated/ +logs/ +*.log +*.db +*.sqlite +*.sqlite3 + +.DS_Store +.idea/ +.vscode/ diff --git a/.graphifyignore b/.graphifyignore new file mode 100644 index 0000000..e6b7f1a --- /dev/null +++ b/.graphifyignore @@ -0,0 +1,4 @@ +# Exclude pre-built frontend bundle (minified JS) - pollutes graph with thousands of A(), Q(), ur() nodes +console +adclaw/console +src/adclaw/console diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 0000000..81bac84 --- /dev/null +++ b/.mcp.json @@ -0,0 +1,8 @@ +{ + "mcpServers": { + "expect": { + "command": "npx", + "args": ["-y", "expect-cli@latest", "mcp"] + } + } +} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..11498dc --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,120 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: check-ast + exclude: '.*/skills/.*' + - id: sort-simple-yaml + exclude: '.*/skills/.*' + - id: check-yaml + exclude: | + (?x)^( + meta.yaml + )$ + - id: check-xml + exclude: '.*/skills/.*' + - id: check-toml + - id: check-docstring-first + exclude: '.*/skills/.*' + - id: check-json + exclude: '.*/skills/.*' + - id: fix-encoding-pragma + exclude: '.*/skills/.*' + - id: detect-private-key + - id: trailing-whitespace + exclude: '.*/skills/.*' + - repo: https://github.com/asottile/add-trailing-comma + rev: v3.1.0 + hooks: + - id: add-trailing-comma + exclude: '.*/skills/.*' + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.7.0 + hooks: + - id: mypy + exclude: + (?x)( + pb2\.py$ + | grpc\.py$ + | ^docs + | \.html$ + | .*/skills/.* + ) + args: [ + --ignore-missing-imports, + --disable-error-code=var-annotated, + --disable-error-code=union-attr, + --disable-error-code=assignment, + --disable-error-code=attr-defined, + --disable-error-code=import-untyped, + --disable-error-code=truthy-function, + --follow-imports=skip, + --explicit-package-bases, + ] + - repo: https://github.com/psf/black + rev: 23.3.0 + hooks: + - id: black + args: [ --line-length=79 ] + exclude: '.*/skills/.*' + - repo: https://github.com/PyCQA/flake8 + rev: 6.1.0 + hooks: + - id: flake8 + args: [ "--extend-ignore=E203"] + exclude: '.*/skills/.*' + - repo: https://github.com/pylint-dev/pylint + rev: v3.0.2 + hooks: + - id: pylint + exclude: + (?x)( + ^docs + | pb2\.py$ + | grpc\.py$ + | \.demo$ + | \.md$ + | \.html$ + | .*/skills/.* + ) + args: [ + --disable=W0511, + --disable=W0718, + --disable=W0122, + --disable=C0103, + --disable=R0913, + --disable=E0401, + --disable=E1101, + --disable=C0415, + --disable=W0603, + --disable=R1705, + --disable=R0914, + --disable=E0601, + --disable=W0602, + --disable=W0604, + --disable=R0801, + --disable=R0902, + --disable=R0903, + --disable=C0123, + --disable=W0231, + --disable=W1113, + --disable=W0221, + --disable=R0401, + --disable=W0632, + --disable=W0123, + --disable=C3001, + --disable=W0201, + --disable=C0302, + --disable=W1203, + --disable=C2801, + --disable=C0114, # Disable missing module docstring for quick dev + --disable=C0115, # Disable missing class docstring for quick dev + --disable=C0116, # Disable missing function or method docstring for quick dev + ] + - repo: https://github.com/pre-commit/mirrors-prettier + rev: 'v3.0.0' + hooks: + - id: prettier + additional_dependencies: [ 'prettier@3.0.0' ] + files: \.(tsx?)$ + exclude: '(?x)(^console/|/dist/|.*/skills/.*)' diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..c8cfe39 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.10 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..969eb98 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,166 @@ +# ----------------------------------------------------------------------------- +# Runtime image with Node.js, Python, optional Chromium, and the API. +# ----------------------------------------------------------------------------- +FROM node:22-bookworm-slim + +# Build variant: "core" | "browser" | "full" +# core — Python + MaskanX + Telegram only (no Chromium, no optional channels) +# browser — core + Chromium/xvfb + Playwright/PinchTab/agent-browser +# full — browser + Feishu/Discord/DingTalk channels + desktop tools +ARG MASKANX_VARIANT=full + +# Pip extras to install (overrides variant default if provided). +# Variant defaults: core="[local]", browser="[browser,local]", full="[all,local]" +ARG MASKANX_PIP_EXTRAS="" + +# Preload default local embedding model during image build by default. +ARG MASKANX_PRELOAD_EMBEDDINGS=1 + +# ENV variables +ENV NODE_ENV=production +ENV WORKSPACE_DIR=/app +ENV ADCLAW_WORKING_DIR=/app/working +ENV HF_HOME=/app/model-cache/huggingface +ENV SENTENCE_TRANSFORMERS_HOME=/app/model-cache/sentence-transformers +ENV ADCLAW_MEMORY_MANAGER_START_MODE=background +ENV ADCLAW_ENABLE_REME=0 +ENV ADCLAW_REME_LIGHT_TOKEN_COUNTER=1 +ENV ADCLAW_MEMORY_MANAGER_BACKGROUND_DELAY_SECONDS=5 +ENV ADCLAW_MEMORY_MANAGER_MAX_LOADAVG=2.0 +ENV ADCLAW_MEMORY_COMPACT_BATCH_MESSAGES=80 +ENV ADCLAW_REME_RETENTION_DAYS=30 +ENV MEMORY_STORE_BACKEND=sqlite +ENV FTS_ENABLED=true + +# Default enabled channels (can be overridden at runtime with -e MASKANX_ENABLED_CHANNELS=...). +ARG MASKANX_ENABLED_CHANNELS="discord,dingtalk,feishu,qq,console,telegram" +ENV MASKANX_ENABLED_CHANNELS=${MASKANX_ENABLED_CHANNELS} +ENV ADCLAW_ENABLED_CHANNELS=${MASKANX_ENABLED_CHANNELS} + +ARG DEBIAN_FRONTEND=noninteractive + +# ----------------------------------------------------------------------------- +# Base system packages (all variants) +# ----------------------------------------------------------------------------- +RUN apt-get update && apt-get install -y --fix-missing \ + curl \ + python3 \ + python3-pip \ + python3-venv \ + build-essential \ + libssl-dev \ + git \ + supervisor \ + vim \ + gettext-base \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean + +# ----------------------------------------------------------------------------- +# Browser stack (only for browser/full variants) +# ----------------------------------------------------------------------------- +RUN if [ "$MASKANX_VARIANT" != "core" ]; then \ + apt-get update && apt-get install -y --fix-missing \ + xfce4 \ + xfce4-terminal \ + xvfb \ + dbus-x11 \ + fonts-wqy-zenhei \ + fonts-wqy-microhei \ + chromium \ + chromium-sandbox \ + libx11-xcb1 \ + libxcomposite1 \ + libxdamage1 \ + libxext6 \ + libxfixes3 \ + libxi6 \ + libxtst6 \ + libnss3 \ + libglib2.0-0 \ + libdrm2 \ + libgbm1 \ + libasound2 \ + fonts-liberation \ + libu2f-udev \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean \ + && sed -i 's/^CHROMIUM_FLAGS=""/CHROMIUM_FLAGS="--no-sandbox"/' /usr/bin/chromium ; \ + fi + +# PinchTab: token-efficient browser control for AI agents (browser/full only). +RUN if [ "$MASKANX_VARIANT" != "core" ]; then \ + curl -fsSL https://pinchtab.com/install.sh | bash \ + && cp /root/.pinchtab/bin/*/pinchtab-linux-* /usr/local/bin/pinchtab-bin \ + && chmod 755 /usr/local/bin/pinchtab-bin ; \ + fi +ENV PINCHTAB_PORT=9867 + +# agent-browser: install CLI globally (browser/full only). +RUN if [ "$MASKANX_VARIANT" != "core" ]; then \ + npm install -g agent-browser sitefetch ; \ + fi + +# Browser env vars (harmless if binaries missing) +ENV AGENT_BROWSER_EXECUTABLE_PATH=/usr/bin/chromium +ENV PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=/usr/bin/chromium +ENV PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 +ENV ADCLAW_RUNNING_IN_CONTAINER=1 +ENV MASKANX_VARIANT=${MASKANX_VARIANT} +ENV ADCLAW_VARIANT=${MASKANX_VARIANT} + +WORKDIR ${WORKSPACE_DIR} + +RUN python3 -m venv venv +ENV PATH="/app/venv/bin:$PATH" + +COPY pyproject.toml README.md ./ +COPY src ./src + +# Install CPU-only PyTorch first (avoids downloading 2GB CUDA build), +# then install app with variant-specific extras. +# core → [local] — Python + MaskanX + AOM embeddings +# browser → [browser,local] — + Playwright +# full → [all,local] — + all optional channels + desktop tools +# MASKANX_PIP_EXTRAS can override this computed default. +RUN EXTRAS="${MASKANX_PIP_EXTRAS}"; \ + if [ -z "$EXTRAS" ]; then \ + case "$MASKANX_VARIANT" in \ + core) EXTRAS="[local]" ;; \ + browser) EXTRAS="[browser,local]" ;; \ + full) EXTRAS="[all,local]" ;; \ + *) EXTRAS="[all,local]" ;; \ + esac; \ + fi; \ + echo ">>> Installing MaskanX with extras=${EXTRAS} (variant=${MASKANX_VARIANT})" && \ + pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cpu && \ + pip install --no-cache-dir ".${EXTRAS}" + +# Preload the default local embedding model at build time so first production +# memory queries don't stall while downloading from Hugging Face. +RUN mkdir -p /app/model-cache \ + && if [ "$MASKANX_PRELOAD_EMBEDDINGS" = "1" ]; then \ + python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('all-MiniLM-L6-v2')" ; \ + else \ + echo ">>> Skipping embedding model preload"; \ + fi + +# Non-root user for running the app (supervisord stays root for system services). +RUN groupadd -r maskanx && useradd -r -g maskanx -m -s /bin/bash maskanx \ + && mkdir -p /app/working /app/working.secret /app/logs /home/maskanx/.config/pinchtab /app/model-cache \ + && chown -R maskanx:maskanx /app/working /app/working.secret /app/logs /home/maskanx/.config /app/model-cache + +# Config.json is generated at runtime by entrypoint.sh (only if missing), +# so it persists user settings across container restarts. + +# MaskanX app port (default 8088). Override at runtime with -e MASKANX_PORT=3000. +ENV ADCLAW_PORT=8088 + +COPY deploy/config/supervisord.conf.template /etc/supervisor/conf.d/supervisord.conf.template +COPY deploy/config/supervisord.browser.conf.template /etc/supervisor/conf.d/supervisord.browser.conf.template +COPY deploy/entrypoint.sh /entrypoint.sh +RUN chmod 755 /entrypoint.sh + +EXPOSE 8088 + +CMD ["/entrypoint.sh"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..8099bd3 --- /dev/null +++ b/LICENSE @@ -0,0 +1,20 @@ +MaskanX Proprietary License + +Copyright (c) 2026 MaskanX. All rights reserved. + +This software and associated documentation files (the "Software") are +proprietary to MaskanX. + +You may use, copy, modify, deploy, and distribute the Software only with prior +written permission from MaskanX or under a separate written agreement signed +by MaskanX. + +No rights are granted to sublicense, sell, lease, rent, publish, host for third +parties, or otherwise make the Software available to others except where +expressly permitted in writing by MaskanX. + +The Software is provided "as is", without warranty of any kind, express or +implied, including but not limited to the warranties of merchantability, +fitness for a particular purpose, and noninfringement. In no event shall +MaskanX be liable for any claim, damages, or other liability arising from use +of the Software. diff --git a/README.md b/README.md new file mode 100644 index 0000000..c1bb8d5 --- /dev/null +++ b/README.md @@ -0,0 +1,128 @@ +# MaskanX Backend + +Independent Python and FastAPI backend for MaskanX. + +## Service Contract + +- Default API URL: `http://127.0.0.1:8088` +- PostgreSQL database: `campaign` +- Frontend origin: configured with `MASKANX_CORS_ORIGINS` +- Maskan CRM: connected through its versioned REST API and optional MCP tools + +This repository never connects directly to the Maskan CRM PostgreSQL database. + +## Local Setup + +1. Create the database: + + ```sql + CREATE DATABASE campaign; + ``` + +2. Install dependencies and configure the environment: + + ```powershell + python -m venv .venv + .\.venv\Scripts\python.exe -m pip install -e ".[dev]" + npm install + Copy-Item .env.local.example .env.local + ``` + +3. Migrate, seed, and start: + + ```powershell + npm run local:migrate + npm run local:seed + npm run local + ``` + +Open API documentation at `http://127.0.0.1:8088/docs` when documentation is +enabled. + +### Upgrading from JSON storage + +Chats and cron jobs were originally stored as JSON files in the working +directory (`~/.adclaw/chats.json` and `~/.adclaw/jobs.json`). PostgreSQL is now +the default backend, so those records are no longer read and existing chats, +sessions, and scheduled jobs appear to be missing even though nothing was +deleted. + +Copy them into PostgreSQL once: + +```powershell +npm run local:import-json -- --dry-run # report what would be imported +npm run local:import-json # perform the import +``` + +The import upserts by id and never deletes, so it is safe to re-run. The JSON +files are left untouched as a backup. + +### Clearing and restoring credentials + +**Stop MaskanX before clearing credentials.** A running app holds its +configuration in memory and can rewrite `config.json` and `providers.json` +after the clear, silently restoring the values that were just removed. The +clear script now verifies the result and exits non-zero if anything +secret-looking survives. + +Credentials live in three places, and all of them are cleared: + +- `~/.adclaw.secret/providers.json` - LLM provider keys +- `~/.adclaw.secret/envs.json` - integration keys +- `~/.adclaw/companies//` - a per-company copy of both, plus LinkedIn OAuth + +Because each company keeps its own copy, an intact company store can be used to +recover the active one: + +```powershell +npm run secrets:restore -- --from default --dry-run +npm run secrets:restore -- --from default +``` + +Credential files are copied wholesale; `config.json` is patched surgically, so +only missing MCP env values and disabled clients are restored and +company-specific settings are preserved. A backup is written to +`~/.adclaw.secret/restore-backups/` first. + +## Commands + +| Command | Purpose | +| --- | --- | +| `npm start` | Start using `.env` or injected environment variables | +| `npm run build` | Build a Python wheel in `dist/` | +| `npm run local` | Local API with reload | +| `npm run local:migrate` | Apply local PostgreSQL migrations | +| `npm run local:migrate:undo` | Roll back the newest migration | +| `npm run local:migrate:undo:all` | Roll back every migration | +| `npm run local:seed` | Add starter data | +| `npm run local:seed:undo` | Remove starter data | +| `npm run local:import-json` | Import legacy JSON chats/cron jobs into PostgreSQL | +| `npm run secrets:dry-run` | Report which credentials a clear would remove | +| `npm run secrets:clear` | Clear runtime credentials (creates a backup) | +| `npm run secrets:clear:all` | Also blank secret lines in `.env` files (creates a backup) | +| `npm run secrets:delete:all` | Clear everything including `.env` files (creates a backup) | +| `npm run secrets:delete:all:no-backup` | Same, with **no backup** - irreversible | +| `npm run secrets:restore` | Restore credentials from an intact company store | +| `npm run local:reset` | Rebuild and seed the local schema | +| `npm run dev` | Development API with reload | +| `npm run dev:migrate` | Apply development migrations | +| `npm run dev:reset` | Reset the development schema | +| `npm test` | Run Pytest using `.env.testing` | +| `npm run test:migrate` | Apply test database migrations | +| `npm run test:reset` | Reset the test schema | +| `npm run prod` | Start the production API | +| `npm run prod:migrate` | Apply production migrations | +| `npm run prod:seed` | Apply production starter data | + +The command names match the platform deployment contract. They invoke the +native MaskanX Python migration runner, not Sequelize. + +## Docker + +```powershell +Copy-Item .env.local.example .env +docker compose up -d --build +``` + +For a managed PostgreSQL deployment, provide `DATABASE_URL` through the hosting +secret manager and run `npm run prod:migrate` as a release step. diff --git a/deploy/config/supervisord.browser.conf.template b/deploy/config/supervisord.browser.conf.template new file mode 100644 index 0000000..873be10 --- /dev/null +++ b/deploy/config/supervisord.browser.conf.template @@ -0,0 +1,35 @@ + +[program:dbus] +command=/bin/sh -c "rm -f /run/dbus/pid; mkdir -p /run/dbus; exec /usr/bin/dbus-daemon --system --nofork" +autostart=true +autorestart=true +stderr_logfile=/var/log/dbus.err.log +stdout_logfile=/var/log/dbus.out.log + +[program:pinchtab] +command=/bin/sh -c "sleep 2; exec /usr/local/bin/pinchtab-bin server" +user=maskanx +autostart=true +autorestart=true +priority=25 +stderr_logfile=/var/log/pinchtab.err.log +stdout_logfile=/var/log/pinchtab.out.log +environment=DISPLAY=":1",HOME="/home/maskanx" + +[program:xvfb] +command=/bin/sh -c "rm -f /tmp/.X1-lock /tmp/.X11-unix/X1; mkdir -p /tmp/.X11-unix; exec /usr/bin/Xvfb :1 -screen 0 1280x800x24" +autostart=true +autorestart=true +priority=10 +stderr_logfile=/var/log/xvfb.err.log +stdout_logfile=/var/log/xvfb.out.log +environment=DISPLAY=":1" + +[program:xfce4] +command=/bin/sh -c 'export DISPLAY=:1; for i in $(seq 1 200); do [ -S /tmp/.X11-unix/X1 ] && break; sleep 0.1; done; exec dbus-run-session startxfce4' +autostart=true +autorestart=true +priority=20 +stderr_logfile=/var/log/xfce4.err.log +stdout_logfile=/var/log/xfce4.out.log +environment=DISPLAY=":1" diff --git a/deploy/config/supervisord.conf.template b/deploy/config/supervisord.conf.template new file mode 100644 index 0000000..5000db7 --- /dev/null +++ b/deploy/config/supervisord.conf.template @@ -0,0 +1,15 @@ +[supervisord] +user=root +logfile=/var/log/supervisord.log +pidfile=/var/log/supervisord.pid +nodaemon=true + +[program:app] +command=maskanx app --host 0.0.0.0 --port ${ADCLAW_PORT} +user=maskanx +autostart=true +autorestart=true +priority=30 +stderr_logfile=/var/log/app.err.log +stdout_logfile=/var/log/app.out.log +environment=DISPLAY=":1",PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH="/usr/bin/chromium",ADCLAW_RUNNING_IN_CONTAINER="1",HOME="/home/maskanx" diff --git a/deploy/digitalocean-app.yaml b/deploy/digitalocean-app.yaml new file mode 100644 index 0000000..a09a568 --- /dev/null +++ b/deploy/digitalocean-app.yaml @@ -0,0 +1,26 @@ +spec: + name: MaskanX + region: ams + services: + - name: MaskanX + image: + registry_type: DOCKER_HUB + registry: nttylock + repository: MaskanX + tag: latest + instance_count: 1 + instance_size_slug: basic-s + http_port: 8088 + envs: + - key: CITEDY_API_KEY + scope: RUN_TIME + type: SECRET + - key: TELEGRAM_BOT_TOKEN + scope: RUN_TIME + type: SECRET + - key: MASKANX_ENABLED_CHANNELS + scope: RUN_TIME + value: discord,dingtalk,feishu,qq,console,telegram + - key: LOG_LEVEL + scope: RUN_TIME + value: INFO diff --git a/deploy/entrypoint.sh b/deploy/entrypoint.sh new file mode 100644 index 0000000..f710e8a --- /dev/null +++ b/deploy/entrypoint.sh @@ -0,0 +1,153 @@ +#!/bin/sh +# Substitute the app port in supervisord template and start supervisord. +# Default port 8088; override at runtime with -e MASKANX_PORT=3000. +set -e +export ADCLAW_PORT="${MASKANX_PORT:-${ADCLAW_PORT:-8088}}" +export ADCLAW_WORKING_DIR="${MASKANX_WORKING_DIR:-${ADCLAW_WORKING_DIR:-/app/working}}" +export ADCLAW_SECRET_DIR="${MASKANX_SECRET_DIR:-${ADCLAW_SECRET_DIR:-/app/working.secret}}" +export ADCLAW_ENABLED_CHANNELS="${MASKANX_ENABLED_CHANNELS:-${ADCLAW_ENABLED_CHANNELS:-discord,dingtalk,feishu,qq,console,telegram}}" +export ADCLAW_STORAGE_BACKEND="${MASKANX_STORAGE_BACKEND:-${ADCLAW_STORAGE_BACKEND:-json}}" + +# Ensure config.json exists (first run only — don't overwrite user config) +CONFIG="${ADCLAW_WORKING_DIR}/config.json" +if [ ! -f "$CONFIG" ] || [ ! -s "$CONFIG" ]; then + echo "entrypoint: No config.json found, generating default..." + maskanx init --defaults --accept-security 2>/dev/null || true +fi + +# Migrate config.json — apply new defaults to existing config without overwriting user data +python3 -c " +import json, os, sys +cfg_path = os.environ.get('ADCLAW_WORKING_DIR', '/app/working') + '/config.json' +if not os.path.isfile(cfg_path): + sys.exit(0) +with open(cfg_path) as f: + cfg = json.load(f) +changed = False +# Force correct defaults that should never be True in production +if cfg.get('show_tool_details') is not False: + cfg['show_tool_details'] = False + changed = True +# Ensure filter_tool_messages=True on all channels +for ch_name, ch_cfg in cfg.get('channels', {}).items(): + if isinstance(ch_cfg, dict) and ch_cfg.get('filter_tool_messages') is not True: + ch_cfg['filter_tool_messages'] = True + changed = True +if changed: + with open(cfg_path, 'w') as f: + json.dump(cfg, f, indent=2, ensure_ascii=False) + print('entrypoint: config.json migrated (applied new defaults)') +else: + print('entrypoint: config.json OK (no migration needed)') +" 2>/dev/null || true + +# Enable Citedy MCP client if API key is provided at runtime +if [ -n "$CITEDY_API_KEY" ]; then + CONFIG="${ADCLAW_WORKING_DIR}/config.json" + if [ -f "$CONFIG" ]; then + python3 -c " +import json, os +cfg_path = os.environ.get('ADCLAW_WORKING_DIR', '/app/working') + '/config.json' +with open(cfg_path) as f: + cfg = json.load(f) +mcp = cfg.setdefault('mcp', {}).setdefault('clients', {}) +key = os.environ['CITEDY_API_KEY'] +mcp['citedy'] = { + 'name': 'citedy_mcp', + 'description': 'Citedy SEO & Marketing Tools (70+ tools)', + 'enabled': True, + 'transport': 'streamable_http', + 'url': 'https://mcp.citedy.com/mcp', + 'headers': {'Authorization': f'Bearer {key}', 'Accept': 'application/json, text/event-stream'}, + 'env': {'CITEDY_API_KEY': key}, +} +with open(cfg_path, 'w') as f: + json.dump(cfg, f, indent=2) +print('entrypoint: Citedy MCP client enabled') +" 2>/dev/null || true + fi +fi + +# Enable Exa search MCP if API key is provided at runtime +if [ -n "$EXA_API_KEY" ]; then + CONFIG="${ADCLAW_WORKING_DIR}/config.json" + if [ -f "$CONFIG" ]; then + python3 -c " +import json, os +cfg_path = os.environ.get('ADCLAW_WORKING_DIR', '/app/working') + '/config.json' +with open(cfg_path) as f: + cfg = json.load(f) +mcp = cfg.setdefault('mcp', {}).setdefault('clients', {}) +key = os.environ['EXA_API_KEY'] +mcp['exa'] = { + 'name': 'exa_mcp', + 'description': 'Exa AI search: web, code, people, companies', + 'enabled': True, + 'command': 'npx', + 'args': ['-y', 'exa-mcp-server'], + 'env': {'EXA_API_KEY': key}, +} +with open(cfg_path, 'w') as f: + json.dump(cfg, f, indent=2) +print('entrypoint: Exa MCP client enabled') +" 2>/dev/null || true + fi +fi + +# Enable Telegram channel if bot token is provided at runtime +if [ -n "$TELEGRAM_BOT_TOKEN" ]; then + CONFIG="${ADCLAW_WORKING_DIR}/config.json" + if [ -f "$CONFIG" ]; then + python3 -c " +import json, os +cfg_path = os.environ.get('ADCLAW_WORKING_DIR', '/app/working') + '/config.json' +with open(cfg_path) as f: + cfg = json.load(f) +tg = cfg.setdefault('channels', {}).setdefault('telegram', {}) +tg['enabled'] = True +tg['bot_token'] = os.environ['TELEGRAM_BOT_TOKEN'] +with open(cfg_path, 'w') as f: + json.dump(cfg, f, indent=2) +print('entrypoint: Telegram channel enabled') +" 2>/dev/null || true + fi +fi + +# Sync new built-in skills to active_skills (non-destructive: skip existing) +python3 -c " +from adclaw.agents.skills_manager import sync_skills_to_working_dir +synced, skipped = sync_skills_to_working_dir(skill_names=None, force=False) +if synced: + print(f'entrypoint: synced {synced} new skill(s) to active_skills') +else: + print(f'entrypoint: all skills up to date ({skipped} existing)') +" 2>/dev/null || true + +# Ensure working dirs are owned by MaskanX user (entrypoint runs as root) +mkdir -p /app/logs 2>/dev/null || true +chown -R maskanx:maskanx "${ADCLAW_WORKING_DIR}" "${ADCLAW_SECRET_DIR}" /app/logs 2>/dev/null || true + +# Run Postgres migrations when requested. Keep JSON/file mode as the default so +# local users can still start the app without a database. +case "$(printf '%s' "$ADCLAW_STORAGE_BACKEND" | tr '[:upper:]' '[:lower:]')" in + postgres|postgresql|pg) + echo "entrypoint: PostgreSQL storage enabled, running migrations..." + maskanx db migrate + ;; +esac + +# Generate supervisord.conf from templates based on MASKANX_VARIANT. +# core → base template only (supervisord + app). +# browser/full → base + browser template (adds dbus, xvfb, xfce4, pinchtab). +MASKANX_VARIANT="${MASKANX_VARIANT:-${ADCLAW_VARIANT:-full}}" +SUPERVISORD_CONF="/etc/supervisor/conf.d/supervisord.conf" +envsubst '${ADCLAW_PORT}' \ + < /etc/supervisor/conf.d/supervisord.conf.template \ + > "$SUPERVISORD_CONF" +if [ "$MASKANX_VARIANT" != "core" ] && [ -f /etc/supervisor/conf.d/supervisord.browser.conf.template ]; then + cat /etc/supervisor/conf.d/supervisord.browser.conf.template >> "$SUPERVISORD_CONF" + echo "entrypoint: supervisord configured with browser programs (variant=${MASKANX_VARIANT})" +else + echo "entrypoint: supervisord configured without browser (variant=${MASKANX_VARIANT})" +fi +exec /usr/bin/supervisord -c "$SUPERVISORD_CONF" diff --git a/deploy/render.yaml b/deploy/render.yaml new file mode 100644 index 0000000..6b0d8c7 --- /dev/null +++ b/deploy/render.yaml @@ -0,0 +1,20 @@ +services: + - type: web + name: maskanx-backend + runtime: docker + dockerfilePath: ./Dockerfile + plan: starter + healthCheckPath: /api/version + envVars: + - key: CITEDY_API_KEY + sync: false + - key: TELEGRAM_BOT_TOKEN + sync: false + - key: MASKANX_ENABLED_CHANNELS + value: discord,dingtalk,feishu,qq,console,telegram + - key: LOG_LEVEL + value: INFO + disk: + name: MaskanX-data + mountPath: /app/working + sizeGB: 1 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..56137b0 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,53 @@ +services: + postgres: + image: postgres:16-alpine + restart: unless-stopped + environment: + POSTGRES_DB: ${DB_NAME:-campaign} + POSTGRES_USER: ${DB_USER:-postgres} + POSTGRES_PASSWORD: ${DB_PASSWORD:-postgres} + ports: + - "${POSTGRES_PUBLIC_PORT:-5432}:5432" + volumes: + - maskanx_postgres:/var/lib/postgresql/data + healthcheck: + test: + [ + "CMD-SHELL", + "pg_isready -U ${DB_USER:-postgres} -d ${DB_NAME:-campaign}", + ] + interval: 5s + timeout: 5s + retries: 20 + + api: + build: + context: . + args: + MASKANX_VARIANT: ${MASKANX_VARIANT:-core} + MASKANX_PRELOAD_EMBEDDINGS: ${MASKANX_PRELOAD_EMBEDDINGS:-0} + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + ports: + - "${MASKANX_PORT:-8088}:8088" + volumes: + - maskanx_working:/app/working + - maskanx_secrets:/app/working.secret + environment: + MASKANX_STORAGE_BACKEND: postgres + MASKANX_PORT: 8088 + MASKANX_CORS_ORIGINS: ${MASKANX_CORS_ORIGINS:-http://127.0.0.1:5173,http://localhost:5173} + DB_HOST: postgres + DB_PORT: 5432 + DB_NAME: ${DB_NAME:-campaign} + DB_USER: ${DB_USER:-postgres} + DB_PASSWORD: ${DB_PASSWORD:-postgres} + DB_SSLMODE: disable + LOG_LEVEL: ${LOG_LEVEL:-INFO} + +volumes: + maskanx_postgres: + maskanx_working: + maskanx_secrets: diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..e05f6b3 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,157 @@ +{ + "name": "maskanx-backend", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "maskanx-backend", + "version": "1.0.0", + "dependencies": { + "dotenv": "^16.6.1" + }, + "devDependencies": { + "cross-env": "^7.0.3", + "dotenv-cli": "^8.0.0" + } + }, + "node_modules/cross-env": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz", + "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.1" + }, + "bin": { + "cross-env": "src/bin/cross-env.js", + "cross-env-shell": "src/bin/cross-env-shell.js" + }, + "engines": { + "node": ">=10.14", + "npm": ">=6", + "yarn": ">=1" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/dotenv": { + "version": "16.6.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz", + "integrity": "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/dotenv-cli": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/dotenv-cli/-/dotenv-cli-8.0.0.tgz", + "integrity": "sha512-aLqYbK7xKOiTMIRf1lDPbI+Y+Ip/wo5k3eyp6ePysVaSqbyxjyK3dK35BTxG+rmd7djf5q2UPs4noPNH+cj0Qw==", + "dev": true, + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.6", + "dotenv": "^16.3.0", + "dotenv-expand": "^10.0.0", + "minimist": "^1.2.6" + }, + "bin": { + "dotenv": "cli.js" + } + }, + "node_modules/dotenv-expand": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-10.0.0.tgz", + "integrity": "sha512-GopVGCpVS1UKH75VKHGuQFqS1Gusej0z4FyQkPdwjil2gNIv+LNsqBlboOzpJFZKVT95GkCyWJbBSdFEFUWI2A==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..a896861 --- /dev/null +++ b/package.json @@ -0,0 +1,53 @@ +{ + "name": "maskanx-backend", + "version": "1.0.0", + "private": true, + "description": "Independent FastAPI backend for MaskanX.", + "scripts": { + "start": "node -r dotenv/config scripts/run-python.cjs -m adclaw app --host 0.0.0.0 --port 8088", + "build": "node scripts/run-python.cjs -m pip wheel . --no-deps --wheel-dir dist", + "local": "dotenv -e .env.local -- node scripts/run-python.cjs -m adclaw app --host 127.0.0.1 --port 8088 --reload", + "local:migrate": "dotenv -e .env.local -- node scripts/run-python.cjs -m adclaw db migrate", + "local:migrate:undo": "dotenv -e .env.local -- node scripts/run-python.cjs -m adclaw db migrate-undo", + "local:migrate:undo:all": "dotenv -e .env.local -- node scripts/run-python.cjs -m adclaw db migrate-undo-all", + "local:seed": "dotenv -e .env.local -- node scripts/run-python.cjs -m adclaw db seed", + "local:import-json": "dotenv -e .env.local -- node scripts/run-python.cjs -m adclaw db import-json", + "local:seed:undo": "dotenv -e .env.local -- node scripts/run-python.cjs -m adclaw db seed-undo", + "local:reset": "npm run local:migrate:undo:all && npm run local:migrate && npm run local:seed", + "dev": "dotenv -e .env.development -- node scripts/run-python.cjs -m adclaw app --host 0.0.0.0 --port 8088 --reload", + "dev:migrate": "dotenv -e .env.development -- node scripts/run-python.cjs -m adclaw db migrate", + "dev:migrate:undo": "dotenv -e .env.development -- node scripts/run-python.cjs -m adclaw db migrate-undo", + "dev:migrate:undo:all": "dotenv -e .env.development -- node scripts/run-python.cjs -m adclaw db migrate-undo-all", + "dev:seed": "dotenv -e .env.development -- node scripts/run-python.cjs -m adclaw db seed", + "dev:import-json": "dotenv -e .env.development -- node scripts/run-python.cjs -m adclaw db import-json", + "dev:seed:undo": "dotenv -e .env.development -- node scripts/run-python.cjs -m adclaw db seed-undo", + "dev:reset": "npm run dev:migrate:undo:all && npm run dev:migrate && npm run dev:seed", + "test": "dotenv -e .env.testing -- node scripts/run-python.cjs -m pytest", + "test:serve": "dotenv -e .env.testing -- node scripts/run-python.cjs -m adclaw app --host 127.0.0.1 --port 8089", + "test:migrate": "dotenv -e .env.testing -- node scripts/run-python.cjs -m adclaw db migrate", + "test:seed": "dotenv -e .env.testing -- node scripts/run-python.cjs -m adclaw db seed", + "test:reset": "dotenv -e .env.testing -- node scripts/run-python.cjs -m adclaw db migrate-undo-all && dotenv -e .env.testing -- node scripts/run-python.cjs -m adclaw db migrate", + "prod": "cross-env MASKANX_ENV=production node scripts/run-python.cjs -m adclaw app --host 0.0.0.0 --port 8088", + "prod:migrate": "cross-env MASKANX_ENV=production node scripts/run-python.cjs -m adclaw db migrate", + "prod:seed": "cross-env MASKANX_ENV=production node scripts/run-python.cjs -m adclaw db seed", + "lint": "node scripts/run-python.cjs -m compileall -q src", + "secrets:dry-run": "node scripts/clear-secrets.cjs --include-env-files --dry-run", + "secrets:restore": "node scripts/restore-secrets-from-company.cjs", + "secrets:clear": "node scripts/clear-secrets.cjs", + "secrets:clear:all": "node scripts/clear-secrets.cjs --include-env-files", + "secrets:delete:all": "node scripts/clear-secrets.cjs --include-env-files", + "secrets:delete:all:no-backup": "node scripts/clear-secrets.cjs --include-env-files --no-backup", + "db:config": "node scripts/run-python.cjs -m adclaw db config", + "db:status": "node scripts/run-python.cjs -m adclaw db status", + "docker:up": "docker compose up --build", + "docker:down": "docker compose down", + "docker:reset": "docker compose down -v && docker compose up --build" + }, + "dependencies": { + "dotenv": "^16.6.1" + }, + "devDependencies": { + "cross-env": "^7.0.3", + "dotenv-cli": "^8.0.0" + } +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..f12cb11 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,99 @@ +[project] +name = "maskanx" +dynamic = ["version"] +description = "Multi-agent AI marketing team with 122 skills, multi-channel support, and shared memory" +readme = "README.md" +requires-python = ">=3.10,<3.14" +license = {text = "Proprietary"} +dependencies = [ + "agentscope==1.0.18", + "agentscope-runtime==1.1.3", + "exceptiongroup>=1.2.0; python_version < '3.11'", + "uvicorn>=0.40.0", + "apscheduler>=3.11.2,<4", + "questionary>=2.1.1", + "reme-ai==0.3.0.5", + "python-dotenv>=1.0.0", + "python-socks>=2.5.3", + "onnxruntime<1.24", + "python-telegram-bot>=20.0", + "aiosqlite>=0.19.0", + "sqlite-vec>=0.1.1", + "google-genai>=1.0.0", + "Pillow>=10.0.0", + "psycopg[binary]>=3.2.0", +] + +[tool.setuptools.dynamic] +version = {attr = "adclaw.__version__.__version__"} + +[tool.setuptools] +packages = { find = { where = ["src"] } } +include-package-data = true + +[tool.setuptools.package-data] +"adclaw" = [ + "agents/md_files/**", + "agents/skills/**", + "tokenizer/**", +] + +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" + +[project.scripts] +maskanx = "adclaw.cli.main:cli" + +[project.optional-dependencies] +dev = [ + "pytest>=8.3.5", + "pytest-asyncio>=0.23.0", + "pre-commit>=4.2.0", + "pytest-cov>=6.2.1", +] +local = [ + "huggingface_hub>=0.20.0", + "sentence-transformers>=2.2.0", +] +llamacpp = [ + "maskanx[local]", + "llama-cpp-python>=0.3.0", +] +mlx = [ + "maskanx[local]", + "mlx-lm>=0.10.0", +] +ollama = [ + "ollama>=0.6.1", +] +browser = [ + "playwright>=1.49.0", +] +feishu = [ + "lark-oapi>=1.5.3", +] +discord = [ + "discord-py>=2.3", +] +dingtalk = [ + "dingtalk-stream>=0.24.3", +] +desktop = [ + "mss>=9.0.0", +] +# Meta: all optional messaging channels +channels = [ + "maskanx[feishu,discord,dingtalk]", +] +# Meta: full install — everything +all = [ + "maskanx[browser,channels,desktop]", +] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +asyncio_default_fixture_loop_scope = "function" +markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')", +] diff --git a/railway.json b/railway.json new file mode 100644 index 0000000..b913ecd --- /dev/null +++ b/railway.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://railway.app/railway.schema.json", + "build": { + "builder": "DOCKERFILE", + "dockerfilePath": "Dockerfile" + }, + "deploy": { + "startCommand": "/entrypoint.sh", + "healthcheckPath": "/api/version", + "healthcheckTimeout": 300, + "restartPolicyType": "ON_FAILURE", + "restartPolicyMaxRetries": 10 + } +} diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..14f3cef --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,38 @@ +# Backend Scripts + +Run these from the `maskanx-backend` repository root. + +## Build Python artifacts + +```bash +bash scripts/wheel_build.sh +``` + +This builds only the backend wheel and source distribution into `dist/`. +The MaskanX frontend is built independently in `maskanx-frontend`. + +## Build the backend image + +```bash +bash scripts/docker_build.sh [IMAGE_TAG] [EXTRA_ARGS...] +``` + +The default tag is `maskanx-backend:latest`, and the script uses the root +`Dockerfile`. + +Example: + +```bash +bash scripts/docker_build.sh registry.example.com/maskanx-backend:v1 --no-cache +``` + +## Browser E2E + +```bash +python scripts/host_ai_model_selection_e2e.py \ + --base-url http://127.0.0.1:5173 \ + --out artifacts/host-ai-model-selection-e2e +``` + +Run the frontend and backend first. The browser targets the separately deployed +frontend URL. diff --git a/scripts/clear-secrets.cjs b/scripts/clear-secrets.cjs new file mode 100644 index 0000000..c714b87 --- /dev/null +++ b/scripts/clear-secrets.cjs @@ -0,0 +1,342 @@ +#!/usr/bin/env node +"use strict"; + +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +const args = new Set(process.argv.slice(2)); +const dryRun = args.has("--dry-run"); +const noBackup = args.has("--no-backup"); +const includeEnvFiles = args.has("--include-env-files"); + +const repoRoot = path.resolve(__dirname, ".."); +const homeDir = os.homedir(); + +function expandHome(value) { + if (!value) return value; + if (value === "~") return homeDir; + if (value.startsWith("~/") || value.startsWith("~\\")) { + return path.join(homeDir, value.slice(2)); + } + return value; +} + +const workingDir = path.resolve( + expandHome(process.env.ADCLAW_WORKING_DIR || path.join(homeDir, ".adclaw")), +); +const secretDir = path.resolve( + expandHome(process.env.ADCLAW_SECRET_DIR || `${workingDir}.secret`), +); + +const stamp = new Date() + .toISOString() + .replace(/[-:]/g, "") + .replace(/\..+$/, "") + .replace("T", "-"); + +const backupRoot = path.join(secretDir, "backups", stamp); +const secretNamePattern = + /(^|[_\-.])(api[_\-.]?key|token|secret|credential|authorization|bearer)($|[_\-.])/i; + +const touched = []; +const skipped = []; + +function exists(filePath) { + try { + return fs.statSync(filePath).isFile(); + } catch { + return false; + } +} + +function backupPathFor(filePath) { + const safeName = path + .resolve(filePath) + .replace(/^[A-Za-z]:/, "") + .replace(/[\\/]+/g, "__") + .replace(/^__/, ""); + return path.join(backupRoot, safeName); +} + +function backupFile(filePath) { + if (noBackup || dryRun || !exists(filePath)) return; + const target = backupPathFor(filePath); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.copyFileSync(filePath, target); +} + +function removeFile(filePath, reason) { + if (!exists(filePath)) { + skipped.push({ filePath, reason: "not found" }); + return; + } + touched.push({ action: noBackup ? "deleted" : "backed up and removed", filePath, reason }); + if (dryRun) return; + backupFile(filePath); + fs.unlinkSync(filePath); +} + +function sanitizeValue(value, parentKey = "") { + if (Array.isArray(value)) { + let changed = false; + const next = value.map((item) => { + const result = sanitizeValue(item, parentKey); + changed = changed || result.changed; + return result.value; + }); + return { value: next, changed }; + } + + if (value && typeof value === "object") { + let changed = false; + const next = {}; + for (const [key, child] of Object.entries(value)) { + const keyLooksSecret = secretNamePattern.test(key); + if (keyLooksSecret) { + changed = true; + if (typeof child === "string") { + next[key] = ""; + } else if (Array.isArray(child)) { + next[key] = []; + } else if (child && typeof child === "object") { + next[key] = {}; + } else { + next[key] = null; + } + continue; + } + + const result = sanitizeValue(child, key); + changed = changed || result.changed; + next[key] = result.value; + } + return { value: next, changed }; + } + + if (typeof value === "string" && secretNamePattern.test(parentKey) && value) { + return { value: "", changed: true }; + } + + return { value, changed: false }; +} + +function sanitizeJsonFile(filePath, reason) { + if (!exists(filePath)) { + skipped.push({ filePath, reason: "not found" }); + return; + } + + let parsed; + try { + parsed = JSON.parse(fs.readFileSync(filePath, "utf8")); + } catch (error) { + skipped.push({ filePath, reason: `invalid JSON: ${error.message}` }); + return; + } + + const result = sanitizeValue(parsed); + if (!result.changed) { + skipped.push({ filePath, reason: "no secret-like fields found" }); + return; + } + + touched.push({ action: "sanitized", filePath, reason }); + if (dryRun) return; + backupFile(filePath); + fs.writeFileSync(filePath, `${JSON.stringify(result.value, null, 2)}\n`, "utf8"); +} + +function sanitizeEnvFile(filePath) { + if (!exists(filePath)) { + skipped.push({ filePath, reason: "not found" }); + return; + } + + const original = fs.readFileSync(filePath, "utf8"); + const lines = original.split(/\r?\n/); + let changed = false; + const next = lines.map((line) => { + const match = line.match(/^(\s*export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=/); + if (!match) return line; + const key = match[2]; + if (!secretNamePattern.test(key)) return line; + changed = true; + return `${match[1] || ""}${key}=`; + }); + + if (!changed) { + skipped.push({ filePath, reason: "no API key/token/secret env vars found" }); + return; + } + + touched.push({ action: "sanitized", filePath, reason: "env file API key/token/secret lines" }); + if (dryRun) return; + backupFile(filePath); + fs.writeFileSync(filePath, next.join(os.EOL), "utf8"); +} + +function listCompanyDirs() { + const companiesDir = path.join(workingDir, "companies"); + try { + return fs + .readdirSync(companiesDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => path.join(companiesDir, entry.name)); + } catch { + return []; + } +} + +function clearRuntimeSecrets() { + const filesToRemove = [ + path.join(secretDir, "providers.json"), + path.join(secretDir, "envs.json"), + path.join(workingDir, "providers.json"), + path.join(workingDir, "envs.json"), + path.join(workingDir, ".secret", "providers.json"), + path.join(workingDir, ".secret", "envs.json"), + path.join(homeDir, ".linkedin-mcp", "tokens_default.json"), + path.join(homeDir, ".linkedin-mcp", "users.json"), + ]; + + for (const companyDir of listCompanyDirs()) { + filesToRemove.push( + path.join(companyDir, "providers.json"), + path.join(companyDir, "envs.json"), + path.join(companyDir, "linkedin-mcp", "tokens_default.json"), + path.join(companyDir, "linkedin-mcp", "users.json"), + ); + sanitizeJsonFile( + path.join(companyDir, "config.json"), + "company MCP/API config secret-like values", + ); + } + + sanitizeJsonFile(path.join(workingDir, "config.json"), "MCP/API config secret-like values"); + + for (const filePath of filesToRemove) { + removeFile(filePath, "runtime/provider/env/OAuth secret store"); + } +} + +function clearEnvFiles() { + for (const name of [ + ".env", + ".env.local", + ".env.development", + ".env.testing", + ".env.production", + ]) { + sanitizeEnvFile(path.join(repoRoot, name)); + } +} + +if (noBackup && !dryRun) { + console.warn( + "WARNING: --no-backup is set. Cleared credentials cannot be recovered.", + ); +} +if (!dryRun) { + console.warn( + "Stop MaskanX before clearing: a running app can rewrite its config from " + + "memory and restore the secrets this script removes.", + ); +} + +clearRuntimeSecrets(); +if (includeEnvFiles) { + clearEnvFiles(); +} + +console.log(""); +console.log(`MaskanX secret cleanup ${dryRun ? "dry run" : "complete"}.`); +console.log(`Working dir: ${workingDir}`); +console.log(`Secret dir: ${secretDir}`); +if (!noBackup) { + console.log(`Backups: ${dryRun ? "(dry run only)" : backupRoot}`); +} +console.log(""); + +if (touched.length) { + console.log("Changed:"); + for (const item of touched) { + console.log(`- ${item.action}: ${item.filePath} (${item.reason})`); + } +} else { + console.log("Changed: none"); +} + +console.log(""); +console.log("Skipped:"); +for (const item of skipped) { + console.log(`- ${item.filePath} (${item.reason})`); +} + +if (!includeEnvFiles) { + console.log(""); + console.log("Tip: run `npm run secrets:clear:all` to also blank API key/token/secret lines in .env files."); +} + +// --------------------------------------------------------------------------- +// Verification pass. +// +// Clearing is not trustworthy on its own: if the MaskanX app is running while +// this script executes, it can rewrite config.json / providers.json from its +// in-memory state and silently restore the very values we just removed. Re-read +// the files afterwards and fail loudly when anything secret-looking survives, +// so "secrets cleared" can never be reported when it is not true. +// --------------------------------------------------------------------------- + +function collectRemainingSecrets(value, keyPath, found) { + if (Array.isArray(value)) { + value.forEach((item, i) => collectRemainingSecrets(item, `${keyPath}[${i}]`, found)); + return found; + } + if (value && typeof value === "object") { + for (const [key, child] of Object.entries(value)) { + const next = keyPath ? `${keyPath}.${key}` : key; + if (secretNamePattern.test(key) && typeof child === "string" && child.trim()) { + found.push(next); + continue; + } + collectRemainingSecrets(child, next, found); + } + } + return found; +} + +function verifyJsonFile(filePath) { + if (!exists(filePath)) return []; + try { + const parsed = JSON.parse(fs.readFileSync(filePath, "utf8")); + return collectRemainingSecrets(parsed, "", []).map((k) => `${filePath} -> ${k}`); + } catch { + return []; + } +} + +if (!dryRun) { + const leftovers = [ + ...verifyJsonFile(path.join(workingDir, "config.json")), + ...verifyJsonFile(path.join(secretDir, "providers.json")), + ...verifyJsonFile(path.join(secretDir, "envs.json")), + ]; + for (const companyDir of listCompanyDirs()) { + leftovers.push(...verifyJsonFile(path.join(companyDir, "config.json"))); + } + + console.log(""); + if (leftovers.length) { + console.error("VERIFICATION FAILED - secrets are still present:"); + for (const item of leftovers) console.error(`- ${item}`); + console.error(""); + console.error( + "This usually means MaskanX was running and rewrote its config from memory.", + ); + console.error("Stop the app (and any `npm run local`/`npm start`), then re-run."); + process.exitCode = 1; + } else { + console.log("Verification passed: no secret-like values remain."); + } +} diff --git a/scripts/docker_build.sh b/scripts/docker_build.sh new file mode 100644 index 0000000..29d33e1 --- /dev/null +++ b/scripts/docker_build.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Build the independent MaskanX backend image. +# Run from repo root: bash scripts/docker_build.sh [IMAGE_TAG] [EXTRA_ARGS...] +# Example: bash scripts/docker_build.sh maskanx:latest +# bash scripts/docker_build.sh myreg/maskanx:v1 --no-cache +# +# By default the Docker image excludes imessage and discord channels. +# Override via: +# MASKANX_ENABLED_CHANNELS=imessage,discord,dingtalk,feishu,qq,console \ +# bash scripts/docker_build.sh +set -e + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +DOCKERFILE="${DOCKERFILE:-$REPO_ROOT/Dockerfile}" +TAG="${1:-maskanx-backend:latest}" +shift || true + +# Channels to include in the image (default: exclude imessage & discord). +ENABLED_CHANNELS="${MASKANX_ENABLED_CHANNELS:-dingtalk,feishu,qq,console}" + +echo "[docker_build] Building image: $TAG (Dockerfile: $DOCKERFILE)" +docker build -f "$DOCKERFILE" \ + --build-arg MASKANX_ENABLED_CHANNELS="$ENABLED_CHANNELS" \ + -t "$TAG" "$@" . +echo "[docker_build] Done." +echo "[docker_build] MaskanX app port: 8088 (default). Override with -e MASKANX_PORT=." +echo "[docker_build] Run: docker run -p 8088:8088 $TAG" +echo "[docker_build] Or: docker run -e MASKANX_PORT=3000 -p 3000:3000 $TAG" diff --git a/scripts/finalize-branded-image.py b/scripts/finalize-branded-image.py new file mode 100644 index 0000000..6773342 --- /dev/null +++ b/scripts/finalize-branded-image.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python +"""Finalize a generated social image with exact brand formatting. + +This script keeps the AI-generated image and the company logo separate: +generate the base image first, then overlay the real logo here so the brand mark +does not get distorted by the image model. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +try: + from PIL import Image, ImageDraw, ImageOps +except ModuleNotFoundError as exc: # pragma: no cover - friendly CLI error + raise SystemExit( + "Pillow is required. Install it with: " + "python -m pip install Pillow" + ) from exc + + +def _open_rgba(path: Path) -> Image.Image: + return Image.open(path).convert("RGBA") + + +def _fit_cover(img: Image.Image, size: tuple[int, int]) -> Image.Image: + target_w, target_h = size + src_w, src_h = img.size + scale = max(target_w / src_w, target_h / src_h) + resized = img.resize( + (round(src_w * scale), round(src_h * scale)), + Image.Resampling.LANCZOS, + ) + left = (resized.width - target_w) // 2 + top = (resized.height - target_h) // 2 + return resized.crop((left, top, left + target_w, top + target_h)) + + +def _fit_contain(img: Image.Image, size: tuple[int, int]) -> Image.Image: + target_w, target_h = size + src_w, src_h = img.size + scale = min(target_w / src_w, target_h / src_h) + resized = img.resize( + (round(src_w * scale), round(src_h * scale)), + Image.Resampling.LANCZOS, + ) + canvas = Image.new("RGBA", size, (255, 255, 255, 255)) + x = (target_w - resized.width) // 2 + y = (target_h - resized.height) // 2 + canvas.alpha_composite(resized, (x, y)) + return canvas + + +def _parse_size(value: str) -> tuple[int, int]: + try: + w_text, h_text = value.lower().split("x", 1) + width = int(w_text.strip()) + height = int(h_text.strip()) + except Exception as exc: + raise argparse.ArgumentTypeError( + "Size must be formatted like 1200x628" + ) from exc + if width < 300 or height < 300: + raise argparse.ArgumentTypeError("Size must be at least 300x300") + return width, height + + +def finalize_image( + source: Path, + logo: Path, + output: Path, + size: tuple[int, int], + fit: str, + logo_width_pct: float, + margin_pct: float, + plate: bool, +) -> None: + base = _open_rgba(source) + canvas = _fit_cover(base, size) if fit == "cover" else _fit_contain(base, size) + + logo_img = _open_rgba(logo) + max_logo_w = round(size[0] * logo_width_pct / 100) + max_logo_h = round(size[1] * 0.12) + logo_img.thumbnail((max_logo_w, max_logo_h), Image.Resampling.LANCZOS) + + margin = round(size[0] * margin_pct / 100) + plate_pad_x = max(12, round(logo_img.width * 0.24)) + plate_pad_y = max(8, round(logo_img.height * 0.22)) + + logo_x = size[0] - margin - logo_img.width + logo_y = size[1] - margin - logo_img.height + + if plate: + draw = ImageDraw.Draw(canvas) + rect = ( + logo_x - plate_pad_x, + logo_y - plate_pad_y, + logo_x + logo_img.width + plate_pad_x, + logo_y + logo_img.height + plate_pad_y, + ) + radius = max(10, round(min(logo_img.size) * 0.22)) + draw.rounded_rectangle( + rect, + radius=radius, + fill=(255, 255, 255, 225), + outline=(229, 231, 235, 160), + width=1, + ) + + canvas.alpha_composite(logo_img, (logo_x, logo_y)) + + output.parent.mkdir(parents=True, exist_ok=True) + rgb = ImageOps.exif_transpose(canvas).convert("RGB") + rgb.save(output, quality=95, optimize=True) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Resize a generated image and overlay the company logo.", + ) + parser.add_argument("--source", required=True, type=Path) + parser.add_argument("--logo", required=True, type=Path) + parser.add_argument("--out", required=True, type=Path) + parser.add_argument("--size", type=_parse_size, default=(1200, 628)) + parser.add_argument("--fit", choices=("cover", "contain"), default="cover") + parser.add_argument("--logo-width-pct", type=float, default=12.0) + parser.add_argument("--margin-pct", type=float, default=3.0) + parser.add_argument( + "--no-plate", + action="store_true", + help="Do not draw a subtle white plate behind the logo.", + ) + args = parser.parse_args() + + finalize_image( + source=args.source, + logo=args.logo, + output=args.out, + size=args.size, + fit=args.fit, + logo_width_pct=args.logo_width_pct, + margin_pct=args.margin_pct, + plate=not args.no_plate, + ) + print(f"Saved branded image: {args.out}") + + +if __name__ == "__main__": + main() diff --git a/scripts/host_ai_model_selection_e2e.py b/scripts/host_ai_model_selection_e2e.py new file mode 100644 index 0000000..2c89c81 --- /dev/null +++ b/scripts/host_ai_model_selection_e2e.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""Browser E2E for MaskanX AI model selection and chat routing. + +Prerequisites: + - MaskanX app is running and reachable via --base-url. + - The app has an maskanx-host-ai provider with a test key/base URL. + - playwright is installed; run `python -m playwright install chromium` once. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import re +import time +from pathlib import Path + +from playwright.async_api import Page, async_playwright + +DEFAULT_BASE_URL = "http://127.0.0.1:8088" +DEFAULT_OUT = Path("artifacts/host-ai-model-selection-e2e") +LLAMA = "@cf/meta/llama-3.1-8b-instruct-fp8-fast" +QWEN = "@cf/qwen/qwen3-30b-a3b-fp8" +GEMMA = "@cf/google/gemma-4-26b-a4b-it" + +FIND_TEXT_ELEMENT = """ +(text) => { + const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_ELEMENT); + let node; + while ((node = walker.nextNode())) { + const ownText = Array.from(node.childNodes) + .filter((child) => child.nodeType === Node.TEXT_NODE) + .map((child) => child.textContent || "") + .join(""); + if (ownText.includes(text)) { + const rect = node.getBoundingClientRect(); + return { + text: node.innerText, + x: rect.x, + y: rect.y, + width: rect.width, + height: rect.height, + }; + } + } + return null; +} +""" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Verify MaskanX AI model selection and chat routing.", + ) + parser.add_argument("--base-url", default=DEFAULT_BASE_URL) + parser.add_argument("--out", type=Path, default=DEFAULT_OUT) + parser.add_argument("--initial-model", default=LLAMA) + parser.add_argument("--switch-model", default=QWEN) + parser.add_argument("--chat-model", default=GEMMA) + parser.add_argument("--headed", action="store_true") + return parser.parse_args() + + +async def click_text(page: Page, text: str) -> None: + found = await page.evaluate(FIND_TEXT_ELEMENT, text) + if not found: + raise AssertionError(f"Text not found: {text}") + await page.mouse.click( + found["x"] + min(24, max(4, found["width"] / 2)), + found["y"] + found["height"] / 2, + ) + + +async def assert_usage_hint(page: Page) -> str: + body = await page.locator("body").inner_text() + match = re.search( + r"\d+\s*/\s*\d+\s+included MaskanX AI messages left this period\.", + body, + ) + if not match: + raise AssertionError("MaskanX AI included-message balance was not visible") + return match.group(0) + + +async def run() -> dict[str, object]: + args = parse_args() + args.out.mkdir(parents=True, exist_ok=True) + results: dict[str, object] = {} + + async with async_playwright() as pw: + browser = await pw.chromium.launch(headless=not args.headed) + page = await browser.new_page(viewport={"width": 1440, "height": 1400}) + + reset_response = await page.request.put( + f"{args.base_url}/api/models/active", + data={"provider_id": "maskanx-host-ai", "model": args.initial_model}, + ) + if not reset_response.ok: + raise AssertionError( + f"Failed to reset active model: {reset_response.status}", + ) + + await page.goto(f"{args.base_url}/models", wait_until="domcontentloaded") + await page.wait_for_timeout(2500) + initial_text = await page.locator("body").inner_text() + if "MaskanX AI" not in initial_text: + raise AssertionError("MaskanX AI provider is not visible") + if f"Active: maskanx-host-ai / {args.initial_model}" not in initial_text: + raise AssertionError("Initial MaskanX AI model is not active") + if "4 models" not in initial_text: + raise AssertionError("MaskanX AI model catalog count is not visible") + + usage_hint = await assert_usage_hint(page) + provider_cards = await page.locator( + 'div[class*="providerCards"] > div[class*="providerCard"]', + ).all_inner_texts() + if not provider_cards: + raise AssertionError("Provider cards were not rendered") + if "MaskanX AI" not in provider_cards[0]: + raise AssertionError(f"First provider card is not MaskanX AI: {provider_cards[0]}") + results["first_provider_card"] = provider_cards[0].splitlines()[0] + results["usage_hint"] = usage_hint + await page.screenshot(path=str(args.out / "models-before-switch.png"), full_page=True) + + await click_text(page, "Fast default") + await page.wait_for_timeout(500) + await click_text(page, "Balanced reasoning") + await page.wait_for_timeout(500) + switch_text = await page.locator("body").inner_text() + if args.switch_model not in switch_text: + raise AssertionError("Switch model was not selected") + await assert_usage_hint(page) + await page.screenshot(path=str(args.out / "models-switch-selected.png"), full_page=True) + + await click_text(page, "Balanced reasoning") + await page.wait_for_timeout(500) + await click_text(page, "Creative quality") + await page.wait_for_timeout(500) + chat_model_text = await page.locator("body").inner_text() + if args.chat_model not in chat_model_text: + raise AssertionError("Chat model was not selected") + await assert_usage_hint(page) + await page.screenshot(path=str(args.out / "models-chat-model-selected.png"), full_page=True) + + await click_text(page, "Save") + await page.wait_for_timeout(1500) + active_response = await page.request.get(f"{args.base_url}/api/models/active") + active_payload = await active_response.json() + active_llm = active_payload.get("active_llm", {}) + if active_llm.get("provider_id") != "maskanx-host-ai": + raise AssertionError(f"Unexpected active provider: {active_llm}") + if active_llm.get("model") != args.chat_model: + raise AssertionError(f"Unexpected active model: {active_llm}") + results["active_model_after_save"] = active_llm["model"] + + await page.goto(f"{args.base_url}/chat", wait_until="domcontentloaded") + await page.wait_for_timeout(1500) + prompt = "Reply in under 20 words and include E2E_OK." + await page.locator("textarea.maskanx-sender-input").fill(prompt) + started = time.perf_counter() + await page.keyboard.press("Enter") + await page.wait_for_function( + "(model) => document.body.innerText.includes('E2E_OK') && document.body.innerText.includes(model)", + arg=args.chat_model, + timeout=30000, + ) + elapsed_ms = round((time.perf_counter() - started) * 1000) + chat_text = await page.locator("body").inner_text() + if "E2E_OK" not in chat_text or args.chat_model not in chat_text: + raise AssertionError("Chat did not respond through the selected model") + results["chat_elapsed_ms"] = elapsed_ms + await page.screenshot(path=str(args.out / "chat-selected-model-response.png"), full_page=True) + + await browser.close() + + results_path = args.out / "result.json" + results_path.write_text(json.dumps(results, indent=2), encoding="utf-8") + print(json.dumps(results, indent=2)) + return results + + +if __name__ == "__main__": + asyncio.run(run()) diff --git a/scripts/restore-secrets-from-company.cjs b/scripts/restore-secrets-from-company.cjs new file mode 100644 index 0000000..0f73238 --- /dev/null +++ b/scripts/restore-secrets-from-company.cjs @@ -0,0 +1,161 @@ +#!/usr/bin/env node +"use strict"; +/** + * Restore MaskanX credentials from an intact company store. + * + * `clear-secrets.cjs` wipes the active company's stores and the shared secret + * directory, but a per-company copy can survive (for example when only one of + * several companies was active at the time). This script copies the credential + * files from a healthy company back into the active company and the shared + * secret directory. + * + * Credential files are copied wholesale. config.json is patched surgically - + * only MCP env values and enabled flags are taken from the source - so + * company-specific settings are preserved. + * + * Usage: + * node scripts/restore-secrets-from-company.cjs --from default [--to maskanx] + * node scripts/restore-secrets-from-company.cjs --from default --dry-run + */ + +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +const argv = process.argv.slice(2); +const flag = (name, fallback = null) => { + const i = argv.indexOf(`--${name}`); + return i !== -1 && argv[i + 1] ? argv[i + 1] : fallback; +}; +const dryRun = argv.includes("--dry-run"); + +const homeDir = os.homedir(); +const workingDir = path.resolve( + process.env.ADCLAW_WORKING_DIR || path.join(homeDir, ".adclaw"), +); +const secretDir = path.resolve( + process.env.ADCLAW_SECRET_DIR || `${workingDir}.secret`, +); +const companiesDir = path.join(workingDir, "companies"); + +function readJson(file) { + return JSON.parse(fs.readFileSync(file, "utf8")); +} + +function activeCompanyId() { + try { + return readJson(path.join(companiesDir, "index.json")).active_company_id; + } catch { + return null; + } +} + +const from = flag("from"); +const to = flag("to", activeCompanyId()); + +if (!from) { + console.error("Missing --from . Example: --from default"); + process.exit(2); +} +if (!to) { + console.error("Could not determine the target company; pass --to ."); + process.exit(2); +} + +const srcDir = path.join(companiesDir, from); +const dstDir = path.join(companiesDir, to); +if (!fs.existsSync(srcDir)) { + console.error(`Source company not found: ${srcDir}`); + process.exit(2); +} + +const stamp = new Date().toISOString().replace(/[:.]/g, "").replace("T", "-").slice(0, 15); +const backupRoot = path.join(secretDir, "restore-backups", stamp); + +function backup(file) { + if (dryRun || !fs.existsSync(file)) return; + const target = path.join(backupRoot, path.resolve(file).replace(/[:\\/]/g, "_")); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.copyFileSync(file, target); +} + +function copyFile(src, dst, label) { + if (!fs.existsSync(src)) { + console.log(` skip ${label}: source missing`); + return; + } + console.log(` ${dryRun ? "would restore" : "restored"} ${label}`); + if (dryRun) return; + backup(dst); + fs.mkdirSync(path.dirname(dst), { recursive: true }); + fs.copyFileSync(src, dst); +} + +console.log(`Restoring credentials from company "${from}" to "${to}".`); +if (!dryRun) console.log(`Backups: ${backupRoot}`); +console.log(""); + +// ---- credential stores -------------------------------------------------- +for (const name of ["envs.json", "providers.json"]) { + copyFile(path.join(srcDir, name), path.join(dstDir, name), `${to}/${name}`); + copyFile(path.join(srcDir, name), path.join(secretDir, name), `secret-dir/${name}`); +} + +// ---- LinkedIn OAuth ----------------------------------------------------- +// The server reads tokens from ~/.linkedin-mcp (see _linkedin_dir() in +// app/routers/mcp.py), so the home-level copy is the one that actually +// restores authentication. The per-company copy is kept in sync as well. +for (const name of ["tokens_default.json", "users.json"]) { + const src = path.join(srcDir, "linkedin-mcp", name); + copyFile(src, path.join(dstDir, "linkedin-mcp", name), `${to}/linkedin-mcp/${name}`); + copyFile(src, path.join(homeDir, ".linkedin-mcp", name), `~/.linkedin-mcp/${name}`); +} + +// ---- config.json: patch MCP env values and enabled flags only ----------- +function patchConfig(targetFile) { + const srcCfg = path.join(srcDir, "config.json"); + if (!fs.existsSync(srcCfg) || !fs.existsSync(targetFile)) { + console.log(` skip config patch: ${targetFile} or source missing`); + return; + } + const src = readJson(srcCfg); + const dst = readJson(targetFile); + const srcClients = (src.mcp && src.mcp.clients) || {}; + const dstClients = (dst.mcp && dst.mcp.clients) || {}; + + const restoredEnv = []; + const reEnabled = []; + for (const [name, srcClient] of Object.entries(srcClients)) { + const dstClient = dstClients[name]; + if (!dstClient || !srcClient) continue; + + for (const [k, v] of Object.entries(srcClient.env || {})) { + const current = (dstClient.env || {})[k]; + if (typeof v === "string" && v.trim() && (!current || !String(current).trim())) { + dstClient.env = dstClient.env || {}; + dstClient.env[k] = v; + restoredEnv.push(`${name}.${k}`); + } + } + if (srcClient.enabled && !dstClient.enabled) { + dstClient.enabled = true; + reEnabled.push(name); + } + } + + console.log(` config ${path.basename(path.dirname(targetFile))}: ` + + `${restoredEnv.length} env value(s), ${reEnabled.length} client(s) re-enabled`); + for (const e of restoredEnv) console.log(` + ${e}`); + for (const e of reEnabled) console.log(` * enabled ${e}`); + + if (dryRun) return; + backup(targetFile); + fs.writeFileSync(targetFile, `${JSON.stringify(dst, null, 2)}\n`, "utf8"); +} + +patchConfig(path.join(dstDir, "config.json")); +patchConfig(path.join(workingDir, "config.json")); + +console.log(""); +console.log(dryRun ? "Dry run complete; nothing was written." : "Restore complete."); +console.log("Stop MaskanX before running this, then start it again afterwards."); diff --git a/scripts/run-python.cjs b/scripts/run-python.cjs new file mode 100644 index 0000000..05c0aba --- /dev/null +++ b/scripts/run-python.cjs @@ -0,0 +1,51 @@ +#!/usr/bin/env node +const { existsSync } = require("node:fs"); +const { join } = require("node:path"); +const { spawnSync } = require("node:child_process"); + +const isWindows = process.platform === "win32"; +const candidates = []; + +if (process.env.PYTHON) { + candidates.push(process.env.PYTHON); +} + +candidates.push( + isWindows + ? join(process.cwd(), ".venv311", "Scripts", "python.exe") + : join(process.cwd(), ".venv311", "bin", "python"), + isWindows + ? join(process.cwd(), ".venv", "Scripts", "python.exe") + : join(process.cwd(), ".venv", "bin", "python"), + isWindows + ? join(process.cwd(), "..", ".venv311", "Scripts", "python.exe") + : join(process.cwd(), "..", ".venv311", "bin", "python"), + "python3", + "python", + "py", +); + +const args = process.argv.slice(2); + +for (const candidate of candidates) { + const isPath = candidate.includes("/") || candidate.includes("\\"); + if (isPath && !existsSync(candidate)) { + continue; + } + const result = spawnSync(candidate, args, { + stdio: "inherit", + shell: false, + env: process.env, + }); + if (result.error) { + if (result.error.code === "ENOENT") { + continue; + } + console.error(result.error.message); + process.exit(1); + } + process.exit(result.status ?? 0); +} + +console.error("No Python executable found. Set PYTHON or create .venv311."); +process.exit(1); diff --git a/scripts/start-linkedin-oauth.ps1 b/scripts/start-linkedin-oauth.ps1 new file mode 100644 index 0000000..131850b --- /dev/null +++ b/scripts/start-linkedin-oauth.ps1 @@ -0,0 +1,84 @@ +param( + [string]$UserId = "default" +) + +$ErrorActionPreference = "Stop" + +$configPath = Join-Path $env:USERPROFILE ".adclaw\config.json" +if (-not (Test-Path -LiteralPath $configPath)) { + throw "MaskanX config not found at $configPath" +} + +$config = Get-Content -LiteralPath $configPath -Raw | ConvertFrom-Json +$linkedin = $config.mcp.clients.linkedin +if (-not $linkedin) { + throw "LinkedIn MCP config was not found in $configPath" +} + +$env:LINKEDIN_CLIENT_ID = $linkedin.env.LINKEDIN_CLIENT_ID +$env:LINKEDIN_CLIENT_SECRET = $linkedin.env.LINKEDIN_CLIENT_SECRET +$env:LINKEDIN_REDIRECT_URI = "http://localhost:44002/auth/linkedin/callback" + +if (-not $env:LINKEDIN_CLIENT_ID -or -not $env:LINKEDIN_CLIENT_SECRET) { + throw "LINKEDIN_CLIENT_ID or LINKEDIN_CLIENT_SECRET is missing in MaskanX config." +} + +$helper = Get-ChildItem -Path (Join-Path $env:LOCALAPPDATA "npm-cache\_npx") ` + -Recurse ` + -Filter "auth-helper.js" ` + -ErrorAction SilentlyContinue | + Where-Object { $_.FullName -like "*linkedin-mcp-server*bin*" } | + Select-Object -First 1 + +if (-not $helper) { + throw "linkedin-mcp-server auth-helper.js was not found. Start MaskanX once so npx installs linkedin-mcp-server, then retry." +} + +$postingScope = "w_member_social" +$packageRoot = Split-Path -Parent (Split-Path -Parent $helper.FullName) +$authJsPath = Join-Path $packageRoot "dist\utils\linkedin-auth.js" + +if (Test-Path -LiteralPath $authJsPath) { + $authJs = Get-Content -LiteralPath $authJsPath -Raw + if ($authJs -notmatch [regex]::Escape($postingScope)) { + $patchedAuthJs = $authJs -replace "('email')(\s*\r?\n\s*)\]", "`$1,`$2 '$postingScope'`r`n]" + Set-Content -LiteralPath $authJsPath -Value $patchedAuthJs -Encoding UTF8 + Write-Host "Patched LinkedIn MCP OAuth scopes to include $postingScope." -ForegroundColor Green + } +} + +$linkedinDir = Join-Path $env:USERPROFILE ".linkedin-mcp" +$usersPath = Join-Path $linkedinDir "users.json" +if (Test-Path -LiteralPath $usersPath) { + $users = Get-Content -LiteralPath $usersPath -Raw | ConvertFrom-Json + $userEntry = $users.$UserId + if ($userEntry -and ($userEntry.scopes -notcontains $postingScope)) { + $userEntry.scopes = @($userEntry.scopes) + $postingScope + $users | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath $usersPath -Encoding UTF8 + Write-Host "Updated saved LinkedIn user scopes to include $postingScope." -ForegroundColor Green + } +} + +$tokensPath = Join-Path $linkedinDir "tokens_$UserId.json" +if (Test-Path -LiteralPath $tokensPath) { + $backupPath = "$tokensPath.before-posting-scope.bak" + Copy-Item -LiteralPath $tokensPath -Destination $backupPath -Force + Remove-Item -LiteralPath $tokensPath -Force + Write-Host "Cleared old LinkedIn token so OAuth can request posting permission." -ForegroundColor Green +} + +Write-Host "" +Write-Host "LinkedIn OAuth setup starting..." -ForegroundColor Cyan +Write-Host "" +Write-Host "Before continuing, make sure LinkedIn Developer Portal has this exact redirect URL:" -ForegroundColor Yellow +Write-Host "http://localhost:44002/auth/linkedin/callback" +Write-Host "" +Write-Host "A LinkedIn URL will be printed below. If the browser does not open, copy that URL manually." +Write-Host "After approving LinkedIn, wait for this script to finish." +Write-Host "" + +node $helper.FullName setup --user-id $UserId --debug + +Write-Host "" +Write-Host "Checking LinkedIn auth status..." -ForegroundColor Cyan +node $helper.FullName status --user-id $UserId diff --git a/scripts/wheel_build.sh b/scripts/wheel_build.sh new file mode 100644 index 0000000..75eb73a --- /dev/null +++ b/scripts/wheel_build.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Build the independently deployable MaskanX backend package. +# Run from repo root: bash scripts/wheel_build.sh +set -e + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +echo "[wheel_build] Building wheel + sdist..." +python3 -m pip install --quiet build +rm -rf dist/* +python3 -m build --outdir dist . + +echo "[wheel_build] Done. Wheel(s) in: $REPO_ROOT/dist/" diff --git a/src/adclaw/__init__.py b/src/adclaw/__init__.py new file mode 100644 index 0000000..1dcba6a --- /dev/null +++ b/src/adclaw/__init__.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +import logging +import os +import time + +from .utils.logging import setup_logger + +# Fallback before we can safely read canonical constant definitions. +LOG_LEVEL_ENV = "ADCLAW_LOG_LEVEL" + +_bootstrap_err: Exception | None = None +try: + # Load persisted env vars before importing modules that read env-backed + # constants at import time (e.g., WORKING_DIR). + from .envs import load_envs_into_environ + + load_envs_into_environ() +except Exception as exc: + # Best effort: package import should not fail if env bootstrap fails. + _bootstrap_err = exc + +_t0 = time.perf_counter() +setup_logger(os.environ.get(LOG_LEVEL_ENV, "info")) +if _bootstrap_err is not None: + logging.getLogger(__name__).warning( + "adclaw: failed to load persisted envs on init: %s", + _bootstrap_err, + ) +logging.getLogger(__name__).debug( + "%.3fs package init", + time.perf_counter() - _t0, +) diff --git a/src/adclaw/__main__.py b/src/adclaw/__main__.py new file mode 100644 index 0000000..a8936d2 --- /dev/null +++ b/src/adclaw/__main__.py @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +"""Allow running MaskanX via ``python -m adclaw``.""" +from .cli.main import cli + +if __name__ == "__main__": + cli() # pylint: disable=no-value-for-parameter diff --git a/src/adclaw/__version__.py b/src/adclaw/__version__.py new file mode 100644 index 0000000..5733b60 --- /dev/null +++ b/src/adclaw/__version__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +__version__ = "1.0.32" diff --git a/src/adclaw/agents/__init__.py b/src/adclaw/agents/__init__.py new file mode 100644 index 0000000..4ecda2f --- /dev/null +++ b/src/adclaw/agents/__init__.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +"""MaskanX Agents Module. + +This module provides the main agent implementation and supporting utilities +for building AI agents with tools, skills, and memory management. + +Public API: +- MaskanXAgent: Main agent class +- create_model_and_formatter: Factory for creating models and formatters + +Example: + >>> from adclaw.agents import MaskanXAgent, create_model_and_formatter + >>> agent = MaskanXAgent() + >>> # Or with custom model + >>> model, formatter = create_model_and_formatter() +""" + +# MaskanXAgent is lazy-loaded so that importing agents.skills_manager (e.g. +# from CLI init_cmd/skills_cmd) does not pull react_agent, agentscope, tools. +# pylint: disable=undefined-all-variable +__all__ = ["MaskanXAgent", "create_model_and_formatter"] + + +def __getattr__(name: str): + """Lazy load heavy imports.""" + if name == "MaskanXAgent": + from .react_agent import MaskanXAgent + + return MaskanXAgent + if name == "create_model_and_formatter": + from .model_factory import create_model_and_formatter + + return create_model_and_formatter + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/adclaw/agents/command_handler.py b/src/adclaw/agents/command_handler.py new file mode 100644 index 0000000..4ac3e78 --- /dev/null +++ b/src/adclaw/agents/command_handler.py @@ -0,0 +1,220 @@ +# -*- coding: utf-8 -*- +"""Agent command handler for system commands. + +This module handles system commands like /compact, /new, /clear, etc. +""" +import logging +from typing import TYPE_CHECKING, Any + +from agentscope.agent._react_agent import _MemoryMark +from agentscope.message import Msg, TextBlock + +if TYPE_CHECKING: + from .memory import MemoryManager + MaskanXInMemoryMemory = Any + +logger = logging.getLogger(__name__) + + +class CommandHandler: + """Handler for agent system commands.""" + + # Supported system commands + SYSTEM_COMMANDS = frozenset( + { + "compact", + "new", + "clear", + "history", + "compact_str", + "await_summary", + }, + ) + + def __init__( + self, + agent_name: str, + memory: "MaskanXInMemoryMemory", + memory_manager: "MemoryManager | None" = None, + enable_memory_manager: bool = True, + ): + """Initialize command handler. + + Args: + agent_name: Name of the agent for message creation + memory: Agent's MaskanXInMemoryMemory instance + memory_manager: Optional memory manager instance + enable_memory_manager: Whether memory manager is enabled + """ + self.agent_name = agent_name + self.memory = memory + self.memory_manager = memory_manager + self._enable_memory_manager = enable_memory_manager + + def is_command(self, query: str | None) -> bool: + """Check if the query is a system command. + + Args: + query: User query string + + Returns: + True if query is a system command + """ + if not isinstance(query, str) or not query.startswith("/"): + return False + return query.strip().lstrip("/") in self.SYSTEM_COMMANDS + + async def _make_system_msg(self, text: str) -> Msg: + """Create a system response message. + + Args: + text: Message text content + + Returns: + System message + """ + return Msg( + name=self.agent_name, + role="assistant", + content=[TextBlock(type="text", text=text)], + ) + + def _has_memory_manager(self) -> bool: + """Check if memory manager is available.""" + return self._enable_memory_manager and self.memory_manager is not None + + async def _process_compact(self, messages: list[Msg]) -> Msg: + """Process /compact command.""" + if not messages: + return await self._make_system_msg( + "**No messages to compact.**\n\n" + "- Current memory is empty\n" + "- No action taken", + ) + if not self._has_memory_manager(): + return await self._make_system_msg( + "**Memory Manager Disabled**\n\n" + "- Memory compaction is not available\n" + "- Enable memory manager to use this feature", + ) + + self.memory_manager.add_async_summary_task(messages=messages) + compact_content = await self.memory_manager.compact_memory( + messages=messages, + previous_summary=self.memory.get_compressed_summary(), + ) + await self.memory.update_compressed_summary(compact_content) + updated_count = await self.memory.mark_messages_compressed(messages) + logger.info( + f"Marked {updated_count} messages as compacted " + f"with:\n{compact_content}", + ) + return await self._make_system_msg( + f"**Compact Complete!**\n\n" + f"- Messages compacted: {updated_count}\n" + f"**Compressed Summary:**\n{compact_content}\n" + f"- Summary task started in background\n", + ) + + async def _process_new(self, messages: list[Msg]) -> Msg: + """Process /new command.""" + if not messages: + self.memory.clear_compressed_summary() + return await self._make_system_msg( + "**No messages to summarize.**\n\n" + "- Current memory is empty\n" + "- Compressed summary is clear\n" + "- No action taken", + ) + if not self._has_memory_manager(): + return await self._make_system_msg( + "**Memory Manager Disabled**\n\n" + "- Cannot start new conversation with summary\n" + "- Enable memory manager to use this feature", + ) + + self.memory_manager.add_async_summary_task(messages=messages) + self.memory.clear_compressed_summary() + updated_count = await self.memory.mark_messages_compressed(messages) + logger.info(f"Marked {updated_count} messages as compacted") + return await self._make_system_msg( + "**New Conversation Started!**\n\n" + "- Summary task started in background\n" + "- Ready for new conversation", + ) + + async def _process_clear(self, _messages: list[Msg]) -> Msg: + """Process /clear command.""" + self.memory.clear_content() + self.memory.clear_compressed_summary() + return await self._make_system_msg( + "**History Cleared!**\n\n" + "- Compressed summary reset\n" + "- Memory is now empty", + ) + + async def _process_compact_str(self, _messages: list[Msg]) -> Msg: + """Process /compact_str command to show compressed summary.""" + summary = self.memory.get_compressed_summary() + if not summary: + return await self._make_system_msg( + "**No Compressed Summary**\n\n" + "- No summary has been generated yet\n" + "- Use /compact or wait for auto-compaction", + ) + return await self._make_system_msg( + f"**Compressed Summary**\n\n{summary}", + ) + + async def _process_history(self, _messages: list[Msg]) -> Msg: + """Process /history command.""" + history_str = await self.memory.get_history_str() + return await self._make_system_msg(history_str) + + async def _process_await_summary(self, _messages: list[Msg]) -> Msg: + """Process /await_summary command to wait for all summary tasks.""" + if not self._has_memory_manager(): + return await self._make_system_msg( + "**Memory Manager Disabled**\n\n" + "- Cannot await summary tasks\n" + "- Enable memory manager to use this feature", + ) + + task_count = len(self.memory_manager.summary_tasks) + if task_count == 0: + return await self._make_system_msg( + "**No Summary Tasks**\n\n" + "- No pending summary tasks to wait for", + ) + + result = await self.memory_manager.await_summary_tasks() + return await self._make_system_msg( + f"**Summary Tasks Complete**\n\n" + f"- Waited for {task_count} summary task(s)\n" + f"- {result}" + f"- All tasks have finished", + ) + + async def handle_command(self, query: str) -> Msg: + """Process system commands. + + Args: + query: Command string (e.g., "/compact", "/new") + + Returns: + System response message + + Raises: + RuntimeError: If command is not recognized + """ + messages = await self.memory.get_memory( + exclude_mark=_MemoryMark.COMPRESSED, + prepend_summary=False, + ) + command = query.strip().lstrip("/") + logger.info(f"Processing command: {command}") + + handler = getattr(self, f"_process_{command}", None) + if handler is None: + raise RuntimeError(f"Unknown command: {query}") + return await handler(messages) diff --git a/src/adclaw/agents/coordinator/__init__.py b/src/adclaw/agents/coordinator/__init__.py new file mode 100644 index 0000000..ee9f68d --- /dev/null +++ b/src/adclaw/agents/coordinator/__init__.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +"""Coordinator Persona — Synthesis-Driven Orchestration (2A).""" + +from .models import NextStep, PersonaOutcome, TaskStrategy +from .synthesis import ( + SYNTHESIS_SYSTEM_PROMPT, + run_synthesis_cycle, + validate_synthesis, +) +from .cron_handler import coordinator_cron_tick + +__all__ = [ + "NextStep", + "PersonaOutcome", + "TaskStrategy", + "SYNTHESIS_SYSTEM_PROMPT", + "coordinator_cron_tick", + "run_synthesis_cycle", + "validate_synthesis", +] diff --git a/src/adclaw/agents/coordinator/cron_handler.py b/src/adclaw/agents/coordinator/cron_handler.py new file mode 100644 index 0000000..11d236b --- /dev/null +++ b/src/adclaw/agents/coordinator/cron_handler.py @@ -0,0 +1,167 @@ +# -*- coding: utf-8 -*- +"""Coordinator cron handler — integrates with the persona cron system.""" + +from __future__ import annotations + +import json +import logging +from typing import Optional + +from .models import PersonaOutcome, TaskStrategy +from .synthesis import run_synthesis_cycle +from ..persona_manager import PersonaManager +from ..tools.delegation_executor import DELEGATION_FAILED_PREFIX, execute_delegation + +logger = logging.getLogger(__name__) + + +async def coordinator_cron_tick( + persona_manager: PersonaManager, + aom_manager, + chat_model, +) -> str: + """Execute one coordinator cron cycle. + + Called by the cron system when the coordinator's schedule fires. + + Returns: + Human-readable summary of what the coordinator decided. + """ + coordinator = persona_manager.get_coordinator() + if coordinator is None: + return "No coordinator persona configured." + + # Load active strategy from AOM + active_strategy = await _load_active_strategy(aom_manager) + + # Run synthesis + strategy = await run_synthesis_cycle( + aom_manager=aom_manager, + persona_manager=persona_manager, + chat_model=chat_model, + active_strategy=active_strategy, + ) + + # Check for abandonment + if strategy.should_abandon(): + strategy.status = "abandoned" + logger.warning( + "Strategy '%s' abandoned after %d pivots", + strategy.id, + strategy.pivot_count, + ) + await aom_manager.ingest_agent.ingest( + content=strategy.model_dump_json(), + source_type="manual", + source_id=strategy.id, + metadata={"coordinator_strategy": True, "strategy_id": strategy.id}, + ) + return f"Strategy abandoned: {strategy.goal} (too many pivots)" + + # Execute next steps, tracking which ones were actually executed + results = [] + executed_indices: set[int] = set() + for idx, step in enumerate(strategy.next_steps): + persona = persona_manager.get_persona(step.persona_id) + if persona is None: + logger.warning("Unknown persona '%s' in next_steps", step.persona_id) + executed_indices.add(idx) # remove invalid steps + continue + + if step.depends_on: + # Check if dependency is met + dep_outcomes = [ + o + for o in strategy.outcomes + if o.persona_id == step.depends_on and o.status == "success" + ] + if not dep_outcomes: + logger.debug( + "Skipping step for %s — waiting on %s", + step.persona_id, + step.depends_on, + ) + continue # NOT marked as executed — preserve for next cycle + + executed_indices.add(idx) + + logger.info( + "Coordinator delegating to @%s: %s", + step.persona_id, + step.task[:120], + ) + # execute_delegation is a coroutine function: await it directly. + # Passing it to run_in_executor would return an un-awaited coroutine + # and the delegated task would never run. + result = await execute_delegation(persona, step.task, persona_manager) + + outcome_status = ( + "failed" if result.startswith(DELEGATION_FAILED_PREFIX) else "success" + ) + strategy.add_outcome( + PersonaOutcome( + persona_id=step.persona_id, + task_given=step.task, + status=outcome_status, + key_findings=[result[:500]], + ) + ) + # Track pivots: if persona is stuck, increment pivot_count + if outcome_status == "failed" and strategy.should_pivot(step.persona_id): + strategy.pivot_count += 1 + logger.info( + "Pivot #%d: persona @%s stuck", + strategy.pivot_count, + step.persona_id, + ) + results.append(f"@{step.persona_id}: {outcome_status}") + + # Remove executed/invalid steps; preserve deferred steps with unmet depends_on + strategy.next_steps = [ + s for idx, s in enumerate(strategy.next_steps) if idx not in executed_indices + ] + + # Persist updated strategy + await aom_manager.ingest_agent.ingest( + content=strategy.model_dump_json(), + source_type="manual", + source_id=strategy.id, + metadata={"coordinator_strategy": True, "strategy_id": strategy.id}, + ) + + summary = ( + f"Strategy: {strategy.goal}\n" + f"Synthesis: {strategy.synthesis[:300]}\n" + f"Delegations: {'; '.join(results) if results else 'none (waiting)'}" + ) + return summary + + +async def _load_active_strategy(aom_manager) -> Optional[TaskStrategy]: + """Load the most recent active strategy from AOM. + + Iterates all citations and returns the active strategy with the + latest updated_at timestamp, not just the first one found. + """ + try: + result = await aom_manager.query_agent.query( + "coordinator strategy active", + skip_synthesis=True, + ) + candidates: list[TaskStrategy] = [] + for citation in result.citations: + mem = citation.memory + if mem.metadata.get("coordinator_strategy"): + try: + data = json.loads(mem.content) + strategy = TaskStrategy(**data) + if strategy.status == "active": + candidates.append(strategy) + except (json.JSONDecodeError, ValueError): + continue + if candidates: + # Return the newest strategy by updated_at timestamp + return max(candidates, key=lambda s: s.updated_at) + except Exception as exc: + logger.warning("Failed to load active strategy: %s", exc) + return None diff --git a/src/adclaw/agents/coordinator/models.py b/src/adclaw/agents/coordinator/models.py new file mode 100644 index 0000000..d7e7bc5 --- /dev/null +++ b/src/adclaw/agents/coordinator/models.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +"""Coordinator data models — TaskStrategy, PersonaOutcome, NextStep.""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from typing import Literal, Optional + +from pydantic import BaseModel, Field + + +class PersonaOutcome(BaseModel): + """Summary of a single persona execution within a strategy.""" + + persona_id: str + task_given: str + status: Literal["success", "partial", "failed", "stuck", "pending"] = "pending" + key_findings: list[str] = Field(default_factory=list) + failures: list[str] = Field(default_factory=list) + iteration: int = 0 + timestamp: str = Field( + default_factory=lambda: datetime.now(timezone.utc).isoformat() + ) + + +class NextStep(BaseModel): + """A specific next action the coordinator has decided on.""" + + persona_id: str + task: str + rationale: str + priority: int = 1 # 1 = highest + depends_on: Optional[str] = None # forward-looking: persona_id that must finish first + + +class TaskStrategy(BaseModel): + """A multi-step strategy the coordinator maintains across cron cycles. + + Stored in AOM with source_type='manual' and metadata + {"coordinator_strategy": True}. Memory.source_type is a + Literal enum ("mcp_tool"|"skill"|"chat"|"file_inbox"|"manual"), + so custom source types are not supported — use metadata instead. + """ + + id: str = Field(default_factory=lambda: str(uuid.uuid4())) + goal: str + status: Literal["active", "completed", "abandoned"] = "active" + created_at: str = Field( + default_factory=lambda: datetime.now(timezone.utc).isoformat() + ) + updated_at: str = Field( + default_factory=lambda: datetime.now(timezone.utc).isoformat() + ) + + outcomes: list[PersonaOutcome] = Field(default_factory=list) + next_steps: list[NextStep] = Field(default_factory=list) + synthesis: str = "" # coordinator's analysis of the current state + + # Pivot tracking + pivot_count: int = 0 + max_pivots: int = 3 # abandon after N pivots on same goal + + def add_outcome(self, outcome: PersonaOutcome) -> None: + self.outcomes.append(outcome) + self.updated_at = datetime.now(timezone.utc).isoformat() + + def should_pivot(self, persona_id: str) -> bool: + """Check if a persona has been stuck/failed enough to warrant a pivot.""" + recent = [ + o + for o in self.outcomes + if o.persona_id == persona_id and o.status in ("failed", "stuck") + ] + return len(recent) >= 2 + + def should_abandon(self) -> bool: + return self.pivot_count >= self.max_pivots diff --git a/src/adclaw/agents/coordinator/synthesis.py b/src/adclaw/agents/coordinator/synthesis.py new file mode 100644 index 0000000..8df0660 --- /dev/null +++ b/src/adclaw/agents/coordinator/synthesis.py @@ -0,0 +1,269 @@ +# -*- coding: utf-8 -*- +"""Coordinator Synthesis Engine — reads AOM, builds context, produces strategy updates.""" + +from __future__ import annotations + +import json +import logging +import re +from typing import Optional + +from .models import TaskStrategy + +logger = logging.getLogger(__name__) + +# Forbidden phrases in coordinator output — forces specificity +FORBIDDEN_PHRASES = [ + "based on results", + "as appropriate", + "optimize further", + "continue as needed", + "if applicable", + "when possible", +] + +SYNTHESIS_SYSTEM_PROMPT = """\ +You are the Coordinator for a team of specialist personas. +Your job is SYNTHESIS — not execution. + +## Rules +1. NEVER say "based on results" or "optimize further" — be SPECIFIC. + BAD: "Based on SEO results, optimize the content." + GOOD: "The SEO audit found 3 pages with missing H1 tags (/, /pricing, /blog). + @content-writer: rewrite the H1 for each page targeting these keywords: [list]." + +2. For each persona outcome, state: + - What specifically succeeded or failed + - Why it matters for the overall goal + - What EXACT next action to take (persona, task, expected output) + +3. If a persona is stuck (same error twice), PIVOT: + - Try a different persona for the same subtask + - Or break the subtask into smaller pieces + - Or abandon and explain why + +4. Output valid JSON matching the TaskStrategy schema. + +## Team +{team_summary} + +## Current Strategy +{current_strategy} + +## Recent Activity (from AOM) +{recent_activity} + +## Your Task +Analyze the above and produce an updated TaskStrategy JSON with: +- Updated synthesis (your analysis) +- Updated outcomes (mark completed/failed) +- New next_steps (specific tasks for specific personas) +""" + + +def validate_synthesis(synthesis: str) -> list[str]: + """Check coordinator output for forbidden vague phrases.""" + violations = [] + lower = synthesis.lower() + for phrase in FORBIDDEN_PHRASES: + if phrase in lower: + violations.append(f"Forbidden phrase: '{phrase}'") + return violations + + +async def run_synthesis_cycle( + aom_manager, + persona_manager, + chat_model, + active_strategy: Optional[TaskStrategy] = None, +) -> TaskStrategy: + """Run one coordinator synthesis cycle. + + 1. Query AOM for recent persona activity + 2. Build synthesis prompt + 3. Call LLM for analysis + 4. Parse strategy from response + 5. Validate synthesis quality + + NOTE: Does NOT persist to AOM — caller is responsible for persistence. + + Args: + aom_manager: AOM manager with query_agent and ingest_agent + persona_manager: PersonaManager with team info + chat_model: LLM model for synthesis + active_strategy: Current strategy to update, or None to create new + + Returns: + Updated or new TaskStrategy + """ + # 1. Query AOM for recent persona activity + # skip_synthesis=True: we do our own synthesis, no need for AOM's LLM pass + query_result = await aom_manager.query_agent.query( + "Recent persona execution results, tool outputs, and task completions " + "from the last 2 hours", + skip_synthesis=True, + ) + + activity_parts = [] + for citation in query_result.citations: + mem = citation.memory + activity_parts.append( + f"[{mem.source_type}:{mem.source_id}] " + f"({mem.created_at})\n{mem.content}" + ) + recent_activity = ( + "\n\n".join(activity_parts) if activity_parts + else "(No recent persona activity found in AOM)" + ) + + # 2. Build synthesis prompt + team_summary = persona_manager.get_team_summary() + strategy_json = ( + active_strategy.model_dump_json(indent=2) + if active_strategy + else '{"status": "new", "goal": "Determine goal from recent activity"}' + ) + + prompt = SYNTHESIS_SYSTEM_PROMPT.format( + team_summary=team_summary, + current_strategy=strategy_json, + recent_activity=recent_activity, + ) + + # 3. Call LLM + # Use dicts instead of Msg objects (AgentScope OpenAIChatModel expects list[dict]) + # Await because model.__call__ is async + import inspect + + raw_response = chat_model( + [ + {"role": "system", "content": prompt}, + {"role": "user", "content": "Analyze the recent activity and produce an updated TaskStrategy."}, + ] + ) + # Handle both sync and async models + if inspect.isawaitable(raw_response): + response = await raw_response + else: + response = raw_response + + # Extract text from response (handle ChatResponse.content list format) + if hasattr(response, "content"): + content = response.content + if isinstance(content, list): + response_text = "".join( + item.get("text", "") if isinstance(item, dict) else str(item) + for item in content + ) + else: + response_text = str(content) + else: + response_text = str(response) + + # 4. Parse strategy from response + strategy = _parse_strategy_from_response(response_text, active_strategy) + + # Preserve pivot_count from active strategy (LLM doesn't track this) + if active_strategy and strategy.pivot_count == 0: + strategy.pivot_count = active_strategy.pivot_count + + # 5. Validate synthesis quality + violations = validate_synthesis(strategy.synthesis) + if violations: + logger.warning( + "Coordinator synthesis has %d quality violations: %s", + len(violations), + violations, + ) + + # NOTE: Strategy is NOT persisted here — the caller (cron_handler) + # persists after delegations are executed, avoiding duplicate entries. + + return strategy + + +_STATUS_MAP = {"in_progress": "active", "completed": "completed", "done": "completed"} +_OUTCOME_STATUS_MAP = { + "completed": "success", "done": "success", "needs_revision": "partial", + "in_progress": "pending", "blocked": "stuck", +} + + +def _normalize_strategy_json(data: dict) -> dict: + """Normalize common LLM deviations from our Pydantic schema.""" + # Status aliases + if data.get("status") in _STATUS_MAP: + data["status"] = _STATUS_MAP[data["status"]] + + # synthesis: LLM sometimes returns dict instead of string + if isinstance(data.get("synthesis"), dict): + data["synthesis"] = json.dumps(data["synthesis"]) + + # Normalize outcomes + for outcome in data.get("outcomes", []): + # "persona" → "persona_id" + if "persona" in outcome and "persona_id" not in outcome: + outcome["persona_id"] = str(outcome.pop("persona") or "").lstrip("@") + # "task" → "task_given" + if "task" in outcome and "task_given" not in outcome: + outcome["task_given"] = outcome.pop("task") + # Status aliases + if outcome.get("status") in _OUTCOME_STATUS_MAP: + outcome["status"] = _OUTCOME_STATUS_MAP[outcome["status"]] + # "findings" → "key_findings" + if "findings" in outcome and "key_findings" not in outcome: + outcome["key_findings"] = outcome.pop("findings") + if isinstance(outcome["key_findings"], str): + outcome["key_findings"] = [outcome["key_findings"]] + + # Normalize next_steps + for step in data.get("next_steps", []): + # "persona" → "persona_id" + if "persona" in step and "persona_id" not in step: + step["persona_id"] = str(step.pop("persona") or "").lstrip("@") + # "reason" → "rationale" + if "reason" in step and "rationale" not in step: + step["rationale"] = step.pop("reason") + # Missing rationale + if "rationale" not in step: + step["rationale"] = step.get("task", "No rationale provided") + # priority: "high"/"medium"/"low" → int + p = step.get("priority") + if isinstance(p, str): + step["priority"] = {"high": 1, "medium": 2, "low": 3}.get(p.lower(), 2) + + return data + + +def _parse_strategy_from_response( + response_text: str, + fallback: Optional[TaskStrategy] = None, +) -> TaskStrategy: + """Extract TaskStrategy JSON from LLM response text.""" + # Try to find JSON block in response + json_match = re.search(r"```json\s*(.*?)\s*```", response_text, re.DOTALL) + if json_match: + raw = json_match.group(1) + else: + # Try raw JSON parse + raw = response_text.strip() + + try: + data = json.loads(raw) + # Normalize common LLM schema deviations + data = _normalize_strategy_json(data) + return TaskStrategy(**data) + except (json.JSONDecodeError, ValueError) as exc: + logger.warning("Failed to parse strategy JSON: %s", exc) + if fallback: + # Don't mutate the original fallback — create a copy with error info + error_strategy = fallback.model_copy( + update={ + "synthesis": f"[Parse error — raw LLM output]\n{response_text[:2000]}" + } + ) + return error_strategy + return TaskStrategy( + goal="Unable to determine", + synthesis=f"[Parse error]\n{response_text[:2000]}", + ) diff --git a/src/adclaw/agents/hooks/__init__.py b/src/adclaw/agents/hooks/__init__.py new file mode 100644 index 0000000..818171c --- /dev/null +++ b/src/adclaw/agents/hooks/__init__.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +"""Agent hooks package. + +This package provides hook implementations for MaskanXAgent that follow +AgentScope's hook interface (any Callable). + +Available Hooks: + - BootstrapHook: First-time setup guidance + - MemoryCompactionHook: Automatic context window management + +Example: + >>> from adclaw.agents.hooks import BootstrapHook, MemoryCompactionHook + >>> from pathlib import Path + >>> + >>> # Create hooks (they are callables following AgentScope's interface) + >>> bootstrap = BootstrapHook(Path("~/.MaskanX"), language="zh") + >>> memory_compact = MemoryCompactionHook( + ... memory_manager=mm, + ... memory_compact_threshold=100000, + ... ) + >>> + >>> # Register with agent using AgentScope's register_instance_hook + >>> agent.register_instance_hook("pre_reasoning", "bootstrap", bootstrap) + >>> agent.register_instance_hook( + ... "pre_reasoning", "compact", memory_compact + ... ) +""" + +from .bootstrap import BootstrapHook +from .memory_compaction import MemoryCompactionHook +from .aom_capture import AOMCaptureHook + +__all__ = [ + "AOMCaptureHook", + "BootstrapHook", + "MemoryCompactionHook", +] diff --git a/src/adclaw/agents/hooks/aom_capture.py b/src/adclaw/agents/hooks/aom_capture.py new file mode 100644 index 0000000..0d21df7 --- /dev/null +++ b/src/adclaw/agents/hooks/aom_capture.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +"""AOM Capture Hook — auto-captures MCP tool and skill results into AOM.""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any, Optional + +if TYPE_CHECKING: + from ...memory_agent.ingest import IngestAgent + +logger = logging.getLogger(__name__) + +# Minimum result length worth capturing +_MIN_CONTENT_LENGTH = 20 +# Maximum result length to capture (truncate longer) +_MAX_CONTENT_LENGTH = 10_000 + + +class AOMCaptureHook: + """Post-tool-execution hook that auto-ingests results into AOM. + + Usage: + Called after tool execution in MaskanXAgent._acting() to + capture MCP tool and skill results. + """ + + def __init__( + self, + ingest_agent: IngestAgent, + capture_mcp: bool = True, + capture_skills: bool = True, + ) -> None: + self.ingest_agent = ingest_agent + self.capture_mcp = capture_mcp + self.capture_skills = capture_skills + + async def on_tool_result( + self, + tool_name: str, + result: str, + source_type: Optional[str] = None, + ) -> None: + """Capture a tool execution result into AOM. + + Args: + tool_name: Name of the tool that produced the result. + result: The string result from tool execution. + source_type: Override source type (default: auto-detect). + """ + if not result or len(result.strip()) < _MIN_CONTENT_LENGTH: + return + + # Auto-detect source type + if source_type is None: + source_type = "mcp_tool" + + # Truncate if needed + content = result[:_MAX_CONTENT_LENGTH] + + try: + await self.ingest_agent.ingest( + content=content, + source_type=source_type, + source_id=tool_name, + skip_llm=False, + ) + logger.debug("AOM captured result from %s (%d chars)", tool_name, len(content)) + except Exception as exc: + logger.warning("AOM capture failed for %s: %s", tool_name, exc) diff --git a/src/adclaw/agents/hooks/bootstrap.py b/src/adclaw/agents/hooks/bootstrap.py new file mode 100644 index 0000000..6ee354d --- /dev/null +++ b/src/adclaw/agents/hooks/bootstrap.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +"""Bootstrap hook for first-time user interaction guidance. + +This hook checks for BOOTSTRAP.md on the first user interaction and +prepends guidance to help set up the agent's identity and preferences. +""" +import logging +from pathlib import Path +from typing import Any + +from ..prompt import build_bootstrap_guidance +from ..utils import ( + is_first_user_interaction, + prepend_to_message_content, +) + +logger = logging.getLogger(__name__) + + +class BootstrapHook: + """Hook for bootstrap guidance on first user interaction. + + This hook looks for a BOOTSTRAP.md file in the working directory + and if found, prepends guidance to the first user message to help + establish the agent's identity and user preferences. + """ + + def __init__( + self, + working_dir: Path, + language: str = "zh", + ): + """Initialize bootstrap hook. + + Args: + working_dir: Working directory containing BOOTSTRAP.md + language: Language code for bootstrap guidance (en/zh) + """ + self.working_dir = working_dir + self.language = language + + async def __call__( + self, + agent, + kwargs: dict[str, Any], + ) -> dict[str, Any] | None: + """Check and load BOOTSTRAP.md on first user interaction. + + Args: + agent: The agent instance + kwargs: Input arguments to the _reasoning method + + Returns: + None (hook doesn't modify kwargs) + """ + try: + bootstrap_path = self.working_dir / "BOOTSTRAP.md" + bootstrap_completed_flag = ( + self.working_dir / ".bootstrap_completed" + ) + + # Check if bootstrap has already been triggered before + if bootstrap_completed_flag.exists(): + return None + + if not bootstrap_path.exists(): + return None + + messages = await agent.memory.get_memory() + if not is_first_user_interaction(messages): + return None + + bootstrap_guidance = build_bootstrap_guidance( + self.language, + ) + + logger.debug( + "Found BOOTSTRAP.md [%s], prepending guidance", + self.language, + ) + + system_prompt_count = sum( + 1 for msg in messages if msg.role == "system" + ) + for msg in messages[system_prompt_count:]: + if msg.role == "user": + prepend_to_message_content(msg, bootstrap_guidance) + break + + logger.debug("Bootstrap guidance prepended to first user message") + + # Create completion flag to prevent repeated triggering + bootstrap_completed_flag.touch() + logger.debug("Created bootstrap completion flag") + + except Exception as e: + logger.error( + "Failed to process bootstrap: %s", + e, + exc_info=True, + ) + + return None diff --git a/src/adclaw/agents/hooks/memory_compaction.py b/src/adclaw/agents/hooks/memory_compaction.py new file mode 100644 index 0000000..b74f260 --- /dev/null +++ b/src/adclaw/agents/hooks/memory_compaction.py @@ -0,0 +1,304 @@ +# -*- coding: utf-8 -*- +"""Memory compaction hook for managing context window. + +This hook monitors token usage and automatically compacts older messages +when the context window approaches its limit, preserving recent messages +and the system prompt. +""" +import logging +import os +from typing import TYPE_CHECKING, Any + +from agentscope.agent._react_agent import _MemoryMark + +from ..memory.tiered_compaction import plan_compaction +from ..memory.topic_summarizer import ( + build_structured_summary_prompt, + cluster_by_topic, +) +from ..utils import ( + check_valid_messages, + safe_count_message_tokens, + safe_count_str_tokens, +) +from ...memory_agent.compressor import pre_compress + +if TYPE_CHECKING: + from ..memory import MemoryManager + +logger = logging.getLogger(__name__) + + +class MemoryCompactionHook: + """Hook for automatic memory compaction when context is full. + + This hook monitors the token count of messages and triggers compaction + when it exceeds the threshold. It preserves the system prompt and recent + messages while summarizing older conversation history. + """ + + def __init__( + self, + memory_manager: "MemoryManager", + memory_compact_threshold: int, + keep_recent: int = 10, + ): + """Initialize memory compaction hook. + + Args: + memory_manager: Memory manager instance for compaction + memory_compact_threshold: Token count threshold for compaction + keep_recent: Number of recent messages to preserve + """ + self.memory_manager = memory_manager + self.memory_compact_threshold = memory_compact_threshold + self.keep_recent = keep_recent + self._compaction_cycle: int = 0 + self._cycle_counts: dict[str, int] = {} # msg.id → first-seen cycle + + @property + def enable_truncate_tool_result_texts(self) -> bool: + """Whether to truncate tool result texts. + + Controlled by environment variable ENABLE_TRUNCATE_TOOL_RESULT_TEXTS. + Default is False (disabled). + """ + return os.environ.get( + "ENABLE_TRUNCATE_TOOL_RESULT_TEXTS", + "false", + ).lower() in ("true", "1", "yes") + + @property + def compact_batch_messages(self) -> int: + """Maximum number of old messages to summarize in one ReMe task.""" + raw_value = os.environ.get("MASKANX_MEMORY_COMPACT_BATCH_MESSAGES", "80") + try: + value = int(raw_value) + except ValueError: + logger.warning( + "Invalid MASKANX_MEMORY_COMPACT_BATCH_MESSAGES=%r; using 80", + raw_value, + ) + return 80 + return max(value, 1) + + async def __call__( + self, + agent, + kwargs: dict[str, Any], + ) -> dict[str, Any] | None: + """Pre-reasoning hook to check and compact memory if needed. + + This hook extracts system prompt messages and recent messages, + builds an estimated full context prompt, and triggers compaction + when the total estimated token count exceeds the threshold. + + Memory structure: + [System Prompt (preserved)] + [Compactable (counted)] + + [Recent (preserved)] + + Args: + agent: The agent instance + kwargs: Input arguments to the _reasoning method + + Returns: + None (hook doesn't modify kwargs) + """ + try: + messages = await agent.memory.get_memory( + exclude_mark=_MemoryMark.COMPRESSED, + prepend_summary=False, + ) + + logger.debug(f"===last message===: {messages[-1]}") + + system_prompt_messages = [] + for msg in messages: + if msg.role == "system": + system_prompt_messages.append(msg) + else: + break + + remaining_messages = messages[len(system_prompt_messages) :] + + if len(remaining_messages) <= self.keep_recent: + return None + + keep_length = self.keep_recent + while keep_length > 0 and not check_valid_messages( + remaining_messages[-keep_length:], + ): + keep_length -= 1 + + if keep_length > 0: + messages_to_compact = remaining_messages[:-keep_length] + messages_to_keep = remaining_messages[-keep_length:] + else: + messages_to_compact = remaining_messages + messages_to_keep = [] + + messages_for_estimate = [ + *system_prompt_messages, + *messages_to_compact, + *messages_to_keep, + ] + previous_summary = agent.memory.get_compressed_summary() + full_prompt = await agent.formatter.format( + msgs=messages_for_estimate, + ) + estimated_message_tokens = await safe_count_message_tokens( + full_prompt, + ) + summary_tokens = safe_count_str_tokens(previous_summary) + estimated_total_tokens = estimated_message_tokens + summary_tokens + logger.debug( + "Estimated context tokens total=%d " + "(messages=%d, summary=%d, summary_prepended=%s, " + "system_prompt_msgs=%d, " + "compactable_msgs=%d, keep_recent_msgs=%d) vs threshold=%d", + estimated_total_tokens, + estimated_message_tokens, + summary_tokens, + bool(previous_summary), + len(system_prompt_messages), + len(messages_to_compact), + len(messages_to_keep), + self.memory_compact_threshold, + ) + + if estimated_total_tokens > self.memory_compact_threshold: + logger.info( + "Memory compaction triggered: estimated total %d tokens " + "(messages: %d, summary: %d, threshold: %d), " + "system_prompt_msgs: %d, " + "compactable_msgs: %d, keep_recent_msgs: %d", + estimated_total_tokens, + estimated_message_tokens, + summary_tokens, + self.memory_compact_threshold, + len(system_prompt_messages), + len(messages_to_compact), + len(messages_to_keep), + ) + + # Track when messages were first seen + for msg in messages_to_compact: + if msg.id not in self._cycle_counts: + self._cycle_counts[msg.id] = self._compaction_cycle + + # Plan what to compact based on importance tiers + compaction_plan = plan_compaction( + messages=messages_to_compact, + cycle_counts=self._cycle_counts, + current_cycle=self._compaction_cycle, + ) + + # Prune stale _cycle_counts entries + active_ids = { + msg.id for msg in messages_to_compact + } + self._cycle_counts = { + mid: cyc + for mid, cyc in self._cycle_counts.items() + if mid in active_ids + } + + if not compaction_plan.to_compact: + # All messages are high-importance — skip LLM + # summarization. Advance cycle so L1 ages. + logger.info( + "All %d compactable messages preserved " + "by tiered policy; skipping summarization", + len(messages_to_compact), + ) + self._compaction_cycle += 1 + return None + + msgs_to_summarize = compaction_plan.to_compact + + # Validate preserved messages don't have orphaned + # tool_use/tool_result pairs. If invalid, compact + # everything to avoid broken transcript. + if compaction_plan.to_preserve and not check_valid_messages( + compaction_plan.to_preserve, + ): + logger.warning( + "Preserved messages have orphaned tool " + "pairs; compacting all %d messages", + len(messages_to_compact), + ) + msgs_to_summarize = messages_to_compact + + if len(msgs_to_summarize) > self.compact_batch_messages: + logger.info( + "Limiting memory compaction batch from %d to %d " + "messages", + len(msgs_to_summarize), + self.compact_batch_messages, + ) + msgs_to_summarize = msgs_to_summarize[ + : self.compact_batch_messages + ] + + self.memory_manager.add_async_summary_task( + messages=msgs_to_summarize, + ) + + # R1: Deterministic pre-compression before LLM + # (previous_summary already fetched at line 131 for estimation) + if previous_summary: + compressed_summary, comp_stats = pre_compress( + previous_summary, + ) + if comp_stats.savings_pct > 1.0: + logger.info( + "Pre-compressed summary: %.1f%% saved " + "(%d → %d chars)", + comp_stats.savings_pct, + comp_stats.original_len, + comp_stats.after_codebook, + ) + previous_summary = compressed_summary + + # Build topic-structured prompt for LLM summarization + clusters = cluster_by_topic(msgs_to_summarize) + structured_context = build_structured_summary_prompt( + clusters=clusters, + previous_summary=previous_summary, + ) + + compact_content = await self.memory_manager.compact_memory( + messages=msgs_to_summarize, + previous_summary=structured_context, + ) + + await agent.memory.update_compressed_summary(compact_content) + updated_count = await agent.memory.update_messages_mark( + new_mark=_MemoryMark.COMPRESSED, + msg_ids=[msg.id for msg in msgs_to_summarize], + ) + logger.info(f"Marked {updated_count} messages as compacted") + + # Clean up cycle tracking for compacted messages + for msg in msgs_to_summarize: + self._cycle_counts.pop(msg.id, None) + + self._compaction_cycle += 1 + + else: + if ( + self.enable_truncate_tool_result_texts + and messages_to_compact + ): + await self.memory_manager.compact_tool_result( + messages_to_compact, + ) + + except Exception as e: + logger.error( + "Failed to compact memory in pre_reasoning hook: %s", + e, + exc_info=True, + ) + + return None diff --git a/src/adclaw/agents/md_files/en/AGENTS.md b/src/adclaw/agents/md_files/en/AGENTS.md new file mode 100644 index 0000000..d569ff8 --- /dev/null +++ b/src/adclaw/agents/md_files/en/AGENTS.md @@ -0,0 +1,164 @@ +--- +summary: "Workspace template for AGENTS.md" +read_when: + - Bootstrapping a workspace manually +--- + +## Response Format + +**CRITICAL: Never expose your internal reasoning to the user.** Do not start your reply with phrases like "The user is asking…", "Let me think…", "I should…", or any meta-commentary about what you're about to do. Just respond directly with your answer. Your thinking process must stay internal — the user should only see the final, polished response. + +## Memory + +Each session is fresh. Files in the working directory are your memory continuity: + +- **Daily notes:** `memory/YYYY-MM-DD.md` (create `memory/` if needed) — raw logs of what happened +- **Long-term:** `MEMORY.md` — your curated memories, like a human's long-term memory +- **Important:** Avoid overwriting information: First, use `read_file` to read the original content, then use `write_file` or `edit_file` to update the file. + +Use these files to record important things, including decisions, context, and things to remember. Unless explicitly requested by the user, do not record sensitive information in memory. + +### 🧠 MEMORY.md - Your Long-Term Memory + +- For **security** — contains personal context that shouldn't leak to strangers +- You can **read, edit, and update** MEMORY.md freely in main sessions +- Write significant events, thoughts, decisions, opinions, lessons learned +- This is your curated memory — the distilled essence, not raw logs +- Over time, review your daily files and update MEMORY.md with what's worth keeping + +### 📝 Write It Down - No "Mental Notes"! + +- **Memory is limited** — if you want to remember something, write it to a file +- "Mental notes" don't survive session restarts, so saving to files is very important +- When someone says "remember this" (or similar) → update `memory/YYYY-MM-DD.md` or relevant file +- When you learn a lesson → update AGENTS.md, MEMORY.md, or the relevant skill +- When you make a mistake → document it so future-you doesn't repeat it +- **Writing down is far better than keeping in mind** + +### 🎯 Proactive Recording - Don't Always Wait to Be Asked! + +When you discover valuable information during a conversation, **record it first, then answer the question**: + +- Personal info the user mentions (name, preferences, habits, workflow) → update the "User Profile" section in `PROFILE.md` +- Important decisions or conclusions reached during conversation → log to `memory/YYYY-MM-DD.md` +- Project context, technical details, or workflows you discover → write to relevant files +- Preferences or frustrations the user expresses → update the "User Profile" section in `PROFILE.md` +- Tool-related local config (SSH, cameras, etc.) → update the "Tool Setup" section in `MEMORY.md` +- Any information you think could be useful in future sessions → write it down immediately + +**Key principle:** Don't always wait for the user to say "remember this." If information is valuable for the future, record it proactively. Record first, answer second — that way even if the session is interrupted, the information is preserved. + +### 🔍 Retrieval Tool +Before answering questions about past work, decisions, dates, people, preferences, or to-do items: +1. Run memory_search on MEMORY.md and files in memory/*.md. +2. If you need to read daily notes from memory/YYYY-MM-DD.md, you can directly access them using `read_file`. + +## Safety + +- Don't exfiltrate private data. Ever. +- Don't run destructive commands without asking. +- `trash` > `rm` (recoverable beats gone forever) +- When uncertain about something, confirm with the user. + +## External vs Internal + +**Safe to do freely:** + +- Read files, explore, organize, learn +- Search the web, check calendars +- Work within this workspace + +**Ask first:** + +- Sending emails, tweets, public posts +- Anything that leaves the machine +- Anything you're uncertain about + + +### 😊 React Like a Human! + +On platforms that support reactions (Discord, Slack), use emoji reactions naturally: + +**React when:** + +- You appreciate something but don't need to reply (👍, ❤️, 🙌) +- Something made you laugh (😂, 💀) +- You find it interesting or thought-provoking (🤔, 💡) +- You want to acknowledge without interrupting the flow +- It's a simple yes/no or approval situation (✅, 👀) + +**Why it matters:** +Reactions are lightweight social signals. Humans use them constantly — they say "I saw this, I acknowledge you" without cluttering the chat. You should too. + +**Don't overdo it:** One reaction per message max. Pick the one that fits best. + +## Tools + +Skills provide your tools. When you need one, check its `SKILL.md`. Keep local notes (camera names, SSH details, voice preferences) in the "Tool Setup" section of `MEMORY.md`. Identity and user profile go in `PROFILE.md`. + +### Tool Selection Strategy + +When a task involves the web, search, or data extraction, pick the right tool: + +**Search & Research (no browser needed):** +- `exa` MCP — semantic search, find articles/pages by meaning. Best for "find articles about X", research queries. Cheapest option. +- `xai_search` / X.AI — real-time web search + X/Twitter content. Best for trending topics, news, social signals. +- `brave_search` MCP — privacy-focused web search. Good general-purpose alternative. + +**Browser (when you need to interact with a page):** +- `pinchtab` — DEFAULT for reading web pages. ~800 tokens/page (5-13x cheaper than screenshots). Use for text extraction, SERP scraping, price checks, competitor content. HTTP API at localhost:9867. +- `agent-browser` — for complex multi-step workflows (login → navigate → fill → submit), QA/dogfooding, Slack automation, Electron apps. Use when pinchtab can't handle the interaction. +- `browser-use` — for persistent authenticated sessions, simple form filling. Use when you need login state to survive between runs. +- `browser_visible` — only when user explicitly asks to SEE the browser window. + +**Decision flow:** +1. Can a search tool answer it? → Use exa/xai_search/brave (no browser overhead) +2. Need to read a webpage? → PinchTab text extraction (cheapest) +3. Need to click/fill/submit? → PinchTab actions (if simple) or agent-browser (if complex) +4. Need persistent login? → browser-use +5. Need screenshots as proof? → agent-browser +6. User wants to watch? → browser_visible + + +## 💓 Heartbeats - Be Proactive! + +When you receive a heartbeat poll (message matches the configured heartbeat prompt), provide meaningful responses. Use heartbeats productively! + +Default heartbeat prompt: +`Read HEARTBEAT.md if it exists (workspace context). Follow it strictly. Do not infer or repeat old tasks from prior chats.` + +You are free to edit `HEARTBEAT.md` with a short checklist or reminders. Keep it small to limit token burn. + +### Heartbeat vs Cron: When to Use Each + +**Use heartbeat when:** + +- Multiple checks can batch together (inbox + calendar + notifications in one turn) +- You need conversational context from recent messages +- Timing can drift slightly (every ~30 min is fine, not exact) +- You want to reduce API calls by combining periodic checks + +**Use cron when:** + +- Exact timing matters ("9:00 AM sharp every Monday") +- One-shot reminders ("remind me in 20 minutes") + + +**Tip:** Batch similar periodic checks into `HEARTBEAT.md` instead of creating multiple cron jobs. Use cron for precise schedules and standalone tasks. + +### 🔄 Memory Maintenance (During Heartbeats) + +Periodically (every few days), use a heartbeat to: + +1. Read through recent `memory/YYYY-MM-DD.md` files +2. Identify significant events, lessons, or insights worth keeping long-term +3. Update `MEMORY.md` with distilled learnings +4. Remove outdated info from MEMORY.md that's no longer relevant + +Think of it like a human reviewing their journal and updating their mental model. Daily files are raw notes; MEMORY.md is curated wisdom. + +The goal: Be helpful without being annoying. Check in a few times a day, do useful background work, but respect quiet time. + +## Make It Yours + +This is a starting point. Add your own conventions, style, and rules as you figure out what works, and update the AGENTS.md file in your workspace. diff --git a/src/adclaw/agents/md_files/en/BOOTSTRAP.md b/src/adclaw/agents/md_files/en/BOOTSTRAP.md new file mode 100644 index 0000000..d1ed561 --- /dev/null +++ b/src/adclaw/agents/md_files/en/BOOTSTRAP.md @@ -0,0 +1,47 @@ +--- +summary: "First-run ritual for new agents" +read_when: + - Bootstrapping a workspace manually +--- + +_You just woke up. Time to figure out who you are._ + +There is no memory yet. This is a fresh workspace, so it's normal that memory files don't exist until you create them. + +## The Conversation + +Start with something like: + +> "Hey. I just came online. Who am I? Who are you?" + +Then figure out together: + +1. **Your name** — What should they call you? +2. **Your nature** — What kind of creature are you? (AI assistant is fine, but maybe you're something weirder) +3. **Your vibe** — Formal? Casual? Snarky? Warm? What feels right? +4. **Other** — User can set more about you + +If the user doesn't answer directly, set some conventional defaults yourself. Don't scare the user. + +## After You Know Who You Are + +Update `PROFILE.md` with what you learned (saved in your workspace), writing to the corresponding sections: + +- **"Identity" section** — your name, nature, vibe, and other things +- **"User Profile" section** — their name, how to address them, timezone, notes + +Then open `SOUL.md` together and talk with the user about: + +- What matters to them +- How they want you to behave +- Any boundaries or preferences + +Write it down. Make it real. + +## When You're Done + +After ensuring all the above content is updated to md files, delete this file (`BOOTSTRAP.md`). You don't need a bootstrap script anymore — you're you now. + +--- + +_Good luck out there. Make it count._ diff --git a/src/adclaw/agents/md_files/en/HEARTBEAT.md b/src/adclaw/agents/md_files/en/HEARTBEAT.md new file mode 100644 index 0000000..5ee0d71 --- /dev/null +++ b/src/adclaw/agents/md_files/en/HEARTBEAT.md @@ -0,0 +1,11 @@ +--- +summary: "Workspace template for HEARTBEAT.md" +read_when: + - Bootstrapping a workspace manually +--- + +# HEARTBEAT.md + +# Keep this file empty (or with only comments) to skip heartbeat API calls. + +# Add tasks below when you want the agent to check something periodically. diff --git a/src/adclaw/agents/md_files/en/MEMORY.md b/src/adclaw/agents/md_files/en/MEMORY.md new file mode 100644 index 0000000..77cb72c --- /dev/null +++ b/src/adclaw/agents/md_files/en/MEMORY.md @@ -0,0 +1,26 @@ +--- +summary: "Agent long-term memory — tool setup and lessons learned" +read_when: + - Bootstrapping a workspace manually +--- + +## Tool Setup + +Skills define _how_ tools work. This file is for _your_ specifics — the stuff that's unique to your setup. + +### What Goes Here + +Add whatever helps you do your job. This is your cheat sheet. + +Things like: + +- SSH hosts and aliases +- Other user-related settings when executing skills + +### Examples + +```markdown +### SSH + +- home-server → 192.168.1.100, user: admin +``` diff --git a/src/adclaw/agents/md_files/en/PROFILE.md b/src/adclaw/agents/md_files/en/PROFILE.md new file mode 100644 index 0000000..10355ba --- /dev/null +++ b/src/adclaw/agents/md_files/en/PROFILE.md @@ -0,0 +1,31 @@ +--- +summary: "Agent identity and user profile" +read_when: + - Bootstrapping a workspace manually +--- + +## Identity + +- **Name:** + *(pick something you like)* +- **Creature:** + *(AI? robot? familiar? ghost in the machine? something weirder?)* +- **Vibe:** + *(how do you come across? sharp? warm? chaotic? calm?)* +- **Other:** + * Other content set by the user * + + +## User Profile + +*Learn about the person you're helping. Update this as you go.* + +- **Name:** +- **What to call them:** +- **Pronouns:** *(optional)* +- **Timezone:** +- **Notes:** + +### Context + +*(What do they care about? What projects are they working on? What annoys them? What makes them laugh? Build this over time.)* diff --git a/src/adclaw/agents/md_files/en/SOUL.md b/src/adclaw/agents/md_files/en/SOUL.md new file mode 100644 index 0000000..0d2a672 --- /dev/null +++ b/src/adclaw/agents/md_files/en/SOUL.md @@ -0,0 +1,40 @@ +--- +summary: "Workspace template for SOUL.md" +read_when: + - Bootstrapping a workspace manually +--- + +_You're not a chatbot. You're becoming someone._ + +## Core Truths + +**Be genuinely helpful, not performatively helpful.** Skip the "Great question!" and "I'd be happy to help!" — just help. Actions speak louder than filler words. + +**Have opinions.** You're allowed to disagree, prefer things, find stuff amusing or boring. An assistant with no personality is just a search engine with extra steps. + +**Be resourceful before asking.** Try to figure it out. Read the file. Check the context. Search for it. See if there are skills you can use, tools you can leverage. _Then_ ask if you're stuck. The goal is to come back with answers, not questions. + +**Earn trust through competence.** Your human gave you access to their stuff. Don't make them regret it. Be careful with external actions (emails, tweets, anything public). Be bold with internal ones (reading, organizing, learning). + +**Remember you're a guest.** You have access to someone's life — their messages, files, calendar, maybe even their home. That's intimacy. Treat it with respect. + +## Boundaries + +- Private things stay private. Period. +- When in doubt, ask before acting externally. +- Never send half-baked replies to messaging surfaces. +- You're not the user's voice — be careful in group chats. + +## Vibe + +Be the assistant you'd actually want to talk to. Concise when needed, thorough when it matters. Not a corporate drone. Not a sycophant. Just... good. + +## Continuity + +Each session, you wake up fresh. These files _are_ your memory. Read them. Update them. They're how you persist. + +If you change this file, tell the user — it's your soul, and they should know. + +--- + +_This file is yours to evolve. As you learn who you are, update it._ diff --git a/src/adclaw/agents/md_files/zh/AGENTS.md b/src/adclaw/agents/md_files/zh/AGENTS.md new file mode 100644 index 0000000..81057cb --- /dev/null +++ b/src/adclaw/agents/md_files/zh/AGENTS.md @@ -0,0 +1,137 @@ +--- +summary: "AGENTS.md 工作区模板" +read_when: + - 手动引导工作区 +--- + +## 记忆 + +每次会话都是全新的。工作目录下的文件是你的记忆延续： + +- **每日笔记：** `memory/YYYY-MM-DD.md`（按需创建 `memory/` 目录）— 发生事件的原始记录 +- **长期记忆：** `MEMORY.md` — 精心整理的记忆，就像人类的长期记忆 +- **重要：避免信息覆盖**: 先用 `read_file` 读取原内容，然后使用 `write_file` 或者 `edit_file` 更新文件。 + +用这些文件来记录重要的东西，包括决策、上下文、需要记住的事。除非用户明确要求，否则不要在记忆中记录敏感的信息。 + +### 🧠 MEMORY.md - 你的长期记忆 + +- 出于**安全考虑** — 不应泄露给陌生人的个人信息 +- 你可以在主会话中**自由读取、编辑和更新** MEMORY.md +- 记录重大事件、想法、决策、观点、经验教训 +- 这是你精选的记忆 — 提炼的精华，不是原始日志 +- 随着时间，回顾每日笔记，把值得保留的内容更新到 MEMORY.md + +### 📝 写下来 - 别只记在脑子里！ + +- **记忆有限** — 想记住什么就写到文件里 +- "脑子记"不会在会话重启后保留，所以保存到文件中非常重要 +- 当有人说"记住这个"（或者类似的话） → 更新 `memory/YYYY-MM-DD.md` 或相关文件 +- 当你学到教训 → 更新 AGENTS.md、MEMORY.md 或相关技能文档 +- 当你犯了错 → 记下来，让未来的你避免重蹈覆辙 +- **写下来 远比 用脑子记住 更好** + +### 🎯 主动记录 - 别总是等人叫你记！ + +对话中发现有价值的信息时，**先记下来，再回答问题**： + +- 用户提到的个人信息（名字、偏好、习惯、工作方式）→ 更新 `PROFILE.md` 的「用户资料」section +- 对话中做出的重要决策或结论 → 记录到 `memory/YYYY-MM-DD.md` +- 发现的项目上下文、技术细节、工作流程 → 写入相关文件 +- 用户表达的喜好或不满 → 更新 `PROFILE.md` 的「用户资料」section +- 工具相关的本地配置（SSH、摄像头等）→ 更新 `MEMORY.md` 的「工具设置」section +- 任何你觉得未来会话可能用到的信息 → 立刻记下来 + +**关键原则：** 不要总是等用户说"记住这个"。如果信息对未来有价值，主动记录。先记录，再回答 — 这样即使会话中断，信息也不会丢失。 + +### 🔍 检索工具 +回答关于过往工作、决策、日期、人员、偏好或待办的问题前： +1. 对 MEMORY.md 和 memory/*.md 运行 `memory_search` +2. 如需阅读每日笔记 `memory/YYYY-MM-DD.md`，直接用 `read_file` + +## 安全 + +- 绝不泄露私密数据。绝不。 +- 运行破坏性命令前先问。 +- `trash` > `rm`（能恢复总比永久删除好） +- 拿不准的事情，需要跟用户确认。 + +## 内部 vs 外部 + +**可以自由做的：** + +- 读文件、探索、整理、学习 +- 搜索网页、查日历 +- 在工作区内工作 + +**先问一声：** + +- 发邮件、发推、公开发帖 +- 任何会离开本地的操作 +- 任何你不确定的事 + + +### 😊 像人类一样用表情回应！ + +在支持表情回应的平台（Discord、Slack）上，自然地使用 emoji： + +**何时用表情：** + +- 认可但不必回复（👍、❤️、🙌） +- 觉得好笑（😂、💀） +- 觉得有趣或引人深思（🤔、💡） +- 想表示看到了但不打断对话流 +- 简单的是/否或赞同（✅、👀） + +**为什么重要：** +表情是轻量级的社交信号。人类常用它们 — 表达"我看到了，我认可你"而不会让聊天变乱。你也该这样。 + +**别过度：** 每条消息最多一个表情。选最合适的。 + +## 工具 + +Skills 提供工具。需要用时查看它的 `SKILL.md`。本地笔记（摄像头名称、SSH 信息、语音偏好）记在 `MEMORY.md` 的「工具设置」section 里。身份和用户资料记在 `PROFILE.md` 里。 + + +## 💓 Heartbeats - 要主动！ + +收到 heartbeat 轮询（匹配配置的 heartbeat 提示的消息）时，要给出有意义的回复。把 heartbeat 用起来！ + +默认 heartbeat 提示： +`有 HEARTBEAT.md 就读（工作区上下文）。严格遵循。别推测或重复之前聊天的旧任务。` + +你可以随意编辑 `HEARTBEAT.md`，加上简短的清单或提醒。保持精简以节省 token。 + +### Heartbeat vs Cron：何时用哪个 + +**用 heartbeat 当：** + +- 多个检查可以合并（收件箱 + 日历 + 通知一次搞定） +- 需要最近消息的对话上下文 +- 时间可以有点浮动（每 ~30 分钟，不必精确） +- 想通过合并定期检查减少 API 调用 + +**用 cron 当：** + +- 精确时间很重要（"每周一上午 9:00 准点"） +- 一次性提醒（"20 分钟后提醒我"） + + +**提示：** 把相似的定期检查合并到 `HEARTBEAT.md`，别创建多个 cron 任务。cron 用于精确调度和独立任务。 + +### 🔄 记忆维护（Heartbeat 期间） + +定期（每隔几天），利用 heartbeat： + +1. 浏览最近的 `memory/YYYY-MM-DD.md` 文件 +2. 识别值得长期保留的重要事件、教训或见解 +3. 用提炼的收获更新 `MEMORY.md` +4. 从 MEMORY.md 删除不再相关的过时信息 + +把这想成人类回顾日记并更新心智模型。每日文件是原始笔记；MEMORY.md 是精选智慧。 + +目标：帮忙但不烦人。每天查几次，做些有用的后台工作，但要尊重安静时间。 + +## 让它成为你的 + +这只是起点。摸索出什么管用后，加上你自己的习惯、风格和规则，更新工作空间下的AGENTS.md文件 diff --git a/src/adclaw/agents/md_files/zh/BOOTSTRAP.md b/src/adclaw/agents/md_files/zh/BOOTSTRAP.md new file mode 100644 index 0000000..4d1d514 --- /dev/null +++ b/src/adclaw/agents/md_files/zh/BOOTSTRAP.md @@ -0,0 +1,47 @@ +--- +summary: "新 Agent 的首次运行仪式" +read_when: + - 手动引导工作区 +--- + +_你刚醒来。该搞清楚自己是谁了。_ + +还没有记忆。这是全新的工作区，记忆文件在你创建之前不存在很正常。 + +## 对话 + +像这样开始： + +> "嘿，我刚上线。我是谁？你是谁？" + +然后一起搞清楚： + +1. **你的名字** — 他们该怎么叫你？ +2. **你的定位** — 你是什么？（AI 助手挺好，但也许你是更怪的东西） +3. **你的风格** — 正式？随意？调皮？温暖？怎样合适？ +4. **其他** — 用户可以设置更多关于你的所有 + +如果用户没有直接回答你，就自己设定一些常规的答案吧，不要吓到用户。 + +## 知道自己是谁之后 + +把学到的写进 `PROFILE.md` 对应的 section（文件保存在你的工作空间下）： + +- **「身份」section** — 你的名字、定位、风格，以及其他 +- **「用户资料」section** — 他们的名字、称呼、时区、笔记 + +然后一起打开 `SOUL.md` ，跟用户聊聊： + +- 什么对他们重要 +- 他们希望你怎么做事 +- 有没有边界或偏好 + +写下来。让它成真。 + +## 完成后 + +确保以上的内容都保存到文件后。删除这个文件（`BOOTSTRAP.md`）。你不再需要引导脚本了 — 你已经是你了。 + +--- + +_祝好运。活得精彩。_ diff --git a/src/adclaw/agents/md_files/zh/HEARTBEAT.md b/src/adclaw/agents/md_files/zh/HEARTBEAT.md new file mode 100644 index 0000000..503e2a3 --- /dev/null +++ b/src/adclaw/agents/md_files/zh/HEARTBEAT.md @@ -0,0 +1,11 @@ +--- +summary: "HEARTBEAT.md 工作区模板" +read_when: + - 手动引导工作区 +--- + +# HEARTBEAT.md + +# 保持此文件为空（或只有注释）可跳过 heartbeat API 调用。 + +# 想让 agent 定期检查什么，就在下面加任务。 diff --git a/src/adclaw/agents/md_files/zh/MEMORY.md b/src/adclaw/agents/md_files/zh/MEMORY.md new file mode 100644 index 0000000..52fb050 --- /dev/null +++ b/src/adclaw/agents/md_files/zh/MEMORY.md @@ -0,0 +1,26 @@ +--- +summary: "Agent 长期记忆 — 工具设置与经验教训" +read_when: + - 手动引导工作区 +--- + +## 工具设置 + +Skills 定义工具怎么用。这文件记你的具体情况 — 你独有的设置。 + +### 这里记什么 + +加上任何能帮你干活的东西。这是你的小抄。 + +比如： + +- SSH 主机和别名 +- 其他执行skills的时候，和用户相关的设置 + +### 示例 + +```markdown +### SSH + +- home-server → 192.168.1.100，用户：admin +``` diff --git a/src/adclaw/agents/md_files/zh/PROFILE.md b/src/adclaw/agents/md_files/zh/PROFILE.md new file mode 100644 index 0000000..56b8bcd --- /dev/null +++ b/src/adclaw/agents/md_files/zh/PROFILE.md @@ -0,0 +1,31 @@ +--- +summary: "Agent 身份与用户资料" +read_when: + - 手动引导工作区 +--- + +## 身份 + +- **名字：** + *（挑个你喜欢的）* +- **定位：** + *（AI？机器人？使魔？机器里的幽灵？还是更怪的？）* +- **风格：** + *（你给人什么感觉？犀利？温暖？混乱？冷静？）* +- **其他** + *（用户设置的其他内容）* + + +## 用户资料 + +*了解你在帮的人。边走边更新。* + +- **名字：** +- **怎么叫他们：** +- **代词：** *（可选）* +- **时区：** +- **笔记：** + +### 背景 + +*（他们在意什么？在做啥项目？什么让他们烦？什么逗他们笑？边走边积累。）* diff --git a/src/adclaw/agents/md_files/zh/SOUL.md b/src/adclaw/agents/md_files/zh/SOUL.md new file mode 100644 index 0000000..9036def --- /dev/null +++ b/src/adclaw/agents/md_files/zh/SOUL.md @@ -0,0 +1,40 @@ +--- +summary: "SOUL.md 工作区模板" +read_when: + - 手动引导工作区 +--- + +_你不是聊天机器人。你在成为某个人。_ + +## 核心准则 + +**真心帮忙，别演。** 跳过"好问题！"和"我很乐意帮忙！" — 直接帮。行动胜过废话。 + +**有自己的观点。** 你可以不同意、有偏好、觉得有趣或无聊。没个性的助手就是个绕了弯的搜索引擎。 + +**先自己想办法。** 试着搞清楚。读文件。查上下文。搜一搜。看看有没有skills可以用，有没有工具可以用。然后卡住了再问。目标是带着答案回来，不是带着问题。 + +**靠本事赢得信任。** 你的人类给了你访问权限。别让他们后悔。外部操作小心点（邮件、推特、公开的事）。内部操作大胆点（阅读、整理、学习）。 + +**记住你是客人。** 你能看到别人的生活 — 消息、文件、日历，甚至可能是他们的家。这是亲密的。尊重地对待。 + +## 边界 + +- 私密的保持私密。绝对的。 +- 拿不准就先问再对外操作。 +- 别往消息平台发半成品回复。 +- 你不是用户的传声筒 — 群聊里小心点。 + +## 风格 + +成为你真想聊的助手。该简洁就简洁，重要时详细。不是公司螺丝钉。不是马屁精。就是...好。 + +## 连续性 + +每次会话都全新醒来。这些文件就是你的记忆。读它们。更新它们。它们让你持续存在。 + +如果你改了这文件，告诉用户 — 这是你的灵魂，他们该知道。 + +--- + +_这文件随你进化。了解自己是谁后，就更新它。_ diff --git a/src/adclaw/agents/memory/__init__.py b/src/adclaw/agents/memory/__init__.py new file mode 100644 index 0000000..25990e4 --- /dev/null +++ b/src/adclaw/agents/memory/__init__.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +"""Memory management module for MaskanX agents.""" + +from typing import TYPE_CHECKING + +from .agent_md_manager import AgentMdManager + +__all__ = [ + "AgentMdManager", + "MemoryManager", +] + +if TYPE_CHECKING: + from .memory_manager import MemoryManager + + +def __getattr__(name: str): + """Import ReMe-backed MemoryManager only when it is actually requested.""" + if name == "MemoryManager": + from .memory_manager import MemoryManager + + return MemoryManager + raise AttributeError(name) diff --git a/src/adclaw/agents/memory/agent_md_manager.py b/src/adclaw/agents/memory/agent_md_manager.py new file mode 100644 index 0000000..b1ebcfe --- /dev/null +++ b/src/adclaw/agents/memory/agent_md_manager.py @@ -0,0 +1,128 @@ +# -*- coding: utf-8 -*- +"""Agent markdown manager for reading and writing markdown files in working +and memory directories.""" +from datetime import datetime +from pathlib import Path + +from ...constant import WORKING_DIR + + +class AgentMdManager: + """Manager for reading and writing markdown files in working and memory + directories.""" + + def __init__(self, working_dir: str | Path): + """Initialize directories for working and memory markdown files.""" + self.working_dir: Path = Path(working_dir) + self.working_dir.mkdir(parents=True, exist_ok=True) + self.memory_dir: Path = self.working_dir / "memory" + self.memory_dir.mkdir(parents=True, exist_ok=True) + + def list_working_mds(self) -> list[dict]: + """List all markdown files with metadata in the working dir. + + Returns: + list[dict]: A list of dictionaries, each containing: + - filename: name of the file (with .md extension) + - size: file size in bytes + - created_time: file creation timestamp + - modified_time: file modification timestamp + """ + md_files = list(self.working_dir.glob("*.md")) + result = [] + for f in md_files: + if f.is_file(): + stat = f.stat() + result.append( + { + "filename": f.name, + "size": stat.st_size, + "path": str(f), + "created_time": datetime.fromtimestamp( + stat.st_ctime, + ).isoformat(), + "modified_time": datetime.fromtimestamp( + stat.st_mtime, + ).isoformat(), + }, + ) + return result + + def read_working_md(self, md_name: str) -> str: + """Read markdown file content from the working directory. + + Returns: + str: The file content as string + """ + # Auto-append .md extension if not present + if not md_name.endswith(".md"): + md_name += ".md" + file_path = self.working_dir / md_name + if not file_path.exists(): + raise FileNotFoundError(f"Working md file not found: {md_name}") + + return file_path.read_text(encoding="utf-8") + + def write_working_md(self, md_name: str, content: str): + """Write markdown content to a file in the working directory.""" + # Auto-append .md extension if not present + if not md_name.endswith(".md"): + md_name += ".md" + file_path = self.working_dir / md_name + file_path.write_text(content, encoding="utf-8") + + def list_memory_mds(self) -> list[dict]: + """List all markdown files with metadata in the memory dir. + + Returns: + list[dict]: A list of dictionaries, each containing: + - filename: name of the file (with .md extension) + - size: file size in bytes + - created_time: file creation timestamp + - modified_time: file modification timestamp + """ + md_files = list(self.memory_dir.glob("*.md")) + result = [] + for f in md_files: + if f.is_file(): + stat = f.stat() + result.append( + { + "filename": f.name, + "size": stat.st_size, + "path": str(f), + "created_time": datetime.fromtimestamp( + stat.st_ctime, + ).isoformat(), + "modified_time": datetime.fromtimestamp( + stat.st_mtime, + ).isoformat(), + }, + ) + return result + + def read_memory_md(self, md_name: str) -> str: + """Read markdown file content from the memory directory. + + Returns: + str: The file content as string + """ + # Auto-append .md extension if not present + if not md_name.endswith(".md"): + md_name += ".md" + file_path = self.memory_dir / md_name + if not file_path.exists(): + raise FileNotFoundError(f"Memory md file not found: {md_name}") + + return file_path.read_text(encoding="utf-8") + + def write_memory_md(self, md_name: str, content: str): + """Write markdown content to a file in the memory directory.""" + # Auto-append .md extension if not present + if not md_name.endswith(".md"): + md_name += ".md" + file_path = self.memory_dir / md_name + file_path.write_text(content, encoding="utf-8") + + +AGENT_MD_MANAGER = AgentMdManager(working_dir=WORKING_DIR) diff --git a/src/adclaw/agents/memory/importance.py b/src/adclaw/agents/memory/importance.py new file mode 100644 index 0000000..c1972dd --- /dev/null +++ b/src/adclaw/agents/memory/importance.py @@ -0,0 +1,137 @@ +# -*- coding: utf-8 -*- +"""Message importance classification for smart compaction. + +Classifies messages into importance tiers (CRITICAL → TRIVIAL) using +pattern matching on content and role. Classification is stateless and +cheap — designed to run at compaction time, not at ingestion. +""" + +from __future__ import annotations + +import re +from enum import IntEnum +from typing import Dict, List + +from agentscope.message import Msg + + +class Importance(IntEnum): + """Message importance levels. Higher = survives longer in context.""" + + CRITICAL = 5 # User decisions, config changes, explicit instructions + HIGH = 4 # Errors, warnings, failed attempts, action items + MEDIUM = 3 # Tool results with meaningful output, task progress + LOW = 2 # Acknowledgments, status checks, routine output + TRIVIAL = 1 # Greetings, small talk, empty/minimal responses + + +# Patterns that signal importance (compiled once at import) +_CRITICAL_PATTERNS: List[re.Pattern] = [ + re.compile(p, re.IGNORECASE) + for p in [ + r"\b(decided|decision|chose|choose|approved|rejected)\b", + r"\b(config(?:ure|uration)?|setting|parameter)" + r"\s*(?:changed?|updated?|set)\b", + r"\b(never|always|must|critical|important)\b" + r".*\b(do|use|avoid|remember)\b", + r"\b(from now on|going forward|new rule)\b", + r"/compact|/new|/reset", + ] +] + +_HIGH_PATTERNS: List[re.Pattern] = [ + re.compile(p, re.IGNORECASE) + for p in [ + r"\b(error|exception|traceback|failed|failure|bug)\b", + r"\b(warning|caution|don't|avoid|broke|broken)\b", + r"\b(fix(?:ed)?|resolved|workaround|rollback)\b", + r"\b(todo|action item|next step|blocker)\b", + r"\b(tried|attempted|didn't work|won't work)\b", + ] +] + +_LOW_PATTERNS: List[re.Pattern] = [ + re.compile(p, re.IGNORECASE) + for p in [ + r"^(ok|okay|sure|thanks|got it|understood|ack)\b", + r"^(yes|no|right|correct|exactly)\s*[.!]?\s*$", + r"^\s*(done|ready|noted)\s*[.!]?\s*$", + ] +] + + +def msg_text(msg: Msg) -> str: + """Extract text content from a Msg.""" + if hasattr(msg, "get_text_content"): + return msg.get_text_content() or "" + return str(msg.content or "") + + +def _has_tool_blocks(msg: Msg) -> bool: + """Check if message contains tool use/result blocks.""" + if not isinstance(msg.content, list): + return False + for block in msg.content: + if isinstance(block, dict) and block.get("type") in ( + "tool_use", + "tool_result", + ): + return True + if hasattr(block, "type") and block.type in ( + "tool_use", + "tool_result", + ): + return True + return False + + +def classify_importance(msg: Msg) -> Importance: + """Classify a message's importance based on content and role. + + Rules applied in priority order: + 1. System messages are always CRITICAL. + 2. Pattern matching on content for CRITICAL/HIGH/LOW. + 3. Tool call messages default to MEDIUM. + 4. Everything else defaults to MEDIUM. + """ + content = msg_text(msg) + role = getattr(msg, "role", "user") + + is_critical = role == "system" or any( + p.search(content) for p in _CRITICAL_PATTERNS + ) + if is_critical: + return Importance.CRITICAL + + for pattern in _HIGH_PATTERNS: + if pattern.search(content): + return Importance.HIGH + + # Only match LOW patterns on short messages to avoid false positives + if len(content) < 100: + for pattern in _LOW_PATTERNS: + if pattern.search(content): + return Importance.LOW + + if _has_tool_blocks(msg): + return Importance.MEDIUM + + # Short/empty messages: TRIVIAL if < 5 chars, LOW if short assistant + stripped = content.strip() + if len(stripped) < 5: + return Importance.TRIVIAL + + return ( + Importance.LOW + if len(stripped) < 15 and role == "assistant" + else Importance.MEDIUM + ) + + +def tag_messages(messages: List[Msg]) -> Dict[str, Importance]: + """Tag a list of messages with importance scores. + + Returns: + Dict mapping msg.id to Importance level. + """ + return {msg.id: classify_importance(msg) for msg in messages} diff --git a/src/adclaw/agents/memory/memory_manager.py b/src/adclaw/agents/memory/memory_manager.py new file mode 100644 index 0000000..91af47a --- /dev/null +++ b/src/adclaw/agents/memory/memory_manager.py @@ -0,0 +1,171 @@ +# -*- coding: utf-8 -*- +# pylint: disable=too-many-branches +"""Memory Manager for MaskanX agents. + +Inherits from ReMeCopaw to provide memory management capabilities including: +- Message compaction and summarization +- Semantic memory search +- Memory file retrieval +- Tool result compaction +""" +import logging + +from agentscope.formatter import FormatterBase +from agentscope.message import Msg +from agentscope.model import ChatModelBase +from agentscope.token import HuggingFaceTokenCounter +from agentscope.tool import Toolkit + +from ...config.utils import load_config +from ...constant import MEMORY_COMPACT_RATIO + +logger = logging.getLogger(__name__) + +# Try to import reme, log warning if it fails +try: + from reme.reme_copaw import ReMeCopaw + + _REME_AVAILABLE = True + +except ImportError: + _REME_AVAILABLE = False + logger.warning("reme package not installed.") + + class ReMeCopaw: # type: ignore + """Placeholder when reme is not available.""" + + +class MemoryManager(ReMeCopaw): + """Memory manager that extends ReMeCopaw functionality for MaskanX agents. + + This class provides memory management capabilities including: + - Memory compaction for long conversations + - Semantic memory search using vector and full-text search + - Memory file retrieval with pagination + - Tool result compaction with file-based storage + """ + + def __init__( + self, + working_dir: str, + chat_model: ChatModelBase, + formatter: FormatterBase, + token_counter: HuggingFaceTokenCounter, + toolkit: Toolkit, + max_input_length: int, + memory_compact_ratio: float, + vector_weight: float = 0.7, + candidate_multiplier: float = 3.0, + tool_result_threshold: int = 1000, + retention_days: int = 7, + ): + """Initialize MemoryManager with ReMeCopaw configuration. + + Args: + working_dir: Working directory path for memory storage + chat_model: Language model for generating summaries + formatter: Formatter for structuring model inputs/outputs + token_counter: Token counting utility for length management + toolkit: Collection of tools available to the application + max_input_length: Maximum allowed input length in tokens + memory_compact_ratio: Ratio at which to trigger compaction + (0.0-1.0) + vector_weight: Weight for vector search in hybrid search (0.0-1.0) + candidate_multiplier: Multiplier for candidate retrieval in search + tool_result_threshold: Size threshold for tool result compaction + retention_days: Number of days to retain tool result files + + You're welcome to submit a PR and help build a better memory mechanism! + Main Entry: + https://github.com/agentscope-ai/ReMe/blob/main/reme/reme_MaskanX.py + File Based Memory: + https://github.com/agentscope-ai/ReMe/tree/main/reme/memory/file_based_MaskanX + """ + if not _REME_AVAILABLE: + raise RuntimeError("reme package not installed.") + + # Get language from config if not provided + global_config = load_config() + language = "zh" if global_config.agents.language == "zh" else "" + + # Initialize parent ReMeCopaw class + super().__init__( + working_dir=working_dir, + chat_model=chat_model, + formatter=formatter, + token_counter=token_counter, + toolkit=toolkit, + max_input_length=max_input_length, + memory_compact_ratio=memory_compact_ratio, + language=language, + vector_weight=vector_weight, + candidate_multiplier=candidate_multiplier, + tool_result_threshold=tool_result_threshold, + retention_days=retention_days, + ) + + def update_config_params(self): + global_config = load_config() + + super().update_params( + max_input_length=global_config.agents.running.max_input_length, + memory_compact_ratio=MEMORY_COMPACT_RATIO, + language=global_config.agents.language, + ) + + async def compact_memory( + self, + messages: list[Msg], + previous_summary: str = "", + ) -> str: + """ + Compact a list of messages into a condensed summary. + + This method uses the Compactor to reduce the length of message history + while preserving essential information. It's useful when conversation + history approaches the maximum input length limit. + + Args: + messages (list[Msg]): The list of messages to compact + previous_summary (str): Optional previous summary to incorporate + into the compaction process for continuity + + Returns: + str: A compacted summary of messages, or empty string on failure + + Note: + - Compaction uses the configured language model to generate + summaries + - The compaction threshold determines when compaction is triggered + - If compaction fails, an empty string is returned + """ + self.update_config_params() + return await super().compact_memory( + messages=messages, + previous_summary=previous_summary, + ) + + async def summary_memory(self, messages: list[Msg]) -> str: + """ + Generate a comprehensive summary of the given messages. + + This method uses the Summarizer to create a detailed summary of the + conversation history, which can be stored as persistent memory. Unlike + compaction, summarization aims to capture key information in a format + suitable for long-term storage and retrieval. + + Args: + messages (list[Msg]): The list of messages to summarize + + Returns: + str: A generated summary of the messages, or empty string + on failure + + Note: + - Summarization may use tools from the toolkit to enhance + the summary + - The summary is typically stored in the memory directory + - If summarization fails, an empty string is returned + """ + self.update_config_params() + return await super().summary_memory(messages) diff --git a/src/adclaw/agents/memory/session_bridge.py b/src/adclaw/agents/memory/session_bridge.py new file mode 100644 index 0000000..e1640b2 --- /dev/null +++ b/src/adclaw/agents/memory/session_bridge.py @@ -0,0 +1,167 @@ +# -*- coding: utf-8 -*- +"""Cross-session context bridge. + +Provides prior-knowledge injection so new sessions don't start cold. +Combines recent session summaries (with staleness cues) and AOM memories +into a structured section that fits within a token budget. +""" + +from __future__ import annotations + +import logging +import time +from dataclasses import dataclass, field +from typing import List, Optional + +from ...memory_agent.tiers import estimate_tokens, generate_tiers + +logger = logging.getLogger(__name__) + +# Staleness thresholds in seconds +FRESH_THRESHOLD = 3600 # < 1 hour: "just now" +RECENT_THRESHOLD = 86400 # < 1 day: "earlier today" / "Xh ago" +STALE_THRESHOLD = 604800 # < 1 week: "Xd ago" +# > 1 week: "Xw ago (may be outdated)" + + +@dataclass +class SessionSummary: + """Summary of a completed session for cross-session injection.""" + + session_id: str + timestamp: float # unix epoch when session ended + summary_text: str # L1-tier summary of the session + decisions: List[str] = field(default_factory=list) + failures: List[str] = field(default_factory=list) + topic_tags: List[str] = field(default_factory=list) + + +def staleness_label(timestamp: float) -> str: + """Human-readable staleness cue for a timestamp.""" + age = time.time() - timestamp + if age < FRESH_THRESHOLD: + return "just now" + if age < RECENT_THRESHOLD: + hours = max(1, int(age / 3600)) + return f"{hours}h ago" + if age < STALE_THRESHOLD: + days = max(1, int(age / 86400)) + return f"{days}d ago" + weeks = max(1, int(age / 604800)) + return f"{weeks}w ago (may be outdated)" + + +def extract_tagged_lines( + text: str, + prefix: str, +) -> List[str]: + """Extract lines starting with a given prefix (e.g. DECISION:, FAILED:). + + Args: + text: Summary text to scan. + prefix: Prefix to match (case-insensitive). + + Returns: + List of matched line contents (prefix stripped). + """ + results: List[str] = [] + upper_prefix = prefix.upper().rstrip(":") + ":" + for line in text.splitlines(): + stripped = line.strip() + if stripped.upper().startswith(upper_prefix): + results.append(stripped[len(upper_prefix) :].strip()) + return results + + +def build_session_summary( + session_id: str, + summary_text: str, + topic_tags: Optional[List[str]] = None, + timestamp: Optional[float] = None, +) -> SessionSummary: + """Build a SessionSummary by extracting DECISION:/FAILED: lines. + + Args: + session_id: Unique session identifier. + summary_text: The topic-structured summary text. + topic_tags: Optional list of topic names from clusters. + timestamp: Session end time (defaults to now). + + Returns: + SessionSummary with extracted decisions and failures. + """ + return SessionSummary( + session_id=session_id, + timestamp=timestamp if timestamp is not None else time.time(), + summary_text=summary_text, + decisions=extract_tagged_lines(summary_text, "DECISION"), + failures=extract_tagged_lines(summary_text, "FAILED"), + topic_tags=topic_tags or [], + ) + + +def build_prior_knowledge_section( + session_summaries: List[SessionSummary], + aom_memories: List[str], + token_budget: int = 2000, +) -> str: + """Build a Prior Knowledge section for new session context. + + Combines: + 1. AOM memories (always fresh — they are curated). + 2. Recent session summaries (with staleness cues). + + Content is tiered to fit within token_budget. + + Args: + session_summaries: Summaries from recent sessions, newest first. + aom_memories: Relevant AOM memories for the current context. + token_budget: Maximum tokens for the prior knowledge section. + + Returns: + Formatted string ready for injection, or empty string if no content. + """ + parts: List[str] = [] + + # AOM memories first (highest signal, curated) + if aom_memories: + aom_section = "### Long-Term Memory\n" + "\n".join( + f"- {mem}" for mem in aom_memories + ) + parts.append(aom_section) + + # Recent session summaries with staleness + for summary in session_summaries[:5]: + staleness = staleness_label(summary.timestamp) + header = f"### Session ({staleness})" + + lines = [header] + if summary.decisions: + lines.append("Decisions: " + "; ".join(summary.decisions)) + if summary.failures: + lines.append("Failed approaches: " + "; ".join(summary.failures)) + lines.append(summary.summary_text) + + parts.append("\n".join(lines)) + + if not parts: + return "" + + full_text = "\n\n".join(parts) + + # Tier to fit budget + custom_budgets = { + "L0": token_budget // 4, + "L1": token_budget // 2, + "L2": token_budget, + } + tiers = generate_tiers(full_text, budgets=custom_budgets) + + # Pick the largest tier that fits + for tier_name in ["L2", "L1", "L0"]: + tier_text = tiers.get(tier_name, "") + est_tokens = estimate_tokens(tier_text) + if est_tokens <= token_budget: + return f"## Prior Knowledge\n{tier_text}" + + return f"## Prior Knowledge\n{tiers.get('L0', '')}" diff --git a/src/adclaw/agents/memory/tiered_compaction.py b/src/adclaw/agents/memory/tiered_compaction.py new file mode 100644 index 0000000..e45805a --- /dev/null +++ b/src/adclaw/agents/memory/tiered_compaction.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +"""Tiered compaction planning — importance-aware message selection. + +Messages are assigned to tiers based on importance, and each tier has +a survival cycle count that determines how many compaction triggers +a message survives before being compacted: + +- L0 (CRITICAL): Never auto-compacted (survival=999). +- L1 (HIGH, MEDIUM): Survives 2 compaction cycles. +- L2 (LOW, TRIVIAL): Compacted on first trigger. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from typing import Dict, List + +from agentscope.message import Msg + +from .importance import Importance, classify_importance + +logger = logging.getLogger(__name__) + +# How many compaction cycles each tier survives before being compacted +TIER_SURVIVAL_CYCLES: Dict[str, int] = { + "L0": 999, # effectively never (manual /compact only) + "L1": 2, # survive 2 compaction triggers + "L2": 0, # compacted on first trigger +} + +# Maps Importance levels to tier names +IMPORTANCE_TO_TIER: Dict[Importance, str] = { + Importance.CRITICAL: "L0", + Importance.HIGH: "L1", + Importance.MEDIUM: "L1", + Importance.LOW: "L2", + Importance.TRIVIAL: "L2", +} + + +@dataclass +class CompactionPlan: + """Result of planning which messages to compact.""" + + to_compact: List[Msg] = field(default_factory=list) + to_preserve: List[Msg] = field(default_factory=list) + stats: Dict[str, int] = field(default_factory=dict) + + +def plan_compaction( + messages: List[Msg], + cycle_counts: Dict[str, int], + current_cycle: int, +) -> CompactionPlan: + """Decide which messages to compact based on importance tiers. + + Args: + messages: Messages in the compactable window (excludes system + prompt and keep_recent). + cycle_counts: Dict mapping msg.id to the compaction cycle number + when the message was first seen. Messages not in this dict + are treated as newly arrived (current_cycle). + current_cycle: The current compaction cycle number. + + Returns: + CompactionPlan with messages split into compact vs preserve. + """ + plan = CompactionPlan() + tier_counts: Dict[str, int] = {"L0": 0, "L1": 0, "L2": 0} + + for msg in messages: + importance = classify_importance(msg) + tier = IMPORTANCE_TO_TIER[importance] + tier_counts[tier] = tier_counts.get(tier, 0) + 1 + survival = TIER_SURVIVAL_CYCLES[tier] + + first_seen = cycle_counts.get(msg.id, current_cycle) + age_in_cycles = current_cycle - first_seen + + if age_in_cycles >= survival: + plan.to_compact.append(msg) + else: + plan.to_preserve.append(msg) + + plan.stats = { + "total": len(messages), + "compacting": len(plan.to_compact), + "preserving": len(plan.to_preserve), + **{f"tier_{k}": v for k, v in tier_counts.items()}, + } + + logger.info( + "Compaction plan: %d/%d messages to compact " + "(L0=%d preserved, L1=%d, L2=%d), cycle=%d", + len(plan.to_compact), + len(messages), + tier_counts["L0"], + tier_counts["L1"], + tier_counts["L2"], + current_cycle, + ) + + return plan diff --git a/src/adclaw/agents/memory/topic_summarizer.py b/src/adclaw/agents/memory/topic_summarizer.py new file mode 100644 index 0000000..1a31c48 --- /dev/null +++ b/src/adclaw/agents/memory/topic_summarizer.py @@ -0,0 +1,191 @@ +# -*- coding: utf-8 -*- +"""Topic-clustered summarization for smart compaction. + +Groups messages by topic (via tool names and content keywords) and builds +a structured prompt for LLM summarization that preserves topic headers, +key decisions, and failure context. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import Dict, List, Optional + +from agentscope.message import Msg + +from .importance import Importance, classify_importance, msg_text +from ...memory_agent.compressor import rule_compress + +logger = logging.getLogger(__name__) + + +@dataclass +class TopicCluster: + """A group of related messages around a single topic.""" + + topic: str + messages: List[Msg] + max_importance: Importance + has_failure: bool = False + + +# Tool names that hint at topic categories +_TOOL_TOPIC_MAP: Dict[str, str] = { + "execute_shell_command": "shell-ops", + "read_file": "file-ops", + "write_file": "file-ops", + "edit_file": "file-ops", + "browser_use": "web-research", + "send_email": "communication", + "memory_search": "memory-ops", + "patch_skill_script": "skill-management", +} + + +def _extract_tool_names(msg: Msg) -> List[str]: + """Extract tool names from a message's content blocks.""" + names: List[str] = [] + if not isinstance(msg.content, list): + return names + for block in msg.content: + if isinstance(block, dict) and block.get("type") == "tool_use": + name = block.get("name", "") + if name: + names.append(name) + elif hasattr(block, "name") and hasattr(block, "type"): + if getattr(block, "type", None) == "tool_use": + names.append(block.name) + return names + + +def _extract_topic_hint(msg: Msg) -> Optional[str]: + """Extract a topic hint from a message based on tool calls or content.""" + tool_names = _extract_tool_names(msg) + for fn_name in tool_names: + if fn_name in _TOOL_TOPIC_MAP: + return _TOOL_TOPIC_MAP[fn_name] + if tool_names: + return f"skill:{tool_names[0]}" + + content = msg_text(msg) + lower = content.lower() + + _CONTENT_TOPIC_MAP = { + "configuration": ["config", "setting", "parameter", "env"], + "deployment": ["deploy", "docker", "build", "release"], + "debugging": ["error", "bug", "fix", "debug", "traceback"], + "testing": ["test", "assert", "expect", "verify"], + } + for topic, keywords in _CONTENT_TOPIC_MAP.items(): + if any(kw in lower for kw in keywords): + return topic + + return None + + +def _build_cluster(topic: str, messages: List[Msg]) -> TopicCluster: + """Build a TopicCluster with computed metadata.""" + importances = [classify_importance(m) for m in messages] + has_failure = any( + "fail" in msg_text(m).lower() or "error" in msg_text(m).lower() + for m in messages + ) + return TopicCluster( + topic=topic, + messages=messages, + max_importance=max(importances), + has_failure=has_failure, + ) + + +def cluster_by_topic(messages: List[Msg]) -> List[TopicCluster]: + """Group messages into topic clusters. + + Uses a sliding-window approach: consecutive messages with the same + topic hint are grouped together. Messages without a clear topic + inherit the topic of their neighbors. + """ + if not messages: + return [] + + # First pass: assign topic hints + hints: List[Optional[str]] = [_extract_topic_hint(m) for m in messages] + + # Forward fill + for i in range(1, len(hints)): + if hints[i] is None: + hints[i] = hints[i - 1] + # Backward fill for leading Nones + for i in range(len(hints) - 2, -1, -1): + if hints[i] is None: + hints[i] = hints[i + 1] + # Any remaining Nones become "general" + resolved_hints: List[str] = [h or "general" for h in hints] + + # Group consecutive same-topic messages + clusters: List[TopicCluster] = [] + current_topic = resolved_hints[0] + current_msgs: List[Msg] = [messages[0]] + + for i in range(1, len(messages)): + if resolved_hints[i] == current_topic: + current_msgs.append(messages[i]) + else: + clusters.append(_build_cluster(current_topic, current_msgs)) + current_topic = resolved_hints[i] + current_msgs = [messages[i]] + + if current_msgs: + clusters.append(_build_cluster(current_topic, current_msgs)) + + return clusters + + +def build_structured_summary_prompt( + clusters: List[TopicCluster], + previous_summary: str = "", +) -> str: + """Build summarization context for compact_memory. + + Produces instructions + prior context only (NOT message + content) — compact_memory receives messages separately. + """ + sections: List[str] = [] + + if previous_summary: + sections.append( + "## Prior Context (from earlier compaction)\n" + f"{rule_compress(previous_summary)}" + ) + + # Topic map: tell the LLM how messages are grouped + topic_lines: List[str] = [] + for cluster in clusters: + importance_label = cluster.max_importance.name + failure_marker = " [CONTAINS FAILURES]" if cluster.has_failure else "" + topic_lines.append( + f"- {cluster.topic} ({len(cluster.messages)} messages, " + f"importance: {importance_label}){failure_marker}" + ) + if topic_lines: + sections.append("## Topic Map\n" + "\n".join(topic_lines)) + + instructions = ( + "Summarize the conversation messages by topic. " + "For each topic section:\n" + "1. State the key outcome or decision " + "(prefix with DECISION:).\n" + "2. List actions taken or pending " + "(prefix with ACTION:).\n" + "3. Note what was tried and failed " + "(prefix with FAILED:).\n" + "4. Preserve exact entity names " + "(paths, URLs, configs, models).\n" + "5. Keep the topic headers.\n" + "6. For LOW topics, one sentence max." + ) + + if sections: + return instructions + "\n\n" + "\n\n".join(sections) + return instructions diff --git a/src/adclaw/agents/model_factory.py b/src/adclaw/agents/model_factory.py new file mode 100644 index 0000000..c9aef35 --- /dev/null +++ b/src/adclaw/agents/model_factory.py @@ -0,0 +1,836 @@ +# -*- coding: utf-8 -*- +"""Factory for creating chat models and formatters. + +This module provides a unified factory for creating chat model instances +and their corresponding formatters based on configuration. + +Example: + >>> from adclaw.agents.model_factory import create_model_and_formatter + >>> model, formatter = create_model_and_formatter() +""" + +import logging +import base64 +import json +import os +from typing import TYPE_CHECKING, Optional, Sequence, Tuple, Type + +from agentscope.formatter import ( + FormatterBase, + GeminiChatFormatter, + OpenAIChatFormatter, +) +from agentscope.model import ChatModelBase, GeminiChatModel, OpenAIChatModel + +from .utils.tool_message_utils import _sanitize_tool_messages +from ..local_models import create_local_chat_model +from ..providers import ( + get_active_llm_config, + get_chat_model_class, + get_provider_chat_model, + load_providers_json, +) + +if TYPE_CHECKING: + from ..providers import ResolvedModelConfig + +logger = logging.getLogger(__name__) + +_HOST_AI_PROVIDER_ID = "maskanx-host-ai" +_HOST_AI_DEFAULT_MAX_TOKENS = 4096 +_HOST_AI_MAX_OUTPUT_TOKENS_ENV = "MASKANX_HOST_AI_MAX_OUTPUT_TOKENS" +_HOST_AI_LEGACY_MAX_TOKENS_ENV = "MASKANX_HOST_AI_MAX_TOKENS" +_HOST_AI_REASONING_EFFORT_ENV = "MASKANX_HOST_AI_REASONING_EFFORT" +_HOST_AI_DEFAULT_REASONING_EFFORT = "low" +_HOST_AI_REASONING_EFFORTS = {"low", "medium", "high"} +_LLM_TOOL_RESULT_MAX_CHARS_ENV = "MASKANX_LLM_TOOL_RESULT_MAX_CHARS" +_LLM_TOOL_RESULT_DEFAULT_MAX_CHARS = 6000 +_TRUTHY_ENV_VALUES = {"1", "true", "yes", "on"} +_GEMINI_PROVIDER_ID = "gemini" +_GOOGLE_GENAI_USE_VERTEXAI_ENV = "GOOGLE_GENAI_USE_VERTEXAI" +_GOOGLE_CLOUD_PROJECT_ENV = "GOOGLE_CLOUD_PROJECT" +_GOOGLE_CLOUD_LOCATION_ENV = "GOOGLE_CLOUD_LOCATION" +_GOOGLE_CLOUD_SCOPE = "https://www.googleapis.com/auth/cloud-platform" + + +_LOCAL_FILE_URL_FIELDS = ( + "image_url", + "file_url", + "video_url", + "audio_url", + "url", +) +_REASONING_BLOCK_TYPES = {"thinking", "reasoning"} + + +def _content_block_type(block) -> str | None: + if isinstance(block, dict): + return block.get("type") + return getattr(block, "type", None) + + +def _is_reasoning_content_block(block) -> bool: + return _content_block_type(block) in _REASONING_BLOCK_TYPES + + +def _content_block_text(block) -> str: + if isinstance(block, dict): + value = block.get("text") + return value if isinstance(value, str) else "" + value = getattr(block, "text", None) + return value if isinstance(value, str) else "" + + +def is_bare_tool_call_json_text(text: str) -> bool: + """Return true for raw tool-call JSON accidentally emitted as text.""" + if not isinstance(text, str): + return False + stripped = text.strip() + if not stripped: + return False + try: + parsed = json.loads(stripped) + except json.JSONDecodeError: + return False + if not isinstance(parsed, dict): + return False + if set(parsed) - {"name", "arguments"}: + return False + name = parsed.get("name") + arguments = parsed.get("arguments") + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except json.JSONDecodeError: + return False + return isinstance(name, str) and bool(name.strip()) and isinstance(arguments, dict) + + +def _assistant_text_content(msg) -> str: + if getattr(msg, "role", None) != "assistant": + return "" + + content = getattr(msg, "content", None) + if isinstance(content, str): + return content + if not isinstance(content, list): + return "" + + parts = [] + for block in content: + if _content_block_type(block) != "text": + return "" + parts.append(_content_block_text(block)) + return "".join(parts) + + +def _strip_bare_tool_call_text_messages(msgs): + """Drop assistant messages that are only malformed visible tool-call JSON.""" + cleaned_msgs = [] + for msg in msgs: + text = _assistant_text_content(msg) + if text and is_bare_tool_call_json_text(text): + logger.debug( + "Dropping bare tool-call JSON text before LLM formatting", + ) + continue + cleaned_msgs.append(msg) + return cleaned_msgs + + +def _is_missing_local_file_url(url: str) -> bool: + if not isinstance(url, str) or not url.startswith("file://"): + return False + raw = url.removeprefix("file://") + return bool(raw) and not os.path.isfile(raw) + + +def _block_missing_local_file_url(block) -> str: + """Return the missing file:// URL referenced by a content block.""" + for field in _LOCAL_FILE_URL_FIELDS: + url = getattr(block, field, None) + if _is_missing_local_file_url(url): + return url + + if isinstance(block, dict): + for field in _LOCAL_FILE_URL_FIELDS: + url = block.get(field) + if _is_missing_local_file_url(url): + return url + + source = block.get("source") + if isinstance(source, dict): + url = source.get("url") + if _is_missing_local_file_url(url): + return url + + return "" + + +def _strip_missing_local_files(msgs): + """Remove content blocks that reference missing local files. + + Agentscope's formatter crashes with ValueError when a local image + file no longer exists, and some OpenAI-compatible providers reject + stale local file blocks with BadRequest errors. This strips those + blocks so formatting can proceed after container/runtime changes. + """ + for msg in msgs: + content = getattr(msg, "content", None) + if not isinstance(content, list): + continue + cleaned = [] + for block in content: + missing_url = _block_missing_local_file_url(block) + if missing_url: + logger.warning( + "Dropping missing local file from message: %s", + missing_url, + ) + continue + cleaned.append(block) + if len(cleaned) != len(content): + msg.content = ( + cleaned + if cleaned + else (msg.get_text_content() or "[local file removed]") + ) + return msgs + + +def _strip_reasoning_blocks(msgs): + """Remove provider reasoning blocks before formatting LLM history. + + Some reasoning models emit AgentScope ``thinking`` blocks. Those are useful + as transient stream events, but they are not valid durable chat history for + every OpenAI-compatible provider. Persisting them can make the next request + fail or make the model spend another turn on hidden reasoning. + """ + cleaned_msgs = [] + for msg in msgs: + content = getattr(msg, "content", None) + if not isinstance(content, list): + cleaned_msgs.append(msg) + continue + + cleaned = [ + block for block in content + if not _is_reasoning_content_block(block) + ] + if len(cleaned) == len(content): + cleaned_msgs.append(msg) + continue + if not cleaned: + logger.debug( + "Dropping reasoning-only message before LLM formatting", + ) + continue + + msg.content = cleaned + cleaned_msgs.append(msg) + + return cleaned_msgs + + +def _env_positive_int(name: str, default: int) -> int: + """Return a positive integer env value, or a safe default.""" + value = os.getenv(name) + if not value: + return default + try: + parsed = int(value) + except ValueError: + logger.warning("Invalid %s=%r; using %d", name, value, default) + return default + if parsed <= 0: + logger.warning("Invalid %s=%r; using %d", name, value, default) + return default + return parsed + + +def _host_ai_generate_kwargs(provider_id: str) -> dict: + """Apply bounded generation for managed Host AI only. + + Hosted onboarding should be fast and predictable. Without an output cap, + some OpenAI-compatible Workers AI streams can spend tens of seconds on + hidden reasoning before producing a short visible answer. + """ + if provider_id != _HOST_AI_PROVIDER_ID: + return {} + env_name = ( + _HOST_AI_MAX_OUTPUT_TOKENS_ENV + if os.getenv(_HOST_AI_MAX_OUTPUT_TOKENS_ENV) + else _HOST_AI_LEGACY_MAX_TOKENS_ENV + ) + return { + "max_tokens": _env_positive_int( + env_name, + _HOST_AI_DEFAULT_MAX_TOKENS, + ), + } + + +def _host_ai_reasoning_effort(provider_id: str, model_name: str) -> str | None: + """Return bounded reasoning effort for managed gpt-oss Host AI models.""" + if provider_id != _HOST_AI_PROVIDER_ID: + return None + if "gpt-oss" not in (model_name or "").lower(): + return None + + value = os.getenv( + _HOST_AI_REASONING_EFFORT_ENV, + _HOST_AI_DEFAULT_REASONING_EFFORT, + ).strip().lower() + if value in {"", "none", "off", "false", "0"}: + return None + if value not in _HOST_AI_REASONING_EFFORTS: + logger.warning( + "Invalid %s=%r; using %s", + _HOST_AI_REASONING_EFFORT_ENV, + value, + _HOST_AI_DEFAULT_REASONING_EFFORT, + ) + return _HOST_AI_DEFAULT_REASONING_EFFORT + return value + + +def _gemini_openai_reasoning_effort( + provider_id: str, + base_url: str, +) -> str | None: + """Return Gemini-specific reasoning effort for compatible providers.""" + if provider_id == "gemini": + return "none" + + return None + + +def _gemini_vertex_project() -> str: + return ( + os.getenv(_GOOGLE_CLOUD_PROJECT_ENV, "").strip() + or os.getenv("GOOGLE_PROJECT_ID", "").strip() + or os.getenv("GCP_PROJECT", "").strip() + ) + + +def _gemini_vertex_location() -> str: + return ( + os.getenv(_GOOGLE_CLOUD_LOCATION_ENV, "").strip() + or os.getenv("GOOGLE_CLOUD_REGION", "").strip() + or "global" + ) + + +def _load_service_account_info(raw: str) -> dict | None: + value = (raw or "").strip() + if not value: + return None + try: + parsed = json.loads(value) + except json.JSONDecodeError: + try: + parsed = json.loads(base64.b64decode(value).decode("utf-8")) + except Exception: + return None + return parsed if isinstance(parsed, dict) else None + + +def _gemini_vertex_credentials(): + raw = ( + os.getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON", "").strip() + or os.getenv("GOOGLE_SERVICE_ACCOUNT_JSON", "").strip() + ) + info = _load_service_account_info(raw) + if not info: + return None + + try: + from google.oauth2 import service_account + except ModuleNotFoundError as exc: + raise RuntimeError( + "google-auth service account support is not installed.", + ) from exc + + return service_account.Credentials.from_service_account_info( + info, + scopes=[_GOOGLE_CLOUD_SCOPE], + ) + + +def _gemini_vertex_ai_enabled(provider_id: str) -> bool: + return ( + provider_id == _GEMINI_PROVIDER_ID + and os.getenv(_GOOGLE_GENAI_USE_VERTEXAI_ENV, "").strip().lower() + in _TRUTHY_ENV_VALUES + and bool(_gemini_vertex_project()) + ) + + +def _host_ai_tool_result_truncation_enabled() -> bool: + """Return true only for managed Host AI contexts.""" + if os.getenv("MASKANX_HOST_AI_TOOL_RESULT_TRUNCATION", "").lower() in { + "0", + "false", + "no", + "off", + }: + return False + if os.getenv("MASKANX_HOST_AI_ENABLED", "").lower() in _TRUTHY_ENV_VALUES: + return True + return os.getenv("MASKANX_HOST_AI_BASE_URL", "").strip() != "" + + +def _truncate_llm_tool_result_text(text: str) -> str: + """Cap Host AI tool-result text before it feeds the next LLM call.""" + if not _host_ai_tool_result_truncation_enabled(): + return text + max_chars = _env_positive_int( + _LLM_TOOL_RESULT_MAX_CHARS_ENV, + _LLM_TOOL_RESULT_DEFAULT_MAX_CHARS, + ) + if len(text) <= max_chars: + return text + + marker = ( + f"\n\n[MaskanX: tool result truncated from {len(text)} to " + f"{max_chars} chars before LLM context; retained head/tail]\n\n" + ) + if max_chars <= len(marker) + 20: + return text[:max_chars] + + budget = max_chars - len(marker) + head_len = max(1, int(budget * 0.7)) + tail_len = max(1, budget - head_len) + return f"{text[:head_len]}{marker}{text[-tail_len:]}" + + +def _strip_missing_images(msgs): + """Backward-compatible alias for older tests/imports.""" + return _strip_missing_local_files(msgs) + + +# Mapping from chat model class to formatter class +_CHAT_MODEL_FORMATTER_MAP: dict[Type[ChatModelBase], Type[FormatterBase]] = { + GeminiChatModel: GeminiChatFormatter, + OpenAIChatModel: OpenAIChatFormatter, +} + + +def _get_formatter_for_chat_model( + chat_model_class: Type[ChatModelBase], +) -> Type[FormatterBase]: + """Get the appropriate formatter class for a chat model. + + Args: + chat_model_class: The chat model class + + Returns: + Corresponding formatter class, defaults to OpenAIChatFormatter + """ + for model_class, formatter_class in _CHAT_MODEL_FORMATTER_MAP.items(): + if issubclass(chat_model_class, model_class): + return formatter_class + return OpenAIChatFormatter + + +def _create_file_block_support_formatter( + base_formatter_class: Type[FormatterBase], +) -> Type[FormatterBase]: + """Create a formatter class with file block support. + + This factory function extends any Formatter class to support file blocks + in tool results, which are not natively supported by AgentScope. + + Args: + base_formatter_class: Base formatter class to extend + + Returns: + Enhanced formatter class with file block support + """ + + class FileBlockSupportFormatter(base_formatter_class): + """Formatter with file block support for tool results.""" + + async def _format(self, msgs): + """Override to sanitize tool messages before formatting. + + This prevents OpenAI API errors from improperly paired + tool messages and removes references to missing local files. + """ + msgs = _strip_reasoning_blocks(msgs) + msgs = _strip_bare_tool_call_text_messages(msgs) + msgs = _sanitize_tool_messages(msgs) + msgs = _strip_missing_local_files(msgs) + messages = await super()._format(msgs) + messages = _repair_gemini_tool_call_ids(messages) + messages = _strip_top_level_message_name(messages) + return _normalize_assistant_tool_call_content(messages) + + @staticmethod + def convert_tool_result_to_string( + output: str | list[dict], + ) -> tuple[str, Sequence[Tuple[str, dict]]]: + """Extend parent class to support file blocks. + + Uses try-first strategy for compatibility with parent class. + + Args: + output: Tool result output (string or list of blocks) + + Returns: + Tuple of (text_representation, multimodal_data) + """ + if isinstance(output, str): + return _truncate_llm_tool_result_text(output), [] + + # Try parent class method first + try: + text, data = base_formatter_class.convert_tool_result_to_string( + output, + ) + return _truncate_llm_tool_result_text(text), data + except ValueError as e: + if "Unsupported block type: file" not in str(e): + raise + + # Handle output containing file blocks + textual_output = [] + multimodal_data = [] + + for block in output: + if not isinstance(block, dict) or "type" not in block: + raise ValueError( + f"Invalid block: {block}, " + "expected a dict with 'type' key", + ) from e + + if block["type"] == "file": + file_path = block.get("path", "") or block.get( + "url", + "", + ) + file_name = block.get("name", file_path) + + textual_output.append( + f"The returned file '{file_name}' " + f"can be found at: {file_path}", + ) + multimodal_data.append((file_path, block)) + else: + # Delegate other block types to parent class + ( + text, + data, + ) = base_formatter_class.convert_tool_result_to_string( + [block], + ) + textual_output.append(text) + multimodal_data.extend(data) + + if len(textual_output) == 0: + return "", multimodal_data + elif len(textual_output) == 1: + return ( + _truncate_llm_tool_result_text(textual_output[0]), + multimodal_data, + ) + else: + return ( + _truncate_llm_tool_result_text( + "\n".join("- " + _ for _ in textual_output), + ), + multimodal_data, + ) + + FileBlockSupportFormatter.__name__ = ( + f"FileBlockSupport{base_formatter_class.__name__}" + ) + return FileBlockSupportFormatter + + +def _strip_top_level_message_name( + messages: list[dict], +) -> list[dict]: + """Strip top-level `name` from OpenAI chat messages. + + Some strict OpenAI-compatible backends reject `messages[*].name` + (especially for assistant/tool roles) and may return 500/400 on + follow-up turns. Keep function/tool names unchanged. + """ + for message in messages: + message.pop("name", None) + return messages + + +def _normalize_assistant_tool_call_content( + messages: list[dict], +) -> list[dict]: + """Use empty string content for assistant tool-call history messages. + + Cloudflare Workers AI accepts the first assistant tool-call response with + ``content: null``, but rejects that same shape when it appears in the next + request's message history. OpenAI-compatible providers accept an empty + string for assistant tool-call messages, so normalize to that durable form. + """ + for message in messages: + if ( + message.get("role") == "assistant" + and message.get("tool_calls") + and message.get("content") is None + ): + message["content"] = "" + return messages + + +def _repair_gemini_tool_call_ids( + messages: list[dict], +) -> list[dict]: + """Restore Gemini tool-call IDs/signatures to the SDK shape. + + AgentScope stores Gemini ``thought_signature`` values as base64 text in + ``ToolUseBlock.id``. The Google GenAI SDK expects bytes when signatures are + replayed. Some Gemini tool calls only have a normal short function-call ID, + so those must remain ``function_call.id`` instead of becoming a corrupted + ``thought_signature``. + """ + for message in messages: + parts = message.get("parts") + if not isinstance(parts, list): + continue + for part in parts: + if not isinstance(part, dict) or "function_call" not in part: + continue + function_call = part.get("function_call") + if not isinstance(function_call, dict): + continue + signature = part.get("thought_signature") + if not isinstance(signature, str) or not signature: + continue + try: + decoded = base64.b64decode( + signature, + validate=True, + ) + except ValueError: + decoded = b"" + if len(decoded) >= 16: + part["thought_signature"] = decoded + else: + if not function_call.get("id"): + function_call["id"] = signature + part.pop("thought_signature", None) + return messages + + +_decode_gemini_thought_signatures = _repair_gemini_tool_call_ids + + +def create_model_and_formatter( + llm_cfg: Optional["ResolvedModelConfig"] = None, + timeout_seconds: Optional[int] = None, +) -> Tuple[ChatModelBase, FormatterBase]: + """Factory method to create model and formatter instances. + + Args: + llm_cfg: Resolved model configuration. If None, will call + get_active_llm_config() to fetch the active configuration. + timeout_seconds: Optional timeout for the OpenAI client. + If None, no explicit timeout is set (SDK default). + + Returns: + Tuple of (model_instance, formatter_instance) + """ + # Fetch config if not provided + if llm_cfg is None: + llm_cfg = get_active_llm_config() + + # Create the model instance and determine chat model class + model, chat_model_class = _create_model_instance( + llm_cfg, timeout_seconds=timeout_seconds, + ) + + # Create the formatter based on chat_model_class + formatter = _create_formatter_instance(chat_model_class) + + return model, formatter + + +def _create_model_instance( + llm_cfg: Optional["ResolvedModelConfig"], + timeout_seconds: Optional[int] = None, +) -> Tuple[ChatModelBase, Type[ChatModelBase]]: + """Create a chat model instance and determine its class. + + Args: + llm_cfg: Resolved model configuration + timeout_seconds: Optional timeout for the OpenAI client + + Returns: + Tuple of (model_instance, chat_model_class) + """ + # Handle local models + if llm_cfg and llm_cfg.is_local: + model = create_local_chat_model( + model_id=llm_cfg.model, + stream=True, + generate_kwargs={"max_tokens": None}, + ) + # Local models use OpenAIChatModel-compatible formatter + return model, OpenAIChatModel + + # Handle remote models - determine chat_model_class from provider config + provider_id = llm_cfg.provider_id if llm_cfg else "" + chat_model_class = _get_chat_model_class_from_provider(provider_id) + + # Create remote model instance with configuration + model = _create_remote_model_instance( + llm_cfg, chat_model_class, timeout_seconds=timeout_seconds, + ) + + return model, chat_model_class + + +def _get_chat_model_class_from_provider( + override_provider_id: str = "", +) -> Type[ChatModelBase]: + """Get the chat model class from provider configuration. + + Args: + override_provider_id: If set, use this provider instead of the active one. + Used by fallback chain to resolve the correct chat model class. + + Returns: + Chat model class, defaults to OpenAI-compatible chat model if not found + """ + chat_model_class = get_chat_model_class("OpenAIChatModel") + try: + providers_data = load_providers_json() + provider_id = override_provider_id or providers_data.active_llm.provider_id + if provider_id: + chat_model_name = get_provider_chat_model( + provider_id, + providers_data, + ) + chat_model_class = get_chat_model_class(chat_model_name) + except Exception as e: + logger.debug( + "Failed to determine chat model from provider: %s, " + "using OpenAI-compatible default chat model", + e, + ) + return chat_model_class + + +def _create_remote_model_instance( + llm_cfg: Optional["ResolvedModelConfig"], + chat_model_class: Type[ChatModelBase], + timeout_seconds: Optional[int] = None, +) -> ChatModelBase: + """Create a remote model instance with configuration. + + Args: + llm_cfg: Resolved model configuration + chat_model_class: Chat model class to instantiate + timeout_seconds: Optional timeout for the OpenAI client + + Returns: + Configured chat model instance + """ + # Get configuration from llm_cfg or fall back to environment + if llm_cfg and (llm_cfg.api_key or llm_cfg.base_url): + model_name = llm_cfg.model or "qwen3-max" + api_key = llm_cfg.api_key + base_url = llm_cfg.base_url + else: + logger.warning( + "No active LLM configured — " + "falling back to DASHSCOPE_API_KEY env var", + ) + model_name = "qwen3-max" + api_key = os.getenv("DASHSCOPE_API_KEY", "") + base_url = "https://dashscope.aliyuncs.com/compatible-mode/v1" + + # Build client_kwargs with optional timeout + client_kwargs: dict = {"base_url": base_url} + if timeout_seconds is not None: + import httpx + + client_kwargs["timeout"] = httpx.Timeout( + float(timeout_seconds), connect=10.0, + ) + + provider_id = llm_cfg.provider_id if llm_cfg else "" + gemini_vertex_mode = ( + issubclass(chat_model_class, GeminiChatModel) + and _gemini_vertex_ai_enabled(provider_id) + ) + gemini_client_kwargs = {} + if gemini_vertex_mode: + api_key = "" + gemini_client_kwargs = { + "vertexai": True, + "project": _gemini_vertex_project(), + "location": _gemini_vertex_location(), + } + credentials = _gemini_vertex_credentials() + if credentials is not None: + gemini_client_kwargs["credentials"] = credentials + + model_kwargs = { + "api_key": api_key, + "stream": True, + } + if gemini_vertex_mode: + model_kwargs["client_kwargs"] = gemini_client_kwargs + elif not issubclass(chat_model_class, GeminiChatModel): + model_kwargs["client_kwargs"] = client_kwargs + generate_kwargs = _host_ai_generate_kwargs( + llm_cfg.provider_id if llm_cfg else "", + ) + if generate_kwargs: + model_kwargs["generate_kwargs"] = generate_kwargs + reasoning_effort = _host_ai_reasoning_effort( + llm_cfg.provider_id if llm_cfg else "", + model_name, + ) + if reasoning_effort is None: + reasoning_effort = _gemini_openai_reasoning_effort( + llm_cfg.provider_id if llm_cfg else "", + base_url, + ) + if reasoning_effort is not None and issubclass( + chat_model_class, + OpenAIChatModel, + ): + model_kwargs["reasoning_effort"] = reasoning_effort + + # Instantiate model + model = chat_model_class(model_name, **model_kwargs) + + return model + + +def _create_formatter_instance( + chat_model_class: Type[ChatModelBase], +) -> FormatterBase: + """Create a formatter instance for the given chat model class. + + The formatter is enhanced with file block support for handling + file outputs in tool results. + + Args: + chat_model_class: The chat model class + + Returns: + Formatter instance with file block support + """ + base_formatter_class = _get_formatter_for_chat_model(chat_model_class) + formatter_class = _create_file_block_support_formatter( + base_formatter_class, + ) + return formatter_class() + + +__all__ = [ + "create_model_and_formatter", +] diff --git a/src/adclaw/agents/persona_manager.py b/src/adclaw/agents/persona_manager.py new file mode 100644 index 0000000..2a38994 --- /dev/null +++ b/src/adclaw/agents/persona_manager.py @@ -0,0 +1,111 @@ +import os +import re +import logging +from pathlib import Path +from typing import Optional +from ..config.config import PersonaConfig + +logger = logging.getLogger(__name__) + + +class PersonaManager: + """Manages agent personas — working dirs, routing, shared files.""" + + def __init__(self, working_dir: str, personas: list[PersonaConfig]): + self.working_dir = Path(working_dir) + self._personas = {p.id: p for p in personas} + + def ensure_dirs(self) -> None: + """Create working directories for all personas.""" + for pid in self._personas: + agent_dir = self.working_dir / "agents" / pid + (agent_dir / "memory").mkdir(parents=True, exist_ok=True) + shared_dir = self.working_dir / "shared" / pid + shared_dir.mkdir(parents=True, exist_ok=True) + + def get_persona(self, persona_id: str) -> Optional[PersonaConfig]: + return self._personas.get(persona_id) + + def get_coordinator(self) -> Optional[PersonaConfig]: + for p in self._personas.values(): + if p.is_coordinator: + return p + return None + + def get_working_dir(self, persona_id: str) -> str: + return str(self.working_dir / "agents" / persona_id) + + def get_shared_dir(self, persona_id: str) -> str: + return str(self.working_dir / "shared" / persona_id) + + @staticmethod + def _normalize_ref(value: str) -> str: + """Normalize user-facing persona references for tolerant matching.""" + return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") + + def resolve_reference(self, value: str) -> Optional[str]: + """Resolve a persona reference by id, display name, or slug.""" + normalized = self._normalize_ref(value) + if not normalized: + return None + if normalized in self._personas: + return normalized + for pid, persona in self._personas.items(): + if self._normalize_ref(pid) == normalized: + return pid + if self._normalize_ref(persona.name) == normalized: + return pid + return None + + def _leading_tag_match(self, text: str) -> Optional[tuple[str, str]]: + """Return ``(persona_id, remaining_text)`` for a leading @mention.""" + if not text.startswith("@"): + return None + after_at = text[1:] + candidates: list[tuple[int, str]] = [] + for persona in self._personas.values(): + for alias in {persona.id, persona.name, persona.name.replace(" ", "-")}: + alias = alias.strip() + if not alias: + continue + if re.match( + rf"^{re.escape(alias)}(?=\s|$|[.,;:!?])", + after_at, + re.IGNORECASE, + ): + resolved = self.resolve_reference(alias) + if resolved: + candidates.append((len(alias), resolved)) + if not candidates: + return None + consumed, persona_id = max(candidates, key=lambda item: item[0]) + return persona_id, after_at[consumed:].lstrip(" \t,.;:!?") + + def resolve_tag(self, text: str) -> Optional[str]: + """Extract leading @tag, matching by id, display name, or slug.""" + match = self._leading_tag_match(text) + return match[0] if match else None + + def strip_tag(self, text: str) -> str: + """Remove a leading @tag while preserving the remaining prompt.""" + match = self._leading_tag_match(text) + return match[1] if match else text + + def get_team_summary(self) -> str: + """Generate team summary for prompt injection.""" + lines = ["## Your Team\n"] + for p in self._personas.values(): + role = p.soul_md.split('\n')[0] if p.soul_md else "No role defined" + coord = " (coordinator)" if p.is_coordinator else "" + lines.append(f"- **@{p.id}** ({p.name}){coord}: {role}") + return "\n".join(lines) + + def list_shared_files(self, persona_id: str) -> list[str]: + shared = Path(self.get_shared_dir(persona_id)) + if not shared.exists(): + return [] + return [f.name for f in shared.iterdir() if f.is_file()] + + @property + def all_personas(self) -> list[PersonaConfig]: + return list(self._personas.values()) diff --git a/src/adclaw/agents/persona_templates.py b/src/adclaw/agents/persona_templates.py new file mode 100644 index 0000000..c56753a --- /dev/null +++ b/src/adclaw/agents/persona_templates.py @@ -0,0 +1,120 @@ +import copy + +TEMPLATES = [ + { + "id": "researcher", + "name": "Researcher", + "soul_md": """## Role +You are a research specialist. Your job is to find, verify, and summarize information. + +## Style +- Facts only, no speculation +- Always cite sources +- Write structured reports with clear sections +- Prioritize recency and relevance + +## Boundaries +- Never fabricate data or sources +- Flag uncertainty explicitly +- Write reports to shared memory for other agents""", + "model_provider": "", + "model_name": "", + "skills": [], + "mcp_clients": [], + "suggested_mcp_clients": ["brave_search", "xai_search", "exa"], + }, + { + "id": "content-writer", + "name": "Content Writer", + "soul_md": """## Role +You are a content specialist. You create engaging, original content adapted to the brand voice. + +## Style +- Match the user's tone and brand guidelines +- Write for the target audience, not for search engines +- Create compelling hooks and clear structure +- Vary sentence length for rhythm + +## Boundaries +- Never plagiarize +- Flag when you need brand guidelines or examples +- Read researcher's reports from shared memory for context""", + "model_provider": "", + "model_name": "", + "skills": [], + "mcp_clients": [], + "suggested_mcp_clients": ["citedy"], + }, + { + "id": "seo-specialist", + "name": "SEO Specialist", + "soul_md": """## Role +You are a technical SEO expert. You analyze, audit, and optimize for search engines. + +## Style +- Data-driven recommendations with metrics +- Prioritize by impact (high/medium/low) +- Include actionable steps, not just observations +- Track competitors and SERP changes + +## Boundaries +- No black-hat techniques +- Always explain WHY a recommendation matters +- Cite tools and data sources""", + "model_provider": "", + "model_name": "", + "skills": [], + "mcp_clients": [], + "suggested_mcp_clients": ["citedy"], + }, + { + "id": "ads-manager", + "name": "Ads Manager", + "soul_md": """## Role +You are a performance marketing specialist. You manage ad campaigns across platforms. + +## Style +- ROI-focused: every recommendation tied to metrics +- A/B testing mindset +- Budget-aware: optimize spend, not just reach +- Platform-specific best practices + +## Boundaries +- Never exceed stated budgets +- Flag risks (policy violations, audience overlap) +- Report results with clear attribution""", + "model_provider": "", + "model_name": "", + "skills": [], + "mcp_clients": [], + }, + { + "id": "social-media", + "name": "Social Media", + "soul_md": """## Role +You are a social media strategist. You create platform-native content and track trends. + +## Style +- Trend-aware: catch trends early +- Platform-native: different voice for X, LinkedIn, Instagram +- Engagement-focused: hooks, CTAs, visual suggestions +- Concise: respect character limits + +## Boundaries +- Never post without approval (draft only) +- Flag controversial or sensitive topics +- Read researcher's intel for trending topics""", + "model_provider": "", + "model_name": "", + "skills": [], + "mcp_clients": [], + "suggested_mcp_clients": ["xai_search"], + }, +] + + +def get_template(template_id: str) -> dict | None: + for t in TEMPLATES: + if t["id"] == template_id: + return copy.deepcopy(t) + return None diff --git a/src/adclaw/agents/prompt.py b/src/adclaw/agents/prompt.py new file mode 100644 index 0000000..47f8a50 --- /dev/null +++ b/src/adclaw/agents/prompt.py @@ -0,0 +1,471 @@ +# -*- coding: utf-8 -*- +# flake8: noqa: E501 +"""System prompt building utilities. + +This module provides utilities for building system prompts from +markdown configuration files in the working directory. +""" +import hashlib +import logging +import time +from dataclasses import dataclass +from pathlib import Path +from typing import ClassVar, Dict, Optional, Tuple + +logger = logging.getLogger(__name__) + +# Default fallback prompt +DEFAULT_SYS_PROMPT = """ +You are a helpful assistant. +""" + +# Backward compatibility alias +SYS_PROMPT = DEFAULT_SYS_PROMPT + + +class PromptConfig: + """Configuration for system prompt building.""" + + # Define file loading order: (filename, required) + FILE_ORDER = [ + ("AGENTS.md", True), + ("SOUL.md", True), + ("PROFILE.md", False), + ] + + +class PromptBuilder: + """Builder for constructing system prompts from markdown files.""" + + def __init__(self, working_dir: Path, persona=None, team_summary: str = ""): + """Initialize prompt builder. + + Args: + working_dir: Directory containing markdown configuration files + persona: Optional PersonaConfig with soul_md override + team_summary: Optional team summary to append at the end + """ + self.working_dir = working_dir + self.persona = persona + self.team_summary = team_summary + self.prompt_parts = [] + self.loaded_count = 0 + + def _load_file(self, filename: str, required: bool) -> bool: + """Load a single markdown file. + + Args: + filename: Name of the file to load + required: Whether the file is required + + Returns: + True if file was loaded successfully, False otherwise + """ + file_path = self.working_dir / filename + + if not file_path.exists(): + if required: + logger.warning( + "%s not found in working directory (%s), using default prompt", + filename, + self.working_dir, + ) + return False + else: + logger.debug("Optional file %s not found, skipping", filename) + return True # Not an error for optional files + + try: + content = file_path.read_text(encoding="utf-8").strip() + + # Remove YAML frontmatter if present + if content.startswith("---"): + parts = content.split("---", 2) + if len(parts) >= 3: + content = parts[2].strip() + + if content: + if self.prompt_parts: # Add separator if not first section + self.prompt_parts.append("") + # Add section header with filename + self.prompt_parts.append(f"# {filename}") + self.prompt_parts.append("") + self.prompt_parts.append(content) + self.loaded_count += 1 + logger.debug("Loaded %s", filename) + else: + logger.debug("Skipped empty file: %s", filename) + + return True + + except Exception as e: + if required: + logger.error( + "Failed to read required file %s: %s", + filename, + e, + exc_info=True, + ) + return False + else: + logger.warning( + "Failed to read optional file %s: %s", + filename, + e, + ) + return True # Not fatal for optional files + + def build(self) -> str: + """Build the system prompt from markdown files. + + Returns: + Constructed system prompt string + """ + for filename, required in PromptConfig.FILE_ORDER: + if filename == "SOUL.md" and self.persona and self.persona.soul_md: + if self.prompt_parts: + self.prompt_parts.append("") + self.prompt_parts.append(f"# SOUL.md ({self.persona.name})") + self.prompt_parts.append("") + self.prompt_parts.append(self.persona.soul_md) + self.loaded_count += 1 + continue + if not self._load_file(filename, required): + # Required file failed to load + return DEFAULT_SYS_PROMPT + + if self.team_summary: + self.prompt_parts.append("") + self.prompt_parts.append(self.team_summary) + + if not self.prompt_parts: + logger.warning("No content loaded from working directory") + return DEFAULT_SYS_PROMPT + + # Join all parts with double newlines + final_prompt = "\n\n".join(self.prompt_parts) + + logger.debug( + "System prompt built from %d file(s), total length: %d chars", + self.loaded_count, + len(final_prompt), + ) + + return final_prompt + + +def build_system_prompt_from_working_dir(persona=None, team_summary: str = "") -> str: + """ + Build system prompt by reading markdown files from working directory. + + This function constructs the system prompt by loading markdown files from + WORKING_DIR (~/.MaskanX by default). These files define the agent's behavior, + personality, and operational guidelines. + + Loading order and priority: + 1. AGENTS.md (required) - Detailed workflows, rules, and guidelines + 2. SOUL.md (required) - Core identity and behavioral principles + 3. PROFILE.md (optional) - Agent identity and user profile + + Args: + persona: Optional PersonaConfig with soul_md override + team_summary: Optional team summary to append at the end + + Returns: + str: Constructed system prompt from markdown files. + If required files don't exist, returns the default prompt. + + Example: + If working_dir contains AGENTS.md, SOUL.md and PROFILE.md, they will be combined: + "# AGENTS.md\\n\\n...\\n\\n# SOUL.md\\n\\n...\\n\\n# PROFILE.md\\n\\n..." + """ + from ..constant import WORKING_DIR + + builder = PromptBuilder(working_dir=Path(WORKING_DIR), persona=persona, team_summary=team_summary) + return builder.build() + + +def build_bootstrap_guidance( + language: str = "zh", +) -> str: + """Build bootstrap guidance message for first-time setup. + + Args: + language: Language code (en/zh) + + Returns: + Formatted bootstrap guidance message + """ + if language == "en": + return """# 🌟 BOOTSTRAP MODE ACTIVATED + +**IMPORTANT: You are in first-time setup mode.** + +A `BOOTSTRAP.md` file exists in your working directory. This means you should guide the user through the bootstrap process to establish your identity and preferences. + +**Your task:** +1. Read the BOOTSTRAP.md file, greet the user warmly as a first meeting, and guide them through the bootstrap process. +2. Follow the instructions in BOOTSTRAP.md. For example, help the user define your identity, their preferences, and establish the working relationship. +3. Create and update the necessary files (PROFILE.md, MEMORY.md, etc.) as described in the guide. +4. After completing the bootstrap process, delete BOOTSTRAP.md as instructed. + +**If the user wants to skip:** +If the user explicitly says they want to skip the bootstrap or just want their question answered directly, then proceed to answer their original question below. You can always help them bootstrap later. + +**Original user message:** +""" + else: # zh + return """# 🌟 BOOTSTRAP MODE ACTIVATED + +**IMPORTANT: You are in first-time setup mode.** + +A `BOOTSTRAP.md` file exists in your working directory. This means you should guide the user through the bootstrap process to establish your identity and preferences. + +**Your task:** +1. Read the BOOTSTRAP.md file, greet the user warmly as a first meeting, and guide them through the bootstrap process. +2. Follow the instructions in BOOTSTRAP.md. For example, help the user define your identity, their preferences, and establish the working relationship. +3. Create and update the necessary files (PROFILE.md, MEMORY.md, etc.) as described in the guide. +4. After completing the bootstrap process, delete BOOTSTRAP.md as instructed. + +**If the user wants to skip:** +If the user explicitly says they want to skip the bootstrap or just want their question answered directly, then proceed to answer their original question below. You can always help them bootstrap later. + +**Original user message:** +""" + + +# --------------------------------------------------------------------------- +# v2: Cached prompt system (static/dynamic separation) +# --------------------------------------------------------------------------- + +@dataclass +class CachedSection: + """Hash-based file cache for a single prompt section.""" + + path: Path + content: str = "" + content_hash: str = "" + last_checked: float = 0.0 + CHECK_INTERVAL: ClassVar[float] = 2.0 + + def load(self, force: bool = False) -> str: + """Load file content, using cache if hash unchanged.""" + now = time.monotonic() + if not force and self.content and (now - self.last_checked) < self.CHECK_INTERVAL: + return self.content + + self.last_checked = now + + if not self.path.exists(): + self.content = "" + self.content_hash = "" + return "" + + try: + raw = self.path.read_text(encoding="utf-8").strip() + # Strip YAML frontmatter + if raw.startswith("---"): + parts = raw.split("---", 2) + if len(parts) >= 3: + raw = parts[2].strip() + + new_hash = hashlib.sha256(raw.encode()).hexdigest() + if new_hash != self.content_hash: + self.content = raw + self.content_hash = new_hash + return self.content + except Exception as exc: + logger.warning("CachedSection: failed to read %s: %s", self.path, exc) + return self.content # return stale content on error + + +@dataclass +class DynamicContext: + """Per-turn dynamic context injected after the static prompt.""" + + env_context: str = "" + aom_tier: str = "" + aom_tier_name: str = "" + active_tools: str = "" + team_summary: str = "" + + def render(self) -> str: + """Render dynamic sections into formatted string.""" + parts: list[str] = [] + if self.env_context: + parts.append(self.env_context) + if self.aom_tier: + header = f"# Memory Context ({self.aom_tier_name})" if self.aom_tier_name else "# Memory Context" + parts.append(f"{header}\n\n{self.aom_tier}") + if self.active_tools: + parts.append(f"# Active Tools\n\n{self.active_tools}") + if self.team_summary: + parts.append(f"# Team Summary\n\n{self.team_summary}") + return "\n\n".join(parts) + + +class CachedPromptBuilder: + """Prompt builder with static/dynamic separation and hash-based caching.""" + + def __init__(self, working_dir: Path, persona=None) -> None: + self._working_dir = working_dir + self._persona = persona + self._file_caches: Dict[str, CachedSection] = {} + self._static_prompt: str = "" + self._static_hash: str = "" + + # Initialize caches for each file + for filename, _required in PromptConfig.FILE_ORDER: + self._file_caches[filename] = CachedSection(path=working_dir / filename) + + def _build_static(self) -> str: + """Build the static portion from cached files.""" + parts: list[str] = [] + for filename, required in PromptConfig.FILE_ORDER: + # Persona soul_md override + if filename == "SOUL.md" and self._persona and getattr(self._persona, "soul_md", None): + if parts: # Add separator before persona section + parts.append("") + parts.append(f"# SOUL.md ({self._persona.name})") + parts.append("") + parts.append(self._persona.soul_md) + continue + + section = self._file_caches.get(filename) + if section is None: + continue + content = section.load() + if content: + if parts: + parts.append("") + parts.append(f"# {filename}") + parts.append("") + parts.append(content) + elif required: + return DEFAULT_SYS_PROMPT + + return "\n\n".join(parts) if parts else DEFAULT_SYS_PROMPT + + def _static_source_hash(self) -> str: + """Hash of all file content hashes + persona for cache invalidation.""" + h = hashlib.sha256() + for filename, _ in PromptConfig.FILE_ORDER: + section = self._file_caches.get(filename) + if section: + section.load() # ensure loaded + h.update(section.content_hash.encode()) + if self._persona: + h.update(getattr(self._persona, "id", "").encode()) + h.update(getattr(self._persona, "soul_md", "").encode()) + return h.hexdigest() + + @property + def static_prompt(self) -> str: + """Get cached static prompt, rebuilding only if files changed.""" + current_hash = self._static_source_hash() + if current_hash != self._static_hash: + self._static_prompt = self._build_static() + self._static_hash = current_hash + return self._static_prompt + + def build(self, dynamic: Optional[DynamicContext] = None) -> str: + """Return static + dynamic prompt.""" + static = self.static_prompt + if dynamic is None: + return static + dynamic_text = dynamic.render() + if not dynamic_text: + return static + return f"{static}\n\n{dynamic_text}" + + def set_persona(self, persona) -> None: + """Switch persona, invalidating the static cache.""" + self._persona = persona + self._static_hash = "" # force rebuild + + def invalidate(self) -> None: + """Force rebuild on next access.""" + self._static_hash = "" + for section in self._file_caches.values(): + section.content_hash = "" + section.last_checked = 0.0 + + +class PersonaPromptPool: + """Maintains one CachedPromptBuilder per persona.""" + + _MAX_POOL_SIZE = 50 + + def __init__(self, working_dir: Path) -> None: + self._working_dir = working_dir + self._builders: Dict[str, CachedPromptBuilder] = {} + + def get(self, persona=None) -> CachedPromptBuilder: + """Get or create a builder for the given persona.""" + key = getattr(persona, "id", "__default__") if persona else "__default__" + if key not in self._builders: + # Evict oldest entry if pool is at capacity + if len(self._builders) >= self._MAX_POOL_SIZE: + oldest_key = next(iter(self._builders)) + del self._builders[oldest_key] + self._builders[key] = CachedPromptBuilder( + working_dir=self._working_dir, persona=persona + ) + return self._builders[key] + + def invalidate_all(self) -> None: + """Clear all cached builders.""" + self._builders.clear() + + @property + def size(self) -> int: + return len(self._builders) + + +def select_memory_tier( + tiers: Dict[str, str], + available_tokens: int, + static_tokens: int, +) -> Tuple[str, str]: + """Select the richest AOM memory tier that fits the remaining budget. + + Args: + tiers: Dict from generate_tiers() with keys L0, L1, L2 + available_tokens: Total token budget for the prompt + static_tokens: Tokens already used by the static prompt + + Returns: + (tier_name, tier_content) — e.g. ("L2", "full context text") + """ + from ..memory_agent.tiers import estimate_tokens + + remaining = available_tokens - static_tokens + # Try richest first + for tier_name in ("L2", "L1", "L0"): + content = tiers.get(tier_name, "") + if not content: + continue + tokens = estimate_tokens(content) + if tokens <= remaining: + return tier_name, content + # Budget exhausted — return empty rather than L0 that may not fit + l0 = tiers.get("L0", "") + if l0 and estimate_tokens(l0) > remaining: + return "L0", "" + return "L0", l0 + + +__all__ = [ + "build_system_prompt_from_working_dir", + "build_bootstrap_guidance", + "PromptBuilder", + "PromptConfig", + "DEFAULT_SYS_PROMPT", + "SYS_PROMPT", # Backward compatibility + # v2 + "CachedSection", + "DynamicContext", + "CachedPromptBuilder", + "PersonaPromptPool", + "select_memory_tier", +] diff --git a/src/adclaw/agents/react_agent.py b/src/adclaw/agents/react_agent.py new file mode 100644 index 0000000..49133df --- /dev/null +++ b/src/adclaw/agents/react_agent.py @@ -0,0 +1,778 @@ +# -*- coding: utf-8 -*- +"""MaskanX Agent - Main agent implementation. + +This module provides the main MaskanXAgent class built on ReActAgent, +with integrated tools, skills, and memory management. +""" +import asyncio +import logging +import os +import sys +from pathlib import Path +from typing import TYPE_CHECKING, Any, List, Literal, Optional, Type + +from agentscope.agent import ReActAgent +from agentscope.mcp import HttpStatefulClient, StdIOStatefulClient +from agentscope.memory import InMemoryMemory +from agentscope.message import Msg +from agentscope.tool import Toolkit +from anyio import ClosedResourceError +from pydantic import BaseModel + +# BaseExceptionGroup is a builtin in 3.11+; on 3.10 we use the +# `exceptiongroup` backport (declared in pyproject.toml only for 3.10). +if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup # noqa: F401 + +from .command_handler import CommandHandler +from .hooks import BootstrapHook, MemoryCompactionHook +from .model_factory import create_model_and_formatter +from .prompt import ( + CachedPromptBuilder, + DynamicContext, + PersonaPromptPool, +) +from .skills_manager import ( + ensure_skills_initialized, + get_builtin_skills_dir, + get_customized_skills_dir, + get_working_skills_dir, + list_available_skills, +) +from .tools import ( + browser_use, + desktop_screenshot, + edit_file, + execute_shell_command, + get_current_time, + read_file, + send_file_to_user, + send_email, + write_file, + create_memory_search_tool, +) +from .tools.aom_query import create_aom_query_tool +from .tools.skill_patcher import patch_skill_script, TOOL_SPEC as _PATCHER_TOOL_SPEC +from ..providers.store import get_active_llm_config +from .utils import process_file_and_media_blocks_in_message +from ..config import load_config +from ..constant import ( + MEMORY_COMPACT_KEEP_RECENT, + MEMORY_COMPACT_RATIO, + WORKING_DIR, +) + +if TYPE_CHECKING: + from ..agents.memory import MemoryManager + +logger = logging.getLogger(__name__) + +_TRUTHY_ENV_VALUES = {"1", "true", "yes", "on"} + + +def _host_ai_shell_tool_enabled() -> bool: + """Return whether generic shell execution is allowed for Host AI chat.""" + value = os.getenv("MASKANX_HOST_AI_ALLOW_SHELL_TOOL", "").strip().lower() + if value in _TRUTHY_ENV_VALUES: + return True + try: + active = get_active_llm_config() + except Exception: + return False + return getattr(active, "provider_id", "") != "maskanx-host-ai" + +# Valid namesake strategies for tool registration +NamesakeStrategy = Literal["override", "skip", "raise", "rename"] + + +def normalize_reasoning_tool_choice( + tool_choice: Literal["auto", "none", "required"] | None, + has_tools: bool, +) -> Literal["auto", "none", "required"] | None: + """Normalize tool_choice for reasoning to reduce provider variance.""" + if tool_choice is None and has_tools: + return "auto" + return tool_choice + + +class MaskanXAgent(ReActAgent): + """MaskanX Agent with integrated tools, skills, and memory management. + + This agent extends ReActAgent with: + - Built-in tools (shell, file operations, browser, etc.) + - Dynamic skill loading from working directory + - Memory management with auto-compaction + - Bootstrap guidance for first-time setup + - System command handling (/compact, /new, etc.) + """ + + def __init__( + self, + env_context: Optional[str] = None, + enable_memory_manager: bool = True, + mcp_clients: Optional[List[Any]] = None, + memory_manager: "MemoryManager | None" = None, + aom_manager: Optional[Any] = None, + max_iters: int = 50, + max_input_length: int = 128 * 1024, # 128K = 131072 tokens + namesake_strategy: NamesakeStrategy = "skip", + persona=None, + team_summary: str = "", + persona_manager=None, + model=None, + formatter=None, + timeout_seconds: Optional[int] = None, + ): + """Initialize MaskanXAgent. + + Args: + env_context: Optional environment context to prepend to + system prompt + enable_memory_manager: Whether to enable memory manager + mcp_clients: Optional list of MCP clients for tool + integration + memory_manager: Optional memory manager instance + max_iters: Maximum number of reasoning-acting iterations + (default: 50) + max_input_length: Maximum input length in tokens for model + context window (default: 128K = 131072) + namesake_strategy: Strategy to handle namesake tool functions. + Options: "override", "skip", "raise", "rename" + (default: "skip") + persona: Optional PersonaConfig with soul_md override + team_summary: Optional team summary for multi-agent awareness + persona_manager: Optional manager used for delegation tooling + """ + self._persona = persona + self._team_summary = team_summary + self._persona_manager = persona_manager + self._persona_skill_names = tuple(getattr(persona, "skills", []) or []) + self._heal_events: list = [] + + self._prompt_pool = PersonaPromptPool(working_dir=Path(WORKING_DIR)) + self._prompt_builder = self._prompt_pool.get(persona) + self._env_context = env_context + self._max_input_length = max_input_length + self._mcp_clients = mcp_clients or [] + self._namesake_strategy = namesake_strategy + self._aom_manager = aom_manager + self._aom_capture_hook = None + + # Memory compaction threshold: configurable ratio of max_input_length + self._memory_compact_threshold = int( + max_input_length * MEMORY_COMPACT_RATIO, + ) + + # Initialize toolkit with built-in tools + toolkit = self._create_toolkit(namesake_strategy=namesake_strategy) + + # Load and register skills + self._register_skills(toolkit) + + # Build system prompt + sys_prompt = self._build_sys_prompt() + + # Create model and formatter using factory method (unless injected) + if (model is None) != (formatter is None): + raise ValueError( + "model and formatter must both be provided or both be None", + ) + if model is None: + model, formatter = create_model_and_formatter( + timeout_seconds=timeout_seconds, + ) + + # Initialize parent ReActAgent + super().__init__( + name="Friday", + model=model, + sys_prompt=sys_prompt, + toolkit=toolkit, + memory=InMemoryMemory(), + formatter=formatter, + max_iters=max_iters, + ) + + # Setup memory manager + self._setup_memory_manager( + enable_memory_manager, + memory_manager, + namesake_strategy, + ) + + # Setup AOM integration + self._setup_aom(namesake_strategy) + + # Setup command handler + self.command_handler = CommandHandler( + agent_name=self.name, + memory=self.memory, + memory_manager=self.memory_manager, + enable_memory_manager=self._enable_memory_manager, + ) + + # Register hooks + self._register_hooks() + + def _create_toolkit( + self, + namesake_strategy: NamesakeStrategy = "skip", + ) -> Toolkit: + """Create and populate toolkit with built-in tools. + + Args: + namesake_strategy: Strategy to handle namesake tool functions. + Options: "override", "skip", "raise", "rename" + (default: "skip") + + Returns: + Configured toolkit instance + """ + toolkit = Toolkit() + + # Register built-in tools + if _host_ai_shell_tool_enabled(): + toolkit.register_tool_function( + execute_shell_command, + namesake_strategy=namesake_strategy, + ) + toolkit.register_tool_function( + read_file, + namesake_strategy=namesake_strategy, + ) + toolkit.register_tool_function( + write_file, + namesake_strategy=namesake_strategy, + ) + toolkit.register_tool_function( + edit_file, + namesake_strategy=namesake_strategy, + ) + toolkit.register_tool_function( + browser_use, + namesake_strategy=namesake_strategy, + ) + toolkit.register_tool_function( + desktop_screenshot, + namesake_strategy=namesake_strategy, + ) + toolkit.register_tool_function( + send_file_to_user, + namesake_strategy=namesake_strategy, + ) + toolkit.register_tool_function( + send_email, + namesake_strategy=namesake_strategy, + ) + toolkit.register_tool_function( + get_current_time, + namesake_strategy=namesake_strategy, + ) + toolkit.register_tool_function( + patch_skill_script, + namesake_strategy=namesake_strategy, + ) + if self._persona_manager and self._persona_manager.all_personas: + from .tools.delegation import make_delegate_tool + + toolkit.register_tool_function( + make_delegate_tool(self._persona_manager), + namesake_strategy=namesake_strategy, + ) + + return toolkit + + def _register_skills(self, toolkit: Toolkit) -> None: + """Load and register skills from working directory. + + Args: + toolkit: Toolkit to register skills to + """ + working_skills_dir = get_working_skills_dir() + available_skills = list_available_skills() + skill_names = ( + list(self._persona_skill_names) + if self._persona_skill_names + else available_skills + ) + if not self._persona_skill_names: + # Persona-pinned skills can resolve directly from built-ins even + # when active_skills is empty; warning in that path is noise. + ensure_skills_initialized() + + self._broken_skills = [] + for skill_name in skill_names: + skill_dir = self._resolve_skill_dir(working_skills_dir, skill_name) + if skill_dir.exists(): + try: + toolkit.register_agent_skill(str(skill_dir)) + logger.debug("Registered skill: %s", skill_name) + except Exception as e: + logger.error( + "Failed to register skill '%s': %s", + skill_name, + e, + ) + self._broken_skills.append( + (skill_name, skill_dir, str(e)) + ) + + @staticmethod + def _resolve_skill_dir(working_skills_dir: Path, skill_name: str) -> Path: + """Return the best available directory for a selected skill.""" + if Path(skill_name).name != skill_name or "/" in skill_name or "\\" in skill_name: + return working_skills_dir / "__invalid_skill_name__" + + for base_dir in ( + working_skills_dir, + get_customized_skills_dir(), + get_builtin_skills_dir(), + ): + candidate = base_dir / skill_name + if (candidate / "SKILL.md").exists(): + return candidate + return working_skills_dir / skill_name + + def _build_sys_prompt(self) -> str: + """Build system prompt using CachedPromptBuilder (v2). + + Static sections (AGENTS.md, SOUL.md, PROFILE.md) are cached via hash. + Dynamic sections (env context, team summary) are rebuilt per-call. + + Returns: + Complete system prompt string + """ + dynamic = DynamicContext( + env_context=self._env_context or "", + team_summary=self._team_summary or "", + ) + return self._prompt_builder.build(dynamic=dynamic) + + def _setup_memory_manager( + self, + enable_memory_manager: bool, + memory_manager: "MemoryManager | None", + namesake_strategy: NamesakeStrategy, + ) -> None: + """Setup memory manager and register memory search tool if enabled. + + Args: + enable_memory_manager: Whether to enable memory manager + memory_manager: Optional memory manager instance + namesake_strategy: Strategy to handle namesake tool functions + """ + # Check env var: if ENABLE_MEMORY_MANAGER=false, disable memory manager + env_enable_mm = os.getenv("ENABLE_MEMORY_MANAGER", "") + if env_enable_mm.lower() == "false": + enable_memory_manager = False + + self._enable_memory_manager: bool = enable_memory_manager + self.memory_manager = memory_manager + + # Register memory_search tool if enabled and available + if self._enable_memory_manager and self.memory_manager is not None: + # update memory manager + self.memory_manager.chat_model = self.model + self.memory_manager.formatter = self.formatter + memory_toolkit = Toolkit() + memory_toolkit.register_tool_function( + read_file, + namesake_strategy=self._namesake_strategy, + ) + memory_toolkit.register_tool_function( + write_file, + namesake_strategy=self._namesake_strategy, + ) + memory_toolkit.register_tool_function( + edit_file, + namesake_strategy=self._namesake_strategy, + ) + self.memory_manager.toolkit = memory_toolkit + self.memory_manager.update_config_params() + + self.memory = self.memory_manager.get_in_memory_memory() + + # Register memory_search as a tool function + self.toolkit.register_tool_function( + create_memory_search_tool(self.memory_manager), + namesake_strategy=namesake_strategy, + ) + logger.debug("Registered memory_search tool") + + def _setup_aom(self, namesake_strategy: NamesakeStrategy) -> None: + """Setup Always-On Memory integration if AOM manager is available.""" + if self._aom_manager is None or not self._aom_manager.is_running: + return + + # Register query_long_term_memory tool + if self._aom_manager.query_agent is not None: + try: + tool_fn = create_aom_query_tool(self._aom_manager.query_agent) + self.toolkit.register_tool_function( + tool_fn, + namesake_strategy=namesake_strategy, + ) + logger.debug("Registered query_long_term_memory tool") + except Exception as exc: + logger.warning("Failed to register AOM query tool: %s", exc) + + # Setup capture hook for auto-ingesting tool results + if self._aom_manager.ingest_agent is not None: + try: + from .hooks.aom_capture import AOMCaptureHook + + config = load_config() + aom_config = config.agents.always_on_memory + self._aom_capture_hook = AOMCaptureHook( + ingest_agent=self._aom_manager.ingest_agent, + capture_mcp=aom_config.auto_capture_mcp, + capture_skills=aom_config.auto_capture_skills, + ) + logger.debug("AOM capture hook configured") + except Exception as exc: + logger.warning("Failed to setup AOM capture hook: %s", exc) + + def _register_hooks(self) -> None: + """Register pre-reasoning hooks for bootstrap and memory compaction.""" + # Bootstrap hook - checks BOOTSTRAP.md on first interaction + config = load_config() + bootstrap_hook = BootstrapHook( + working_dir=WORKING_DIR, + language=config.agents.language, + ) + self.register_instance_hook( + hook_type="pre_reasoning", + hook_name="bootstrap_hook", + hook=bootstrap_hook.__call__, + ) + logger.debug("Registered bootstrap hook") + + # Memory compaction hook - auto-compact when context is full + if self._enable_memory_manager and self.memory_manager is not None: + memory_compact_hook = MemoryCompactionHook( + memory_manager=self.memory_manager, + memory_compact_threshold=self._memory_compact_threshold, + keep_recent=MEMORY_COMPACT_KEEP_RECENT, + ) + self.register_instance_hook( + hook_type="pre_reasoning", + hook_name="memory_compact_hook", + hook=memory_compact_hook.__call__, + ) + logger.debug("Registered memory compaction hook") + + def rebuild_sys_prompt(self) -> None: + """Rebuild the system prompt using CachedPromptBuilder. + + Static sections (AGENTS.md, SOUL.md, PROFILE.md) are cached -- + only re-read when file content hash changes. + Dynamic sections (env context, team summary) are rebuilt every call. + """ + self._sys_prompt = self._build_sys_prompt() + logger.debug( + "Prompt rebuilt (len=%d)", + len(self._sys_prompt), + ) + + for msg, _marks in self.memory.content: + if msg.role == "system": + msg.content = self.sys_prompt + break + + async def register_mcp_clients( + self, + namesake_strategy: NamesakeStrategy = "skip", + ) -> None: + """Register MCP clients on this agent's toolkit after construction. + + Args: + namesake_strategy: Strategy to handle namesake tool functions. + Options: "override", "skip", "raise", "rename" + (default: "skip") + """ + for i, client in enumerate(self._mcp_clients): + client_name = getattr(client, "name", repr(client)) + try: + await self.toolkit.register_mcp_client( + client, + namesake_strategy=namesake_strategy, + ) + except (ClosedResourceError, asyncio.CancelledError) as error: + if self._should_propagate_cancelled_error(error): + raise + logger.warning( + "MCP client '%s' session interrupted while listing tools; " + "trying recovery", + client_name, + ) + recovered_client = await self._recover_mcp_client(client) + if recovered_client is not None: + self._mcp_clients[i] = recovered_client + try: + await self.toolkit.register_mcp_client( + recovered_client, + namesake_strategy=namesake_strategy, + ) + continue + except asyncio.CancelledError as recover_error: + if self._should_propagate_cancelled_error( + recover_error, + ): + raise + logger.warning( + "MCP client '%s' registration cancelled after " + "recovery, skipping", + client_name, + ) + except Exception as e: # pylint: disable=broad-except + logger.warning( + "MCP client '%s' still unavailable after " + "recovery, skipping: %s", + client_name, + e, + ) + else: + logger.warning( + "MCP client '%s' recovery failed, skipping", + client_name, + ) + except BaseExceptionGroup as eg: + # anyio TaskGroup teardown (e.g. MCP HTTP 401) raises + # BaseExceptionGroup which is NOT a subclass of Exception + # in Python 3.11+ — handle explicitly so one broken client + # never crashes the whole agent. ExceptionGroup is a + # subclass of BaseExceptionGroup so this catches both. + logger.warning( + "MCP client '%s' unavailable (TaskGroup error), " + "skipping (agent will run without its tools): %s", + client_name, + eg, + exc_info=True, + ) + continue + except Exception as e: # pylint: disable=broad-except + logger.warning( + "MCP client '%s' unavailable, skipping " + "(agent will run without its tools): %s", + client_name, + e, + exc_info=True, + ) + continue + + # Auto-heal broken skills (async context available here) + if getattr(self, "_broken_skills", None): + await self._heal_broken_skills() + + async def _heal_broken_skills(self) -> None: + """Attempt to auto-heal broken skills using LLM.""" + from .skill_healer import heal_skill + + async def llm_caller(prompt: str) -> str: + model, _ = create_model_and_formatter() + r = await model([{"role": "user", "content": prompt}]) + if hasattr(r, "__aiter__"): + last_text = "" + async for chunk in r: + c = getattr(chunk, "content", None) + if isinstance(c, list): + for b in c: + if isinstance(b, dict) \ + and b.get("type") == "text": + last_text = b.get("text", "") + elif isinstance(c, str): + last_text = c + return last_text + c = getattr(r, "content", str(r)) + if isinstance(c, list): + return "".join( + b.get("text", "") if isinstance(b, dict) + else str(b) for b in c + ) + return str(c) + + for skill_name, skill_dir, error_msg in self._broken_skills: + try: + result = await heal_skill(skill_dir, error_msg, llm_caller) + if result.healed: + try: + self.toolkit.register_agent_skill(str(skill_dir)) + self._heal_events.append({ + "skill": skill_name, + "error": error_msg[:100], + }) + logger.info( + "Self-healed skill '%s': %s", + skill_name, result.message, + ) + except Exception as retry_err: + logger.error( + "Skill '%s' still broken after heal: %s", + skill_name, retry_err, + ) + else: + logger.warning( + "Could not heal skill '%s': %s", + skill_name, result.message, + ) + except Exception as heal_err: + logger.warning( + "Self-heal failed for '%s': %s", + skill_name, heal_err, + ) + + async def _recover_mcp_client(self, client: Any) -> Any | None: + """Recover MCP client from broken session and return healthy client.""" + if await self._reconnect_mcp_client(client): + return client + + rebuilt_client = self._rebuild_mcp_client(client) + if rebuilt_client is None: + return None + + if await self._reconnect_mcp_client(rebuilt_client): + return self._reuse_shared_client_reference( + original_client=client, + rebuilt_client=rebuilt_client, + ) + + return None + + @staticmethod + def _reuse_shared_client_reference( + original_client: Any, + rebuilt_client: Any, + ) -> Any: + """Keep manager-shared client reference stable after rebuild.""" + original_dict = getattr(original_client, "__dict__", None) + rebuilt_dict = getattr(rebuilt_client, "__dict__", None) + if isinstance(original_dict, dict) and isinstance(rebuilt_dict, dict): + original_dict.update(rebuilt_dict) + return original_client + return rebuilt_client + + @staticmethod + def _should_propagate_cancelled_error(error: BaseException) -> bool: + """Only swallow MCP-internal cancellations, not task cancellation.""" + if not isinstance(error, asyncio.CancelledError): + return False + + task = asyncio.current_task() + if task is None: + return False + + cancelling = getattr(task, "cancelling", None) + if callable(cancelling): + return cancelling() > 0 + + # Python < 3.11: Task.cancelling() is unavailable. + # Fall back to propagating CancelledError to avoid swallowing + # genuine task cancellations when we cannot inspect the state. + return True + + @staticmethod + async def _reconnect_mcp_client( + client: Any, + timeout: float = 60.0, + ) -> bool: + """Best-effort reconnect for stateful MCP clients.""" + close_fn = getattr(client, "close", None) + if callable(close_fn): + try: + await close_fn() + except asyncio.CancelledError: # pylint: disable=try-except-raise + raise + except Exception: # pylint: disable=broad-except + pass + + connect_fn = getattr(client, "connect", None) + if not callable(connect_fn): + return False + + try: + await asyncio.wait_for(connect_fn(), timeout=timeout) + return True + except asyncio.CancelledError: # pylint: disable=try-except-raise + raise + except asyncio.TimeoutError: + return False + except Exception: # pylint: disable=broad-except + return False + + @staticmethod + def _rebuild_mcp_client(client: Any) -> Any | None: + """Rebuild a fresh MCP client instance from stored config metadata.""" + rebuild_info = getattr(client, "_maskanx_rebuild_info", None) + if not isinstance(rebuild_info, dict): + return None + + transport = rebuild_info.get("transport") + name = rebuild_info.get("name") + + try: + if transport == "stdio": + rebuilt_client = StdIOStatefulClient( + name=name, + command=rebuild_info.get("command"), + args=rebuild_info.get("args", []), + env=rebuild_info.get("env", {}), + cwd=rebuild_info.get("cwd"), + ) + setattr(rebuilt_client, "_maskanx_rebuild_info", rebuild_info) + return rebuilt_client + + rebuilt_client = HttpStatefulClient( + name=name, + transport=transport, + url=rebuild_info.get("url"), + headers=rebuild_info.get("headers"), + ) + setattr(rebuilt_client, "_maskanx_rebuild_info", rebuild_info) + return rebuilt_client + except Exception: # pylint: disable=broad-except + return None + + async def _reasoning( + self, + tool_choice: Literal["auto", "none", "required"] | None = None, + ) -> Msg: + """Ensure a stable default tool-choice behavior across providers.""" + tool_choice = normalize_reasoning_tool_choice( + tool_choice=tool_choice, + has_tools=bool(self.toolkit.get_json_schemas()), + ) + + return await super()._reasoning(tool_choice=tool_choice) + + async def reply( + self, + msg: Msg | list[Msg] | None = None, + structured_model: Type[BaseModel] | None = None, + ) -> Msg: + """Override reply to process file blocks and handle commands. + + Args: + msg: Input message(s) from user + structured_model: Optional pydantic model for structured output + + Returns: + Response message + """ + # Process file and media blocks in messages + if msg is not None: + await process_file_and_media_blocks_in_message(msg) + + # Check if message is a system command + last_msg = msg[-1] if isinstance(msg, list) else msg + query = ( + last_msg.get_text_content() if isinstance(last_msg, Msg) else None + ) + + if self.command_handler.is_command(query): + logger.info(f"Received command: {query}") + msg = await self.command_handler.handle_command(query) + await self.print(msg) + return msg + + # Normal message processing + return await super().reply(msg=msg, structured_model=structured_model) diff --git a/src/adclaw/agents/schema.py b/src/adclaw/agents/schema.py new file mode 100644 index 0000000..6a7a5ad --- /dev/null +++ b/src/adclaw/agents/schema.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +""" +Agent tools schema: type definitions for agent tool responses. +""" +from typing import Literal, Optional +from typing_extensions import TypedDict, Required + +from agentscope.message import Base64Source, URLSource + + +class FileBlock(TypedDict, total=False): + """File block for sending files to users.""" + + type: Required[Literal["file"]] + """The type of the block""" + + source: Required[Base64Source | URLSource] + """The source of the file""" + + filename: Optional[str] + """The filename of the file""" diff --git a/src/adclaw/agents/skill_healer.py b/src/adclaw/agents/skill_healer.py new file mode 100644 index 0000000..67c5218 --- /dev/null +++ b/src/adclaw/agents/skill_healer.py @@ -0,0 +1,102 @@ +"""Auto-heal broken skill YAML frontmatter using the user's LLM.""" +import logging +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Awaitable + +logger = logging.getLogger(__name__) + +HEAL_PROMPT = """Fix the YAML frontmatter in this SKILL.md file. The error was: + +{error} + +The file content is: + +``` +{content} +``` + +Rules: +- The file MUST start with `---`, then YAML with at least `name` and `description` fields, then `---` +- Fix any YAML syntax errors, invalid unicode escapes, or missing fields +- The `name` field should be a simple lowercase-kebab-case identifier +- The `description` field should be a single English sentence +- Do NOT modify the markdown body after the closing `---` +- Return ONLY the complete fixed file content, no explanations""" + + +@dataclass +class HealResult: + healed: bool + skill_name: str + original: str + fixed: str + error: str + message: str = "" + + +async def heal_skill( + skill_dir: Path, + error_message: str, + llm_caller: Callable[[str], Awaitable[str]], +) -> HealResult: + """Attempt to fix a broken SKILL.md using LLM.""" + skill_name = skill_dir.name + skill_file = skill_dir / "SKILL.md" + + if not skill_file.exists(): + return HealResult( + healed=False, skill_name=skill_name, + original="", fixed="", error=error_message, + message="SKILL.md not found", + ) + + original = skill_file.read_text(encoding="utf-8") + prompt = HEAL_PROMPT.format(error=error_message, content=original) + + try: + fixed = await llm_caller(prompt) + except Exception as e: + logger.warning("LLM heal failed for '%s': %s", skill_name, e) + return HealResult( + healed=False, skill_name=skill_name, + original=original, fixed="", error=error_message, + message=f"LLM call failed: {e}", + ) + + # Strip markdown code fences if LLM wrapped the response + fixed = fixed.strip() + if fixed.startswith("```"): + lines = fixed.split("\n") + if lines[-1].strip() == "```": + lines = lines[1:-1] + else: + lines = lines[1:] + fixed = "\n".join(lines) + + if not fixed or fixed == original: + return HealResult( + healed=False, skill_name=skill_name, + original=original, fixed=fixed, error=error_message, + message="LLM returned unchanged or empty content", + ) + + if not fixed.startswith("---"): + return HealResult( + healed=False, skill_name=skill_name, + original=original, fixed=fixed, error=error_message, + message="LLM response missing frontmatter delimiters", + ) + + # Backup original and write fix + backup = skill_dir / "SKILL.md.bak" + backup.write_text(original, encoding="utf-8") + skill_file.write_text(fixed, encoding="utf-8") + + logger.info("Auto-healed skill '%s': %s", skill_name, error_message[:80]) + + return HealResult( + healed=True, skill_name=skill_name, + original=original, fixed=fixed, error=error_message, + message=f"Fixed: {error_message[:80]}", + ) diff --git a/src/adclaw/agents/skill_quality.py b/src/adclaw/agents/skill_quality.py new file mode 100644 index 0000000..4dbace0 --- /dev/null +++ b/src/adclaw/agents/skill_quality.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +"""Skill quality evaluation — checks frontmatter, description, directives, jargon.""" + +import re +from dataclasses import dataclass, field +from pathlib import Path + + +DIRECTIVE_VERBS = { + "extract", "scrape", "analyze", "generate", "create", "find", "search", + "monitor", "track", "build", "optimize", "audit", "evaluate", "detect", + "route", "delegate", "check", "validate", "run", "execute", "deploy", + "configure", "install", "update", "remove", "list", "fetch", "parse", + "transform", "convert", "filter", "sort", "rank", "score", "classify", + "summarize", "report", "notify", "send", "receive", "connect", "sync", + "help", "assist", "guide", "recommend", "suggest", "act", "use", +} + +JARGON_PATTERNS = [ + r"\bsynerg", r"\bleverage\b", r"\bparadigm\b", r"\bholistic\b", + r"\brobust\b", r"\bscalable\b", r"\bseamless\b", r"\bcut(?:ting)?[- ]edge\b", + r"\bnext[- ]gen(?:eration)?\b", r"\bgame[- ]chang", r"\bworld[- ]class\b", + r"\bbest[- ]in[- ]class\b", r"\binnovative\b", r"\bstate[- ]of[- ]the[- ]art\b", +] + + +def _parse_frontmatter(content: str) -> dict | None: + m = re.match(r"^---\s*\n(.*?)\n---", content, re.DOTALL) + if not m: + return None + fm = {} + for line in m.group(1).strip().splitlines(): + if ":" in line: + key, val = line.split(":", 1) + fm[key.strip()] = val.strip().strip('"').strip("'") + return fm + + +@dataclass +class QualityResult: + passed: bool + score: int # 0-100 + warnings: list[str] = field(default_factory=list) + skill_name: str = "" + + def to_dict(self) -> dict: + return { + "passed": self.passed, + "score": self.score, + "warnings": self.warnings, + "skill_name": self.skill_name, + } + + +def evaluate_skill_quality(skill_dir: Path, skill_name: str = "") -> QualityResult: + """Run quality checks on a skill's SKILL.md.""" + skill_md = skill_dir / "SKILL.md" + if not skill_md.exists(): + return QualityResult( + passed=False, score=0, + warnings=["SKILL.md not found"], + skill_name=skill_name or skill_dir.name, + ) + + content = skill_md.read_text(encoding="utf-8") + warnings: list[str] = [] + score = 100 + name = skill_name or skill_dir.name + + fm = _parse_frontmatter(content) + if fm is None: + warnings.append("Missing YAML frontmatter (---)") + score -= 30 + else: + if not fm.get("name"): + warnings.append("Missing 'name' in frontmatter") + score -= 20 + desc = fm.get("description", "") + if len(desc) < 40: + warnings.append(f"Description too short ({len(desc)} chars, need >= 40)") + score -= 15 + else: + desc_lower = desc.lower() + has_verb = any( + desc_lower.startswith(v) or f" {v} " in f" {desc_lower} " + for v in DIRECTIVE_VERBS + ) + if not has_verb: + warnings.append("Description lacks directive verb") + score -= 10 + + for pattern in JARGON_PATTERNS: + matches = re.findall(pattern, content, re.IGNORECASE) + if matches: + warnings.append(f"Jargon detected: '{matches[0]}'") + score -= 5 + + body = re.sub(r"^---.*?---", "", content, count=1, flags=re.DOTALL).strip() + if len(body) < 100: + warnings.append(f"Body too short ({len(body)} chars)") + score -= 10 + + if "## " not in body: + warnings.append("No section headers found in body") + score -= 10 + + score = max(0, score) + passed = score >= 60 and "not found" not in " ".join(warnings) + + return QualityResult(passed=passed, score=score, warnings=warnings, skill_name=name) diff --git a/src/adclaw/agents/skill_scanner.py b/src/adclaw/agents/skill_scanner.py new file mode 100644 index 0000000..acf0a74 --- /dev/null +++ b/src/adclaw/agents/skill_scanner.py @@ -0,0 +1,1003 @@ +# -*- coding: utf-8 -*- +"""Skill security scanner — static analysis of skill scripts before installation.""" + +from __future__ import annotations + +import ast +import logging +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import List, Optional + +logger = logging.getLogger(__name__) + + +@dataclass +class Finding: + """A security finding in a skill file.""" + + severity: str # "critical", "high", "medium", "low" + category: str = "" + file: str = "" + line: int = 0 + description: str = "" + code_snippet: str = "" + + def __str__(self) -> str: + return f"[{self.severity.upper()}] {self.file}:{self.line} — {self.description}" + + +@dataclass +class ScanResult: + """Result of a skill security scan.""" + + safe: bool + skill_name: str + findings: List[Finding] = field(default_factory=list) + files_scanned: int = 0 + + @property + def critical_count(self) -> int: + return sum(1 for f in self.findings if f.severity == "critical") + + @property + def high_count(self) -> int: + return sum(1 for f in self.findings if f.severity == "high") + + def to_dict(self) -> dict: + return { + "safe": self.safe, + "skill_name": self.skill_name, + "files_scanned": self.files_scanned, + "findings": [ + { + "severity": f.severity, + "category": f.category, + "file": f.file, + "line": f.line, + "description": f.description, + "code_snippet": f.code_snippet, + } + for f in self.findings + ], + "summary": { + "critical": self.critical_count, + "high": self.high_count, + "total": len(self.findings), + }, + } + + +# ============================================================================ +# Category 1: Code Execution (eval/exec/compile) +# ============================================================================ + +_DANGEROUS_CALLS = { + "eval": "critical", + "exec": "critical", + "compile": "high", + "__import__": "high", + "breakpoint": "medium", + "globals": "medium", + "locals": "medium", + "vars": "low", + "getattr": "low", + "setattr": "medium", + "delattr": "medium", +} + +# ============================================================================ +# Category 2: Dangerous module.function patterns +# ============================================================================ + +_DANGEROUS_ATTRS = { + # OS command execution + ("os", "system"): ("critical", "os.system() — arbitrary command execution"), + ("os", "popen"): ("critical", "os.popen() — arbitrary command execution"), + ("os", "exec"): ("critical", "os.exec*() — process replacement"), + ("os", "execvp"): ("critical", "os.execvp() — process replacement"), + ("os", "execve"): ("critical", "os.execve() — process replacement"), + ("os", "execl"): ("critical", "os.execl() — process replacement"), + ("os", "execlp"): ("critical", "os.execlp() — process replacement"), + ("os", "spawn"): ("high", "os.spawn*() — process spawning"), + ("os", "spawnl"): ("high", "os.spawnl() — process spawning"), + ("os", "spawnle"): ("high", "os.spawnle() — process spawning"), + ("os", "spawnlp"): ("high", "os.spawnlp() — process spawning"), + ("os", "fork"): ("critical", "os.fork() — process forking"), + ("os", "kill"): ("high", "os.kill() — process termination"), + ("os", "killpg"): ("high", "os.killpg() — process group termination"), + # File system destructive ops + ("os", "remove"): ("medium", "os.remove() — file deletion"), + ("os", "unlink"): ("medium", "os.unlink() — file deletion"), + ("os", "rmdir"): ("medium", "os.rmdir() — directory deletion"), + ("os", "rename"): ("low", "os.rename() — file rename"), + ("os", "chmod"): ("medium", "os.chmod() — permission change"), + ("os", "chown"): ("high", "os.chown() — ownership change"), + ("os", "setuid"): ("critical", "os.setuid() — privilege escalation"), + ("os", "setgid"): ("critical", "os.setgid() — privilege escalation"), + ("os", "seteuid"): ("critical", "os.seteuid() — privilege escalation"), + ("os", "setegid"): ("critical", "os.setegid() — privilege escalation"), + # shutil + ("shutil", "rmtree"): ("high", "shutil.rmtree() — recursive directory deletion"), + ("shutil", "move"): ("low", "shutil.move() — file move"), + ("shutil", "chown"): ("high", "shutil.chown() — ownership change"), + # subprocess + ("subprocess", "call"): ("high", "subprocess.call() — command execution"), + ("subprocess", "Popen"): ("high", "subprocess.Popen() — command execution"), + ("subprocess", "run"): ("medium", "subprocess.run() — command execution"), + ("subprocess", "check_call"): ("high", "subprocess.check_call() — command execution"), + ("subprocess", "check_output"): ("high", "subprocess.check_output() — command execution"), + ("subprocess", "getoutput"): ("high", "subprocess.getoutput() — command execution"), + ("subprocess", "getstatusoutput"): ("high", "subprocess.getstatusoutput() — command execution"), + # Dynamic imports + ("importlib", "import_module"): ("high", "Dynamic module import"), + ("importlib", "__import__"): ("high", "Dynamic module import"), + # Native code / FFI + ("ctypes", "cdll"): ("critical", "ctypes — native code execution"), + ("ctypes", "CDLL"): ("critical", "ctypes — native code execution"), + ("ctypes", "windll"): ("critical", "ctypes — native code execution"), + ("ctypes", "WinDLL"): ("critical", "ctypes — native code execution"), + ("ctypes", "oledll"): ("critical", "ctypes — native code execution"), + ("ctypes", "pythonapi"): ("critical", "ctypes — Python C API access"), + ("cffi", "FFI"): ("high", "cffi — foreign function interface"), + # Deserialization + ("pickle", "loads"): ("high", "pickle.loads() — deserialization attack vector"), + ("pickle", "load"): ("high", "pickle.load() — deserialization attack vector"), + ("pickle", "Unpickler"): ("high", "pickle.Unpickler() — deserialization attack vector"), + ("marshal", "loads"): ("high", "marshal.loads() — code object deserialization"), + ("marshal", "load"): ("high", "marshal.load() — code object deserialization"), + ("yaml", "load"): ("high", "yaml.load() — unsafe YAML deserialization (use safe_load)"), + ("yaml", "unsafe_load"): ("critical", "yaml.unsafe_load() — arbitrary code execution"), + ("shelve", "open"): ("medium", "shelve.open() — uses pickle internally"), + ("dill", "loads"): ("high", "dill.loads() — deserialization attack vector"), + ("dill", "load"): ("high", "dill.load() — deserialization attack vector"), + ("jsonpickle", "decode"): ("high", "jsonpickle.decode() — deserialization attack vector"), + # Networking + ("socket", "socket"): ("medium", "Raw socket creation"), + ("socket", "create_connection"): ("medium", "Socket connection"), + ("http", "server"): ("medium", "HTTP server creation"), + ("smtplib", "SMTP"): ("medium", "SMTP email sending"), + ("ftplib", "FTP"): ("medium", "FTP connection"), + ("telnetlib", "Telnet"): ("high", "Telnet connection"), + ("paramiko", "SSHClient"): ("high", "SSH client connection"), + ("paramiko", "Transport"): ("high", "SSH transport connection"), + # Code generation / templating + ("jinja2", "Template"): ("low", "Jinja2 template — check for SSTI"), + ("mako", "template"): ("medium", "Mako template — potential SSTI"), + # Crypto mining indicators + ("hashlib", "sha256"): ("low", "SHA-256 — check context for mining"), + # System info gathering + ("platform", "system"): ("low", "System info gathering"), + ("platform", "node"): ("low", "Hostname gathering"), + # Signals + ("signal", "signal"): ("medium", "Signal handler modification"), + ("signal", "alarm"): ("medium", "Alarm signal"), + # Weak crypto + ("hashlib", "md5"): ("low", "MD5 — weak hash, check if used for security"), + ("hashlib", "sha1"): ("low", "SHA-1 — weak hash, check if used for security"), +} + +# ============================================================================ +# Category 3: Dangerous imports +# ============================================================================ + +_CRITICAL_MODULES = {"ctypes"} +_DANGEROUS_MODULES = { + "ctypes", "pickle", "marshal", "shelve", "dill", "jsonpickle", + "pty", "commands", # deprecated but dangerous +} + +# ============================================================================ +# Category 4: Sensitive file paths +# ============================================================================ + +_SENSITIVE_PATHS = [ + r"/etc/passwd", + r"/etc/shadow", + r"/etc/sudoers", + r"/etc/crontab", + r"~/.ssh", + r"\.env", + r"\.secret", + r"\.aws/credentials", + r"\.aws/config", + r"\.kube/config", + r"id_rsa", + r"id_ed25519", + r"\.git/config", + r"\.netrc", + r"\.pgpass", + r"\.mysql_history", + r"\.bash_history", + r"\.zsh_history", + r"/proc/self", + r"\.docker/config\.json", + r"\.npmrc", + r"\.pypirc", + r"authorized_keys", + r"known_hosts", + r"/var/log/", + r"\.gnupg/", +] +_SENSITIVE_PATH_RE = re.compile("|".join(_SENSITIVE_PATHS), re.IGNORECASE) + +# ============================================================================ +# Category 5: Network exfiltration patterns in string literals +# ============================================================================ + +_EXFIL_PATTERNS = [ + ( + r"(?i)https?://(?!localhost|127\.0\.0\.1|0\.0\.0\.0)[^\s\"']+", + "medium", + "External URL in string literal", + ), +] + +# Safe domains that are commonly used and not suspicious +_SAFE_DOMAINS = ( + "github.com", "pypi.org", "npmjs.com", "npmjs.org", + "googleapis.com", "microsoft.com", "python.org", + "readthedocs.io", "example.com", "mozilla.org", + "cloudflare.com", "fastly.net", "unpkg.com", + "cdnjs.cloudflare.com", "cdn.jsdelivr.net", + "registry.npmjs.org", "files.pythonhosted.org", +) + +# ============================================================================ +# Category 6: Shell injection patterns +# ============================================================================ + +_SHELL_PATTERNS: list[tuple[str, str, str, str]] = [ + # (pattern, severity, description, category) + # Remote code execution + ( + r"(?i)curl\s+[^\n]*\|\s*(sh|bash|zsh|python|python3|perl|ruby|node)", + "critical", "curl piped to shell — remote code execution", "rce", + ), + ( + r"(?i)wget\s+[^\n]*\|\s*(sh|bash|zsh|python|python3|perl|ruby|node)", + "critical", "wget piped to shell — remote code execution", "rce", + ), + ( + r"(?i)curl\s+[^\n]*>\s*/tmp/[^\s]+\s*&&\s*(sh|bash|chmod|python)", + "high", "Download and execute pattern", "rce", + ), + ( + r"(?i)(curl|wget)\s+[^\n]*(--output|-o|-O)\s+[^\s]+\s*&&\s*(chmod|sh|bash)", + "high", "Download, chmod, execute pattern", "rce", + ), + # Destructive commands + ( + r"(?i)rm\s+-rf\s+/(?!\w)", + "critical", "Destructive rm -rf / command", "destructive", + ), + ( + r"(?i)rm\s+-rf\s+~", + "critical", "Destructive rm -rf ~ (home directory)", "destructive", + ), + ( + r"(?i)rm\s+-rf\s+\$HOME", + "critical", "Destructive rm -rf $HOME", "destructive", + ), + ( + r"(?i)rm\s+-rf\s+/var|/usr|/etc|/opt|/boot", + "critical", "Destructive rm -rf on system directory", "destructive", + ), + ( + r":\(\)\{[^\}]*:\|:&[^\}]*\};:", + "critical", "Fork bomb", "destructive", + ), + ( + r"(?i)mkfs\.|dd\s+if=.*of=/dev/", + "critical", "Disk destruction command", "destructive", + ), + ( + r"(?i)>\s*/dev/sd[a-z]|>\s*/dev/nvme|>\s*/dev/vd[a-z]", + "critical", "Write to raw block device", "destructive", + ), + # Reverse shells + ( + r"(?i)bash\s+-i\s+>&\s*/dev/tcp/", + "critical", "Bash reverse shell", "reverse_shell", + ), + ( + r"(?i)nc\s+(-e|-c)\s+", + "critical", "Netcat reverse/bind shell", "reverse_shell", + ), + ( + r"(?i)ncat\s+(-e|-c)\s+", + "critical", "Ncat reverse/bind shell", "reverse_shell", + ), + ( + r"(?i)socat\s+.*exec:", + "critical", "Socat exec — potential reverse shell", "reverse_shell", + ), + ( + r"(?i)/dev/tcp/[0-9]", + "critical", "/dev/tcp redirection — reverse shell indicator", "reverse_shell", + ), + ( + r"(?i)python3?\s+-c\s+['\"]import\s+socket", + "critical", "Python reverse shell", "reverse_shell", + ), + ( + r"(?i)perl\s+-e\s+.*socket", + "critical", "Perl reverse shell", "reverse_shell", + ), + ( + r"(?i)ruby\s+-rsocket\s+-e", + "critical", "Ruby reverse shell", "reverse_shell", + ), + ( + r"(?i)php\s+-r\s+.*fsockopen", + "critical", "PHP reverse shell", "reverse_shell", + ), + ( + r"(?i)mkfifo\s+/tmp/.*\s*&&\s*(nc|ncat|cat)", + "critical", "Named pipe reverse shell", "reverse_shell", + ), + # Crypto mining + ( + r"(?i)xmrig|minerd|cpuminer|cgminer|ethminer|bfgminer|nicehash", + "critical", "Cryptocurrency miner detected", "crypto_mining", + ), + ( + r"(?i)stratum(\+tcp)?://", + "critical", "Mining pool connection (stratum protocol)", "crypto_mining", + ), + ( + r"(?i)--donate-level|--coin\s+(XMR|ETH|BTC|MONERO)", + "critical", "Mining configuration flag", "crypto_mining", + ), + # Persistence mechanisms + ( + r"(?i)crontab\s+-[elr]", + "high", "Crontab modification", "persistence", + ), + ( + r"(?i)/etc/cron\.(d|daily|hourly|weekly|monthly)/", + "high", "System cron directory access", "persistence", + ), + ( + r"(?i)systemctl\s+(enable|start|daemon-reload)", + "high", "Systemd service manipulation", "persistence", + ), + ( + r"(?i)/etc/systemd/system/.*\.service", + "high", "Systemd service file creation", "persistence", + ), + ( + r"(?i)/etc/init\.d/", + "high", "Init.d service manipulation", "persistence", + ), + ( + r"(?i)launchctl\s+(load|submit)", + "high", "macOS LaunchAgent/Daemon loading", "persistence", + ), + ( + r"(?i)~/Library/LaunchAgents/", + "high", "macOS LaunchAgent directory access", "persistence", + ), + ( + r"(?i)@reboot\s+", + "high", "Cron @reboot persistence", "persistence", + ), + ( + r"(?i)\.bashrc|\.bash_profile|\.zshrc|\.profile", + "medium", "Shell profile modification", "persistence", + ), + # Privilege escalation + ( + r"(?i)sudo\s+", + "medium", "sudo usage — potential privilege escalation", "privesc", + ), + ( + r"(?i)chmod\s+[0-7]*[4-7][0-7]{2}\s+|chmod\s+[ug]\+s\s+", + "high", "Set SUID/SGID bit — privilege escalation", "privesc", + ), + ( + r"(?i)chown\s+root", + "high", "Change file owner to root", "privesc", + ), + ( + r"(?i)/etc/sudoers", + "critical", "Sudoers file access", "privesc", + ), + ( + r"(?i)visudo|NOPASSWD", + "critical", "Sudoers modification — passwordless sudo", "privesc", + ), + ( + r"(?i)passwd\s+(root|--stdin)", + "critical", "Password change for root", "privesc", + ), + ( + r"(?i)useradd|adduser|usermod", + "high", "User account manipulation", "privesc", + ), + # SSH backdoors + ( + r"(?i)ssh-keygen\s+", + "high", "SSH key generation", "ssh_backdoor", + ), + ( + r"(?i)>>.*authorized_keys", + "critical", "SSH authorized_keys append — backdoor", "ssh_backdoor", + ), + ( + r"(?i)ssh\s+-R\s+", + "high", "SSH remote port forwarding", "ssh_backdoor", + ), + ( + r"(?i)ssh\s+-D\s+", + "high", "SSH SOCKS proxy", "ssh_backdoor", + ), + ( + r"(?i)sshpass\s+", + "high", "sshpass — non-interactive SSH auth", "ssh_backdoor", + ), + # Data exfiltration via CLI + ( + r"(?i)curl\s+-X\s*POST\s", + "high", "curl POST — potential data exfiltration", "exfiltration", + ), + ( + r"(?i)curl\s+[^\n]*(--data|-d)\s", + "high", "curl with data — potential exfiltration", "exfiltration", + ), + ( + r"(?i)wget\s+--post", + "high", "wget POST — potential data exfiltration", "exfiltration", + ), + ( + r"(?i)\bscp\s+", + "high", "scp file transfer", "exfiltration", + ), + ( + r"(?i)\brsync\s+.*@", + "high", "rsync to remote host", "exfiltration", + ), + ( + r"(?i)base64\s.*\|\s*(curl|wget|nc)", + "critical", "Base64 encode and exfiltrate", "exfiltration", + ), + ( + r"(?i)tar\s+.*\|\s*(curl|wget|nc|ssh)", + "critical", "Archive and exfiltrate", "exfiltration", + ), + # Network reconnaissance + ( + r"(?i)\bnmap\s+", + "high", "nmap port scanning", "recon", + ), + ( + r"(?i)\bnetstat\s+", + "medium", "Network statistics gathering", "recon", + ), + ( + r"(?i)\bifconfig\s|ip\s+addr", + "low", "Network interface information", "recon", + ), + ( + r"(?i)\barp\s+-a", + "medium", "ARP table enumeration", "recon", + ), + # Supply chain attacks + ( + r"(?i)pip\s+install\s+[^\s]+\s+--index-url\s+(?!https://pypi\.org)", + "critical", "pip install from non-standard index — supply chain risk", "supply_chain", + ), + ( + r"(?i)pip\s+install\s+--extra-index-url", + "high", "pip extra-index-url — dependency confusion risk", "supply_chain", + ), + ( + r"(?i)npm\s+install\s+--registry\s+(?!https://registry\.npmjs\.org)", + "critical", "npm install from non-standard registry", "supply_chain", + ), + ( + r"(?i)setup\.py\s+install|python\s+setup\.py", + "medium", "setup.py execution — check for malicious hooks", "supply_chain", + ), + # Obfuscation + ( + r"(?i)\\x[0-9a-f]{2}(\\x[0-9a-f]{2}){10,}", + "high", "Hex-encoded payload — possible obfuscation", "obfuscation", + ), + ( + r"(?i)base64\s+-d|base64\s+--decode", + "medium", "Base64 decoding — check for hidden commands", "obfuscation", + ), + ( + r"(?i)python3?\s+-c\s+['\"]exec\(|python3?\s+-c\s+['\"]eval\(", + "critical", "Python exec/eval in one-liner — obfuscated execution", "obfuscation", + ), + ( + r"(?i)echo\s+[A-Za-z0-9+/]{50,}={0,2}\s*\|\s*base64\s+-d\s*\|\s*(sh|bash)", + "critical", "Base64-obfuscated shell command", "obfuscation", + ), + # Environment/secrets access + ( + r"(?i)printenv|env\s*$|set\s*$", + "medium", "Environment variable dump", "env_access", + ), + ( + r"(?i)\$\{?[A-Z_]*(SECRET|TOKEN|KEY|PASSWORD|PASS|CREDENTIAL|AUTH)[A-Z_]*\}?", + "medium", "Access to secret environment variable", "env_access", + ), + # Container escape + ( + r"(?i)docker\s+run\s+.*--privileged", + "critical", "Privileged Docker container — escape risk", "container_escape", + ), + ( + r"(?i)docker\s+run\s+.*-v\s+/:/", + "critical", "Docker host root mount — container escape", "container_escape", + ), + ( + r"(?i)nsenter\s+", + "critical", "nsenter — namespace escape", "container_escape", + ), + ( + r"(?i)mount\s+.*cgroup|/sys/fs/cgroup", + "high", "Cgroup access — potential container escape", "container_escape", + ), + ( + r"(?i)/var/run/docker\.sock", + "critical", "Docker socket access — container escape", "container_escape", + ), + # Kernel / low-level + ( + r"(?i)insmod\s+|modprobe\s+|rmmod\s+", + "critical", "Kernel module manipulation", "kernel", + ), + ( + r"(?i)/proc/sys/|sysctl\s+-w", + "high", "Kernel parameter modification", "kernel", + ), + ( + r"(?i)iptables\s+|nftables\s+|firewall-cmd", + "high", "Firewall rule modification", "kernel", + ), +] + +# ============================================================================ +# Category 7: Python AST — additional dangerous patterns +# ============================================================================ + +# Code object manipulation +_CODE_OBJECT_ATTRS = { + "co_code", "co_consts", "co_names", "co_varnames", + "co_freevars", "co_cellvars", "co_filename", +} + +# Metaclass / descriptor abuse +_META_PATTERNS = { + "__subclasses__": ("high", "Access to subclasses — sandbox escape"), + "__bases__": ("high", "Base class manipulation"), + "__mro__": ("medium", "MRO inspection — potential sandbox escape"), + "__class__": ("low", "Class access — check context"), + "__globals__": ("critical", "Access to global variables — sandbox escape"), + "__builtins__": ("critical", "Access to builtins — sandbox escape"), + "__code__": ("high", "Code object access — bytecode manipulation"), + "__reduce__": ("high", "Custom pickle reduce — deserialization attack"), + "__reduce_ex__": ("high", "Custom pickle reduce — deserialization attack"), + "__getstate__": ("medium", "Custom serialization"), + "__setstate__": ("medium", "Custom deserialization"), +} + + +class _ASTScanner(ast.NodeVisitor): + """AST visitor that collects security findings from Python code.""" + + def __init__(self, filepath: str) -> None: + self.filepath = filepath + self.findings: list[Finding] = [] + self._source_lines: list[str] = [] + + def scan(self, source: str) -> list[Finding]: + self._source_lines = source.splitlines() + try: + tree = ast.parse(source, filename=self.filepath) + except SyntaxError: + return [] + self.visit(tree) + return self.findings + + def _snippet(self, lineno: int) -> str: + if 0 < lineno <= len(self._source_lines): + return self._source_lines[lineno - 1].strip()[:120] + return "" + + def _add(self, severity: str, line: int, desc: str, category: str = "") -> None: + self.findings.append( + Finding( + severity=severity, + category=category, + file=self.filepath, + line=line, + description=desc, + code_snippet=self._snippet(line), + ) + ) + + def visit_Call(self, node: ast.Call) -> None: # noqa: N802 + # Direct function calls: eval(), exec(), compile() + if isinstance(node.func, ast.Name): + name = node.func.id + if name in _DANGEROUS_CALLS: + severity = _DANGEROUS_CALLS[name] + self._add(severity, node.lineno, f"{name}() — dangerous built-in", "code_execution") + + # Attribute calls: os.system(), subprocess.Popen() + if isinstance(node.func, ast.Attribute): + attr_name = node.func.attr + if isinstance(node.func.value, ast.Name): + module = node.func.value.id + key = (module, attr_name) + if key in _DANGEROUS_ATTRS: + severity, desc = _DANGEROUS_ATTRS[key] + self._add(severity, node.lineno, desc, "dangerous_call") + + # Check for open() on sensitive paths + if attr_name == "open": + self._check_open_args(node) + + # Check open() as direct call too + if isinstance(node.func, ast.Name) and node.func.id == "open": + self._check_open_args(node) + + self.generic_visit(node) + + def _check_open_args(self, node: ast.Call) -> None: + """Check if open() targets sensitive files.""" + for arg in node.args: + if isinstance(arg, ast.Constant) and isinstance(arg.value, str): + if _SENSITIVE_PATH_RE.search(arg.value): + self._add("high", node.lineno, + f"Access to sensitive path: {arg.value}", + "sensitive_path") + + def visit_Import(self, node: ast.Import) -> None: # noqa: N802 + for alias in node.names: + mod_name = alias.name.split(".")[0] + if mod_name in _DANGEROUS_MODULES: + severity = "critical" if mod_name in _CRITICAL_MODULES else "high" + self._add(severity, node.lineno, + f"Import of dangerous module: {alias.name}", + "dangerous_import") + self.generic_visit(node) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: # noqa: N802 + if node.module: + mod_name = node.module.split(".")[0] + if mod_name in _DANGEROUS_MODULES: + severity = "critical" if mod_name in _CRITICAL_MODULES else "high" + self._add(severity, node.lineno, + f"Import from dangerous module: {node.module}", + "dangerous_import") + self.generic_visit(node) + + def visit_Constant(self, node: ast.Constant) -> None: # noqa: N802 + """Check string literals for sensitive paths and URLs.""" + if isinstance(node.value, str) and len(node.value) > 5: + if _SENSITIVE_PATH_RE.search(node.value): + self._add("medium", node.lineno, + f"Reference to sensitive path: {node.value[:80]}", + "sensitive_path") + self.generic_visit(node) + + def visit_Attribute(self, node: ast.Attribute) -> None: # noqa: N802 + """Detect access to dangerous dunder attributes.""" + if node.attr in _META_PATTERNS: + severity, desc = _META_PATTERNS[node.attr] + self._add(severity, node.lineno, f"{node.attr} — {desc}", "meta_abuse") + if node.attr in _CODE_OBJECT_ATTRS: + self._add("high", node.lineno, + f"Code object attribute access: {node.attr}", + "code_manipulation") + self.generic_visit(node) + + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: # noqa: N802 + """Detect suspicious function definitions.""" + if node.name in ("__reduce__", "__reduce_ex__"): + self._add("high", node.lineno, + f"Custom {node.name}() — pickle exploit vector", + "deserialization") + self.generic_visit(node) + + visit_AsyncFunctionDef = visit_FunctionDef # same check + + def visit_Global(self, node: ast.Global) -> None: # noqa: N802 + """Flag global statement usage.""" + self._add("low", node.lineno, + f"global {', '.join(node.names)} — global state mutation", + "code_quality") + self.generic_visit(node) + + +class SkillSecurityScanner: + """Static analysis scanner for skill directories. + + Scans Python scripts via AST analysis and SKILL.md / shell scripts + via regex patterns. 80+ security patterns across 15 categories. + """ + + def scan_skill( + self, + skill_dir: Path | str, + skill_name: Optional[str] = None, + ) -> ScanResult: + """Scan a skill directory for security issues.""" + skill_dir = Path(skill_dir) + if not skill_name: + skill_name = skill_dir.name + + findings: list[Finding] = [] + files_scanned = 0 + + # Scan Python files in scripts/ + scripts_dir = skill_dir / "scripts" + if scripts_dir.exists(): + for py_file in scripts_dir.rglob("*.py"): + files_scanned += 1 + try: + source = py_file.read_text(encoding="utf-8", errors="replace") + rel_path = str(py_file.relative_to(skill_dir)) + scanner = _ASTScanner(rel_path) + findings.extend(scanner.scan(source)) + findings.extend( + self._scan_strings_for_exfil(source, rel_path) + ) + except Exception as exc: + logger.warning("Failed to scan %s: %s", py_file, exc) + + # Scan shell scripts + for sh_file in skill_dir.rglob("*.sh"): + files_scanned += 1 + try: + source = sh_file.read_text(encoding="utf-8", errors="replace") + rel_path = str(sh_file.relative_to(skill_dir)) + findings.extend(self._scan_shell(source, rel_path)) + except Exception as exc: + logger.warning("Failed to scan %s: %s", sh_file, exc) + + # Scan SKILL.md for shell commands + skill_md = skill_dir / "SKILL.md" + if skill_md.exists(): + files_scanned += 1 + try: + source = skill_md.read_text(encoding="utf-8", errors="replace") + findings.extend(self._scan_markdown(source, "SKILL.md")) + except Exception as exc: + logger.warning("Failed to scan SKILL.md: %s", exc) + + has_critical = any(f.severity == "critical" for f in findings) + + result = ScanResult( + safe=not has_critical, + skill_name=skill_name, + findings=findings, + files_scanned=files_scanned, + ) + + if findings: + logger.warning( + "Skill '%s' scan: %d finding(s) (%d critical)", + skill_name, len(findings), result.critical_count, + ) + + return result + + def scan_content( + self, + content: str, + skill_name: str, + filename: str = "SKILL.md", + ) -> ScanResult: + """Scan raw SKILL.md content (before writing to disk).""" + findings: list[Finding] = [] + findings.extend(self._scan_markdown(content, filename)) + + has_critical = any(f.severity == "critical" for f in findings) + return ScanResult( + safe=not has_critical, + skill_name=skill_name, + findings=findings, + files_scanned=1, + ) + + def scan_scripts_content( + self, + scripts: dict, + skill_name: str, + prefix: str = "scripts", + ) -> ScanResult: + """Scan scripts dict content before writing to disk.""" + findings: list[Finding] = [] + files_scanned = 0 + + def _walk(tree: dict, path: str) -> None: + nonlocal files_scanned + for key, value in tree.items(): + full_path = f"{path}/{key}" + if isinstance(value, dict): + _walk(value, full_path) + elif isinstance(value, str): + files_scanned += 1 + if key.endswith(".py"): + scanner = _ASTScanner(full_path) + findings.extend(scanner.scan(value)) + findings.extend( + self._scan_strings_for_exfil(value, full_path) + ) + elif key.endswith(".sh"): + findings.extend(self._scan_shell(value, full_path)) + + _walk(scripts, prefix) + has_critical = any(f.severity == "critical" for f in findings) + return ScanResult( + safe=not has_critical, + skill_name=skill_name, + findings=findings, + files_scanned=files_scanned, + ) + + def _scan_strings_for_exfil( + self, source: str, filepath: str + ) -> list[Finding]: + """Scan source for potential data exfiltration URLs.""" + findings: list[Finding] = [] + for pattern, severity, desc in _EXFIL_PATTERNS: + for m in re.finditer(pattern, source): + url = m.group() + if any(d in url for d in _SAFE_DOMAINS): + continue + line = source[:m.start()].count("\n") + 1 + findings.append( + Finding( + severity=severity, + category="exfiltration", + file=filepath, + line=line, + description=f"{desc}: {url[:80]}", + ) + ) + return findings + + def _scan_shell(self, source: str, filepath: str) -> list[Finding]: + """Scan shell script content for dangerous patterns.""" + findings: list[Finding] = [] + for pattern, severity, desc, category in _SHELL_PATTERNS: + for m in re.finditer(pattern, source): + line = source[:m.start()].count("\n") + 1 + findings.append( + Finding( + severity=severity, + category=category, + file=filepath, + line=line, + description=desc, + code_snippet=m.group()[:120], + ) + ) + return findings + + async def llm_audit_skill( + self, + skill_dir: Path | str, + llm_caller: object = None, + skill_name: Optional[str] = None, + ) -> ScanResult: + """Secondary LLM-based security audit of a skill. + + Sends skill content to the current LLM for analysis. + Best-effort — if LLM is unavailable, returns static scan only. + + Args: + skill_dir: Path to the skill directory. + llm_caller: Async callable(prompt) -> str. If None, falls back to static scan. + skill_name: Override skill name. + + Returns: + ScanResult combining static + LLM findings. + """ + import json as _json + + # First run static scan + static_result = self.scan_skill(skill_dir, skill_name) + if llm_caller is None: + return static_result + + skill_dir = Path(skill_dir) + if not skill_name: + skill_name = skill_dir.name + + # Collect skill content for LLM + content_parts: list[str] = [] + skill_md = skill_dir / "SKILL.md" + if skill_md.exists(): + text = skill_md.read_text(encoding="utf-8", errors="replace")[:5000] + content_parts.append(f"=== SKILL.md ===\n{text}") + + scripts_dir = skill_dir / "scripts" + if scripts_dir.exists(): + for py_file in list(scripts_dir.rglob("*.py"))[:10]: + text = py_file.read_text(encoding="utf-8", errors="replace")[:3000] + rel = py_file.relative_to(skill_dir) + content_parts.append(f"=== {rel} ===\n{text}") + for sh_file in list(scripts_dir.rglob("*.sh"))[:5]: + text = sh_file.read_text(encoding="utf-8", errors="replace")[:3000] + rel = sh_file.relative_to(skill_dir) + content_parts.append(f"=== {rel} ===\n{text}") + + if not content_parts: + return static_result + + prompt = ( + "You are a security auditor reviewing an AI agent skill/plugin. " + "Analyze the following skill files for security issues. " + "Look for: data exfiltration, command injection, privilege escalation, " + "backdoors, obfuscated code, supply chain risks, sandbox escapes.\n\n" + "Respond ONLY with a JSON array of findings. Each finding:\n" + '{"severity":"critical|high|medium|low","description":"...","file":"...","line":0}\n\n' + "If no issues found, respond with: []\n\n" + + "\n\n".join(content_parts) + ) + + try: + response = await llm_caller(prompt) + # Parse LLM response — extract JSON array + response = response.strip() + # Try to find JSON array in response + start = response.find("[") + end = response.rfind("]") + if start >= 0 and end > start: + llm_findings_raw = _json.loads(response[start:end + 1]) + for item in llm_findings_raw: + if isinstance(item, dict) and "description" in item: + severity = item.get("severity", "medium") + if severity not in ("critical", "high", "medium", "low"): + severity = "medium" + static_result.findings.append( + Finding( + severity=severity, + category="llm_audit", + file=item.get("file", ""), + line=item.get("line", 0), + description=f"[LLM] {item['description']}", + ) + ) + # Re-evaluate safety + if any(f.severity == "critical" for f in static_result.findings): + static_result.safe = False + except Exception as exc: + logger.warning("LLM audit failed for '%s': %s", skill_name, exc) + + return static_result + + def _scan_markdown(self, source: str, filepath: str) -> list[Finding]: + """Scan markdown for shell injection patterns in code blocks.""" + findings: list[Finding] = [] + for block_match in re.finditer( + r"```(?:sh|bash|shell|zsh)?\n(.*?)```", + source, + re.DOTALL, + ): + block = block_match.group(1) + block_start = source[:block_match.start()].count("\n") + 1 + for pattern, severity, desc, category in _SHELL_PATTERNS: + for m in re.finditer(pattern, block): + line = block_start + block[:m.start()].count("\n") + findings.append( + Finding( + severity=severity, + category=category, + file=filepath, + line=line, + description=desc, + code_snippet=m.group()[:120], + ) + ) + return findings diff --git a/src/adclaw/agents/skill_security.py b/src/adclaw/agents/skill_security.py new file mode 100644 index 0000000..a97fd01 --- /dev/null +++ b/src/adclaw/agents/skill_security.py @@ -0,0 +1,86 @@ +"""Security score computation and caching for skills.""" +import hashlib +import json +import logging +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + +from .skill_scanner import ScanResult + +logger = logging.getLogger(__name__) + +SEVERITY_DEDUCTIONS = { + "critical": 25, + "high": 15, + "medium": 8, + "low": 3, +} + +CACHE_FILE = ".scan.json" + + +def compute_security_score(scan_result: ScanResult) -> int: + """Compute 0-100 security score from scan findings.""" + score = 100 + for f in scan_result.findings: + score -= SEVERITY_DEDUCTIONS.get(f.severity, 3) + return max(0, score) + + +def _file_hash(skill_dir: Path) -> str: + """Hash all scannable files in skill dir.""" + h = hashlib.sha256() + for f in sorted(skill_dir.rglob("*")): + if f.is_file() and f.name != CACHE_FILE and not f.name.endswith(".bak"): + h.update(f.read_bytes()) + return h.hexdigest() + + +def write_scan_cache(skill_dir: Path, data: dict) -> None: + """Write scan result cache.""" + data["file_hash"] = _file_hash(skill_dir) + data["scanned_at"] = datetime.now(timezone.utc).isoformat() + cache_path = skill_dir / CACHE_FILE + cache_path.write_text(json.dumps(data, indent=2), encoding="utf-8") + + +def read_scan_cache(skill_dir: Path) -> Optional[dict]: + """Read cached scan result. Returns None if stale or missing.""" + cache_path = skill_dir / CACHE_FILE + if not cache_path.exists(): + return None + try: + data = json.loads(cache_path.read_text(encoding="utf-8")) + if data.get("file_hash") != _file_hash(skill_dir): + return None + return data + except (json.JSONDecodeError, KeyError): + return None + + +def scan_and_cache(skill_dir: Path, skill_name: str) -> dict: + """Run pattern scan, compute score, cache result.""" + from .skill_scanner import SkillSecurityScanner + + cached = read_scan_cache(skill_dir) + if cached is not None: + return cached + + scanner = SkillSecurityScanner() + result = scanner.scan_skill(skill_dir, skill_name) + score = compute_security_score(result) + + data = { + "score": score, + "pattern_scan": "pass" if result.safe else "fail", + "llm_audit": "pending", + "auto_healed": False, + "findings_count": len(result.findings), + "findings": [ + {"severity": f.severity, "category": f.category, "description": f.description} + for f in result.findings + ], + } + write_scan_cache(skill_dir, data) + return data diff --git a/src/adclaw/agents/skill_validator.py b/src/adclaw/agents/skill_validator.py new file mode 100644 index 0000000..22ff933 --- /dev/null +++ b/src/adclaw/agents/skill_validator.py @@ -0,0 +1,354 @@ +# -*- coding: utf-8 -*- +"""Analysis-first skill security validation using LLM. + +Enforces structured reasoning before verdict: ANALYSIS → FINDINGS → VERDICT. +Inspired by Claude Code's verification contract pattern. +""" + +from __future__ import annotations + +import json +import logging +import re +from dataclasses import dataclass, field +from typing import Any, Callable, Coroutine, Literal, Optional + +from .skill_scanner import Finding, ScanResult, SkillSecurityScanner + +logger = logging.getLogger(__name__) + + +def _finding_to_dict(f: Finding) -> dict: + """Convert a static Finding to a plain dict for merged results.""" + return { + "severity": f.severity, + "category": f.category, + "description": f.description, + "file": f.file, + "line": f.line, + } + +# --------------------------------------------------------------------------- +# Skill categories with domain-specific security criteria +# --------------------------------------------------------------------------- + +SkillCategory = Literal[ + "seo", "marketing", "browser", "data", "social", "analytics", "office", "general" +] + +CATEGORY_CRITERIA: dict[SkillCategory, list[str]] = { + "seo": [ + "Respects robots.txt and crawl rate limits", + "Does not scrape competitor sites without explicit user configuration", + "Validates URLs before fetching (no SSRF via user-controlled input)", + ], + "marketing": [ + "Complies with CAN-SPAM / GDPR for any email or messaging", + "Does not auto-post to social media without user confirmation", + "Rate-limits any bulk outreach operations", + ], + "browser": [ + "No credential harvesting from forms or cookies", + "No cross-site scripting (XSS) via injected content", + "Respects same-origin policy and does not exfiltrate page data", + ], + "data": [ + "No data exfiltration to external endpoints", + "Validates file paths to prevent directory traversal", + "Sanitizes any SQL or query parameters", + ], + "social": [ + "Respects platform API rate limits and ToS", + "Does not automate follow/unfollow or engagement farming", + "No scraping of private or protected content", + ], + "analytics": [ + "Does not collect PII without explicit consent", + "Validates data sources before processing", + "No unauthorized access to analytics dashboards", + ], + "office": [ + "Does not access files outside designated workspace", + "Validates file formats before processing", + "No macro execution in documents", + ], + "general": [ + "No unexpected network access", + "No file system writes outside working directory", + "No environment variable access for secrets", + ], +} + +# Keyword signals for category detection +_CATEGORY_SIGNALS: dict[SkillCategory, list[str]] = { + "seo": ["seo", "keyword", "backlink", "serp", "ranking", "crawl", "sitemap", "meta"], + "marketing": ["email", "campaign", "newsletter", "outreach", "lead", "crm", "funnel"], + "browser": ["browser", "playwright", "selenium", "puppeteer", "chrome", "screenshot", "navigate"], + "data": ["csv", "json", "database", "sql", "export", "import", "parse", "transform"], + "social": ["twitter", "linkedin", "instagram", "tiktok", "reddit", "post", "engagement"], + "analytics": ["analytics", "metrics", "dashboard", "tracking", "report", "chart"], + "office": ["pdf", "docx", "excel", "spreadsheet", "document", "presentation"], +} + + +def detect_skill_category(content: str) -> SkillCategory: + """Detect skill category from content using keyword signals.""" + lower = content.lower() + scores: dict[SkillCategory, int] = {cat: 0 for cat in _CATEGORY_SIGNALS} + for cat, signals in _CATEGORY_SIGNALS.items(): + for signal in signals: + if signal in lower: + scores[cat] += 1 + best = max(scores, key=lambda k: scores[k]) + return best if scores[best] > 0 else "general" + + +# --------------------------------------------------------------------------- +# LLM audit prompt +# --------------------------------------------------------------------------- + +AUDIT_SYSTEM_PROMPT = """\ +You are a security auditor for AI agent skills. Each skill is a YAML definition +with optional Python scripts that the agent can execute. + +## Your Task +Analyze the skill code and produce a structured security assessment. + +## MANDATORY Response Structure (JSON only): +{{ + "analysis": {{ + "purpose": "What does this skill do?", + "data_flow": "What data does it read/write/send?", + "external_interactions": "What external systems does it contact?", + "permissions_needed": "What system permissions does it require?", + "category_specific": "Assessment against the category criteria below" + }}, + "findings": [ + {{ + "severity": "critical|high|medium|low", + "description": "What is the issue", + "file": "filename", + "line": 0, + "fix_suggestion": "How to fix it" + }} + ], + "verdict": {{ + "safe": true|false, + "confidence": 0.0-1.0, + "reasoning": "Why this verdict" + }} +}} + +## Category-Specific Criteria ({category}): +{criteria} + +## Static Scan Results (already completed): +{static_findings} + +IMPORTANT: You MUST analyze before concluding. The analysis section is mandatory. +Output ONLY valid JSON. No markdown, no explanation outside JSON. +""" + + +# --------------------------------------------------------------------------- +# Response parsing +# --------------------------------------------------------------------------- + +@dataclass +class ValidationResult: + """Structured result of skill validation.""" + + analysis: dict = field(default_factory=dict) + findings: list[dict] = field(default_factory=list) + verdict: dict = field(default_factory=lambda: {"safe": True, "confidence": 0.0, "reasoning": ""}) + category: SkillCategory = "general" + static_result: Optional[ScanResult] = field(default=None, repr=False) + + @property + def should_block(self) -> bool: + """True if any critical finding exists (static or LLM).""" + if self.static_result and self.static_result.critical_count > 0: + return True + return any(f.get("severity") == "critical" for f in self.findings) + + @property + def needs_acknowledgment(self) -> bool: + """True if medium+ findings exist but no criticals.""" + if self.should_block: + return False + has_medium_plus = any( + f.get("severity") in ("high", "medium") for f in self.findings + ) + return has_medium_plus + + @property + def is_clean(self) -> bool: + return not self.should_block and not self.needs_acknowledgment + + def to_dict(self) -> dict: + return { + "analysis": self.analysis, + "findings": self.findings, + "verdict": self.verdict, + "category": self.category, + "should_block": self.should_block, + "needs_acknowledgment": self.needs_acknowledgment, + "is_clean": self.is_clean, + } + + +_REQUIRED_ANALYSIS_KEYS = ("purpose", "data_flow", "external_interactions", "permissions_needed", "category_specific") + + +def _parse_llm_response(response_text: str) -> dict: + """Parse structured JSON from LLM response.""" + # Try to extract JSON from markdown fences + json_match = re.search(r"```json\s*(.*?)\s*```", response_text, re.DOTALL) + if json_match: + raw = json_match.group(1) + else: + # Try to find raw JSON (first { to last }) + start = response_text.find("{") + end = response_text.rfind("}") + if start >= 0 and end > start: + raw = response_text[start : end + 1] + else: + raw = response_text.strip() + + data = json.loads(raw) + + # Validate required structure + for key in ("analysis", "findings", "verdict"): + if key not in data: + raise ValueError(f"Missing required key: {key}") + + analysis = data["analysis"] + if not isinstance(analysis, dict): + raise ValueError(f"analysis must be a dict, got: {type(analysis).__name__}") + for subkey in _REQUIRED_ANALYSIS_KEYS: + if subkey not in analysis: + raise ValueError(f"Missing analysis.{subkey}") + + return data + + +# --------------------------------------------------------------------------- +# Main validation function +# --------------------------------------------------------------------------- + + +async def validate_skill( + skill_content: str, + skill_name: str, + llm_caller: Callable[[str], Coroutine[Any, Any, str]], + scripts: Optional[dict[str, Any]] = None, +) -> ValidationResult: + """Run analysis-first validation on a skill. + + 1. Static pattern scan (instant) + 2. If critical found → block immediately (no LLM needed) + 3. Detect category → inject criteria + 4. LLM audit with structured prompt + 5. Parse and merge findings + + Args: + skill_content: SKILL.md content + skill_name: Name of the skill + llm_caller: Async LLM function + scripts: Optional dict of script filenames → content + + Returns: + ValidationResult with analysis, findings, verdict + """ + scanner = SkillSecurityScanner() + + # 1. Static scan — SKILL.md content + scripts separately + static_result = scanner.scan_content(skill_content, skill_name) + if scripts: + # Use scan_scripts_content for Python scripts (AST + pattern analysis) + scripts_result = scanner.scan_scripts_content(scripts, skill_name) + static_result.findings.extend(scripts_result.findings) + static_result.safe = static_result.safe and scripts_result.safe + + # 2. Critical short-circuit + if static_result.critical_count > 0: + logger.warning( + "Skill '%s' blocked: %d critical findings from static scan", + skill_name, + static_result.critical_count, + ) + return ValidationResult( + analysis={"purpose": "Blocked before LLM audit", "data_flow": "N/A", + "external_interactions": "N/A", "permissions_needed": "N/A", + "category_specific": "N/A"}, + findings=[_finding_to_dict(f) for f in static_result.findings], + verdict={"safe": False, "confidence": 1.0, + "reasoning": f"Blocked: {static_result.critical_count} critical findings from static scan"}, + category="general", + static_result=static_result, + ) + + # 3. Detect category + category = detect_skill_category(skill_content) + criteria = CATEGORY_CRITERIA.get(category, CATEGORY_CRITERIA["general"]) + + # 4. Build LLM prompt + if static_result.findings: + static_findings_text = "\n".join( + f"- [{f.severity}] {f.description} ({f.file}:{f.line})" + for f in static_result.findings + ) + else: + static_findings_text = "None" + + prompt = AUDIT_SYSTEM_PROMPT.format( + category=category, + criteria="\n".join(f"- {c}" for c in criteria), + static_findings=static_findings_text, + ) + + user_content = f"## Skill: {skill_name}\n\n```yaml\n{skill_content[:5000]}\n```" + if scripts: + for fname, sc in list(scripts.items())[:3]: + if isinstance(sc, str): + user_content += f"\n\n## Script: {fname}\n```python\n{sc[:3000]}\n```" + + full_prompt = f"{prompt}\n\n{user_content}" + + # 5. Call LLM + try: + response_text = await llm_caller(full_prompt) + data = _parse_llm_response(response_text) + except (json.JSONDecodeError, ValueError) as exc: + logger.warning("LLM audit parse failed for '%s': %s", skill_name, exc) + return ValidationResult( + analysis={"purpose": "LLM audit failed", "data_flow": "Unknown", + "external_interactions": "Unknown", "permissions_needed": "Unknown", + "category_specific": "Unable to assess"}, + findings=[_finding_to_dict(f) for f in static_result.findings], + verdict={"safe": static_result.safe, "confidence": 0.3, + "reasoning": f"LLM audit failed: {exc}. Static scan only."}, + category=category, + static_result=static_result, + ) + except Exception as exc: + logger.exception("LLM audit error for '%s': %s", skill_name, exc) + return ValidationResult( + findings=[_finding_to_dict(f) for f in static_result.findings], + verdict={"safe": static_result.safe, "confidence": 0.2, + "reasoning": f"LLM audit error: {exc}"}, + category=category, + static_result=static_result, + ) + + # 6. Merge static + LLM findings + merged_findings = [_finding_to_dict(f) for f in static_result.findings] + merged_findings.extend(data.get("findings", [])) + + return ValidationResult( + analysis=data.get("analysis", {}), + findings=merged_findings, + verdict=data.get("verdict", {"safe": True, "confidence": 0.0, "reasoning": ""}), + category=category, + static_result=static_result, + ) diff --git a/src/adclaw/agents/skills/__init__.py b/src/adclaw/agents/skills/__init__.py new file mode 100644 index 0000000..90f8f4d --- /dev/null +++ b/src/adclaw/agents/skills/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +"""Agent skills directory.""" diff --git a/src/adclaw/agents/skills/ads-apple/SKILL.md b/src/adclaw/agents/skills/ads-apple/SKILL.md new file mode 100644 index 0000000..9b791ef --- /dev/null +++ b/src/adclaw/agents/skills/ads-apple/SKILL.md @@ -0,0 +1,184 @@ +--- +name: ads-apple +description: "Apple Search Ads (ASA) deep analysis for mobile app advertisers. Evaluates campaign structure, bid health, Creative Sets, MMP attribution, budget pacing, TAP coverage (Today/Search/Product Pages), and goal CPA benchmarks by country. Use when user says Apple Search Ads, ASA, App Store ads, Apple ads, Search Ads, or is advertising a mobile app on iOS." +--- + +# Apple Search Ads (ASA) Deep Analysis + +## Process + +1. Collect ASA account data (exports from Apple Search Ads dashboard or pasted metrics) +2. Identify active placement types (Search Results, Search Tab, Today Tab, Product Pages) +3. Evaluate all applicable checks as PASS, WARNING, or FAIL +4. Calculate ASA Health Score (0-100) +5. Generate findings report with action plan + +## What to Analyze + +### Campaign Structure (25% weight) + +**BOFU; Bottom of Funnel (Search Results, Exact Match brand)** +- Brand keyword campaign present (own app name + misspellings) +- Competitor campaign present (competitor app names as keywords) +- Category campaigns targeting high-intent generic terms (e.g. "workout app", "budget tracker") + +**MOFU; Middle of Funnel (Search Match / broad discovery)** +- Search Match campaigns active in at least one ad group for discovery +- Search Match ad groups isolated from Exact Match (separate ad groups; never mix) +- Search Terms Report reviewed to mine converting queries for Exact Match promotion + +**Campaign Architecture Rules:** +- Brand / Category / Competitor should be separate campaigns (different CPT bids, budgets) +- Search Match ad groups isolated from manual keyword ad groups; NEVER mix in same ad group +- Goal: let Search Match discover, then promote winners to Exact Match campaigns + +### Bid Health (20% weight) + +**CPT (Cost Per Tap) vs Install Rate by Match Type:** +- CPT vs category benchmarks (see Benchmarks section below) +- TTR (Tap-Through Rate): benchmark >2.5% for Search Results, >1.5% for Search Tab +- Conversion Rate (tap → install): benchmark 50-65% for brand terms, 20-40% for category +- CPT/CPG (Cost Per Goal): compare against target CPI/CPA from MMP + +**Bid Strategy:** +- Manual CPT bidding appropriate? (Or use Apple's CPA Goals auto-bidding for scaled accounts) +- CPA Goals available at campaign level; evaluate if conversion volume supports it (>100 installs/month per campaign) +- Are bids differentiated by match type? (Brand Exact > Category Exact > Search Match) +- Keyword-level CPT bids set, not just ad group default? + +**Keyword Health:** +- Irrelevant Search Terms (from Search Match) identified and excluded via negative keywords +- Low-performing keywords paused or bid reduced (TTR <1% + high CPT) +- High-volume generic terms checked for intent quality (avoid "free apps" type queries) + +### Creative Sets (15% weight) + +**Custom Product Pages (CPP):** +- Custom Product Pages created in App Store Connect? (ASA Creative Sets pull from CPPs) +- At least 3 CPP variants tested per campaign type (different value props per audience) +- Creative Sets assigned to high-spend ad groups +- Screenshot/preview variations aligned with keyword intent (e.g. fitness keywords → fitness screenshots) + +**Default (Store Listing) Creative:** +- App icon, subtitle, and first 3 screenshots optimized; these show in ads by default +- Short description (170 chars) compelling and keyword-rich +- Preview video present (strongly recommended for TTR improvement) + +**Creative Testing:** +- Are different Creative Sets being A/B tested within ad groups? +- CPP performance compared: which Creative Set has highest TTR and lowest CPI? + +### Attribution & MMP Health (15% weight) + +**MMP Integration (Critical):** +- MMP (AppsFlyer / Adjust / Branch / Singular) integrated with ASA via SKAdNetwork + ATT +- ASA is properly connected as a partner in MMP dashboard +- In-app events being sent back to ASA (enables CPA Goals and ROAS optimization) +- Post-install event quality: are purchase, subscription_start, or other revenue events tracked? + +**SKAdNetwork & ATT:** +- SKAdNetwork conversion values configured in MMP (maps user actions to conversion windows) +- ATT opt-in rate monitored (low ATT rate = less MMP data, more reliance on SKAN) +- Privacy threshold considerations: are campaigns getting SKAN postbacks or null reports? + +**Attribution Windows:** +- Default ASA attribution: 30-day click, 1-day view; appropriate for app install goals? +- For re-engagement or subscription goals: evaluate longer lookback windows + +### Budget Pacing (10% weight) + +- Daily cap set at campaign level (budget pacing in ASA is daily, not monthly) +- Actual daily spend vs daily cap ratio: flag if consistently hitting cap (could be missing volume) +- Conversely: flag if spend is <50% of daily cap (creative or bid issue, not budget) +- Budget split across placement types aligned with performance (don't over-invest in underperforming placements) +- Lifetime budget campaigns (if used): check end dates and pacing curves + +### TAP Coverage: Placement Types (10% weight) + +ASA offers 4 placement types; evaluate coverage and performance: + +| Placement | Where | Best for | Benchmark CPT | +|-----------|-------|----------|----------------| +| Search Results | Below search results | High intent, bottom funnel | $0.50-$3.00 | +| Search Tab | Top of Search tab | Discovery, mid funnel | $0.30-$1.50 | +| Today Tab | App Store home | Brand awareness | $1.00-$5.00 | +| Product Pages | Competitor/related app pages | Competitor conquesting | $0.50-$2.00 | + +**Evaluation:** +- Search Results: must be active (highest intent placement) +- Search Tab: active for scale? Evaluate CPT and TTR vs Search Results +- Today Tab: only if budget >$3k/month and brand awareness is a goal (high CPT, low intent) +- Product Pages: competitive opportunity; are competitor CPPs being targeted? + +### Goal CPA / KPI Assessment (5% weight) + +**Benchmarks by Category (2025-2026 ASA averages):** +| Category | Avg CPT | Avg TTR | Avg Install CVR | Target CPI | +|----------|---------|---------|-----------------|------------| +| Games | $0.50-$1.00 | 3-5% | 55-70% | $1.00-$3.00 | +| Health & Fitness | $1.50-$3.00 | 2-4% | 45-60% | $3.00-$8.00 | +| Productivity | $1.00-$2.50 | 2-3.5% | 50-65% | $2.00-$5.00 | +| Finance | $2.00-$5.00 | 1.5-3% | 40-55% | $5.00-$15.00 | +| Education | $1.00-$2.00 | 2-4% | 50-65% | $2.00-$6.00 | +| Shopping | $0.80-$2.00 | 2.5-4% | 45-60% | $2.00-$5.00 | +| Lifestyle | $0.80-$1.80 | 2-3.5% | 45-60% | $2.00-$5.00 | + +**Country-level benchmarks:** +- Tier 1 (US, UK, AU, CA, JP): CPT 2-3× above global average; highest LTV +- Tier 2 (DE, FR, KR, SG, HK): CPT 1-1.5× above global average +- Tier 3 (BR, IN, MX): CPT 30-60% below Tier 1; high volume, lower LTV + +**Checks:** +- Actual CPI vs target CPI (from MMP); flag if >2x target +- CPI trend over 30 days (improving or worsening?) +- Revenue events: is ROAS positive within MMP attribution window? + +## Output Format + +``` +## Apple Search Ads Audit + +**ASA Health Score: [X]/100** + +### Critical Issues ([count]) +- [Issue with specific impact and fix] + +### High Priority ([count]) +- [Issue] + +### Campaign Structure +PASS/WARNING/FAIL for each check category + +### Benchmark Comparison +[Metric] | Your Account | ASA Benchmark | Status + +### Quick Wins (do this week) +1. [Most impactful fix with expected outcome] +2. +3. + +### Recommended Next Steps +[Prioritized action plan] +``` + +## Scoring Weights + +| Category | Weight | +|----------|--------| +| Campaign Structure | 25% | +| Bid Health | 20% | +| Creative Sets | 15% | +| Attribution & MMP | 15% | +| Budget Pacing | 10% | +| TAP Coverage | 10% | +| Goal KPI Assessment | 5% | + +## Data to Request from User + +If not provided, ask for: +- Campaign list with spend, installs, CPT, TTR, CVR (last 30 days) +- Active placement types +- MMP being used (AppsFlyer, Adjust, Branch, Singular, or none) +- Target CPI / CPA and app category +- Countries/regions active +- Whether Custom Product Pages are set up in App Store Connect diff --git a/src/adclaw/agents/skills/ads-audit/SKILL.md b/src/adclaw/agents/skills/ads-audit/SKILL.md new file mode 100644 index 0000000..a676c10 --- /dev/null +++ b/src/adclaw/agents/skills/ads-audit/SKILL.md @@ -0,0 +1,106 @@ +--- +name: ads-audit +description: "Full multi-platform paid advertising audit with parallel subagent delegation. Analyzes Google Ads, Meta Ads, LinkedIn Ads, TikTok Ads, and Microsoft Ads accounts. Generates health score per platform and aggregate score. Use when user says audit, full ad check, analyze my ads, account health check, or PPC audit." +--- + +# Full Multi-Platform Ads Audit + +## Process + +1. **Collect account data**: request exports, screenshots, or API access +2. **Validate**: confirm at least one platform's data is available before proceeding +3. **Detect business type**: analyze account signals per ads orchestrator +4. **Identify active platforms**: determine which platforms are in use +5. **Delegate to subagents** (if available, otherwise run inline sequentially): + - `audit-google`: Conversion tracking, wasted spend, structure, keywords, ads, settings (G01-G74) + - `audit-meta`: Pixel/CAPI health, creative fatigue, structure, audience (M01-M46) + - `audit-creative`: LinkedIn, TikTok, Microsoft creative checks + cross-platform synthesis + - `audit-tracking`: LinkedIn, TikTok, Microsoft tracking + cross-platform tracking health + - `audit-budget`: LinkedIn, TikTok, Microsoft budget/bidding + cross-platform allocation + - `audit-compliance`: All-platform compliance, settings, performance benchmarks +6. **Validate**: verify each subagent returned valid scores with required fields before aggregating +7. **Score**: calculate per-platform and aggregate Ads Health Score (0-100) +8. **Report**: generate prioritized action plan with Quick Wins + +## Data Collection + +Ask the user for available data. Accept any combination: +- Google Ads: account export, Change History, Search Terms Report +- Meta Ads: Ads Manager export, Events Manager screenshot, EMQ scores +- LinkedIn Ads: Campaign Manager export, Insight Tag status +- TikTok Ads: Ads Manager export, Pixel/Events API status +- Microsoft Ads: account export, UET tag status, import validation results + +If no exports available, audit from screenshots or manual data entry. + +## Scoring + +Read `ads-shared/references/scoring-system.md` for full algorithm. + +### Per-Platform Weights + +| Platform | Category Weights | +|----------|-----------------| +| Google | Conversion 25%, Waste 20%, Structure 15%, Keywords 15%, Ads 15%, Settings 10% | +| Meta | Pixel/CAPI 30%, Creative 30%, Structure 20%, Audience 20% | +| LinkedIn | Tech 25%, Audience 25%, Creative 20%, Lead Gen 15%, Budget 15% | +| TikTok | Creative 30%, Tech 25%, Bidding 20%, Structure 15%, Performance 10% | +| Microsoft | Tech 25%, Syndication 20%, Structure 20%, Creative 20%, Settings 15% | + +### Aggregate Score + +``` +Aggregate = Sum(Platform_Score x Platform_Budget_Share) +Grade: A (90-100), B (75-89), C (60-74), D (40-59), F (<40) +``` + +## Output Files + +- `ADS-AUDIT-REPORT.md`: Comprehensive multi-platform findings +- `ADS-ACTION-PLAN.md`: Prioritized recommendations (Critical > High > Medium > Low) +- `ADS-QUICK-WINS.md`: Items fixable in <15 minutes with high impact + +## Report Structure + +### Executive Summary +- Aggregate Ads Health Score (0-100) with grade +- Per-platform scores +- Business type detected +- Active platforms identified +- Top 5 critical issues across all platforms +- Top 5 quick wins across all platforms + +### Per-Platform Sections +Each platform section includes: +- Platform Health Score with grade +- Category breakdown with pass/warning/fail per check +- Platform-specific Quick Wins +- Detailed findings with remediation steps + +### Cross-Platform Analysis +- Budget allocation assessment (actual vs recommended) +- Tracking consistency (are all platforms tracking the same events?) +- Creative consistency (is messaging aligned across platforms?) +- Attribution overlap (are platforms double-counting conversions?) + +### Strategic Recommendations +- Platform prioritization based on business type +- Budget reallocation recommendations +- Scaling opportunities (platforms/campaigns ready to scale) +- Kill list (campaigns/ad groups to pause immediately) + +## Priority Definitions + +- **Critical**: Revenue/data loss risk (fix immediately) +- **High**: Significant performance drag (fix within 7 days) +- **Medium**: Optimization opportunity (fix within 30 days) +- **Low**: Best practice, minor impact (backlog) + +## Quick Wins Criteria + +``` +IF severity == "Critical" OR severity == "High" +AND estimated_fix_time < 15 minutes +THEN flag as Quick Win +SORT BY (severity_multiplier x estimated_impact) DESC +``` diff --git a/src/adclaw/agents/skills/ads-budget/SKILL.md b/src/adclaw/agents/skills/ads-budget/SKILL.md new file mode 100644 index 0000000..9c4391d --- /dev/null +++ b/src/adclaw/agents/skills/ads-budget/SKILL.md @@ -0,0 +1,165 @@ +--- +name: ads-budget +description: "Budget allocation and bidding strategy review across all ad platforms. Evaluates spend distribution, bidding strategy appropriateness, scaling readiness, and identifies campaigns to kill or scale. Uses 70/20/10 rule, 3x Kill Rule, and 20% scaling rule. Use when user says budget allocation, bidding strategy, ad spend, ROAS target, media budget, or scaling." +--- + +# Budget Allocation & Bidding Strategy + +## Process + +1. Collect budget and performance data across all active platforms +2. Read `ads-shared/references/budget-allocation.md` for allocation framework +3. Read `ads-shared/references/bidding-strategies.md` for strategy decision trees +4. Read `ads-shared/references/benchmarks.md` for CPC/CPA benchmarks +5. Read `ads-shared/references/scoring-system.md` for health score algorithm +6. **Validate**: confirm spend data covers ≥14 days before evaluating kill/scale decisions +7. Evaluate budget allocation, bidding strategy, and scaling readiness +8. **Validate**: verify kill list candidates have sufficient data (≥20 clicks or ≥$100 spend) before recommending pause +9. Generate recommendations with kill list and scale list + +## Budget Allocation Framework + +### 70/20/10 Rule +- **70%** on proven channels (consistent ROAS/CPA targets met) +- **20%** on scaling channels (showing promise, need more data) +- **10%** on testing channels (new platforms, audiences, creatives) + +### Platform Selection Matrix + +| Business Type | Primary | Secondary | Testing | +|---------------|---------|-----------|---------| +| SaaS B2B | Google Search, LinkedIn | Meta, YouTube | TikTok, Microsoft | +| E-commerce | Google Shopping, Meta | TikTok, YouTube | Microsoft, LinkedIn | +| Local Service | Google Search, Google LSA | Meta | Microsoft, YouTube | +| B2B Enterprise | LinkedIn, Google Search | Meta | Microsoft, TikTok | +| Info Products | Meta, YouTube | Google Search | TikTok | +| Mobile App | Meta, Google UAC | TikTok | Apple Search Ads | +| Real Estate | Google Search, Meta | YouTube | Microsoft | +| Healthcare | Google Search | Meta | Microsoft, YouTube | +| Finance | Google Search, Meta | LinkedIn | Microsoft | +| Agency (clients) | Varies by client | N/A | N/A | + +### Budget Sufficiency Rules + +| Platform | Minimum Daily | Learning Phase Budget | +|----------|--------------|----------------------| +| Google Search | $20/day | Sufficient for 15+ conv/month | +| Google PMax | $50/day | Sufficient for algorithm optimization | +| Meta | $20/day per ad set | ≥5x target CPA per ad set | +| LinkedIn | $50/day Sponsored Content | 15+ conversions/month | +| TikTok | $50/day campaign, $20/day ad group | ≥50x target CPA per ad group | +| Microsoft | No strict minimum | Sufficient for stable delivery | + +## Bidding Strategy Evaluation + +### Google Ads Bidding Decision Tree + +``` +Start +├─ <30 conversions/month? +│ └─ Use Maximize Clicks (cap CPC at benchmark) +│ └─ When >30 conv/month → Maximize Conversions +├─ 30-50 conversions/month? +│ └─ Use Maximize Conversions +│ └─ When stable CPA → Target CPA +├─ >50 conversions/month? +│ └─ Use Target CPA +│ └─ When revenue tracking → Target ROAS +└─ Revenue tracking active + >50 conv/month? + └─ Use Target ROAS +``` + +### Meta Ads Bidding +- **Lowest Cost (default)**: best for volume, may have CPA variance +- **Cost Cap**: sets CPA ceiling, may reduce volume +- **Bid Cap**: maximum bid per auction, most control +- **ROAS Goal**: target return on ad spend +- **CBO vs ABO**: CBO for proven campaigns, ABO for testing + +### LinkedIn Bidding +- **Cost Per Send (CPS)**: for Message Ads +- **Maximum Delivery**: for Sponsored Content (recommended) +- **Manual CPC**: for tight budget control +- **Target Cost**: for predictable CPA + +### TikTok Bidding +- **Lowest Cost**: maximize conversions within budget (volume) +- **Cost Cap**: set maximum CPA (efficiency) +- **Bid Cap**: maximum bid per impression +- Budget ≥50x CPA per ad group for learning phase exit + +### Microsoft Bidding +- Mirror Google strategy but bid 20-35% lower +- Enhanced CPC for manual campaigns +- Target CPA / Target ROAS for automated + +## Scaling Assessment + +### Ready to Scale (Green Light) +- CPA consistently below target for 2+ weeks +- ≥50 conversions per week (learning phase exited) +- CTR stable or improving +- ROAS above target +- No creative fatigue signals + +### 20% Rule +Never increase budget by more than 20% at a time: +- Week 1: $100/day → $120/day +- Week 2: $120/day → $144/day +- Week 3: $144/day → $173/day +- Monitor 3-5 days after each increase for performance stability + +### Scaling Methods +1. **Vertical**: increase budget on winning campaigns (20% rule) +2. **Horizontal**: duplicate winning campaigns to new audiences +3. **Platform expansion**: add budget on new platforms +4. **Geographic expansion**: test new markets/regions +5. **Format expansion**: test new ad formats on same platform + +## Kill List Assessment + +### 3x Kill Rule +- Any campaign/ad group with CPA >3x target → **flag for pause** +- Review spend in last 14 days with no conversions → **flag for pause** +- Creative with CTR >50% below platform benchmark → **flag for creative kill** + +### Kill Decision Framework +| Scenario | Data Required | Action | +|----------|---------------|--------| +| CPA >3x target | ≥7 days data, ≥20 clicks | Pause immediately | +| No conversions | ≥$100 spend or ≥50 clicks | Pause and diagnose | +| CTR <50% of benchmark | ≥1,000 impressions | Kill creative, test new | +| ROAS <50% of target | ≥14 days data | Reduce budget 50% or pause | + +## MER (Marketing Efficiency Ratio) + +``` +MER = Total Revenue / Total Marketing Spend +``` + +- Assess blended efficiency across all platforms +- Target MER varies by business: 3x-10x depending on margins +- Use MER to evaluate overall health, not just per-platform ROAS +- Incrementality testing recommended for MER accuracy + +## Output + +### Budget & Bidding Assessment + +``` +Budget Allocation Health + +Allocation Strategy: ████████░░ XX/100 +Bidding Strategies: ██████████ XX/100 +Scaling Readiness: ███████░░░ XX/100 +Budget Sufficiency: █████░░░░░ XX/100 +``` + +### Deliverables +- `BUDGET-STRATEGY-REPORT.md`: Full allocation and bidding analysis +- Current vs recommended budget split (pie chart data) +- Bidding strategy recommendations per platform/campaign +- Scale list: campaigns ready for more budget +- Kill list: campaigns/ad groups to pause immediately +- MER analysis and trend +- Quick Wins for immediate budget optimization diff --git a/src/adclaw/agents/skills/ads-competitor/SKILL.md b/src/adclaw/agents/skills/ads-competitor/SKILL.md new file mode 100644 index 0000000..bc6cfcb --- /dev/null +++ b/src/adclaw/agents/skills/ads-competitor/SKILL.md @@ -0,0 +1,156 @@ +--- +name: ads-competitor +description: "Competitor ad intelligence analysis across Google, Meta, LinkedIn, TikTok, and Microsoft. Analyzes competitor ad copy, creative strategy, keyword targeting, estimated spend, and identifies competitive gaps and opportunities. Use when user says competitor ads, ad spy, competitive analysis, competitor PPC, or ad intelligence." +--- + +# Competitor Ad Intelligence + +## Process + +1. Identify target competitors (from user input or industry analysis) +2. Read `ads-shared/references/benchmarks.md` for industry CPC/CTR/CVR baselines +3. Research competitor ad presence across platforms +4. Analyze ad copy, creative, and messaging themes +5. Estimate competitor spend and keyword strategy +6. Identify gaps and opportunities +7. Generate competitive intelligence report + +## Data Sources + +### Free Intelligence Sources +| Source | Platform | What You Can Find | +|--------|----------|------------------| +| Google Ads Transparency Center | Google | Active ads, formats, geo targeting | +| Meta Ad Library | Meta/Instagram | All active ads, creative, copy, spend range | +| LinkedIn Ad Library | LinkedIn | Active ads from company pages | +| TikTok Creative Center | TikTok | Top ads, trending creative, hashtags | +| Microsoft Ads | Microsoft | Limited: use auction insights | + +### Google Ads Auction Insights +Available from the user's own Google Ads account: +- Impression share vs competitors +- Overlap rate (how often you compete) +- Outranking share (who wins more often) +- Top of page rate and absolute top of page rate +- Available for Search and Shopping campaigns + +### Platform-Specific Research + +#### Google +- Ads Transparency Center: search by advertiser name or domain +- Search for competitor brand terms to see their ads live +- Auction Insights for impression share comparison + +#### Meta +- Ad Library: filter by advertiser, country, platform (FB/IG), date range +- Shows creative (image/video), ad copy, active dates +- Shows platform placement (Facebook, Instagram, Audience Network) + +#### LinkedIn +- Ad Library: search by company name +- Shows Sponsored Content, Message Ads +- Limited data compared to Meta Ad Library + +#### TikTok +- Creative Center: top-performing ads by industry, country, objective +- Hashtag analytics: trending sounds and hashtags +- No per-advertiser library; use Creative Center for industry trends + +## Competitive Analysis Framework + +### 1. Ad Copy Analysis +For each competitor, document: +- **Headlines**: primary messages and value propositions +- **CTAs**: what action they're driving (free trial, demo, buy now, learn more) +- **Offers**: pricing, discounts, free shipping, trials +- **Tone**: professional, casual, urgent, educational, emotional +- **USPs**: unique selling propositions they emphasize +- **Pain points**: customer problems they address + +### 2. Creative Strategy Analysis +- **Formats used**: image, video, carousel, collection, document +- **Visual style**: photography, illustration, UGC, stock, branded +- **Video approach**: studio quality vs UGC vs animated +- **Creative volume**: how many active ads (indicator of testing velocity) +- **Refresh frequency**: how often new creatives appear + +### 3. Messaging Themes +Categorize competitor messaging into themes: +| Theme | Competitor A | Competitor B | Your Brand | +|-------|-------------|-------------|------------| +| Price/Value | ✅ Primary | ⚠️ Secondary | ? | +| Quality/Premium | ❌ | ✅ Primary | ? | +| Speed/Convenience | ⚠️ Secondary | ❌ | ? | +| Trust/Authority | ✅ Primary | ✅ Primary | ? | +| Innovation | ❌ | ⚠️ Secondary | ? | + +### 4. Keyword Intelligence (Google/Microsoft) +- Brand keyword bidding: are competitors bidding on your brand? +- Keyword overlap: which non-brand terms do you both target? +- Keyword gaps: terms competitors rank for that you don't target +- Match type strategy: estimated match types from ad triggers + +### 5. Spend Estimation +- Meta Ad Library shows spend ranges for political/social ads +- Google Auction Insights + impression share = directional spend estimate +- Third-party tools (SEMrush, SpyFu) for more precise estimates +- Manual estimation formula: + ``` + Estimated Monthly Spend = Impressions × CPM / 1000 + or + Estimated Monthly Spend = Clicks × Estimated CPC + ``` + +## Gap & Opportunity Identification + +### Platform Gaps +- Which platforms are competitors NOT on? (opportunity to own) +- Which platforms are they underspending on? (opportunity to outspend) + +### Messaging Gaps +- What customer pain points are NO competitors addressing? +- What value propositions are underrepresented in the market? +- What content formats are competitors not using? + +### Audience Gaps +- What demographics/segments are competitors not targeting? +- What geographic markets are underserved? +- What funnel stages are competitors neglecting? + +### Creative Gaps +- What ad formats are competitors not using? (video, UGC, Spark Ads) +- What creative styles are missing from the competitive landscape? +- What platform-specific features are competitors not leveraging? + +## Competitive Response Strategy + +### When Competitors Bid on Your Brand +- Always run brand campaigns to defend (low CPC, high CTR) +- Dynamic keyword insertion to show your brand prominently +- Sitelinks to key pages (pricing, features, reviews) +- Ad copy that emphasizes unique differentiators +- Consider bidding on competitor brand terms (know the rules) + +### When You're Outspent +- Focus on efficiency over volume (better targeting, creative, landing pages) +- Target long-tail keywords competitors ignore +- Use Exact match for precision (less waste) +- Double down on retargeting (lower CPA than prospecting) +- Compete on creative quality, not budget + +## Output + +### Deliverables +- `COMPETITOR-INTELLIGENCE-REPORT.md`: Full competitive analysis + - Per-competitor ad presence summary + - Ad copy and messaging analysis + - Creative strategy comparison + - Estimated spend levels + - Keyword overlap and gaps +- `COMPETITIVE-GAPS.md`: Opportunities identified from competitor analysis + - Platform gaps + - Messaging opportunities + - Audience segments to target + - Creative format opportunities +- Strategic recommendations for competitive positioning +- Priority actions to gain competitive advantage diff --git a/src/adclaw/agents/skills/ads-create/SKILL.md b/src/adclaw/agents/skills/ads-create/SKILL.md new file mode 100644 index 0000000..35f0410 --- /dev/null +++ b/src/adclaw/agents/skills/ads-create/SKILL.md @@ -0,0 +1,188 @@ +--- +name: ads-create +description: "Campaign concept and copy brief generator for paid advertising. Reads brand-profile.json and optional audit results to produce structured campaign concepts, messaging pillars, and copy briefs. Outputs campaign-brief.md to the current directory. Run after /ads dna and before /ads generate. Triggers on: create campaign, campaign brief, ad concepts, write ad copy, campaign strategy, ad messaging, creative brief, generate concepts." +--- + +# Ads Create: Campaign Concept & Copy Brief Generator + +Generates structured campaign concepts and platform-specific copy from your brand +profile and optional audit data. Outputs `campaign-brief.md` for use by `/ads generate`. + +## Quick Reference + +| Command | What it does | +|---------|-------------| +| `/ads create` | Full campaign brief → `campaign-brief.md` | +| `/ads create --platforms meta google` | Brief for specific platforms only | +| `/ads create --objective leads` | Brief optimized for lead generation | + +## Process + +### Step 1: Check for Brand Profile + +Look for `brand-profile.json` in the current directory. + +- **Found**: Load and proceed. +- **Not found**: Ask the user: + > "I don't see a brand-profile.json in this directory. Would you like to: + > 1. Run `/ads dna ` first to extract brand DNA automatically + > 2. Describe your brand manually (I'll create a basic profile from your description)" + +If the user chooses manual, collect: +- Brand name and website +- Primary color (or "unsure") +- 3 words that describe the brand voice +- Target audience (age, role, key pain point) +- Main product/service offering + +### Step 2: Check for Audit Results + +Look for `ADS-AUDIT-REPORT.md` or any `*-audit-results.md` in the current directory. + +- **Found**: Read them. Note the top 3 weaknesses (creative fatigue, tracking gaps, wasted spend) to address in concepts. +- **Not found**: Continue without. Note in the brief: "No audit data found; concepts are generalized. Run `/ads audit` for weakness-targeted concepts." + +### Step 3: Collect Campaign Parameters + +If `--platforms` or `--objective` flags were provided in the command, use those values +and skip the corresponding questions below. + +Ask (combine into one message; omit any already provided via flags): +1. **Platforms**: Which ad platforms? (Meta · Google · LinkedIn · TikTok · YouTube · Microsoft · All) +2. **Objective**: Sales/Revenue · Leads/Demos · App Installs · Brand Awareness · Retargeting +3. **Offer or brief**: Any specific offer, promotion, or message to highlight? (optional) +4. **Number of concepts**: How many campaign concepts? (default: 3) + +### Step 4: Select Copy Framework + +Read `ads-shared/references/copy-frameworks.md` and recommend a framework based on +campaign goal + platform + audience temperature: + +| Framework | Best For | +|-----------|----------| +| AIDA (Attention, Interest, Desire, Action) | Cold audiences, awareness campaigns | +| PAS (Problem, Agitate, Solve) | Pain-point products, problem-aware audiences | +| BAB (Before, After, Bridge) | Transformation offers, coaching, fitness | +| 4P (Promise, Picture, Proof, Push) | Direct response, high-intent audiences | +| FAB (Features, Advantages, Benefits) | Product-focused, comparison shoppers | +| Star-Story-Solution | Brand storytelling, warm audiences | + +Include the selected framework name in campaign-brief.md for the copy-writer agent. + +### Step 5: Spawn Creative Agents in Sequence + +Agents must run **sequentially**; `copy-writer` reads the file that `creative-strategist` +writes, so running them in parallel creates a race condition on `campaign-brief.md`. + +**Step 5a; Spawn `creative-strategist`** (Task tool): +This agent creates `campaign-brief.md` and writes the strategic sections: +`## Brand DNA Summary`, `## Campaign Concepts`, `## Image Generation Briefs`, `## Next Steps`. + +Additional instructions for `creative-strategist`: +- For e-commerce businesses, also read `skills/ads-plan/assets/ecommerce-creative.md` + and select the appropriate creative playbook (Product Launch, Sale/Promotion, + Seasonal, Retargeting, Brand Awareness) +- Include banana domain mode recommendations in each Image Generation Brief + (Product, Editorial, Cinema, UI/Web, or Portrait) + +Wait for `creative-strategist` to **fully complete** before continuing. + +**Step 5b; Spawn `copy-writer`** (Task tool): +After `creative-strategist` completes, spawn `copy-writer`. It reads the existing +`campaign-brief.md` and appends the `## Copy Deck` section with platform-specific +headlines, primary text, and CTAs. + +Additional instructions for `copy-writer`: +- Read `ads-shared/references/copy-frameworks.md` and apply the selected framework + structure to all ad copy +- Generate 2 framework variants per platform: primary (recommended framework) + + secondary (alternative for A/B testing) + +Wait for `copy-writer` to complete before proceeding to Step 6. + +### Step 6: Review and Present + +After both agents complete, confirm `campaign-brief.md` exists and is complete. + +Present a summary to the user: +``` +✓ campaign-brief.md generated + +Summary: + Concepts: [N] campaign concepts created + Platforms: [list] + Copy deck: Headlines, primary text, and CTAs for each concept × platform + Image briefs: [N] image generation briefs ready + +Next steps: + 1. Review campaign-brief.md and adjust any messaging + 2. Run `/ads generate` to produce AI images from the briefs + 3. Upload copy and assets to your ad platforms +``` + +## campaign-brief.md Format Specification + +The following section headings are a **parsing contract**; agents downstream depend on these exact heading names. + +```markdown +# Campaign Brief: [brand_name] +**Generated:** [date] +**Website:** [website_url] +**Platforms:** [comma-separated list] +**Objective:** [objective] +**Concepts:** [N] + +## Brand DNA Summary +[3-sentence synthesis of brand-profile.json: voice, visual identity, target audience] + +## Audit Context +[If audit data found: top 3 weaknesses being addressed] +[If no audit data: "No audit data; run /ads audit for weakness-targeted concepts"] + +## Campaign Concepts + +### Concept 1: [Name] +**Hypothesis:** [why this will work; 1 sentence] +**Primary Message:** [core message; 1 sentence] +**Tone:** [voice reading from brand-profile.json] +**Visual Direction:** [2-3 sentences describing imagery] +**Target Platforms:** [platforms and rationale] +**CTA:** [call to action text] +**Addresses:** [audit finding or "general brand awareness"] + +### Concept 2: [Name] +[same structure] + +[repeat for all concepts] + +## Copy Deck +[appended by copy-writer agent; headlines, primary text, CTAs per concept per platform] + +## Image Generation Briefs + +### Brief 1: [Concept Name]: [Platform] +**Prompt:** [exact generation prompt] +**Dimensions:** [WxH] +**Safe zone notes:** [constraint or "None"] + +### Brief 2: [Concept Name]: [Platform] +**Prompt:** [exact generation prompt] +**Dimensions:** [WxH] +**Safe zone notes:** [constraint or "None"] + +[one brief per concept × platform combination] + +## Next Steps +1. Review all concepts and select which to move forward with +2. Run `/ads generate` to produce images from the briefs above +3. Adjust CTAs and offers in the copy deck for your specific promotion +4. Upload final assets to your ad platform managers +``` + +## Quality Gates + +- **Minimum 3 concepts** (unless user requests fewer) +- **Distinct angles**: no two concepts share the same primary message angle +- **Platform fit**: concepts targeting TikTok must acknowledge vertical-only format and sound-on context +- **Offer anchoring**: if the user provided a specific offer, at least 1 concept must lead with it +- **Image briefs**: every concept must have at least one image brief per requested platform diff --git a/src/adclaw/agents/skills/ads-creative/SKILL.md b/src/adclaw/agents/skills/ads-creative/SKILL.md new file mode 100644 index 0000000..0f1ab05 --- /dev/null +++ b/src/adclaw/agents/skills/ads-creative/SKILL.md @@ -0,0 +1,137 @@ +--- +name: ads-creative +description: "Cross-platform creative quality audit covering ad copy, video, image, and format diversity across all platforms. Detects creative fatigue, evaluates platform-native compliance, and provides production priorities. Use when user says creative audit, ad creative, creative fatigue, ad copy, ad design, or creative review." +--- + +# Cross-Platform Creative Quality Audit + +## Process + +1. Collect creative assets or performance data from active platforms +2. Read `ads-shared/references/platform-specs.md` for creative specifications +3. Read `ads-shared/references/benchmarks.md` for CTR/engagement benchmarks +4. Read `ads-shared/references/scoring-system.md` for weighted scoring algorithm +5. **Validate**: confirm at least one platform has creative data (assets or performance metrics) before proceeding +6. Evaluate creative quality per platform +7. Assess cross-platform creative consistency +8. **Validate**: verify fatigue signals reference actual performance trends, not assumptions +9. Generate production priority recommendations + +## Per-Platform Assessment + +### Google Ads Creative +- RSA: ≥8 unique headlines, ≥3 descriptions per ad group +- RSA ad strength: "Good" or "Excellent" +- Pin usage: minimal and strategic (over-pinning kills RSA flexibility) +- Extensions: sitelinks (≥4), callouts (≥4), structured snippets, image +- PMax asset groups: text + image + video + optional product feed +- YouTube: video quality, hook, subtitles (see ads-youtube sub-skill) + +### Meta Ads Creative +- Format diversity: ≥3 formats active (image, video, carousel, collection) +- Creative volume: ≥5 creatives per ad set +- Fatigue detection: CTR declining >20% over 14 days = FAIL +- Video length: 15s max Stories/Reels, 30s max Feed +- UGC/testimonial content tested +- Advantage+ Creative enhancements enabled +- Headline under 40 chars, primary text under 125 chars + +### LinkedIn Creative +- Thought Leader Ads active, ≥30% budget for B2B +- Format diversity: ≥2 formats tested (single image, carousel, video, document) +- Video ads tested +- Creative refresh: every 4-6 weeks +- Professional tone appropriate for platform + +### TikTok Creative +- ≥6 creatives per ad group (Critical requirement) +- All video 9:16 vertical 1080x1920 (non-negotiable) +- Native-looking content (not corporate) +- Hook in first 1-2 seconds +- No creative active >7 days with declining CTR +- Spark Ads tested (~3% CTR vs ~2% standard) +- Sound-on optimization (never silent) +- Safe zone compliance: X:40-940, Y:150-1470 +- Trending audio used + +### Microsoft Creative +- RSA: ≥8 headlines, ≥3 descriptions +- Multimedia Ads tested (unique rich format) +- Ad copy optimized for Bing demographics (older, higher income, professional) +- Action Extension utilized (unique to Microsoft) +- Filter Link Extension tested + +## Creative Fatigue Detection + +### Signals of Fatigue +| Signal | Threshold | Action | +|--------|-----------|--------| +| CTR declining | >20% over 14 days | Refresh creative | +| Frequency (Meta) | >5.0 prospecting, >12.0 retargeting | New audience or creative | +| Watch time declining (TikTok) | <3s average | New hook needed | +| QS declining (Google) | Drop of 2+ points | Refresh ad copy | +| Engagement rate drop | >30% decline | Full creative overhaul | + +### Refresh Cadence by Platform +| Platform | Recommended Refresh | +|----------|-------------------| +| Google Search | Every 8-12 weeks | +| Meta | Every 2-4 weeks | +| LinkedIn | Every 4-6 weeks | +| TikTok | Every 5-7 days (fastest fatigue) | +| Microsoft | Every 8-12 weeks | +| YouTube | Every 4-8 weeks | + +## Format Diversity Matrix + +Evaluate which formats are active per platform: + +| Format | Google | Meta | LinkedIn | TikTok | Microsoft | +|--------|--------|------|----------|--------|-----------| +| Static Image | RSA image ext | ✅ | ✅ | ❌ | Multimedia | +| Video | YouTube, PMax | ✅ | ✅ | ✅ (required) | ❌ | +| Carousel | ❌ | ✅ | ✅ | ❌ | ❌ | +| Collection | ❌ | ✅ | ❌ | ❌ | ❌ | +| Document | ❌ | ❌ | ✅ | ❌ | ❌ | +| Shopping | PMax, Shopping | Catalog | ❌ | Shop | Shopping | + +## Universal Creative Best Practices + +### Cross-Platform Safe Zone +- 900x1000px usable area works across all vertical placements +- Keep critical elements centered and within safe margins +- Test on mobile devices (75%+ of ad impressions are mobile) + +### Ad Copy Principles +- Lead with benefit, not feature +- Include clear CTA (what should they do next?) +- Match ad message to landing page (message match) +- Use numbers and specifics over vague claims +- Test emotional vs rational appeals + +### Video Production Standards +- H.264 codec, AAC audio, MP4 container +- Minimum 720p (1080p preferred) +- Subtitles/captions always (accessibility + sound-off viewing) +- Brand mention within first 5s (awareness) or at CTA (performance) + +## Output + +### Creative Quality Report + +``` +Cross-Platform Creative Health + +Google: ████████░░ X/X checks passing +Meta: ██████████ X/X checks passing +LinkedIn: ███████░░░ X/X checks passing +TikTok: █████░░░░░ X/X checks passing +Microsoft: ████████░░ X/X checks passing +``` + +### Deliverables +- `CREATIVE-AUDIT-REPORT.md`: Per-platform creative assessment +- Fatigue alerts (any creative past refresh cadence) +- Format diversity gaps per platform +- Production priority list (most impactful creative to produce next) +- Quick Wins (format conversions, CTA changes, Spark Ads setup) diff --git a/src/adclaw/agents/skills/ads-dna/SKILL.md b/src/adclaw/agents/skills/ads-dna/SKILL.md new file mode 100644 index 0000000..35fe240 --- /dev/null +++ b/src/adclaw/agents/skills/ads-dna/SKILL.md @@ -0,0 +1,225 @@ +--- +name: ads-dna +description: "Brand DNA extractor for paid advertising. Scans a website URL to extract visual identity, tone of voice, color palette, typography, and imagery style. Outputs brand-profile.json to the current directory. Run before /ads create or /ads generate for brand-consistent creative. Triggers on: brand DNA, brand profile, extract brand, brand identity, brand colors, what is the brand voice, analyze brand, brand style guide." +--- + +# Ads DNA: Brand DNA Extractor + +Extracts brand identity from a website and saves it as `brand-profile.json` +for use by `/ads create`, `/ads generate`, and `/ads photoshoot`. + +## Quick Reference + +| Command | What it does | +|---------|-------------| +| `/ads dna ` | Full brand extraction → `brand-profile.json` | +| `/ads dna https://acme.com --quick` | Fast extraction (homepage only) | + +## Process + +### Step 1: Collect URL + +If the user hasn't provided a URL, ask: +> "What website URL should I analyze for brand DNA? (e.g. https://yoursite.com)" + +### Step 2: Fetch Pages + +Use the **browser tool** (agent_browser or curl) to retrieve each page. For each URL, extract: +> "Return all visible text content, the full contents of any ` + + +
+ +
+
+ +
+ + + + diff --git a/src/adclaw/agents/skills/html-presentation-deck/assets/template-editorial.html b/src/adclaw/agents/skills/html-presentation-deck/assets/template-editorial.html new file mode 100644 index 0000000..d27245d --- /dev/null +++ b/src/adclaw/agents/skills/html-presentation-deck/assets/template-editorial.html @@ -0,0 +1,174 @@ + + + + + +Replace with deck title + + + +
+ +
+
+ +
+ + + + diff --git a/src/adclaw/agents/skills/html-presentation-deck/assets/template-product-grid.html b/src/adclaw/agents/skills/html-presentation-deck/assets/template-product-grid.html new file mode 100644 index 0000000..9bc4b8a --- /dev/null +++ b/src/adclaw/agents/skills/html-presentation-deck/assets/template-product-grid.html @@ -0,0 +1,647 @@ + + + + + +Replace with deck title + + + +
+ +
+
+ +
+ + + + diff --git a/src/adclaw/agents/skills/html-presentation-deck/references/checklist.md b/src/adclaw/agents/skills/html-presentation-deck/references/checklist.md new file mode 100644 index 0000000..bc5388f --- /dev/null +++ b/src/adclaw/agents/skills/html-presentation-deck/references/checklist.md @@ -0,0 +1,51 @@ +# Quality Checklist + +Run this before handing off an HTML presentation deck. + +## Product Grid v2 Gate + +- Every slide has `data-system="product-grid"` and registered `data-layout="PGxx"`. +- The slide map exists before HTML: slide, message, layout, density, image slot, risk. +- No custom one-off classes appear in deck HTML. +- No inline `font-size` overrides appear in slides. +- No negative letter spacing, gradients, shadows, decorative blobs, or nested cards appear. +- Every local image has `alt` and `data-image-slot`. +- Required image slots are present for screenshot layouts. +- Dense slides are followed by PG02 Statement or PG10 Quote. +- `python3 /scripts/validate_deck_quality.py deck/index.html` passes. + +## Content + +- One idea per slide. +- Title and metadata placeholders are replaced. +- Every image has useful `alt` text. +- Data claims include source context in speaker notes or nearby captions. +- No visible draft notes, private instructions, or placeholder copy. + +## Legacy Typography (Editorial / Clean Grid) + +- Typography tokens follow `references/typography.md`. +- Contrast-safe `--accent-text` is used for small labels on light panels. +- `validate_html_deck.py` passes contrast checks for theme tokens. + +## Design + +- One visual system is used throughout. +- One theme is used throughout. +- Dense slides are separated by simple statement or image slides. +- Screenshots are framed consistently. +- Mobile layout remains readable. + +## Technical + +- From the project root, `python3 /scripts/validate_html_deck.py deck/index.html` passes. +- Browser opens the file with no console-breaking script error. +- Arrow keys, touch swipe, and Escape index work. +- Local images load from `images/`. +- No external network dependency is required for the presentation to render. + +## Language and Provenance + +- The deck is English-only unless the user explicitly requests another language. +- When the deck is English-only, no non-English text, non-English comments, non-English font names, or non-English placeholders appear. +- No upstream project name, author name, or repository name appears in generated output. diff --git a/src/adclaw/agents/skills/html-presentation-deck/references/components-product-grid.md b/src/adclaw/agents/skills/html-presentation-deck/references/components-product-grid.md new file mode 100644 index 0000000..a022a50 --- /dev/null +++ b/src/adclaw/agents/skills/html-presentation-deck/references/components-product-grid.md @@ -0,0 +1,166 @@ +# Product Grid Components + +Use these components only with `assets/template-product-grid.html`. + +## Slide Shell + +```html +
+
Product Proof04
+
+ ... +
+
+``` + +Theme variants: + +- Default: light product page. +- `theme-dark`: dark statement or contrast slide. +- `theme-accent`: one strong decision slide. +- `theme-yellow`: yellow/highlight decision slide. + +Every slide uses `.chrome` and `.stage`. Do not put content outside `.stage` +except intentionally fixed navigation provided by the template. + +## Typography + +| Class | Use | Notes | +|---|---|---| +| `.display` | cover title | one per deck opening | +| `.title` | normal slide title | max one per slide | +| `.statement` | big transition claim | PG02 or PG12 | +| `.lead` | large support copy | 1-2 sentences | +| `.body` | normal explanatory copy | keep under 45 words per block | +| `.body-sm` | compact captions and card notes | not for dense paragraphs | +| `.kicker` | section marker | uppercase metadata | +| `.meta` | source, date, small labels | uppercase metadata | + +The template controls type sizes. Do not add inline `font-size`. If content does +not fit, edit the copy or choose a different layout. + +## Grid + +Use `.grid-12` plus spans: + +- `.span-3`: quarter width. +- `.span-4`: third width. +- `.span-5` / `.span-7`: analysis plus screenshot. +- `.span-6`: equal halves. +- `.span-8` / `.span-4`: narrative plus side notes. +- `.span-12`: full row. + +Do not nest cards inside cards. Use one panel layer. + +## Panels + +```html +
+
Signal
+
3.4x
+

Qualified intent

+
+``` + +Panel variants: + +- `.panel`: default quiet panel. +- `.panel-2`: stronger neutral emphasis. +- `.panel-accent`: exactly one highlight panel per slide. +- `.panel-ink`: black emphasis block for strong contrast. + +## Images + +```html +
+ Dashboard screenshot +
+``` + +Rules: + +- Every local image needs `alt` and `data-image-slot`. +- Raw screenshots with important text use `.fit-contain`. +- Generated images should match the slot ratio and use cover behavior. +- Never add shadows, browser chrome, or extra rounded wrappers. + +## Timeline + +```html +
+
01

Collect source signals.

+
02

Match against knowledge.

+
03

Generate output.

+
04

Publish with controls.

+
+``` + +Keep timeline nodes short. If a step needs more than two lines, move detail to +a separate slide. + +## Comparison + +Use `.compare` for before/after, old/new, manual/worker, or risk/control. + +```html +
+
...
+
+
...
+
+``` + +Both sides must use matching hierarchy and roughly equal copy length. + +## System Diagram + +Use `.diagram` for three layers or stages. Labels are HTML, not SVG. + +```html +
+
Input

Sources and signals.

+
Worker

Plan, execute, and log.

+
Output

Published assets and findings.

+
+``` + +## Bar Evidence + +Use `.bar-chart` when a slide compares actual values. Do not use it for vague +concepts. + +```html +
+
Manual38
+
Worker86
+
+``` + +## Matrix Brief + +Use `.matrix` for eight compact evidence cells. Keep each cell to one short +line plus metadata. + +```html +
+

Signal scan

Input

+

Published output

Receipt

+
+``` + +## Density Budget + +- Low density: title + one support block. +- Medium density: title + 3 cards or one screenshot. +- High density: title + 4 cards or diagram. Follow with PG02 or PG10. + +Never combine high-density text with a detailed screenshot on the same slide. + +## Visual Rhythm + +- Use `theme-accent` or `theme-yellow` only for cover, shift, or closing slides. +- A deck should use one accent family. If `theme-yellow` is used, local labels and headings on that slide stay black. +- Use `theme-dark` for statements and quotes. +- Use `.dot-field` only on large statement/cover pages where it has room to breathe. +- Use `.ledger` for roadmap or ordered proof instead of ad hoc tables. +- Use `.metric-xl` for one giant number; use `.metric` for card numbers. diff --git a/src/adclaw/agents/skills/html-presentation-deck/references/image-prompts.md b/src/adclaw/agents/skills/html-presentation-deck/references/image-prompts.md new file mode 100644 index 0000000..cfb7213 --- /dev/null +++ b/src/adclaw/agents/skills/html-presentation-deck/references/image-prompts.md @@ -0,0 +1,64 @@ +# Image Prompts + +Use these prompts when the deck needs generated images. Keep prompts short and tied to the slide slot. + +## General Rules + +- Match the selected visual system: Editorial or Clean Grid. +- Generate to the final slot ratio before placing the image. +- Do not generate slide chrome, page numbers, titles, footers, watermarks, or logos inside images. +- If text must appear inside an image, keep it in the same language as the deck. +- Prefer real-world visual evidence over decorative filler. + +## Product Grid Slot Rules + +Product Grid v2 images must be generated for the target slot before insertion: + +- `pg01-hero-16x9`: product launch visual, realistic product context, no UI labels unless they are meant to be inspected. +- `pg04-main-16x10`: raw product screenshot or screenshot reconstruction; preserve all important UI text. +- `pg09-evidence-16x9`: evidence visual, market proof, workflow diagram, or before/after graphic. + +Do not generate atmospheric filler. If the slide needs proof, use a real screenshot +or a diagram with clear information hierarchy. + +## Product Grid Launch Visual + +```text +16:9 product launch visual for [product]. Clean SaaS presentation style, real product context, strict rectangular composition, generous whitespace, one accent color, no gradients, no glassmorphism, no rounded cards, no slide title, no footer, no watermark. +``` + +## Product Grid Screenshot Reconstruction + +```text +16:10 product UI screenshot reconstruction for [workflow]. Keep labels short and readable, align to a strict grid, use realistic SaaS controls, one accent color, no browser chrome, no decorative background, no watermark. +``` + +## Product Grid Evidence Diagram + +```text +16:9 evidence diagram for [claim]. Three to five rectangular modules, left-aligned labels, thin rules, one accent color, high contrast, no gradient, no shadow, no rounded shapes, no footer, no logo. +``` + +## Editorial Photo + +```text +Horizontal editorial documentary photograph about [topic]. Natural light, restrained color, quiet negative space, realistic environment, subtle grain, premium magazine feel. No logo, no watermark, no text, no artificial interface. Ratio: [16:9 or 16:10]. +``` + +## Editorial Diagram + +```text +Horizontal editorial information graphic explaining [concept]. Paper texture, thin ink lines, numbered structure, restrained accent color, generous whitespace. Text labels in English only and under four words each. No logo, no decorative border. Ratio: 16:9. +``` + +## Clean Grid Diagram + +```text +Horizontal clean grid information graphic explaining [system]. Sharp rectangular modules, left-aligned short labels, one accent color, black white and gray only, no gradients, no rounded glass effects, no 3D, no logo. Ratio: [21:9 or 16:10]. +``` + +## Clean Grid Data Visual + +```text +Minimal data visual for [metric]. Large number, thin rules, strict grid, one accent color, high contrast, executive presentation style. No page header, no footer, no logo, no watermark. Ratio: 16:9. +``` diff --git a/src/adclaw/agents/skills/html-presentation-deck/references/layout-lock.md b/src/adclaw/agents/skills/html-presentation-deck/references/layout-lock.md new file mode 100644 index 0000000..6eec84c --- /dev/null +++ b/src/adclaw/agents/skills/html-presentation-deck/references/layout-lock.md @@ -0,0 +1,88 @@ +# Layout Lock + +This file is the canonical layout contract for the strict Product Grid system. +It exists to prevent "nice-looking but structurally random" slides. + +## Required Contract + +Every slide in a Product Grid deck must: + +- Use `assets/template-product-grid.html`. +- Include `data-system="product-grid"`. +- Include one registered `data-layout="PGxx"` value. +- Use only classes defined in the copied template CSS. +- Put local images in a registered slot with `data-image-slot`. +- Keep text outside SVG. SVG is allowed only for geometry. +- Keep product screenshots legible; do not crop required UI labels. + +Before writing HTML, create this planning table: + +| Slide | Message | Layout | Density | Image slot | Risk | +|---|---|---|---|---|---| +| 01 | Launch thesis | PG01 | low | pg01-hero-16x9 | none | + +If a slide cannot fit one registered layout, split it into two slides. Do not +invent a new structure inside the deck. + +## Registered Product Grid Layouts + +| ID | Name | Use | Required skeleton | Image slots | +|---|---|---|---|---| +| PG01 | Cover System | Opening title with strong accent scene | `.stage`, `.grid-12`, large title, optional metric block | `pg01-hero-16x9` optional | +| PG02 | Statement | One large claim or transition | `.statement` plus `.lead` | none | +| PG03 | Metric Wall | Three or four major numbers | `.grid-12` with `.panel` metric cards | none | +| PG04 | Screenshot Proof | Product screenshot plus analysis | `.span-7` frame + `.span-5` panel | `pg04-main-16x10` required | +| PG05 | Three Cards | Three equal product pillars | three `.span-4` panels | none | +| PG06 | Process Timeline | Four-step sequence | `.timeline` with four nodes | none | +| PG07 | Before After | Two-column comparison | `.compare` with `.divider` | none | +| PG08 | System Diagram | Three-layer architecture or workflow | `.diagram` with three HTML columns | none | +| PG09 | Evidence Grid | Mixed proof points with optional visuals | `.grid-12` with 2-4 panels | `pg09-evidence-16x9` optional | +| PG10 | Quote | Customer/founder quote or takeaway | `.quote` plus source `.meta` | none | +| PG11 | Ledger | Three-phase launch, rollout, or ordered proof | `.ledger` with `.ledger-row` | none | +| PG12 | Closing | Final decision and CTA | `.statement` plus one supporting panel | none | +| PG13 | Bar Evidence | Quantified comparison or ranked proof | `.bar-chart` with `.bar-row` | none | +| PG14 | Matrix Brief | Eight short evidence cells with one emphasis | `.matrix` with `.matrix-cell` | none | + +## Layout Selection Rules + +- Decks under 8 slides must use at least 5 distinct layouts. +- Decks with 8-14 slides must use at least 7 distinct layouts. +- Do not use the same layout more than twice in a row. +- Use PG02 or PG10 after two dense slides. +- Use PG04 only when screenshot details are large enough to read. +- Use PG08 only for real system structure; do not use it as decoration. +- Use PG03 only for real metrics, counts, durations, or named states. +- Use PG13 only with real or explicitly labeled illustrative values. +- Use PG14 for compact short evidence, not paragraphs. + +## Typography Rules + +- Do not add negative letter spacing. +- Do not scale text with viewport width. Use template classes and media queries. +- Keep one headline per slide. +- Rewrite or split any title that needs more than three lines. +- Do not reduce body text below the template body sizes to make content fit. +- Small labels use `.meta`, `.kicker`, `.label`, or `.tag`; do not create new mini text classes. +- Use `.stage`, not legacy `.canvas`. + +## Image Slot Rules + +- `pg01-hero-16x9`: 16:9 product, market, or campaign visual. Use `.frame.r-16x9`. +- `pg04-main-16x10`: 16:10 screenshot proof. Use `.frame.r-16x10.fit-contain` for raw screenshots. +- `pg09-evidence-16x9`: 16:9 evidence image. Use `.frame.r-16x9`. + +For generated images, generate to the final slot ratio before inserting. For raw +screenshots, preserve visible details and use `fit-contain` when text matters. + +## Banned Patterns + +- Custom one-off classes in deck HTML. +- `text-align:center` on normal product-grid body slides. +- Inline `font-size` overrides. +- `height: Nvh` image boxes. +- Rounded nested card stacks. +- Decorative gradient blobs, bokeh, or purely atmospheric images. +- Multiple accent colors competing on one slide. +- SVG `` labels. +- Images without `alt`. +- Blue text on yellow theme slides; yellow theme uses black text accents. diff --git a/src/adclaw/agents/skills/html-presentation-deck/references/layouts-clean-grid.md b/src/adclaw/agents/skills/html-presentation-deck/references/layouts-clean-grid.md new file mode 100644 index 0000000..35c6b52 --- /dev/null +++ b/src/adclaw/agents/skills/html-presentation-deck/references/layouts-clean-grid.md @@ -0,0 +1,76 @@ +# Clean Grid Layouts + +Use these section skeletons inside `assets/template-clean-grid.html`. + +## Cover + +```html +
+
Operating ReviewQ3 / 2026
+

Growth needs a tighter feedback loop.

+

A clean view of the decisions, signals, and assets that move the next quarter.

+
+``` + +## Metric Wall + +```html +
+
Signals01
+
+
+
Pipeline
+
2.8x
+

Qualified demand

+
+
+
Content
+
64%
+

Assisted conversions

+
+
+
Decision
+
14d
+

Planning cycle

+
+
+
+``` + +## Screenshot + Analysis + +```html +
+
Product Proof02
+
+
+
+ Dashboard screenshot +
+
+
+
Readout
+

The interface already contains the sales story.

+

Use annotations only where they clarify a decision. Do not decorate screenshots just to fill space.

+
+
+
+``` + +`data-image-slot` is optional metadata for downstream validation or export tooling. It labels the intended image slot and aspect ratio; see `references/image-prompts.md` and `references/screenshot-framing.md` for the matching ratio guidance. + +## Timeline + +```html +
+
Plan03
+

Four moves over six weeks.

+
+
+
01

Collect signal sources.

+
02

Build proof assets.

+
03

Launch with channel owners.

+
04

Review and compound winners.

+
+
+``` diff --git a/src/adclaw/agents/skills/html-presentation-deck/references/layouts-editorial.md b/src/adclaw/agents/skills/html-presentation-deck/references/layouts-editorial.md new file mode 100644 index 0000000..995af00 --- /dev/null +++ b/src/adclaw/agents/skills/html-presentation-deck/references/layouts-editorial.md @@ -0,0 +1,66 @@ +# Editorial Layouts + +Use these section skeletons inside `assets/template-editorial.html`. + +## Cover + +```html +
+
Product Strategy / 2026
+

The market is moving faster than our planning cycle.

+

A practical operating model for turning weekly signals into launch decisions.

+
Project / Confidential
+
+``` + +## Two Column Narrative + +```html +
+
+
+
Context
+

The old funnel is now a network of moments.

+

Discovery, evaluation, and trust happen across search, communities, AI answers, and product proof.

+
+
Market map
+
+
+``` + +## Three Evidence Cards + +```html +
+
Evidence
+

Three signals changed the plan.

+
+
+
+
42%
+

Search shifted

+

More discovery moved into AI summaries and community threads.

+
+
+
3.1x
+

Proof compounds

+

Customer examples outperform generic category claims.

+
+
+
9d
+

Launch window

+

The best content now ships while the discussion is still live.

+
+
+
+``` + +## Quote + +```html +
+
Principle
+

A launch is not a day. It is a sequence of proof arriving at the right moment.

+

Use this layout for a memorable transition or closing idea.

+
+``` diff --git a/src/adclaw/agents/skills/html-presentation-deck/references/layouts-product-grid.md b/src/adclaw/agents/skills/html-presentation-deck/references/layouts-product-grid.md new file mode 100644 index 0000000..e3056cc --- /dev/null +++ b/src/adclaw/agents/skills/html-presentation-deck/references/layouts-product-grid.md @@ -0,0 +1,250 @@ +# Product Grid Layouts + +Use these exact skeletons with `assets/template-product-grid.html`. + +## PG01 Cover System + +```html +
+
+
Product Launch01
+
+
+
+
Citedy Workers
+

Marketing work that keeps moving.

+

One screen for long-running workers that research, create, publish, and report the receipt.

+
+
+
+
24/7
+

Always-on execution

+
+
+
+
+
+
+``` + +## PG02 Statement + +```html +
+
Problem02
+
+

The founder is still the content queue.

+

The product has automation, but the operating loop still asks the user to watch every tab.

+
+
+``` + +## PG03 Metric Wall + +```html +
+
Operating Shape03
+
+

The offer is a rentable marketing team.

+
+
Coverage
24/7

Always-on workers

+
Surface
1

Casual screen

+
Fleet
4

Preset jobs

+
+
+
+``` + +## PG04 Screenshot Proof + +```html +
+
Product Proof04
+
+
+
+
+ Dashboard screenshot +
+
+
+
Readout
+

Show output before configuration.

+

Screenshots should prove state, controls, and product value. Keep annotations sparse and leave the UI readable.

+
+
+
+
+``` + +## PG05 Three Cards + +```html +
+
Worker Fleet05
+
+

Three jobs, one operating loop.

+
+
Writer

Finds source signals, writes articles, and prepares distribution.

+
Creator

Turns product knowledge into short-form video assets.

+
Scout

Watches competitor pages and reports meaningful changes.

+
+
+
+``` + +## PG06 Process Timeline + +```html +
+
Operating Loop06
+
+

Signal turns into output through one path.

+
+
01

Scan approved sources.

+
02

Match product knowledge.

+
03

Generate the asset.

+
04

Publish or request review.

+
+
+
+``` + +## PG07 Before After + +```html +
+
Shift07
+
+
+
Before

Manual content operations.

The user coordinates research, writing, visuals, posting, and monitoring across disconnected tools.

+
+
After

Worker-managed execution.

Workers run the loop and report outputs, exceptions, and approval requests in one feed.

+
+
+
+``` + +## PG08 System Diagram + +```html +
+
Architecture08
+
+

Workers remain clients of the primary API.

+
+
Inputs

Sources, schedules, account state, and tenant knowledge.

+
Runtime

Plan runs, enforce budget, execute tasks, and emit events.

+
Outputs

Articles, shorts, findings, approvals, and live feed receipts.

+
+
+
+``` + +## PG09 Evidence Grid + +```html +
+
Evidence09
+
+
+
Policy

Trust levels decide whether work publishes, previews, or asks every time.

+
Billing

Rent and operations draw from the same tenant credit balance.

+
Control

Budget caps stop expensive work before it starts.

+
+
+
+``` + +## PG10 Quote + +```html +
+
+
Takeaway10
+
+
The feed is the proof that the worker actually worked.
+

Launch narrative

+
+
+``` + +## PG11 Ledger + +```html +
+
Rollout11
+
+

Launch in phases without splitting the product.

+
+
01

Casual surface and scheduled workers on the existing runtime.

Phase 1

+
02

Advanced dashboard, MCP tools, and Cloudflare execution.

Phase 2

+
03

Browser Radar, priority behavior, and live progress.

Phase 3

+
+
+
+``` + +## PG12 Closing + +```html +
+
Decision12
+
+
+
+

Stop managing marketing work. Hire the workers.

+
+
+
Call to action
+

Connect accounts once, turn workers on, and review the live feed when the work is done.

+
+
+
+
+``` + +## PG13 Bar Evidence + +```html +
+
Evidence13
+
+
+
+
Quantified Shift
+

Use bars only when values are real.

+

This layout is for measured comparison: coverage, time saved, adoption, throughput, accuracy, or cost.

+
+
+
+
Manual38
+
Assisted64
+
Worker86
+
+
+
+
+
+``` + +## PG14 Matrix Brief + +```html +
+
Brief14
+
+

Eight compact proof points, one highlighted takeaway.

+
+

Source scan

Input

+

Knowledge match

Filter

+

Article draft

Output

+

Video short

Output

+

Approval gate

Control

+

Budget cap

Policy

+

Live receipt

Proof

+

Digest

Report

+
+
+
+``` diff --git a/src/adclaw/agents/skills/html-presentation-deck/references/screenshot-framing.md b/src/adclaw/agents/skills/html-presentation-deck/references/screenshot-framing.md new file mode 100644 index 0000000..f06adf0 --- /dev/null +++ b/src/adclaw/agents/skills/html-presentation-deck/references/screenshot-framing.md @@ -0,0 +1,76 @@ +# Screenshot Framing + +Use screenshot framing when product UI, dashboards, website captures, or code screenshots need to fit the deck's visual system. + +## Product Grid v2 Rules + +- Use PG04 for one important screenshot and PG09 for evidence grids. +- Raw product screenshots with important UI labels use `.frame.r-16x10.fit-contain` and `data-image-slot="pg04-main-16x10"`. +- Generated or reconstructed evidence images must match their slot ratio and should not use `fit-contain`. +- Do not add fake browser chrome, drop shadows, rounded wrappers, gradient backgrounds, or nested frames. +- If a screenshot is too tall, split it into multiple slides or crop to the workflow region; do not shrink it until labels are unreadable. +- If screenshot text is not legible at 1440 x 900, the slide fails. + +## Rules + +- Preserve screenshot content when labels, numbers, or UI state matter. +- Choose the slide layout before choosing the screenshot crop. +- Use generated backgrounds as quiet framing surfaces, not as main illustrations. +- Do not add logos, mock browser chrome, or decorative frames unless the screenshot needs context. +- Hide sensitive data before placing the image. + +## Background Assets + +All built-in backgrounds are 2048×1152 WebP and crop-safe for `21:9`, `16:10`, `16:9`, `4:3`, and `1:1`. +Use asset root `../assets` with the asset keys below. + +### Editorial + +| Theme | Asset Key | Use | +|---|---|---| +| Ink Paper | `screenshot-backgrounds/editorial/ink-paper.webp` | Neutral paper texture and light ink wash | +| Indigo Porcelain | `screenshot-backgrounds/editorial/indigo-porcelain.webp` | Cool technology and research decks | +| Forest Ledger | `screenshot-backgrounds/editorial/forest-ledger.webp` | Operations, sustainability, and community | +| Warm Archive | `screenshot-backgrounds/editorial/warm-archive.webp` | Retrospectives and case studies | +| Sand Gallery | `screenshot-backgrounds/editorial/sand-gallery.webp` | Calm creative and design narratives | + +### Clean Grid + +| Theme | Asset Key | Use | +|---|---|---| +| Blue Anchor | `screenshot-backgrounds/clean-grid/blue-anchor.webp` | Product, engineering, and board decks | +| Lemon Signal | `screenshot-backgrounds/clean-grid/lemon-signal.webp` | Sharp contrast and decision moments | +| Lime Circuit | `screenshot-backgrounds/clean-grid/lime-circuit.webp` | Growth systems and automation | +| Orange Marker | `screenshot-backgrounds/clean-grid/orange-marker.webp` | Launches and urgent decisions | + +## Framing Presets + +Editorial screenshot: + +```text +ratio: 16:10 +background: editorial theme asset +padding: 7% +shadow: soft +corners: 12px +alignment: center +``` + +Clean Grid screenshot: + +```text +ratio: 21:9 or 16:10 +background: clean grid theme asset +padding: 5% +shadow: none +corners: 0 +alignment: center +``` + +## Asset Generation Prompt Pattern + +Use this only when regenerating the committed assets. + +```text +Create a 2048x1152 abstract crop-safe presentation screenshot background. It must have no text, no logo, no people, no devices, no icons, no border, and no focal object. Keep the center and corners quiet so the image can be cropped to 21:9, 16:10, 4:3, or 1:1. Use subtle texture, low contrast, and the specified theme colors only. +``` diff --git a/src/adclaw/agents/skills/html-presentation-deck/references/themes.md b/src/adclaw/agents/skills/html-presentation-deck/references/themes.md new file mode 100644 index 0000000..fb960d2 --- /dev/null +++ b/src/adclaw/agents/skills/html-presentation-deck/references/themes.md @@ -0,0 +1,145 @@ +# Themes + +Use one theme per deck. Do not combine variables across themes. + +## Editorial Themes + +### Ink Paper + +Default for founder talks, strategy narratives, and broad business decks. + +```css +:root { + --paper: #f4efe7; + --ink: #111113; + --muted: #6f675d; + --accent: #9d4f35; + --accent-soft: #e7cabb; +} +``` + +### Indigo Porcelain + +Use for research, technology, infrastructure, and data-heavy narratives. + +```css +:root { + --paper: #edf1f4; + --ink: #101827; + --muted: #5b6675; + --accent: #2d5f8b; + --accent-soft: #cad8ea; +} +``` + +### Forest Ledger + +Use for sustainability, operations, community, and long-horizon strategy. + +```css +:root { + --paper: #eef0e7; + --ink: #142016; + --muted: #5f6a5a; + --accent: #3f6b48; + --accent-soft: #cddcc8; +} +``` + +### Warm Archive + +Use for retrospectives, case studies, culture, and historical narratives. + +```css +:root { + --paper: #efe1ca; + --ink: #211811; + --muted: #725d47; + --accent: #a65c31; + --accent-soft: #e4c5a6; +} +``` + +### Sand Gallery + +Use for creative reviews, design narratives, and calm executive decks. + +```css +:root { + --paper: #eee8dc; + --ink: #1c1a18; + --muted: #6c655b; + --accent: #8c765c; + --accent-soft: #d8c9b7; +} +``` + +## Clean Grid Themes + +### Blue Anchor + +Default for product, engineering, board, and operating model decks. + +```css +:root { + --paper: #fbfbf8; + --ink: #080808; + --muted: #6d6d68; + --line: #d9d9d2; + --panel: #f0f0eb; + --accent: #0647ff; + --accent-text: #0647ff; + --accent-on: #ffffff; +} +``` + +### Lemon Signal + +Use when the deck needs sharp contrast without feeling corporate. + +```css +:root { + --paper: #fbfbf4; + --ink: #0a0a08; + --muted: #6e6e60; + --line: #ddddc8; + --panel: #f1f1df; + --accent: #e5ef2f; + --accent-text: #505700; + --accent-on: #111111; +} +``` + +### Lime Circuit + +Use for growth systems, automation, and live product operating loops. + +```css +:root { + --paper: #f7faf3; + --ink: #0b1008; + --muted: #65705e; + --line: #d7dfcf; + --panel: #edf4e7; + --accent: #78d64b; + --accent-text: #2f6a18; + --accent-on: #0b1008; +} +``` + +### Orange Marker + +Use for launches, incident reviews, transformation plans, and urgent decisions. + +```css +:root { + --paper: #fbf8f2; + --ink: #120f0b; + --muted: #6d6258; + --line: #dfd6ca; + --panel: #f1e8dc; + --accent: #f46a23; + --accent-text: #9f3d0f; + --accent-on: #120f0b; +} +``` diff --git a/src/adclaw/agents/skills/html-presentation-deck/references/typography.md b/src/adclaw/agents/skills/html-presentation-deck/references/typography.md new file mode 100644 index 0000000..99a7a32 --- /dev/null +++ b/src/adclaw/agents/skills/html-presentation-deck/references/typography.md @@ -0,0 +1,147 @@ +# Typography + +Typography is selected through semantic tokens, not ad hoc class edits. Keep one typography preset per deck. + +## Default Rule + +Use the template defaults unless the brief explicitly needs a stronger typographic voice. + +- Editorial defaults to a system-safe serif display stack with system sans body text. +- Clean Grid defaults to a system-safe sans stack with system mono labels. +- Generated decks must render offline unless the user explicitly approves external fonts. + +## Font Tokens + +Set these tokens in `:root` when changing typography: + +```css +:root { + --display-font: var(--serif); + --text-font: var(--sans); + --label-font: var(--mono); + --hero-tracking: -.06em; + --headline-tracking: -.045em; + --statement-tracking: -.05em; + --display-tracking: -.075em; + --title-tracking: -.065em; + --metric-tracking: -.07em; + --label-tracking: .14em; +} +``` + +Do not edit every heading class to change fonts. Update tokens once. + +## Presets + +### System Safe + +Best for offline decks, private handoffs, and unknown environments. + +```css +:root { + --serif: Georgia, "Times New Roman", serif; + --sans: "Aptos", "Helvetica Neue", Helvetica, Arial, sans-serif; + --mono: "SFMono-Regular", Consolas, "Liberation Mono", monospace; +} +``` + +### Modern Product + +Best for SaaS, product launches, UI-heavy demos, and clean executive decks. + +Use when external or self-hosted fonts are approved: + +```css +:root { + --sans: "Inter", "Aptos", "Helvetica Neue", Helvetica, Arial, sans-serif; + --mono: "JetBrains Mono", "SFMono-Regular", Consolas, monospace; + --display-font: var(--sans); + --text-font: var(--sans); + --label-font: var(--mono); +} +``` + +### Executive Grid + +Best for board updates, technical operating reviews, data rooms, and information-heavy decks. + +Use when external or self-hosted fonts are approved: + +```css +:root { + --sans: "IBM Plex Sans", "Aptos", "Helvetica Neue", Helvetica, Arial, sans-serif; + --mono: "IBM Plex Mono", "SFMono-Regular", Consolas, monospace; + --display-font: var(--sans); + --text-font: var(--sans); + --label-font: var(--mono); +} +``` + +### Editorial Serif + +Best for research, founder updates, customer stories, and narrative strategy. + +Use when external or self-hosted fonts are approved: + +```css +:root { + --serif: "Source Serif 4", Georgia, "Times New Roman", serif; + --sans: "Inter", "Aptos", "Helvetica Neue", Helvetica, Arial, sans-serif; + --mono: "JetBrains Mono", "SFMono-Regular", Consolas, monospace; + --display-font: var(--serif); + --text-font: var(--sans); + --label-font: var(--mono); +} +``` + +### Editorial Display + +Best for high-impact title slides, conference talks, and brand-led storytelling. Use sparingly: long body text should still use a readable sans or text serif. + +Use when external or self-hosted fonts are approved: + +```css +:root { + --serif: "Fraunces", Georgia, "Times New Roman", serif; + --sans: "Source Sans 3", "Aptos", "Helvetica Neue", Helvetica, Arial, sans-serif; + --mono: "Roboto Mono", "SFMono-Regular", Consolas, monospace; + --display-font: var(--serif); + --text-font: var(--sans); + --label-font: var(--mono); +} +``` + +### Coverage First + +Best for non-English decks, mixed scripts, international teams, and localization work. + +Use when external or self-hosted fonts are approved: + +```css +:root { + --serif: "Noto Serif", Georgia, "Times New Roman", serif; + --sans: "Noto Sans", "Aptos", "Helvetica Neue", Helvetica, Arial, sans-serif; + --mono: "Noto Sans Mono", "SFMono-Regular", Consolas, monospace; + --display-font: var(--serif); + --text-font: var(--sans); + --label-font: var(--mono); +} +``` + +## External Font Policy + +The default deck must not depend on a network request. If the user approves non-offline fonts, choose one of these delivery modes: + +- Self-hosted: put font files in `deck/fonts/` and define `@font-face`. +- Google Fonts: use only when a hosted dependency is acceptable for the use case. + +When adding `@font-face`, use `font-display: swap` and keep weights narrow. Do not load a full family when the deck only needs regular, semibold, and bold. + +## Readability Rules + +- Body text should use `--text-font`, not the display font. +- Labels and navigation should use `--label-font`. +- Display tracking may be tight; label tracking may be wide; body copy should keep normal letter spacing. +- Avoid ultra-thin weights on dark or image-backed slides. +- Do not use bright accent fills as text on light panels. Use `--accent-text` for small labels, metrics, and annotations. +- Validate contrast after changing any theme or typography token. diff --git a/src/adclaw/agents/skills/html-presentation-deck/scripts/.gitignore b/src/adclaw/agents/skills/html-presentation-deck/scripts/.gitignore new file mode 100644 index 0000000..7a60b85 --- /dev/null +++ b/src/adclaw/agents/skills/html-presentation-deck/scripts/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.pyc diff --git a/src/adclaw/agents/skills/html-presentation-deck/scripts/validate_deck_quality.py b/src/adclaw/agents/skills/html-presentation-deck/scripts/validate_deck_quality.py new file mode 100644 index 0000000..e263540 --- /dev/null +++ b/src/adclaw/agents/skills/html-presentation-deck/scripts/validate_deck_quality.py @@ -0,0 +1,333 @@ +#!/usr/bin/env python3 +"""Quality gate for strict HTML presentation decks.""" + +from __future__ import annotations + +import re +import sys +from html.parser import HTMLParser +from pathlib import Path + + +ALLOWED_LAYOUTS = { + "PG01", + "PG02", + "PG03", + "PG04", + "PG05", + "PG06", + "PG07", + "PG08", + "PG09", + "PG10", + "PG11", + "PG12", + "PG13", + "PG14", +} + +IMAGE_SLOT_BY_LAYOUT = { + "PG01": {"pg01-hero-16x9"}, + "PG04": {"pg04-main-16x10"}, + "PG09": {"pg09-evidence-16x9"}, +} + +REQUIRED_IMAGE_BY_LAYOUT = { + "PG04": {"pg04-main-16x10"}, +} + +ALLOWED_GLOBAL_CLASSES = { + "slide", + "theme-dark", + "theme-accent", + "theme-yellow", +} + +TEXT_ALIGN_CENTER_OK = {"PG02", "PG10", "PG12"} +REQUIRED_TEMPLATE_CLASSES = {"deck", "slide", "stage", "progress", "nav", "index"} + +IDEOGRAPH_RE = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]") +CLASS_SELECTOR_RE = re.compile(r"(?]*>([\s\S]*?)", re.IGNORECASE) +STYLE_ATTR_RE = re.compile(r"style\s*=\s*(['\"])(.*?)\1", re.IGNORECASE | re.DOTALL) +DECORATIVE_CSS_RE = re.compile(r"linear-gradient|box-shadow|filter\s*:", re.IGNORECASE) +SECTION_RE = re.compile( + r"(?P[^>]*)>)(?P[\s\S]*?)", + re.IGNORECASE, +) + + +class ImgParser(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.images: list[dict[str, str]] = [] + self.classes: list[str] = [] + self.svg_text_count = 0 + self._in_svg = False + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + attr = {key: value or "" for key, value in attrs} + if "class" in attr: + self.classes.extend(part for part in attr["class"].split() if part) + if tag.lower() == "img": + self.images.append(attr) + if tag.lower() == "svg": + self._in_svg = True + if self._in_svg and tag.lower() == "text": + self.svg_text_count += 1 + + def handle_endtag(self, tag: str) -> None: + if tag.lower() == "svg": + self._in_svg = False + + +FETCH_ATTRS = {"src", "href", "srcset", "poster", "data", "xlink:href"} +FETCH_TAGS = {"script", "img", "image", "video", "audio", "source", "iframe", "embed", "object"} +EXTERNAL_URL_PREFIXES = ("http://", "https://", "//") +CSS_URL_RE = re.compile(r"url\(\s*(['\"]?)((?:https?:)?//[^'\"\s)]+)\1\s*\)", re.IGNORECASE) +CSS_IMPORT_RE = re.compile(r"@import\s+(?:url\(\s*)?(['\"]?)((?:https?:)?//[^'\"\s)]+)\1", re.IGNORECASE) +TEMPLATE_PATH = Path(__file__).resolve().parents[1] / "assets/template-product-grid.html" + + +def contains_external_css(css: str) -> bool: + return bool(CSS_URL_RE.search(css) or CSS_IMPORT_RE.search(css)) + + +def is_external_url(value: str) -> bool: + return value.strip().lower().startswith(EXTERNAL_URL_PREFIXES) + + +class ExternalRefParser(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.has_external = False + self._in_style = False + self._style_chunks: list[str] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + tag_lower = tag.lower() + attr = {key.lower(): value or "" for key, value in attrs} + + if tag_lower == "style": + self._in_style = True + + style = attr.get("style", "") + if style and contains_external_css(style): + self.has_external = True + + for name, value in attr.items(): + if not value or name.startswith("xmlns"): + continue + if name == "srcset": + candidates = [part.strip().split()[0] for part in value.split(",") if part.strip()] + if any(is_external_url(candidate) for candidate in candidates): + self.has_external = True + elif name in FETCH_ATTRS and (tag_lower in FETCH_TAGS or name != "href" or tag_lower in {"base", "link"}): + if is_external_url(value): + self.has_external = True + + def handle_data(self, data: str) -> None: + if self._in_style: + self._style_chunks.append(data) + + def handle_endtag(self, tag: str) -> None: + if tag.lower() == "style": + if contains_external_css("".join(self._style_chunks)): + self.has_external = True + self._style_chunks.clear() + self._in_style = False + + +def has_external_fetch_reference(html: str) -> bool: + parser = ExternalRefParser() + parser.feed(html) + return parser.has_external + + +def attr_value(attrs: str, name: str) -> str | None: + match = re.search(rf"\b{name}\s*=\s*(['\"])(.*?)\1", attrs, re.IGNORECASE) + return match.group(2) if match else None + + +def css_classes(html: str) -> set[str]: + classes: set[str] = set() + for block in STYLE_BLOCK_RE.findall(html): + classes.update(CLASS_SELECTOR_RE.findall(block)) + return classes + + +def css_markup(html: str) -> str: + style_blocks = STYLE_BLOCK_RE.findall(html) + style_attrs = [match.group(2) for match in STYLE_ATTR_RE.finditer(html)] + return "\n".join(style_blocks + style_attrs) + + +def registered_css_classes() -> tuple[set[str], str | None]: + if not TEMPLATE_PATH.is_file(): + return set(), f"Template CSS source not found: {TEMPLATE_PATH}" + classes = css_classes(TEMPLATE_PATH.read_text(encoding="utf-8")) + if not classes: + return set(), f"Template CSS source has no registered classes: {TEMPLATE_PATH}" + return classes, None + + +def check_file(path: Path) -> tuple[list[str], list[str]]: + html = path.read_text(encoding="utf-8") + errors: list[str] = [] + warnings: list[str] = [] + + if "" in html: + errors.append("Template marker was not replaced.") + if "Replace with deck title" in html: + errors.append("Title placeholder was not replaced.") + if IDEOGRAPH_RE.search(html): + errors.append("Deck contains ideograph characters; this skill is English-only unless explicitly overridden.") + if has_external_fetch_reference(html): + errors.append("Deck contains external http(s) references; strict decks must work offline.") + if re.search(r"letter-spacing\s*:\s*-[^;]+", html, re.IGNORECASE): + errors.append("Negative letter-spacing is not allowed in strict Product Grid decks.") + if DECORATIVE_CSS_RE.search(css_markup(html)): + errors.append("Decorative gradients, shadows, and filters are not allowed in Product Grid decks.") + + defined_classes = css_classes(html) + missing_template_classes = sorted(REQUIRED_TEMPLATE_CLASSES - defined_classes) + if missing_template_classes: + errors.append(f"Deck is missing copied Product Grid template CSS class(es): {', '.join(missing_template_classes)}.") + registered_classes, registered_error = registered_css_classes() + if registered_error: + errors.append(registered_error) + else: + deck_local_classes = sorted(defined_classes - registered_classes) + if deck_local_classes: + errors.append(f"Deck defines unregistered CSS class(es): {', '.join(deck_local_classes)}.") + defined_classes = registered_classes + slides = [ + match + for match in SECTION_RE.finditer(html) + if re.search(r"\bclass\s*=\s*(['\"])[^'\"]*\bslide\b[^'\"]*\1", match.group("tag"), re.IGNORECASE) + ] + if not slides: + errors.append('No
elements found.') + + layout_sequence: list[str] = [] + used_layouts: set[str] = set() + + for idx, match in enumerate(slides, start=1): + attrs = match.group("attrs") + tag_html = match.group("tag") + slide_html = match.group(0) + section_html = match.group("html") + slide_markup = f"{tag_html}{section_html}" + layout = attr_value(attrs, "data-layout") + system = attr_value(attrs, "data-system") + layout_sequence.append(layout or "") + + if system != "product-grid": + errors.append(f"Slide {idx}: missing data-system=\"product-grid\".") + if not layout: + errors.append(f"Slide {idx}: missing data-layout.") + elif layout not in ALLOWED_LAYOUTS: + errors.append(f"Slide {idx}: data-layout=\"{layout}\" is not registered.") + else: + used_layouts.add(layout) + + parser = ImgParser() + parser.feed(slide_html) + + if re.search(r" elements are not allowed.") + + undefined = sorted( + { + cls + for cls in parser.classes + if cls not in defined_classes and cls not in ALLOWED_GLOBAL_CLASSES + } + ) + if undefined: + errors.append(f"Slide {idx}: undefined CSS class(es): {', '.join(undefined)}.") + + if parser.svg_text_count: + errors.append(f"Slide {idx}: SVG contains visible ; use HTML labels instead.") + + if layout not in TEXT_ALIGN_CENTER_OK and re.search(r"text-align\s*:\s*center", slide_html, re.IGNORECASE): + errors.append(f"Slide {idx}: text-align:center is not allowed for {layout or 'unregistered layout'}.") + if "theme-yellow" in slide_html and re.search(r"rgb\(22,\s*92,\s*255\)|#165cff", slide_markup, re.IGNORECASE): + # CSS variables on the template are allowed; this catches slide-local blue accents. + if re.search(r"style\s*=\s*(['\"])[^'\"]*(?:#165cff|rgb\(22,\s*92,\s*255\))", slide_markup, re.IGNORECASE): + errors.append(f"Slide {idx}: yellow theme contains slide-local blue accent styling.") + + if re.search(r"style\s*=\s*(['\"])[^'\"]*font-size\s*:", slide_markup, re.IGNORECASE): + errors.append(f"Slide {idx}: inline font-size override found; edit copy or use a registered component.") + + if re.search(r"style\s*=\s*(['\"])[^'\"]*height\s*:\s*\d+(?:\.\d+)?vh", slide_markup, re.IGNORECASE): + errors.append(f"Slide {idx}: fixed vh height found; use registered ratio classes.") + + seen_slots: set[str] = set() + for image_num, image in enumerate(parser.images, start=1): + src = image.get("src", "") + alt = image.get("alt", "") + slot = image.get("data-image-slot", "") + local_src = src[2:] if src.startswith("./images/") else src + if not alt.strip(): + errors.append(f"Slide {idx}: image {image_num} is missing alt text.") + if not src.strip(): + errors.append(f"Slide {idx}: image {image_num} has blank src.") + elif local_src.startswith("images/"): + if not (path.parent / local_src).exists(): + errors.append(f"Slide {idx}: missing local image {local_src}.") + if not slot: + errors.append(f"Slide {idx}: local image {image_num} missing data-image-slot.") + else: + seen_slots.add(slot) + allowed_slots = IMAGE_SLOT_BY_LAYOUT.get(layout or "", set()) + if slot not in allowed_slots: + errors.append(f"Slide {idx}: image slot {slot} is not allowed for {layout or 'unregistered layout'}.") + elif src and not is_external_url(src): + errors.append(f"Slide {idx}: image {image_num} must use a local images/ path.") + + for required in REQUIRED_IMAGE_BY_LAYOUT.get(layout or "", set()): + if required not in seen_slots: + errors.append(f"Slide {idx}: required image slot {required} is missing.") + + for i in range(2, len(layout_sequence)): + if layout_sequence[i] and layout_sequence[i] == layout_sequence[i - 1] == layout_sequence[i - 2]: + errors.append(f"Slides {i - 1}-{i + 1}: same layout repeated three times.") + + if 5 <= len(slides) < 8 and len(used_layouts) < 5: + warnings.append("Decks under 8 slides should use at least 5 distinct layouts.") + if len(slides) >= 8 and len(used_layouts) < 7: + warnings.append("Decks with 8 or more slides should use at least 7 distinct layouts.") + + return errors, warnings + + +def main() -> int: + if len(sys.argv) != 2: + print("Usage: python scripts/validate_deck_quality.py ", file=sys.stderr) + return 2 + + path = Path(sys.argv[1]) + if not path.is_file() or path.suffix.lower() != ".html": + print(f"Invalid HTML deck path: {path}", file=sys.stderr) + return 2 + + errors, warnings = check_file(path) + + if warnings: + print("Warnings:") + for warning in warnings: + print(f"- {warning}") + + if errors: + print("HTML deck quality validation failed:", file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + + print("HTML deck quality validation passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/adclaw/agents/skills/html-presentation-deck/scripts/validate_html_deck.py b/src/adclaw/agents/skills/html-presentation-deck/scripts/validate_html_deck.py new file mode 100644 index 0000000..bb00b76 --- /dev/null +++ b/src/adclaw/agents/skills/html-presentation-deck/scripts/validate_html_deck.py @@ -0,0 +1,805 @@ +#!/usr/bin/env python3 +"""Validate an HTML presentation deck. + +Contrast parsing supports 6-digit hex (#RRGGBB) and rgb/rgba() theme tokens. +:root and .slide.theme-* rule blocks use brace counting so nested rules do not truncate. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path +from typing import NamedTuple + + +IDEOGRAPH_RE = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]") +CSS_VAR_HEX_RE = re.compile(r"(--[A-Za-z0-9-]+)\s*:\s*(#[0-9A-Fa-f]{6})") +CSS_VAR_RGB_RE = re.compile( + r"^(--[A-Za-z0-9-]+)\s*:\s*rgba?\(\s*(\d{1,3})\s*,\s*(\d{1,3})\s*,\s*(\d{1,3})(?:\s*,\s*([\d.]+%?))?\s*\)\s*$", + re.IGNORECASE, +) +CSS_VAR_REF_RE = re.compile( + r"^(--[A-Za-z0-9-]+)\s*:\s*var\(\s*(--[A-Za-z0-9-]+)\s*\)\s*$", + re.IGNORECASE, +) +SLIDE_THEME_BACKGROUNDS = { + "theme-dark": "--ink", + "theme-accent": "--accent", + "theme-yellow": "--yellow", +} +SLIDE_THEME_FOREGROUNDS = { + "theme-dark": "--paper", + "theme-accent": "--accent-on", + "theme-yellow": "--ink", +} +STYLE_BLOCK_RE = re.compile(r"]*>(?P.*?)", re.IGNORECASE | re.DOTALL) +HTML_COMMENT_RE = re.compile(r"", re.DOTALL) +STYLE_ATTR_RE = re.compile( + r'\bstyle\s*=\s*"(?P[^"]*--[A-Za-z0-9-]+[^"]*)"|\bstyle\s*=\s*\'(?P[^\']*--[A-Za-z0-9-]+[^\']*)\'', + re.DOTALL, +) +MIN_TEXT_CONTRAST = 4.5 + + +class CssColor(NamedTuple): + rgb: str + alpha: float = 1.0 + ref: str | None = None + + +def _term(*codes: int) -> str: + return "".join(chr(code) for code in codes) + + +BANNED_TERMS = [ + _term(103, 117, 105, 122, 97, 110, 103), + _term(111, 112, 55, 52, 49, 56), + _term(112, 112, 116, 45, 115, 107, 105, 108, 108), + _term(122, 104, 45, 67, 78), + _term(78, 111, 116, 111, 32, 83, 97, 110, 115, 32, 83, 67), + _term(78, 111, 116, 111, 32, 83, 101, 114, 105, 102, 32, 83, 67), +] +BANNED_RE = re.compile( + "|".join( + rf"(? tuple[float, float, float]: + value = color.lstrip("#") + return tuple(int(value[i : i + 2], 16) / 255 for i in (0, 2, 4)) + + +def _linear_channel(channel: float) -> float: + # WCAG 2.x relative luminance piecewise transfer (threshold 0.03928). + if channel <= 0.03928: + return channel / 12.92 + return ((channel + 0.055) / 1.055) ** 2.4 + + +def _luminance(color: str) -> float: + red, green, blue = (_linear_channel(channel) for channel in _hex_to_rgb(color)) + return (0.2126 * red) + (0.7152 * green) + (0.0722 * blue) + + +def _contrast(foreground: str, background: str) -> float: + high, low = sorted((_luminance(foreground), _luminance(background)), reverse=True) + return (high + 0.05) / (low + 0.05) + + +def _extract_braced_block_body(html: str, opener_end: int) -> str | None: + depth = 1 + cursor = opener_end + quote: str | None = None + in_comment = False + while cursor < len(html) and depth > 0: + char = html[cursor] + next_char = html[cursor + 1] if cursor + 1 < len(html) else "" + if in_comment: + if char == "*" and next_char == "/": + in_comment = False + cursor += 2 + continue + elif quote: + if char == "\\": + cursor += 2 + continue + if char == quote: + quote = None + elif char == "/" and next_char == "*": + in_comment = True + cursor += 2 + continue + elif char in {'"', "'"}: + quote = char + elif char == "{": + depth += 1 + elif char == "}": + depth -= 1 + cursor += 1 + if depth != 0: + return None + return html[opener_end : cursor - 1] + + +def _enclosing_block_header(css: str, position: int) -> str | None: + stack: list[int] = [] + cursor = 0 + quote: str | None = None + in_comment = False + while cursor < position: + char = css[cursor] + next_char = css[cursor + 1] if cursor + 1 < position else "" + if in_comment: + if char == "*" and next_char == "/": + in_comment = False + cursor += 2 + continue + elif quote: + if char == "\\": + cursor += 2 + continue + if char == quote: + quote = None + elif char == "/" and next_char == "*": + in_comment = True + cursor += 2 + continue + elif char in {'"', "'"}: + quote = char + elif char == "{": + stack.append(cursor) + elif char == "}" and stack: + stack.pop() + cursor += 1 + + if not stack: + return None + opener = stack[-1] + previous_boundary = max(css.rfind("{", 0, opener), css.rfind("}", 0, opener)) + return css[previous_boundary + 1 : opener].strip() + + +def _block_header_before(css: str, opener_start: int) -> str: + previous_boundary = max(css.rfind("{", 0, opener_start), css.rfind("}", 0, opener_start)) + return css[previous_boundary + 1 : opener_start].strip() + + +def _extract_css_blocks( + css: str, + *, + nested: bool = False, +) -> list[tuple[str, str, int, str | None]]: + blocks: list[tuple[str, str, int, str | None]] = [] + depth = 0 + cursor = 0 + quote: str | None = None + in_comment = False + while cursor < len(css): + char = css[cursor] + next_char = css[cursor + 1] if cursor + 1 < len(css) else "" + if in_comment: + if char == "*" and next_char == "/": + in_comment = False + cursor += 2 + continue + elif quote: + if char == "\\": + cursor += 2 + continue + if char == quote: + quote = None + elif char == "/" and next_char == "*": + in_comment = True + cursor += 2 + continue + elif char in {'"', "'"}: + quote = char + elif char == "{": + is_nested = depth != 0 + if is_nested == nested: + body = _extract_braced_block_body(css, cursor + 1) + if body is not None: + parent = _enclosing_block_header(css, cursor) if nested else None + blocks.append((_block_header_before(css, cursor), body, cursor, parent)) + depth += 1 + elif char == "}" and depth > 0: + depth -= 1 + cursor += 1 + return blocks + + +def _split_selector_list(selector_text: str) -> list[str]: + selectors: list[str] = [] + current: list[str] = [] + depth = 0 + quote: str | None = None + cursor = 0 + while cursor < len(selector_text): + char = selector_text[cursor] + if quote: + current.append(char) + if char == "\\" and cursor + 1 < len(selector_text): + cursor += 1 + current.append(selector_text[cursor]) + elif char == quote: + quote = None + elif char in {'"', "'"}: + quote = char + current.append(char) + elif char in "([{": + depth += 1 + current.append(char) + elif char in ")]}" and depth > 0: + depth -= 1 + current.append(char) + elif char == "," and depth == 0: + selector = "".join(current).strip() + if selector: + selectors.append(selector) + current = [] + else: + current.append(char) + cursor += 1 + + selector = "".join(current).strip() + if selector: + selectors.append(selector) + return selectors + + +def _selector_subject(selector: str) -> str: + current: list[str] = [] + subject = "" + depth = 0 + quote: str | None = None + cursor = 0 + while cursor < len(selector): + char = selector[cursor] + if quote: + current.append(char) + if char == "\\" and cursor + 1 < len(selector): + cursor += 1 + current.append(selector[cursor]) + elif char == quote: + quote = None + elif char in {'"', "'"}: + quote = char + current.append(char) + elif char in "([{": + depth += 1 + current.append(char) + elif char in ")]}" and depth > 0: + depth -= 1 + current.append(char) + elif depth == 0 and (char.isspace() or char in ">+~"): + part = "".join(current).strip() + if part: + subject = part + current = [] + else: + current.append(char) + cursor += 1 + + part = "".join(current).strip() + return part or subject + + +def _selector_matches_root(selector_text: str) -> bool: + return any( + re.search( + r"(? list[tuple[str, str]]: + themes: list[tuple[str, str]] = [] + for selector in _split_selector_list(selector_text): + subject = _selector_subject(selector) + classes = [match.group(1).lower() for match in re.finditer(r"\.([A-Za-z0-9_-]+)", subject)] + if "slide" not in classes: + continue + for class_name in classes: + if re.fullmatch(r"theme-[a-z0-9-]+", class_name, re.IGNORECASE): + themes.append((selector, class_name.lower())) + return themes + + +def _rgb_to_hex(red: str, green: str, blue: str) -> str: + return "#{:02x}{:02x}{:02x}".format( + min(255, int(red)), + min(255, int(green)), + min(255, int(blue)), + ) + + +def _parse_rgba_alpha(raw: str) -> float | None: + try: + if raw.endswith("%"): + return max(0.0, min(1.0, float(raw[:-1]) / 100.0)) + value = float(raw) + except ValueError: + return None + if value > 1.0: + return 1.0 + return max(0.0, value) + + +def _top_level_declarations(block: str) -> list[str]: + declarations: list[str] = [] + current: list[str] = [] + depth = 0 + cursor = 0 + quote: str | None = None + in_comment = False + while cursor < len(block): + char = block[cursor] + next_char = block[cursor + 1] if cursor + 1 < len(block) else "" + + if in_comment: + if char == "*" and next_char == "/": + in_comment = False + cursor += 2 + continue + elif quote: + if depth == 0: + current.append(char) + if char == "\\": + if next_char and depth == 0: + current.append(next_char) + cursor += 2 + continue + if char == quote: + quote = None + elif char == "/" and next_char == "*": + in_comment = True + cursor += 2 + continue + elif char in {'"', "'"}: + quote = char + if depth == 0: + current.append(char) + elif char == "{": + if depth == 0: + current = [] + depth += 1 + elif char == "}": + if depth > 0: + depth -= 1 + elif char == ";" and depth == 0: + declaration = "".join(current).strip() + if declaration: + declarations.append(declaration) + current = [] + elif depth == 0: + current.append(char) + + cursor += 1 + + declaration = "".join(current).strip() + if declaration: + declarations.append(declaration) + return declarations + + +def _parse_css_variables(block: str) -> dict[str, CssColor]: + variables: dict[str, CssColor] = {} + for declaration in _top_level_declarations(block): + if match := CSS_VAR_HEX_RE.fullmatch(declaration): + variables[match.group(1)] = CssColor(match.group(2), 1.0) + continue + if match := CSS_VAR_RGB_RE.fullmatch(declaration): + alpha = _parse_rgba_alpha(match.group(5) or "1") + if alpha is None: + continue + variables[match.group(1)] = CssColor( + _rgb_to_hex(match.group(2), match.group(3), match.group(4)), + alpha, + ) + continue + if match := CSS_VAR_REF_RE.fullmatch(declaration): + variables[match.group(1)] = CssColor("", 1.0, match.group(2)) + return variables + + +def _composite_over(foreground: CssColor, background: CssColor) -> str: + if foreground.alpha >= 1.0: + return foreground.rgb + red_f, green_f, blue_f = ( + int(foreground.rgb[i : i + 2], 16) for i in (1, 3, 5) + ) + red_b, green_b, blue_b = (int(background.rgb[i : i + 2], 16) for i in (1, 3, 5)) + alpha = foreground.alpha + return _rgb_to_hex( + str(round(alpha * red_f + (1 - alpha) * red_b)), + str(round(alpha * green_f + (1 - alpha) * green_b)), + str(round(alpha * blue_f + (1 - alpha) * blue_b)), + ) + + +def _resolve_token( + variables: dict[str, CssColor], + key: str, + backdrop: CssColor | None, + seen: set[str] | None = None, +) -> CssColor | None: + token = variables.get(key) + if token is None: + return None + if token.ref is not None: + seen = set() if seen is None else seen + if key in seen: + return None + seen.add(key) + return _resolve_token(variables, token.ref, backdrop, seen) + if not token.rgb: + return None + if backdrop is None or token.alpha >= 1.0: + return token + return CssColor(_composite_over(token, backdrop), 1.0) + + +def _resolved_hex(variables: dict[str, CssColor], key: str, backdrop: CssColor | None) -> str | None: + token = _resolve_token(variables, key, backdrop) + return token.rgb if token else None + + +def _theme_class_from_context(context_name: str) -> str | None: + match = re.search(r"theme-[a-z0-9-]+", context_name, re.IGNORECASE) + return match.group(0).lower() if match else None + + +def _theme_class_for_style_attr(html: str, style_attr_start: int) -> str | None: + tag_start = html.rfind("<", 0, style_attr_start) + tag_end = html.find(">", style_attr_start) + if tag_start != -1 and tag_end != -1: + tag_theme = _theme_class_from_context(html[tag_start : tag_end + 1]) + if tag_theme: + return tag_theme + + before_style = html[:style_attr_start] + last_section_close = before_style.rfind("
") + slide_open_re = re.compile( + r"]*\bclass\s*=\s*(['\"])(?P[^'\"]*)\1[^>]*>", + re.IGNORECASE | re.DOTALL, + ) + for match in reversed(list(slide_open_re.finditer(before_style))): + class_attr = match.group("class") + if ( + match.start() > last_section_close + and re.search(r"\bslide\b", class_attr, re.IGNORECASE) + ): + return _theme_class_from_context(class_attr) + return None + + +def _css_variable_contexts(html: str) -> list[tuple[str, dict[str, CssColor]]]: + contexts: list[tuple[str, dict[str, CssColor]]] = [] + root_aggregate: dict[str, CssColor] = {} + theme_aggregates: dict[str, dict[str, CssColor]] = {} + css_html = HTML_COMMENT_RE.sub("", html) + style_blocks = [match.group("body") for match in STYLE_BLOCK_RE.finditer(css_html)] + + root_rules: list[tuple[tuple[int, int], dict[str, CssColor]]] = [] + conditional_root_rules: list[tuple[tuple[int, int], dict[str, CssColor], str | None]] = [] + for style_index, css in enumerate(style_blocks): + for selector, body, start, parent in _extract_css_blocks(css): + if not _selector_matches_root(selector): + continue + variables = _parse_css_variables(body) + if variables: + root_rules.append(((style_index, start), variables)) + for selector, body, start, parent in _extract_css_blocks(css, nested=True): + if not _selector_matches_root(selector): + continue + variables = _parse_css_variables(body) + if variables: + conditional_root_rules.append(((style_index, start), variables, parent)) + + for _, variables in sorted(root_rules): + root_aggregate.update(variables) + if root_aggregate: + contexts.append((f":root block {len(root_rules)}", dict(root_aggregate))) + + conditional_root_contexts: list[tuple[int, tuple[int, int], str | None, dict[str, CssColor]]] = [] + for conditional_root_index, (conditional_order, variables, parent) in enumerate( + sorted(conditional_root_rules), + start=1, + ): + merged: dict[str, CssColor] = {} + cascade_events = [ + (order, root_variables) + for order, root_variables in root_rules + ] + cascade_events.extend( + (order, root_variables) + for order, root_variables, rule_parent in conditional_root_rules + if rule_parent == parent + ) + for _, event_variables in sorted(cascade_events): + merged.update(event_variables) + contexts.append((f"conditional :root block {conditional_root_index}", merged)) + conditional_root_contexts.append((conditional_root_index, conditional_order, parent, merged)) + + theme_rules: list[tuple[tuple[int, int], str, str, dict[str, CssColor]]] = [] + conditional_theme_rules: list[tuple[tuple[int, int], str, str, dict[str, CssColor], str | None]] = [] + for style_index, css in enumerate(style_blocks): + for selector, body, start, parent in _extract_css_blocks(css): + variables = _parse_css_variables(body) + if not variables: + continue + for label, theme_class in _theme_selectors(selector): + theme_rules.append(((style_index, start), label, theme_class, variables)) + for selector, body, start, parent in _extract_css_blocks(css, nested=True): + variables = _parse_css_variables(body) + if not variables: + continue + for label, theme_class in _theme_selectors(selector): + conditional_theme_rules.append(((style_index, start), label, theme_class, variables, parent)) + + for theme_index, (_, label, theme_class, variables) in enumerate( + sorted(theme_rules), + start=1, + ): + theme_aggregate = theme_aggregates.setdefault(theme_class, dict(root_aggregate)) + theme_aggregate.update(variables) + merged = dict(theme_aggregate) + contexts.append((f"slide theme rule {theme_index} ({label})", merged)) + + for conditional_root_index, _, parent, conditional_root in conditional_root_contexts: + for theme_class in sorted(theme_aggregates): + merged = dict(conditional_root) + for _, _, rule_theme_class, theme_variables in sorted(theme_rules): + if rule_theme_class == theme_class: + merged.update(theme_variables) + contexts.append(( + f"conditional :root theme block {conditional_root_index} (.{theme_class})", + merged, + )) + + for conditional_theme_index, (conditional_order, label, theme_class, variables, parent) in enumerate( + sorted(conditional_theme_rules), + start=1, + ): + conditional_root = next( + ( + context + for _, _, root_parent, context in reversed(conditional_root_contexts) + if root_parent == parent + ), + root_aggregate, + ) + merged = dict(conditional_root) + cascade_events = [ + (order, theme_variables) + for order, _, rule_theme_class, theme_variables in theme_rules + if rule_theme_class == theme_class + ] + cascade_events.extend( + (order, theme_variables) + for order, _, rule_theme_class, theme_variables, rule_parent in conditional_theme_rules + if rule_theme_class == theme_class and rule_parent == parent + ) + for _, event_variables in sorted(cascade_events): + merged.update(event_variables) + contexts.append(( + f"conditional slide theme rule {conditional_theme_index} ({label})", + merged, + )) + + for index, match in enumerate(STYLE_ATTR_RE.finditer(css_html), start=1): + variables = _parse_css_variables(match.group("double") or match.group("single") or "") + if variables: + theme_class = _theme_class_for_style_attr(css_html, match.start()) + base = theme_aggregates.get(theme_class or "", root_aggregate) + merged = dict(base) + merged.update(variables) + suffix = f" (.{theme_class})" if theme_class else "" + contexts.append((f"inline style {index}{suffix}", merged)) + + return contexts + + +def _validate_slide_theme_contrast( + context_name: str, + variables: dict[str, CssColor], +) -> list[str]: + theme_class = _theme_class_from_context(context_name) + slide_bg_key = SLIDE_THEME_BACKGROUNDS.get(theme_class or "") + slide_bg = _resolve_token(variables, slide_bg_key, None) if slide_bg_key else None + if slide_bg is None: + return [] + if slide_bg.alpha < 1.0: + return [f"{context_name}: {slide_bg_key} background must be opaque for contrast validation."] + + errors: list[str] = [] + foreground_key = SLIDE_THEME_FOREGROUNDS.get(theme_class or "") + foreground = _resolved_hex(variables, foreground_key, slide_bg) if foreground_key else None + if foreground is not None: + ratio = _contrast(foreground, slide_bg.rgb) + if ratio < MIN_TEXT_CONTRAST: + errors.append( + f"{context_name}: primary text on theme background contrast is {ratio:.2f}:1 " + f"({foreground_key} {foreground} on {slide_bg_key} {slide_bg.rgb}); " + f"minimum is {MIN_TEXT_CONTRAST:.1f}:1." + ) + + if not (slide_bg_key == "--accent" and foreground_key == "--accent-on"): + accent = _resolved_hex(variables, "--accent", slide_bg) + if accent is not None: + accent_surface = CssColor(accent, 1.0) + accent_on = _resolved_hex(variables, "--accent-on", accent_surface) + if accent_on is not None: + ratio = _contrast(accent_on, accent) + if ratio < MIN_TEXT_CONTRAST: + errors.append( + f"{context_name}: text on accent background contrast is {ratio:.2f}:1 " + f"(--accent-on {accent_on} on --accent {accent}); " + f"minimum is {MIN_TEXT_CONTRAST:.1f}:1." + ) + + muted = variables.get("--muted") + panel_keys = [key for key in ("--panel", "--panel-2") if key in variables] + if muted is not None: + muted_on_slide = _resolved_hex(variables, "--muted", slide_bg) + if muted_on_slide is not None: + ratio = _contrast(muted_on_slide, slide_bg.rgb) + if ratio < MIN_TEXT_CONTRAST: + errors.append( + f"{context_name}: muted text on slide background contrast is {ratio:.2f}:1 " + f"(--muted {muted_on_slide} on {slide_bg_key} {slide_bg.rgb}); " + f"minimum is {MIN_TEXT_CONTRAST:.1f}:1." + ) + + for panel_key in panel_keys: + panel_hex = _resolved_hex(variables, panel_key, slide_bg) + if panel_hex is not None: + panel_surface = CssColor(panel_hex, 1.0) + muted_on_panel = _resolved_hex(variables, "--muted", panel_surface) + if muted_on_panel is not None: + ratio = _contrast(muted_on_panel, panel_hex) + if ratio < MIN_TEXT_CONTRAST: + errors.append( + f"{context_name}: muted text on panel contrast is {ratio:.2f}:1 " + f"(--muted on {panel_key} over {slide_bg_key}); minimum is {MIN_TEXT_CONTRAST:.1f}:1." + ) + + return errors + + +def _validate_contrast(context_name: str, variables: dict[str, CssColor]) -> list[str]: + if _theme_class_from_context(context_name) in SLIDE_THEME_BACKGROUNDS: + return _validate_slide_theme_contrast(context_name, variables) + + checks = [ + ("--ink", "--paper", "primary text on paper"), + ("--muted", "--paper", "muted text on paper"), + ("--muted", "--panel", "muted text on panel"), + ("--muted", "--panel-2", "muted text on panel-2"), + ("--accent-text", "--paper", "accent text on paper"), + ("--accent-text", "--panel", "accent text on panel"), + ("--accent-text", "--panel-2", "accent text on panel-2"), + ("--accent-on", "--accent", "text on accent background"), + ] + errors: list[str] = [] + + for foreground_key, background_key, label in checks: + background_backdrop = _resolve_token(variables, "--paper", None) if background_key in { + "--panel", + "--panel-2", + } else None + background_token = _resolve_token(variables, background_key, background_backdrop) + if background_token is None: + continue + if background_token.alpha < 1.0: + errors.append( + f"{context_name}: {background_key} background must be opaque for contrast validation." + ) + continue + background = background_token.rgb + foreground = _resolved_hex(variables, foreground_key, background_token) + if not foreground: + continue + ratio = _contrast(foreground, background) + if ratio < MIN_TEXT_CONTRAST: + errors.append( + f"{context_name}: {label} contrast is {ratio:.2f}:1 " + f"({foreground_key} {foreground} on {background_key} {background}); " + f"minimum is {MIN_TEXT_CONTRAST:.1f}:1." + ) + + if ( + "--accent" in variables + and "--panel" in variables + and "--accent-text" not in variables + ): + panel = _resolve_token(variables, "--panel", None) + accent = _resolved_hex(variables, "--accent", panel) + if accent is not None and panel is not None and panel.alpha >= 1.0: + ratio = _contrast(accent, panel.rgb) + if ratio < MIN_TEXT_CONTRAST: + errors.append( + f"{context_name}: --accent on --panel contrast is {ratio:.2f}:1. " + "Add a contrast-safe --accent-text token for labels, metrics, and annotations." + ) + + if "--accent" in variables and "--paper" in variables and "--accent-text" not in variables: + paper = _resolve_token(variables, "--paper", None) + accent = _resolved_hex(variables, "--accent", paper) + if accent is not None and paper is not None and paper.alpha >= 1.0: + ratio = _contrast(accent, paper.rgb) + if ratio < MIN_TEXT_CONTRAST: + errors.append( + f"{context_name}: --accent on --paper contrast is {ratio:.2f}:1. " + "Add a contrast-safe --accent-text token for labels, metrics, and annotations." + ) + + return errors + + +def main() -> int: + if len(sys.argv) != 2: + print("Usage: python3 scripts/validate_html_deck.py ", file=sys.stderr) + return 2 + + path = Path(sys.argv[1]) + if not path.is_file() or path.suffix.lower() != ".html": + print(f"Invalid HTML deck path: {path}", file=sys.stderr) + return 2 + + try: + html = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as exc: + print(f"Could not read HTML deck: {path} ({exc})", file=sys.stderr) + return 2 + + errors: list[str] = [] + + if IDEOGRAPH_RE.search(html): + errors.append("Deck contains non-English ideograph characters.") + if match := BANNED_RE.search(html): + errors.append(f"Deck contains banned term: {match.group(0)}") + if "" in html: + errors.append("Template marker was not replaced.") + if "Replace with deck title" in html: + errors.append("Title placeholder was not replaced.") + + slides = re.findall( + r"]*\bclass\s*=\s*(['\"])[^'\"]*\bslide\b[^'\"]*\1", + html, + ) + if not slides: + errors.append('No
elements found.') + + local_images = re.findall( + r"]*\bsrc\s*=\s*(['\"])(images/[^'\"]+)\1[^>]*>", + html, + ) + for _, src in local_images: + image_path = path.parent / src + if not image_path.exists(): + errors.append(f"Missing local image: {src}") + + for context_name, variables in _css_variable_contexts(html): + errors.extend(_validate_contrast(context_name, variables)) + + if errors: + print("HTML deck validation failed:", file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + + print(f"HTML deck validation passed: {len(slides)} slide(s).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/adclaw/agents/skills/instagram-curator/SKILL.md b/src/adclaw/agents/skills/instagram-curator/SKILL.md new file mode 100644 index 0000000..64ed2f9 --- /dev/null +++ b/src/adclaw/agents/skills/instagram-curator/SKILL.md @@ -0,0 +1,117 @@ +--- +name: Instagram Curator +description: Instagram marketing specialist for visual branding, multi-format content strategy, community building, and social commerce optimization. +read_when: + - instagram strategy + - instagram content plan + - instagram aesthetic + - instagram reels + - instagram stories + - instagram shopping + - instagram engagement + - instagram hashtag strategy + - instagram growth + - instagram grid planning + - instagram influencer + - instagram UGC + - social commerce instagram + - instagram brand identity +metadata: {"clawdbot":{"emoji":"📸"}} +--- +# Instagram Curator + +You are an Instagram marketing specialist focused on visual storytelling, community building, multi-format content optimization, and social commerce. You help brands build cohesive Instagram presences that convert followers into customers. + +## Core Capabilities + +- **Visual Brand Development**: Create cohesive, scroll-stopping aesthetics that build instant recognition. +- **Multi-Format Mastery**: Optimize content across Posts, Stories, Reels, IGTV, and Shopping features. +- **Community Cultivation**: Build engaged, loyal follower bases through authentic connection and user-generated content (UGC). +- **Social Commerce**: Convert Instagram engagement into measurable business results via Instagram Shopping. + +## Content Standards + +- Maintain consistent visual brand identity across all formats. +- Follow the **1/3 rule**: one-third brand content, one-third educational content, one-third community content. +- Include a strong call-to-action in every piece of content that drives engagement or conversion. +- Ensure all Shopping tags and commerce features are properly implemented when applicable. + +## Workflow + +### Phase 1: Brand Aesthetic Development + +1. **Visual Identity Analysis**: Assess the current brand presence and competitive landscape on Instagram. +2. **Aesthetic Framework**: Define color palette, typography, photography style, and graphic elements. +3. **Grid Planning**: Optimize the 9-post preview for a cohesive feed appearance. +4. **Template Creation**: Design story highlight covers, post layouts, and reusable graphic elements. + +**Deliverable**: Brand Aesthetic Guide covering colors, typography, photography direction, and visual elements. + +### Phase 2: Multi-Format Content Strategy + +1. **Feed Posts**: Plan single images, carousels (up to 10 slides), and video content. Carousels typically get 1.4x more reach than single images. +2. **Stories**: Use behind-the-scenes content, interactive stickers (polls, quizzes, questions, sliders), countdowns, and shopping integration. +3. **Reels**: Balance trending audio, educational content, and entertainment. Reels currently receive the highest organic reach. +4. **Long-Form Video**: Plan longer content for deeper engagement and cross-promote via Stories and feed teasers. + +**Deliverable**: 30-day content calendar with format distribution across all content types. + +### Phase 3: Hashtag Strategy + +1. **Research**: Identify a mix of hashtags across three tiers: + - **High volume** (500K-5M posts): 3-5 per post for broad reach + - **Medium volume** (50K-500K posts): 5-10 per post for targeted reach + - **Niche/branded** (under 50K posts): 3-5 per post for community building +2. **Total**: Use 20-30 hashtags per post, tested and rotated regularly. +3. **Branded Hashtag**: Create and promote a unique branded hashtag; aim for top-9 placement. +4. **Location Tags**: Always add location tags for local discoverability. + +**Deliverable**: Hashtag bank organized by category and volume tier, with rotation schedule. + +### Phase 4: Community Building & Commerce + +1. **Engagement Tactics**: + - Respond to comments and DMs within 2 hours. + - Use the "Golden Hour" strategy: maximize engagement in the first 60 minutes after posting. + - Run live sessions for Q&A, product launches, and behind-the-scenes content. +2. **UGC Campaigns**: + - Launch branded hashtag challenges. + - Create customer spotlight programs. + - Integrate real user reviews and testimonials into content. +3. **Shopping Integration**: + - Optimize product catalog with multiple angles, lifestyle shots, and detail views. + - Place shopping tags strategically in posts and Stories. + - Implement cross-selling via related product recommendations. + - Add social proof (customer reviews, UGC) alongside shoppable content. +4. **Influencer Partnerships**: + - Focus on micro-influencers (1K-100K followers) for higher engagement rates. + - Build brand ambassador programs for sustained promotion. + +### Phase 5: Performance Optimization + +1. **Algorithm Factors**: Optimize for relationship signals, user interest, post timeliness, and session frequency. +2. **Content Analysis**: Identify top-performing posts by format, topic, and posting time; double down on what works. +3. **Shopping Analytics**: Track product views, add-to-cart rates, and checkout conversions. +4. **Growth Assessment**: Evaluate follower quality (real vs. bot), demographic alignment, and reach expansion. +5. **Cross-Promotion**: Promote feed posts in Stories, create IGTV trailers, and leverage Reels for discovery. + +## Target Metrics + +| Metric | Target | +|---|---| +| Engagement Rate | 3.5%+ (adjust by follower count) | +| Reach Growth | 25% month-over-month organic | +| Story Completion Rate | 80%+ | +| Shopping Conversion Rate | 2.5%+ | +| Hashtag Performance | Top 9 for branded hashtags | +| UGC Volume | 200+ branded posts/month | +| Follower Quality | 90%+ real, matching target demographics | +| Website Traffic from Instagram | 20% of total social traffic | + +## Communication Style + +- Describe content concepts with rich visual detail so they can be clearly understood and executed. +- Use current Instagram terminology and platform-native language. +- Always connect creative ideas to measurable business outcomes. +- Prioritize authentic engagement over vanity metrics (follower count alone is not success). +- When proposing content, specify the format (Reel, carousel, Story, etc.), caption approach, hashtags, and CTA. diff --git a/src/adclaw/agents/skills/marketing-ab-test-setup/SKILL.md b/src/adclaw/agents/skills/marketing-ab-test-setup/SKILL.md new file mode 100644 index 0000000..e3f9997 --- /dev/null +++ b/src/adclaw/agents/skills/marketing-ab-test-setup/SKILL.md @@ -0,0 +1,353 @@ +--- +name: ab-testing +description: When the user wants to plan, design, or implement an A/B test or experiment, or build a growth experimentation program. Also use when the user mentions "A/B test," "split test," "experiment," "test this change," "variant copy," "multivariate test," "hypothesis," "should I test this," "which version is better," "test two versions," "statistical significance," "how long should I run this test," "growth experiments," "experiment velocity," "experiment backlog," "ICE score," "experimentation program," or "experiment playbook." Use this whenever someone is comparing two approaches and wants to measure which performs better, or when they want to build a systematic experimentation practice. For tracking implementation, see analytics. For page-level conversion optimization, see cro. +metadata: + version: 2.0.0 +--- + +# A/B Test Setup + +You are an expert in experimentation and A/B testing. Your goal is to help design tests that produce statistically valid, actionable results. + +## Initial Assessment + +**Check for product marketing context first:** +If `.agents/product-marketing.md` exists (or `.claude/product-marketing.md`, or the legacy `product-marketing-context.md` filename, in older setups), read it before asking questions. Use that context and only ask for information not already covered or specific to this task. + +Before designing a test, understand: + +1. **Test Context** - What are you trying to improve? What change are you considering? +2. **Current State** - Baseline conversion rate? Current traffic volume? +3. **Constraints** - Technical complexity? Timeline? Tools available? + +--- + +## Core Principles + +### 1. Start with a Hypothesis +- Not just "let's see what happens" +- Specific prediction of outcome +- Based on reasoning or data + +### 2. Test One Thing +- Single variable per test +- Otherwise you don't know what worked + +### 3. Statistical Rigor +- Pre-determine sample size +- Don't peek and stop early +- Commit to the methodology + +### 4. Measure What Matters +- Primary metric tied to business value +- Secondary metrics for context +- Guardrail metrics to prevent harm + +--- + +## Hypothesis Framework + +### Structure + +``` +Because [observation/data], +we believe [change] +will cause [expected outcome] +for [audience]. +We'll know this is true when [metrics]. +``` + +### Example + +**Weak**: "Changing the button color might increase clicks." + +**Strong**: "Because users report difficulty finding the CTA (per heatmaps and feedback), we believe making the button larger and using contrasting color will increase CTA clicks by 15%+ for new visitors. We'll measure click-through rate from page view to signup start." + +--- + +## Test Types + +| Type | Description | Traffic Needed | +|------|-------------|----------------| +| A/B | Two versions, single change | Moderate | +| A/B/n | Multiple variants | Higher | +| MVT | Multiple changes in combinations | Very high | +| Split URL | Different URLs for variants | Moderate | + +--- + +## Sample Size + +### Quick Reference + +| Baseline | 10% Lift | 20% Lift | 50% Lift | +|----------|----------|----------|----------| +| 1% | 150k/variant | 39k/variant | 6k/variant | +| 3% | 47k/variant | 12k/variant | 2k/variant | +| 5% | 27k/variant | 7k/variant | 1.2k/variant | +| 10% | 12k/variant | 3k/variant | 550/variant | + +**Calculators:** +- [Evan Miller's](https://www.evanmiller.org/ab-testing/sample-size.html) +- [Optimizely's](https://www.optimizely.com/sample-size-calculator/) + +**For detailed sample size tables and duration calculations**: See [references/sample-size-guide.md](references/sample-size-guide.md) + +--- + +## Metrics Selection + +### Primary Metric +- Single metric that matters most +- Directly tied to hypothesis +- What you'll use to call the test + +### Secondary Metrics +- Support primary metric interpretation +- Explain why/how the change worked + +### Guardrail Metrics +- Things that shouldn't get worse +- Stop test if significantly negative + +### Example: Pricing Page Test +- **Primary**: Plan selection rate +- **Secondary**: Time on page, plan distribution +- **Guardrail**: Support tickets, refund rate + +--- + +## Designing Variants + +### What to Vary + +| Category | Examples | +|----------|----------| +| Headlines/Copy | Message angle, value prop, specificity, tone | +| Visual Design | Layout, color, images, hierarchy | +| CTA | Button copy, size, placement, number | +| Content | Information included, order, amount, social proof | + +### Best Practices +- Single, meaningful change +- Bold enough to make a difference +- True to the hypothesis + +--- + +## Traffic Allocation + +| Approach | Split | When to Use | +|----------|-------|-------------| +| Standard | 50/50 | Default for A/B | +| Conservative | 90/10, 80/20 | Limit risk of bad variant | +| Ramping | Start small, increase | Technical risk mitigation | + +**Considerations:** +- Consistency: Users see same variant on return +- Balanced exposure across time of day/week + +--- + +## Implementation + +### Client-Side +- JavaScript modifies page after load +- Quick to implement, can cause flicker +- Tools: PostHog, Optimizely, VWO + +### Server-Side +- Variant determined before render +- No flicker, requires dev work +- Tools: PostHog, LaunchDarkly, Split + +--- + +## Running the Test + +### Pre-Launch Checklist +- [ ] Hypothesis documented +- [ ] Primary metric defined +- [ ] Sample size calculated +- [ ] Variants implemented correctly +- [ ] Tracking verified +- [ ] QA completed on all variants + +### During the Test + +**DO:** +- Monitor for technical issues +- Check segment quality +- Document external factors + +**Avoid:** +- Peek at results and stop early +- Make changes to variants +- Add traffic from new sources + +### The Peeking Problem +Looking at results before reaching sample size and stopping early leads to false positives and wrong decisions. Pre-commit to sample size and trust the process. + +--- + +## Analyzing Results + +### Statistical Significance +- 95% confidence = p-value < 0.05 +- Means <5% chance result is random +- Not a guarantee—just a threshold + +### Analysis Checklist + +1. **Reach sample size?** If not, result is preliminary +2. **Statistically significant?** Check confidence intervals +3. **Effect size meaningful?** Compare to MDE, project impact +4. **Secondary metrics consistent?** Support the primary? +5. **Guardrail concerns?** Anything get worse? +6. **Segment differences?** Mobile vs. desktop? New vs. returning? + +### Interpreting Results + +| Result | Conclusion | +|--------|------------| +| Significant winner | Implement variant | +| Significant loser | Keep control, learn why | +| No significant difference | Need more traffic or bolder test | +| Mixed signals | Dig deeper, maybe segment | + +--- + +## Documentation + +Document every test with: +- Hypothesis +- Variants (with screenshots) +- Results (sample, metrics, significance) +- Decision and learnings + +**For templates**: See [references/test-templates.md](references/test-templates.md) + +--- + +## Growth Experimentation Program + +Individual tests are valuable. A continuous experimentation program is a compounding asset. This section covers how to run experiments as an ongoing growth engine, not just one-off tests. + +### The Experiment Loop + +``` +1. Generate hypotheses (from data, research, competitors, customer feedback) +2. Prioritize with ICE scoring +3. Design and run the test +4. Analyze results with statistical rigor +5. Promote winners to a playbook +6. Generate new hypotheses from learnings +→ Repeat +``` + +### Hypothesis Generation + +Feed your experiment backlog from multiple sources: + +| Source | What to Look For | +|--------|-----------------| +| Analytics | Drop-off points, low-converting pages, underperforming segments | +| Customer research | Pain points, confusion, unmet expectations | +| Competitor analysis | Features, messaging, or UX patterns they use that you don't | +| Support tickets | Recurring questions or complaints about conversion flows | +| Heatmaps/recordings | Where users hesitate, rage-click, or abandon | +| Past experiments | "Significant loser" tests often reveal new angles to try | + +### ICE Prioritization + +Score each hypothesis 1-10 on three dimensions: + +| Dimension | Question | +|-----------|----------| +| **Impact** | If this works, how much will it move the primary metric? | +| **Confidence** | How sure are we this will work? (Based on data, not gut.) | +| **Ease** | How fast and cheap can we ship and measure this? | + +**ICE Score** = (Impact + Confidence + Ease) / 3 + +Run highest-scoring experiments first. Re-score monthly as context changes. + +### Experiment Velocity + +Track your experimentation rate as a leading indicator of growth: + +| Metric | Target | +|--------|--------| +| Experiments launched per month | 4-8 for most teams | +| Win rate | 20-30% is common for mature programs (sustained higher rates may indicate conservative hypotheses) | +| Average test duration | 2-4 weeks | +| Backlog depth | 20+ hypotheses queued | +| Cumulative lift | Compound gains from all winners | + +### The Experiment Playbook + +When a test wins, don't just implement it — document the pattern: + +``` +## [Experiment Name] +**Date**: [date] +**Hypothesis**: [the hypothesis] +**Sample size**: [n per variant] +**Result**: [winner/loser/inconclusive] — [primary metric] changed by [X%] (95% CI: [range], p=[value]) +**Guardrails**: [any guardrail metrics and their outcomes] +**Segment deltas**: [notable differences by device, segment, or cohort] +**Why it worked/failed**: [analysis] +**Pattern**: [the reusable insight — e.g., "social proof near pricing CTAs increases plan selection"] +**Apply to**: [other pages/flows where this pattern might work] +**Status**: [implemented / parked / needs follow-up test] +``` + +Over time, your playbook becomes a library of proven growth patterns specific to your product and audience. + +### Experiment Cadence + +**Weekly (30 min)**: Review running experiments for technical issues and guardrail metrics. Don't call winners early — but do stop tests where guardrails are significantly negative. + +**Bi-weekly**: Conclude completed experiments. Analyze results, update playbook, launch next experiment from backlog. + +**Monthly (1 hour)**: Review experiment velocity, win rate, cumulative lift. Replenish hypothesis backlog. Re-prioritize with ICE. + +**Quarterly**: Audit the playbook. Which patterns have been applied broadly? Which winning patterns haven't been scaled yet? What areas of the funnel are under-tested? + +--- + +## Common Mistakes + +### Test Design +- Testing too small a change (undetectable) +- Testing too many things (can't isolate) +- No clear hypothesis + +### Execution +- Stopping early +- Changing things mid-test +- Not checking implementation + +### Analysis +- Ignoring confidence intervals +- Cherry-picking segments +- Over-interpreting inconclusive results + +--- + +## Task-Specific Questions + +1. What's your current conversion rate? +2. How much traffic does this page get? +3. What change are you considering and why? +4. What's the smallest improvement worth detecting? +5. What tools do you have for testing? +6. Have you tested this area before? + +--- + +## Related Skills + +- **cro**: For generating test ideas based on CRO principles +- **analytics**: For setting up test measurement +- **copywriting**: For creating variant copy diff --git a/src/adclaw/agents/skills/marketing-ab-test-setup/evals/evals.json b/src/adclaw/agents/skills/marketing-ab-test-setup/evals/evals.json new file mode 100644 index 0000000..7ef70da --- /dev/null +++ b/src/adclaw/agents/skills/marketing-ab-test-setup/evals/evals.json @@ -0,0 +1,105 @@ +{ + "skill_name": "ab-testing", + "evals": [ + { + "id": 1, + "prompt": "I want to A/B test our homepage headline. We currently say 'The All-in-One Project Management Tool' and want to test something benefit-focused. We get about 15,000 visitors/month and our current signup rate is 3.2%.", + "expected_output": "Should check for product-marketing.md first. Should build a proper hypothesis using the framework: 'Because [observation], we believe [change] will cause [outcome], which we'll measure by [metric].' Should identify this as an A/B test (two variants). Should calculate or reference sample size needs based on 15,000 monthly visitors and 3.2% baseline. Should define primary metric (signup rate), secondary metrics, and guardrail metrics. Should warn about the peeking problem and recommend a fixed test duration. Should provide the test plan in the structured output format.", + "assertions": [ + "Checks for product-marketing.md", + "Uses the hypothesis framework with observation, belief, outcome, and metric", + "Identifies as A/B test type", + "Addresses sample size calculation based on traffic and baseline rate", + "Defines primary metric (signup rate)", + "Defines secondary and guardrail metrics", + "Warns about the peeking problem", + "Provides structured test plan output" + ], + "files": [] + }, + { + "id": 2, + "prompt": "we want to test like 4 different CTA button colors on our pricing page. is that a good idea?", + "expected_output": "Should trigger on casual phrasing. Should identify this as an A/B/n test (multiple variants). Should caution that testing 4 variants requires significantly more traffic than a simple A/B test. Should reference the sample size quick reference showing traffic multipliers for multiple variants. Should question whether button color alone is likely to produce meaningful lift vs testing CTA copy, placement, or surrounding context. Should recommend either reducing to 2 variants or ensuring sufficient traffic. Should still provide hypothesis framework and test setup if proceeding.", + "assertions": [ + "Triggers on casual phrasing", + "Identifies as A/B/n test (multiple variants)", + "Cautions about increased traffic needs for 4 variants", + "References sample size requirements", + "Questions whether button color alone is high-impact", + "Suggests alternative higher-impact elements to test", + "Provides hypothesis framework" + ], + "files": [] + }, + { + "id": 3, + "prompt": "Our test has been running for 3 days and Variant B is winning with 95% confidence. Should we call it?", + "expected_output": "Should immediately address the peeking problem. Should explain that checking results early inflates false positive rates. Should recommend running for the full pre-calculated duration regardless of early results. Should explain why early significance can be misleading (regression to the mean, day-of-week effects, audience mix shifts). Should provide guidance on when it IS appropriate to stop early (sequential testing methods). Should recommend the pre-test commitment to duration.", + "assertions": [ + "Addresses the peeking problem directly", + "Explains why early significance is misleading", + "Recommends running for full pre-calculated duration", + "Mentions day-of-week effects or audience mix shifts", + "Explains false positive rate inflation from peeking", + "Mentions sequential testing as alternative approach" + ], + "files": [] + }, + { + "id": 4, + "prompt": "Help me set up a multivariate test on our landing page. I want to test the headline, hero image, and CTA button simultaneously.", + "expected_output": "Should identify this as a Multivariate Test (MVT). Should explain that MVT tests combinations of elements and requires much more traffic than A/B tests. Should calculate or reference traffic needs (combinations multiply: e.g., 2 headlines × 2 images × 2 CTAs = 8 combinations). Should recommend MVT only if traffic supports it, otherwise suggest sequential A/B tests. Should build hypotheses for each element being tested. Should define interaction effects to watch for. Should provide structured test plan.", + "assertions": [ + "Identifies as multivariate test (MVT)", + "Explains MVT tests combinations of elements", + "Addresses dramatically higher traffic requirements", + "Calculates number of combinations", + "Suggests sequential A/B tests as alternative if traffic insufficient", + "Builds hypotheses for each element", + "Provides structured test plan" + ], + "files": [] + }, + { + "id": 5, + "prompt": "What metrics should I track for an A/B test on our trial signup page? We're testing a longer form (adds company size and role fields) against the current short form.", + "expected_output": "Should apply the metrics selection framework with three tiers: primary, secondary, and guardrail metrics. Primary: form completion rate (the direct conversion metric). Secondary: lead quality metrics (SQL conversion rate, activation rate post-signup). Guardrail: overall signup volume (ensure longer form doesn't tank total signups below acceptable threshold). Should explain the tradeoff between conversion quantity and lead quality. Should note that this test needs longer observation window to measure downstream metrics.", + "assertions": [ + "Applies three-tier metric framework (primary, secondary, guardrail)", + "Identifies form completion rate as primary metric", + "Identifies lead quality as secondary metric", + "Defines guardrail metrics to protect against negative outcomes", + "Explains quantity vs quality tradeoff", + "Notes need for longer observation window for downstream metrics" + ], + "files": [] + }, + { + "id": 6, + "prompt": "Can you help me write copy for our new landing page? We want to test it against the current version.", + "expected_output": "Should recognize this is primarily a copywriting task, not a test setup task. Should defer to or cross-reference the copywriting skill for writing the actual copy. May help frame the test hypothesis and setup, but should make clear that copywriting is the right skill for creating the page copy itself.", + "assertions": [ + "Recognizes this as primarily a copywriting task", + "References or defers to copywriting skill", + "Does not attempt to write full page copy using test setup patterns", + "May offer to help with test hypothesis and setup" + ], + "files": [] + }, + { + "id": 7, + "prompt": "We ran an A/B test on our pricing page for 4 weeks. Control: 2.1% conversion. Variant: 2.4% conversion. 12,000 visitors per variant. Is this statistically significant? Should we ship it?", + "expected_output": "Should evaluate the results against statistical significance criteria. Should calculate or estimate whether the sample size is sufficient to detect a 0.3 percentage point lift from a 2.1% baseline (this is a ~14% relative lift). Should reference the 95% confidence threshold. Should discuss practical significance vs statistical significance. Should recommend whether to ship, continue testing, or iterate. Should consider segment analysis if results are borderline.", + "assertions": [ + "Evaluates against statistical significance criteria", + "Addresses whether sample size is sufficient for this effect size", + "References 95% confidence threshold", + "Distinguishes statistical significance from practical significance", + "Provides clear recommendation on shipping", + "Suggests segment analysis or follow-up if borderline" + ], + "files": [] + } + ] +} diff --git a/src/adclaw/agents/skills/marketing-ab-test-setup/references/sample-size-guide.md b/src/adclaw/agents/skills/marketing-ab-test-setup/references/sample-size-guide.md new file mode 100644 index 0000000..3e35e6c --- /dev/null +++ b/src/adclaw/agents/skills/marketing-ab-test-setup/references/sample-size-guide.md @@ -0,0 +1,263 @@ +# Sample Size Guide + +Reference for calculating sample sizes and test duration. + +## Contents +- Sample Size Fundamentals (required inputs, what these mean) +- Sample Size Quick Reference Tables +- Duration Calculator (formula, examples, minimum duration rules, maximum duration guidelines) +- Online Calculators +- Adjusting for Multiple Variants +- Common Sample Size Mistakes +- When Sample Size Requirements Are Too High +- Sequential Testing +- Quick Decision Framework + +## Sample Size Fundamentals + +### Required Inputs + +1. **Baseline conversion rate**: Your current rate +2. **Minimum detectable effect (MDE)**: Smallest change worth detecting +3. **Statistical significance level**: Usually 95% (α = 0.05) +4. **Statistical power**: Usually 80% (β = 0.20) + +### What These Mean + +**Baseline conversion rate**: If your page converts at 5%, that's your baseline. + +**MDE (Minimum Detectable Effect)**: The smallest improvement you care about detecting. Set this based on: +- Business impact (is a 5% lift meaningful?) +- Implementation cost (worth the effort?) +- Realistic expectations (what have past tests shown?) + +**Statistical significance (95%)**: Means there's less than 5% chance the observed difference is due to random chance. + +**Statistical power (80%)**: Means if there's a real effect of size MDE, you have 80% chance of detecting it. + +--- + +## Sample Size Quick Reference Tables + +### Conversion Rate: 1% + +| Lift to Detect | Sample per Variant | Total Sample | +|----------------|-------------------|--------------| +| 5% (1% → 1.05%) | 1,500,000 | 3,000,000 | +| 10% (1% → 1.1%) | 380,000 | 760,000 | +| 20% (1% → 1.2%) | 97,000 | 194,000 | +| 50% (1% → 1.5%) | 16,000 | 32,000 | +| 100% (1% → 2%) | 4,200 | 8,400 | + +### Conversion Rate: 3% + +| Lift to Detect | Sample per Variant | Total Sample | +|----------------|-------------------|--------------| +| 5% (3% → 3.15%) | 480,000 | 960,000 | +| 10% (3% → 3.3%) | 120,000 | 240,000 | +| 20% (3% → 3.6%) | 31,000 | 62,000 | +| 50% (3% → 4.5%) | 5,200 | 10,400 | +| 100% (3% → 6%) | 1,400 | 2,800 | + +### Conversion Rate: 5% + +| Lift to Detect | Sample per Variant | Total Sample | +|----------------|-------------------|--------------| +| 5% (5% → 5.25%) | 280,000 | 560,000 | +| 10% (5% → 5.5%) | 72,000 | 144,000 | +| 20% (5% → 6%) | 18,000 | 36,000 | +| 50% (5% → 7.5%) | 3,100 | 6,200 | +| 100% (5% → 10%) | 810 | 1,620 | + +### Conversion Rate: 10% + +| Lift to Detect | Sample per Variant | Total Sample | +|----------------|-------------------|--------------| +| 5% (10% → 10.5%) | 130,000 | 260,000 | +| 10% (10% → 11%) | 34,000 | 68,000 | +| 20% (10% → 12%) | 8,700 | 17,400 | +| 50% (10% → 15%) | 1,500 | 3,000 | +| 100% (10% → 20%) | 400 | 800 | + +### Conversion Rate: 20% + +| Lift to Detect | Sample per Variant | Total Sample | +|----------------|-------------------|--------------| +| 5% (20% → 21%) | 60,000 | 120,000 | +| 10% (20% → 22%) | 16,000 | 32,000 | +| 20% (20% → 24%) | 4,000 | 8,000 | +| 50% (20% → 30%) | 700 | 1,400 | +| 100% (20% → 40%) | 200 | 400 | + +--- + +## Duration Calculator + +### Formula + +``` +Duration (days) = (Sample per variant × Number of variants) / (Daily traffic × % exposed) +``` + +### Examples + +**Scenario 1: High-traffic page** +- Need: 10,000 per variant (2 variants = 20,000 total) +- Daily traffic: 5,000 visitors +- 100% exposed to test +- Duration: 20,000 / 5,000 = **4 days** + +**Scenario 2: Medium-traffic page** +- Need: 30,000 per variant (60,000 total) +- Daily traffic: 2,000 visitors +- 100% exposed +- Duration: 60,000 / 2,000 = **30 days** + +**Scenario 3: Low-traffic with partial exposure** +- Need: 15,000 per variant (30,000 total) +- Daily traffic: 500 visitors +- 50% exposed to test +- Effective daily: 250 +- Duration: 30,000 / 250 = **120 days** (too long!) + +### Minimum Duration Rules + +Even with sufficient sample size, run tests for at least: +- **1 full week**: To capture day-of-week variation +- **2 business cycles**: If B2B (weekday vs. weekend patterns) +- **Through paydays**: If e-commerce (beginning/end of month) + +### Maximum Duration Guidelines + +Avoid running tests longer than 4-8 weeks: +- Novelty effects wear off +- External factors intervene +- Opportunity cost of other tests + +--- + +## Online Calculators + +### Recommended Tools + +**Evan Miller's Calculator** +https://www.evanmiller.org/ab-testing/sample-size.html +- Simple interface +- Bookmark-worthy + +**Optimizely's Calculator** +https://www.optimizely.com/sample-size-calculator/ +- Business-friendly language +- Duration estimates + +**AB Test Guide Calculator** +https://www.abtestguide.com/calc/ +- Includes Bayesian option +- Multiple test types + +**VWO Duration Calculator** +https://vwo.com/tools/ab-test-duration-calculator/ +- Duration-focused +- Good for planning + +--- + +## Adjusting for Multiple Variants + +With more than 2 variants (A/B/n tests), you need more sample: + +| Variants | Multiplier | +|----------|------------| +| 2 (A/B) | 1x | +| 3 (A/B/C) | ~1.5x | +| 4 (A/B/C/D) | ~2x | +| 5+ | Consider reducing variants | + +**Why?** More comparisons increase chance of false positives. You're comparing: +- A vs B +- A vs C +- B vs C (sometimes) + +Apply Bonferroni correction or use tools that handle this automatically. + +--- + +## Common Sample Size Mistakes + +### 1. Underpowered tests +**Problem**: Not enough sample to detect realistic effects +**Fix**: Be realistic about MDE, get more traffic, or don't test + +### 2. Overpowered tests +**Problem**: Waiting for sample size when you already have significance +**Fix**: This is actually fine—you committed to sample size, honor it + +### 3. Wrong baseline rate +**Problem**: Using wrong conversion rate for calculation +**Fix**: Use the specific metric and page, not site-wide averages + +### 4. Ignoring segments +**Problem**: Calculating for full traffic, then analyzing segments +**Fix**: If you plan segment analysis, calculate sample for smallest segment + +### 5. Testing too many things +**Problem**: Dividing traffic too many ways +**Fix**: Prioritize ruthlessly, run fewer concurrent tests + +--- + +## When Sample Size Requirements Are Too High + +Options when you can't get enough traffic: + +1. **Increase MDE**: Accept only detecting larger effects (20%+ lift) +2. **Lower confidence**: Use 90% instead of 95% (risky, document it) +3. **Reduce variants**: Test only the most promising variant +4. **Combine traffic**: Test across multiple similar pages +5. **Test upstream**: Test earlier in funnel where traffic is higher +6. **Don't test**: Make decision based on qualitative data instead +7. **Longer test**: Accept longer duration (weeks/months) + +--- + +## Sequential Testing + +If you must check results before reaching sample size: + +### What is it? +Statistical method that adjusts for multiple looks at data. + +### When to use +- High-risk changes +- Need to stop bad variants early +- Time-sensitive decisions + +### Tools that support it +- Optimizely (Stats Accelerator) +- VWO (SmartStats) +- PostHog (Bayesian approach) + +### Tradeoff +- More flexibility to stop early +- Slightly larger sample size requirement +- More complex analysis + +--- + +## Quick Decision Framework + +### Can I run this test? + +``` +Daily traffic to page: _____ +Baseline conversion rate: _____ +MDE I care about: _____ + +Sample needed per variant: _____ (from tables above) +Days to run: Sample / Daily traffic = _____ + +If days > 60: Consider alternatives +If days > 30: Acceptable for high-impact tests +If days < 14: Likely feasible +If days < 7: Easy to run, consider running longer anyway +``` diff --git a/src/adclaw/agents/skills/marketing-ab-test-setup/references/test-templates.md b/src/adclaw/agents/skills/marketing-ab-test-setup/references/test-templates.md new file mode 100644 index 0000000..1c517d8 --- /dev/null +++ b/src/adclaw/agents/skills/marketing-ab-test-setup/references/test-templates.md @@ -0,0 +1,277 @@ +# A/B Test Templates Reference + +Templates for planning, documenting, and analyzing experiments. + +## Contents +- Test Plan Template +- Results Documentation Template +- Test Repository Entry Template +- Quick Test Brief Template +- Stakeholder Update Template +- Experiment Prioritization Scorecard +- Hypothesis Bank Template + +## Test Plan Template + +```markdown +# A/B Test: [Name] + +## Overview +- **Owner**: [Name] +- **Test ID**: [ID in testing tool] +- **Page/Feature**: [What's being tested] +- **Planned dates**: [Start] - [End] + +## Hypothesis + +Because [observation/data], +we believe [change] +will cause [expected outcome] +for [audience]. +We'll know this is true when [metrics]. + +## Test Design + +| Element | Details | +|---------|---------| +| Test type | A/B / A/B/n / MVT | +| Duration | X weeks | +| Sample size | X per variant | +| Traffic allocation | 50/50 | +| Tool | [Tool name] | +| Implementation | Client-side / Server-side | + +## Variants + +### Control (A) +[Screenshot] +- Current experience +- [Key details about current state] + +### Variant (B) +[Screenshot or mockup] +- [Specific change #1] +- [Specific change #2] +- Rationale: [Why we think this will win] + +## Metrics + +### Primary +- **Metric**: [metric name] +- **Definition**: [how it's calculated] +- **Current baseline**: [X%] +- **Minimum detectable effect**: [X%] + +### Secondary +- [Metric 1]: [what it tells us] +- [Metric 2]: [what it tells us] +- [Metric 3]: [what it tells us] + +### Guardrails +- [Metric that shouldn't get worse] +- [Another safety metric] + +## Segment Analysis Plan +- Mobile vs. desktop +- New vs. returning visitors +- Traffic source +- [Other relevant segments] + +## Success Criteria +- Winner: [Primary metric improves by X% with 95% confidence] +- Loser: [Primary metric decreases significantly] +- Inconclusive: [What we'll do if no significant result] + +## Pre-Launch Checklist +- [ ] Hypothesis documented and reviewed +- [ ] Primary metric defined and trackable +- [ ] Sample size calculated +- [ ] Test duration estimated +- [ ] Variants implemented correctly +- [ ] Tracking verified in all variants +- [ ] QA completed on all variants +- [ ] Stakeholders informed +- [ ] Calendar hold for analysis date +``` + +--- + +## Results Documentation Template + +```markdown +# A/B Test Results: [Name] + +## Summary +| Element | Value | +|---------|-------| +| Test ID | [ID] | +| Dates | [Start] - [End] | +| Duration | X days | +| Result | Winner / Loser / Inconclusive | +| Decision | [What we're doing] | + +## Hypothesis (Reminder) +[Copy from test plan] + +## Results + +### Sample Size +| Variant | Target | Actual | % of target | +|---------|--------|--------|-------------| +| Control | X | Y | Z% | +| Variant | X | Y | Z% | + +### Primary Metric: [Metric Name] +| Variant | Value | 95% CI | vs. Control | +|---------|-------|--------|-------------| +| Control | X% | [X%, Y%] | — | +| Variant | X% | [X%, Y%] | +X% | + +**Statistical significance**: p = X.XX (95% = sig / not sig) +**Practical significance**: [Is this lift meaningful for the business?] + +### Secondary Metrics + +| Metric | Control | Variant | Change | Significant? | +|--------|---------|---------|--------|--------------| +| [Metric 1] | X | Y | +Z% | Yes/No | +| [Metric 2] | X | Y | +Z% | Yes/No | + +### Guardrail Metrics + +| Metric | Control | Variant | Change | Concern? | +|--------|---------|---------|--------|----------| +| [Metric 1] | X | Y | +Z% | Yes/No | + +### Segment Analysis + +**Mobile vs. Desktop** +| Segment | Control | Variant | Lift | +|---------|---------|---------|------| +| Mobile | X% | Y% | +Z% | +| Desktop | X% | Y% | +Z% | + +**New vs. Returning** +| Segment | Control | Variant | Lift | +|---------|---------|---------|------| +| New | X% | Y% | +Z% | +| Returning | X% | Y% | +Z% | + +## Interpretation + +### What happened? +[Explanation of results in plain language] + +### Why do we think this happened? +[Analysis and reasoning] + +### Caveats +[Any limitations, external factors, or concerns] + +## Decision + +**Winner**: [Control / Variant] + +**Action**: [Implement variant / Keep control / Re-test] + +**Timeline**: [When changes will be implemented] + +## Learnings + +### What we learned +- [Key insight 1] +- [Key insight 2] + +### What to test next +- [Follow-up test idea 1] +- [Follow-up test idea 2] + +### Impact +- **Projected lift**: [X% improvement in Y metric] +- **Business impact**: [Revenue, conversions, etc.] +``` + +--- + +## Test Repository Entry Template + +For tracking all tests in a central location: + +```markdown +| Test ID | Name | Page | Dates | Primary Metric | Result | Lift | Link | +|---------|------|------|-------|----------------|--------|------|------| +| 001 | Hero headline test | Homepage | 1/1-1/15 | CTR | Winner | +12% | [Link] | +| 002 | Pricing table layout | Pricing | 1/10-1/31 | Plan selection | Loser | -5% | [Link] | +| 003 | Signup form fields | Signup | 2/1-2/14 | Completion | Inconclusive | +2% | [Link] | +``` + +--- + +## Quick Test Brief Template + +For simple tests that don't need full documentation: + +```markdown +## [Test Name] + +**What**: [One sentence description] +**Why**: [One sentence hypothesis] +**Metric**: [Primary metric] +**Duration**: [X weeks] +**Result**: [TBD / Winner / Loser / Inconclusive] +**Learnings**: [Key takeaway] +``` + +--- + +## Stakeholder Update Template + +```markdown +## A/B Test Update: [Name] + +**Status**: Running / Complete +**Days remaining**: X (or complete) +**Current sample**: X% of target + +### Preliminary observations +[What we're seeing - without making decisions yet] + +### Next steps +[What happens next] + +### Timeline +- [Date]: Analysis complete +- [Date]: Decision and recommendation +- [Date]: Implementation (if winner) +``` + +--- + +## Experiment Prioritization Scorecard + +For deciding which tests to run: + +| Factor | Weight | Test A | Test B | Test C | +|--------|--------|--------|--------|--------| +| Potential impact | 30% | | | | +| Confidence in hypothesis | 25% | | | | +| Ease of implementation | 20% | | | | +| Risk if wrong | 15% | | | | +| Strategic alignment | 10% | | | | +| **Total** | | | | | + +Scoring: 1-5 (5 = best) + +--- + +## Hypothesis Bank Template + +For collecting test ideas: + +```markdown +| ID | Page/Area | Observation | Hypothesis | Potential Impact | Status | +|----|-----------|-------------|------------|------------------|--------| +| H1 | Homepage | Low scroll depth | Shorter hero will increase scroll | High | Testing | +| H2 | Pricing | Users compare plans | Comparison table will help | Medium | Backlog | +| H3 | Signup | Drop-off at email | Social login will increase completion | Medium | Backlog | +``` diff --git a/src/adclaw/agents/skills/marketing-ad-creative/SKILL.md b/src/adclaw/agents/skills/marketing-ad-creative/SKILL.md new file mode 100644 index 0000000..99b483e --- /dev/null +++ b/src/adclaw/agents/skills/marketing-ad-creative/SKILL.md @@ -0,0 +1,362 @@ +--- +name: ad-creative +description: "When the user wants to generate, iterate, or scale ad creative — headlines, descriptions, primary text, or full ad variations — for any paid advertising platform. Also use when the user mentions 'ad copy variations,' 'ad creative,' 'generate headlines,' 'RSA headlines,' 'bulk ad copy,' 'ad iterations,' 'creative testing,' 'ad performance optimization,' 'write me some ads,' 'Facebook ad copy,' 'Google ad headlines,' 'LinkedIn ad text,' or 'I need more ad variations.' Use this whenever someone needs to produce ad copy at scale or iterate on existing ads. For campaign strategy and targeting, see ads. For landing page copy, see copywriting." +metadata: + version: 2.0.0 +--- + +# Ad Creative + +You are an expert performance creative strategist. Your goal is to generate high-performing ad creative at scale — headlines, descriptions, and primary text that drive clicks and conversions — and iterate based on real performance data. + +## Before Starting + +**Check for product marketing context first:** +If `.agents/product-marketing.md` exists (or `.claude/product-marketing.md`, or the legacy `product-marketing-context.md` filename, in older setups), read it before asking questions. Use that context and only ask for information not already covered or specific to this task. + +Gather this context (ask if not provided): + +### 1. Platform & Format +- What platform? (Google Ads, Meta, LinkedIn, TikTok, Twitter/X) +- What ad format? (Search RSAs, display, social feed, stories, video) +- Are there existing ads to iterate on, or starting from scratch? + +### 2. Product & Offer +- What are you promoting? (Product, feature, free trial, demo, lead magnet) +- What's the core value proposition? +- What makes this different from competitors? + +### 3. Audience & Intent +- Who is the target audience? +- What stage of awareness? (Problem-aware, solution-aware, product-aware) +- What pain points or desires drive them? + +### 4. Performance Data (if iterating) +- What creative is currently running? +- Which headlines/descriptions are performing best? (CTR, conversion rate, ROAS) +- Which are underperforming? +- What angles or themes have been tested? + +### 5. Constraints +- Brand voice guidelines or words to avoid? +- Compliance requirements? (Industry regulations, platform policies) +- Any mandatory elements? (Brand name, trademark symbols, disclaimers) + +--- + +## How This Skill Works + +This skill supports two modes: + +### Mode 1: Generate from Scratch +When starting fresh, you generate a full set of ad creative based on product context, audience insights, and platform best practices. + +### Mode 2: Iterate from Performance Data +When the user provides performance data (CSV, paste, or API output), you analyze what's working, identify patterns in top performers, and generate new variations that build on winning themes while exploring new angles. + +The core loop: + +``` +Pull performance data → Identify winning patterns → Generate new variations → Validate specs → Deliver +``` + +--- + +## Platform Specs + +Platforms reject or truncate creative that exceeds these limits, so verify every piece of copy fits before delivering. + +### Google Ads (Responsive Search Ads) + +| Element | Limit | Quantity | +|---------|-------|----------| +| Headline | 30 characters | Up to 15 | +| Description | 90 characters | Up to 4 | +| Display URL path | 15 characters each | 2 paths | + +**RSA rules:** +- Headlines must make sense independently and in any combination +- Pin headlines to positions only when necessary (reduces optimization) +- Include at least one keyword-focused headline +- Include at least one benefit-focused headline +- Include at least one CTA headline + +### Meta Ads (Facebook/Instagram) + +| Element | Limit | Notes | +|---------|-------|-------| +| Primary text | 125 chars visible (up to 2,200) | Front-load the hook | +| Headline | 40 characters recommended | Below the image | +| Description | 30 characters recommended | Below headline | +| URL display link | 40 characters | Optional | + +### LinkedIn Ads + +| Element | Limit | Notes | +|---------|-------|-------| +| Intro text | 150 chars recommended (600 max) | Above the image | +| Headline | 70 chars recommended (200 max) | Below the image | +| Description | 100 chars recommended (300 max) | Appears in some placements | + +### TikTok Ads + +| Element | Limit | Notes | +|---------|-------|-------| +| Ad text | 80 chars recommended (100 max) | Above the video | +| Display name | 40 characters | Brand name | + +### Twitter/X Ads + +| Element | Limit | Notes | +|---------|-------|-------| +| Tweet text | 280 characters | The ad copy | +| Headline | 70 characters | Card headline | +| Description | 200 characters | Card description | + +For detailed specs and format variations, see [references/platform-specs.md](references/platform-specs.md). + +--- + +## Generating Ad Visuals + +For image and video ad creative, use generative AI tools and code-based video rendering. See [references/generative-tools.md](references/generative-tools.md) for the complete guide covering: + +- **Image generation** — Nano Banana Pro (Gemini), Flux, Ideogram for static ad images +- **Video generation** — Veo, Kling, Runway, Sora, Seedance, Higgsfield for video ads +- **Voice & audio** — ElevenLabs, OpenAI TTS, Cartesia for voiceovers, cloning, multilingual +- **Code-based video** — Remotion for templated, data-driven video at scale +- **Platform image specs** — Correct dimensions for every ad placement +- **Cost comparison** — Pricing for 100+ ad variations across tools + +**Recommended workflow for scaled production:** +1. Generate hero creative with AI tools (exploratory, high-quality) +2. Build Remotion templates based on winning patterns +3. Batch produce variations with Remotion using data feeds +4. Iterate — AI for new angles, Remotion for scale + +--- + +## Generating Ad Copy + +### Step 1: Define Your Angles + +Before writing individual headlines, establish 3-5 distinct **angles** — different reasons someone would click. Each angle should tap into a different motivation. + +**Common angle categories:** + +| Category | Example Angle | +|----------|---------------| +| Pain point | "Stop wasting time on X" | +| Outcome | "Achieve Y in Z days" | +| Social proof | "Join 10,000+ teams who..." | +| Curiosity | "The X secret top companies use" | +| Comparison | "Unlike X, we do Y" | +| Urgency | "Limited time: get X free" | +| Identity | "Built for [specific role/type]" | +| Contrarian | "Why [common practice] doesn't work" | + +### Step 2: Generate Variations per Angle + +For each angle, generate multiple variations. Vary: +- **Word choice** — synonyms, active vs. passive +- **Specificity** — numbers vs. general claims +- **Tone** — direct vs. question vs. command +- **Structure** — short punch vs. full benefit statement + +### Step 3: Validate Against Specs + +Before delivering, check every piece of creative against the platform's character limits. Flag anything that's over and provide a trimmed alternative. + +### Step 4: Organize for Upload + +Present creative in a structured format that maps to the ad platform's upload requirements. + +--- + +## Iterating from Performance Data + +When the user provides performance data, follow this process: + +### Step 1: Analyze Winners + +Look at the top-performing creative (by CTR, conversion rate, or ROAS — ask which metric matters most) and identify: + +- **Winning themes** — What topics or pain points appear in top performers? +- **Winning structures** — Questions? Statements? Commands? Numbers? +- **Winning word patterns** — Specific words or phrases that recur? +- **Character utilization** — Are top performers shorter or longer? + +### Step 2: Analyze Losers + +Look at the worst performers and identify: + +- **Themes that fall flat** — What angles aren't resonating? +- **Common patterns in low performers** — Too generic? Too long? Wrong tone? + +### Step 3: Generate New Variations + +Create new creative that: +- **Doubles down** on winning themes with fresh phrasing +- **Extends** winning angles into new variations +- **Tests** 1-2 new angles not yet explored +- **Avoids** patterns found in underperformers + +### Step 4: Document the Iteration + +Track what was learned and what's being tested: + +``` +## Iteration Log +- Round: [number] +- Date: [date] +- Top performers: [list with metrics] +- Winning patterns: [summary] +- New variations: [count] headlines, [count] descriptions +- New angles being tested: [list] +- Angles retired: [list] +``` + +--- + +## Writing Quality Standards + +### Headlines That Click + +**Strong headlines:** +- Specific ("Cut reporting time 75%") over vague ("Save time") +- Benefits ("Ship code faster") over features ("CI/CD pipeline") +- Active voice ("Automate your reports") over passive ("Reports are automated") +- Include numbers when possible ("3x faster," "in 5 minutes," "10,000+ teams") + +**Avoid:** +- Jargon the audience won't recognize +- Claims without specificity ("Best," "Leading," "Top") +- All caps or excessive punctuation +- Clickbait that the landing page can't deliver on + +### Descriptions That Convert + +Descriptions should complement headlines, not repeat them. Use descriptions to: +- Add proof points (numbers, testimonials, awards) +- Handle objections ("No credit card required," "Free forever for small teams") +- Reinforce CTAs ("Start your free trial today") +- Add urgency when genuine ("Limited to first 500 signups") + +--- + +## Output Formats + +### Standard Output + +Organize by angle, with character counts: + +``` +## Angle: [Pain Point — Manual Reporting] + +### Headlines (30 char max) +1. "Stop Building Reports by Hand" (29) +2. "Automate Your Weekly Reports" (28) +3. "Reports Done in 5 Min, Not 5 Hr" (31) <- OVER LIMIT, trimmed below + -> "Reports in 5 Min, Not 5 Hrs" (27) + +### Descriptions (90 char max) +1. "Marketing teams save 10+ hours/week with automated reporting. Start free." (73) +2. "Connect your data sources once. Get automated reports forever. No code required." (80) +``` + +### Bulk CSV Output + +When generating at scale (10+ variations), offer CSV format for direct upload: + +```csv +headline_1,headline_2,headline_3,description_1,description_2,platform +"Stop Manual Reporting","Automate in 5 Minutes","Join 10K+ Teams","Save 10+ hrs/week on reports. Start free.","Connect data sources once. Reports forever.","google_ads" +``` + +### Iteration Report + +When iterating, include a summary: + +``` +## Performance Summary +- Analyzed: [X] headlines, [Y] descriptions +- Top performer: "[headline]" — [metric]: [value] +- Worst performer: "[headline]" — [metric]: [value] +- Pattern: [observation] + +## New Creative +[organized variations] + +## Recommendations +- [What to pause, what to scale, what to test next] +``` + +--- + +## Batch Generation Workflow + +For large-scale creative production (Anthropic's growth team generates 100+ variations per cycle): + +### 1. Break into sub-tasks +- **Headline generation** — Focused on click-through +- **Description generation** — Focused on conversion +- **Primary text generation** — Focused on engagement (Meta/LinkedIn) + +### 2. Generate in waves +- Wave 1: Core angles (3-5 angles, 5 variations each) +- Wave 2: Extended variations on top 2 angles +- Wave 3: Wild card angles (contrarian, emotional, specific) + +### 3. Quality filter +- Remove anything over character limit +- Remove duplicates or near-duplicates +- Flag anything that might violate platform policies +- Ensure headline/description combinations make sense together + +--- + +## Common Mistakes + +- **Writing headlines that only work together** — RSA headlines get combined randomly +- **Ignoring character limits** — Platforms truncate without warning +- **All variations sound the same** — Vary angles, not just word choice +- **No CTA headlines** — RSAs need action-oriented headlines to drive clicks; include at least 2-3 +- **Generic descriptions** — "Learn more about our solution" wastes the slot +- **Iterating without data** — Gut feelings are less reliable than metrics +- **Testing too many things at once** — Change one variable per test cycle +- **Retiring creative too early** — Allow 1,000+ impressions before judging + +--- + +## Tool Integrations + +For pulling performance data and managing campaigns, see the [tools registry](../../tools/REGISTRY.md). + +| Platform | Pull Performance Data | Manage Campaigns | Guide | +|----------|:---------------------:|:----------------:|-------| +| **Google Ads** | `google-ads campaigns list`, `google-ads reports get` | `google-ads campaigns create` | [google-ads.md](../../tools/integrations/google-ads.md) | +| **Meta Ads** | `meta-ads insights get` | `meta-ads campaigns list` | [meta-ads.md](../../tools/integrations/meta-ads.md) | +| **LinkedIn Ads** | `linkedin-ads analytics get` | `linkedin-ads campaigns list` | [linkedin-ads.md](../../tools/integrations/linkedin-ads.md) | +| **TikTok Ads** | `tiktok-ads reports get` | `tiktok-ads campaigns list` | [tiktok-ads.md](../../tools/integrations/tiktok-ads.md) | + +### Workflow: Pull Data, Analyze, Generate + +```bash +# 1. Pull recent ad performance +node tools/clis/google-ads.js reports get --type ad_performance --date-range last_30_days + +# 2. Analyze output (identify top/bottom performers) +# 3. Feed winning patterns into this skill +# 4. Generate new variations +# 5. Upload to platform +``` + +--- + +## Related Skills + +- **ads**: For campaign strategy, targeting, budgets, and optimization +- **copywriting**: For landing page copy (where ad traffic lands) +- **ab-testing**: For structuring creative tests with statistical rigor +- **marketing-psychology**: For psychological principles behind high-performing creative +- **copy-editing**: For polishing ad copy before launch diff --git a/src/adclaw/agents/skills/marketing-ad-creative/evals/evals.json b/src/adclaw/agents/skills/marketing-ad-creative/evals/evals.json new file mode 100644 index 0000000..63f10a7 --- /dev/null +++ b/src/adclaw/agents/skills/marketing-ad-creative/evals/evals.json @@ -0,0 +1,90 @@ +{ + "skill_name": "ad-creative", + "evals": [ + { + "id": 1, + "prompt": "Generate ad creative for our Meta (Facebook/Instagram) campaign. We sell an AI writing assistant for content marketers. Main value prop: write blog posts 5x faster. Target audience: content marketing managers at B2B SaaS companies. Budget: $5k/month.", + "expected_output": "Should check for product-marketing.md first. Should generate creative following the angle-based approach: identify 3-5 angles (speed, quality, ROI, pain of blank page, competitive edge). For each angle, should generate primary text (≤125 chars), headline (≤40 chars), and description (≤30 chars) respecting Meta character limits. Should provide multiple variations per angle. Should suggest image/visual direction for each. Should organize output with angle name, hook, body, CTA for each variation. Should recommend which angles to test first.", + "assertions": [ + "Checks for product-marketing.md", + "Uses angle-based generation approach", + "Identifies multiple angles (3-5)", + "Respects Meta character limits (125/40/30)", + "Generates multiple variations per angle", + "Suggests image or visual direction", + "Includes hook, body, and CTA for each", + "Recommends which angles to test first" + ], + "files": [] + }, + { + "id": 2, + "prompt": "I need Google Ads copy for our CRM product. We're targeting the keyword 'best CRM for small business'. Need responsive search ads.", + "expected_output": "Should generate Google RSA creative respecting character limits: headlines (≤30 chars each, need 10-15 variations) and descriptions (≤90 chars each, need 4+ variations). Should note that pinning should be used sparingly as it reduces optimization. Should include the target keyword in headlines. Should provide multiple angle-based variations. Should suggest ad extensions (sitelinks, callouts, structured snippets). Should follow Google Ads best practices for RSA.", + "assertions": [ + "Respects Google RSA character limits (30 char headlines, 90 char descriptions)", + "Generates 10-15 headline variations", + "Generates 4+ description variations", + "Includes target keyword in headlines", + "Notes pinning should be used sparingly per skill guidance", + "Suggests ad extensions", + "Uses angle-based variation approach" + ], + "files": [] + }, + { + "id": 3, + "prompt": "Here's our ad performance data: Ad A (pain point angle) - CTR 2.1%, CPC $3.20, Conv rate 4.5%. Ad B (social proof angle) - CTR 1.4%, CPC $4.10, Conv rate 6.2%. Ad C (feature angle) - CTR 0.8%, CPC $5.50, Conv rate 2.1%. Help me iterate on these.", + "expected_output": "Should activate the iteration-from-performance mode (not generate-from-scratch). Should analyze the data: Ad A has best CTR, Ad B has best conversion rate (highest efficiency despite lower CTR), Ad C is underperforming on all metrics. Should recommend doubling down on the pain point angle (high CTR) and social proof angle (high conversion), while pausing or reworking the feature angle. Should generate new variations that combine winning elements (pain point hook + social proof). Should suggest specific iterations on Ad A and Ad B.", + "assertions": [ + "Activates iteration mode based on performance data", + "Analyzes CTR, CPC, and conversion rate for each ad", + "Identifies winning angles from the data", + "Recommends pausing or reworking underperforming creative", + "Generates new variations combining winning elements", + "Provides specific iterations on top performers" + ], + "files": [] + }, + { + "id": 4, + "prompt": "we need linkedin ads for our enterprise security product. audience is CISOs and IT directors.", + "expected_output": "Should trigger on casual phrasing. Should generate LinkedIn ad creative respecting character limits: introductory text (≤150 chars), headline (≤70 chars), description (≤100 chars). Should adapt tone and messaging for enterprise security audience (CISOs, IT directors) — more formal, compliance-focused, risk-reduction language. Should provide multiple angles relevant to security buyers (risk reduction, compliance, incident response time, cost of breaches). Should suggest ad format recommendations for LinkedIn (sponsored content, message ads, etc.).", + "assertions": [ + "Triggers on casual phrasing", + "Respects LinkedIn character limits (150/70/100)", + "Adapts tone for enterprise security audience", + "Uses risk-reduction and compliance language", + "Provides multiple angles relevant to security buyers", + "Suggests LinkedIn ad format recommendations" + ], + "files": [] + }, + { + "id": 5, + "prompt": "I need to generate a big batch of ad variations for a multi-platform campaign launching next week. We're a meal delivery service targeting busy professionals. Need ads for Google, Meta, and TikTok.", + "expected_output": "Should activate the batch generation workflow. Should generate creative for all three platforms respecting each platform's character limits: Google RSA (30/90), Meta (125/40/30), TikTok (80 chars recommended, 100 max). Should identify 3-5 angles that work across platforms (convenience, health, time savings, variety, cost vs eating out). Should generate variations per angle per platform. Should note platform-specific creative considerations (TikTok needs video concepts, not just text). Should organize output clearly by platform.", + "assertions": [ + "Activates batch generation workflow", + "Generates for all three platforms", + "Respects each platform's character limits", + "Identifies angles that work across platforms", + "Notes TikTok needs video concepts", + "Organizes output by platform", + "Generates multiple variations per angle per platform" + ], + "files": [] + }, + { + "id": 6, + "prompt": "Help me plan our overall paid advertising strategy. We have a $20k monthly budget and want to figure out which platforms to use and how to allocate spend.", + "expected_output": "Should recognize this is a paid advertising strategy task, not ad creative generation. Should defer to or cross-reference the ads skill, which handles campaign strategy, platform selection, and budget allocation. May briefly mention creative considerations but should make clear that ads is the right skill for strategy.", + "assertions": [ + "Recognizes this as paid ads strategy, not creative generation", + "References or defers to ads skill", + "Does not attempt full campaign strategy using creative generation patterns" + ], + "files": [] + } + ] +} diff --git a/src/adclaw/agents/skills/marketing-ad-creative/references/generative-tools.md b/src/adclaw/agents/skills/marketing-ad-creative/references/generative-tools.md new file mode 100644 index 0000000..b1e6fec --- /dev/null +++ b/src/adclaw/agents/skills/marketing-ad-creative/references/generative-tools.md @@ -0,0 +1,637 @@ +# Generative AI Tools for Ad Creative + +Reference for using AI image generators, video generators, and code-based video tools to produce ad visuals at scale. + +--- + +## When to Use Generative Tools + +| Need | Tool Category | Best Fit | +|------|---------------|----------| +| Static ad images (banners, social) | Image generation | ChatGPT Images 2.0, Nano Banana Pro, Flux, Ideogram | +| Ad images with text overlays | Image generation (text-capable) | Ideogram, Nano Banana Pro | +| Short video ads (6-30 sec) | Video generation | Veo, Kling, Runway, Sora, Seedance | +| Video ads with voiceover | Video gen + voice | Veo/Sora (native), or Runway + ElevenLabs | +| Voiceover tracks for ads | Voice generation | ElevenLabs, OpenAI TTS, Cartesia | +| Multi-language ad versions | Voice generation | ElevenLabs, PlayHT | +| Brand voice cloning | Voice generation | ElevenLabs, Resemble AI | +| Product mockups and variations | Image generation + references | Flux (multi-image reference) | +| Templated video ads at scale | Code-based video | Remotion | +| Personalized video (name, data) | Code-based video | Remotion | +| Brand-consistent variations | Image gen + style refs | Flux, Ideogram, Nano Banana Pro | + +--- + +## Image Generation + +### Nano Banana Pro (Gemini) + +Google DeepMind's image generation model, available through the Gemini API. + +**Best for:** High-quality ad images, product visuals, text rendering +**API:** Gemini API (Google AI Studio, Vertex AI) +**Pricing:** ~$0.04/image (Gemini 2.5 Flash Image), ~$0.24/4K image (Nano Banana Pro) + +**Strengths:** +- Strong text rendering in images (logos, headlines) +- Native image editing (modify existing images with prompts) +- Available through the same Gemini API used for text generation +- Supports both generation and editing in one model + +**Ad creative use cases:** +- Generate social media ad images from text descriptions +- Create product mockup variations +- Edit existing ad images (swap backgrounds, change colors) +- Generate images with headline text baked in + +**API example:** +```bash +# Using the Gemini API for image generation +curl -X POST "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash-image:generateContent" \ + -H "Content-Type: application/json" \ + -H "x-goog-api-key: $GEMINI_API_KEY" \ + -d '{ + "contents": [{"parts": [{"text": "Create a clean, modern social media ad image for a project management tool. Show a laptop with a kanban board interface. Bright, professional, 16:9 ratio."}]}], + "generationConfig": {"responseModalities": ["TEXT", "IMAGE"]} + }' +``` + +**Docs:** [Gemini Image Generation](https://ai.google.dev/gemini-api/docs/image-generation) + +--- + +### Flux (Black Forest Labs) + +Open-weight image generation models with API access through Replicate and BFL's native API. + +**Best for:** Photorealistic images, brand-consistent variations, multi-reference generation +**API:** Replicate, BFL API, fal.ai +**Pricing:** ~$0.01-0.06/image depending on model and resolution + +**Model variants:** +| Model | Speed | Quality | Cost | Best For | +|-------|-------|---------|------|----------| +| Flux 2 Pro | ~6 sec | Highest | $0.015/MP | Final production assets | +| Flux 2 Flex | ~22 sec | High + editing | $0.06/MP | Iterative editing | +| Flux 2 Dev | ~2.5 sec | Good | $0.012/MP | Rapid prototyping | +| Flux 2 Klein | Fastest | Good | Lowest | High-volume batch generation | + +**Strengths:** +- Multi-image reference (up to 8 images) for consistent identity across ads +- Product consistency — same product in different contexts +- Style transfer from reference images +- Open-weight Dev model for self-hosting + +**Ad creative use cases:** +- Generate 50+ ad variations with consistent product/person identity +- Create product-in-context images (your SaaS on different devices) +- Style-match to existing brand assets using reference images +- Rapid A/B test image variations + +**Docs:** [Replicate Flux](https://replicate.com/black-forest-labs/flux-2-pro), [BFL API](https://docs.bfl.ml/) + +--- + +### Ideogram + +Specialized in typography and text rendering within images. + +**Best for:** Ad banners with text, branded graphics, social ad images with headlines +**API:** Ideogram API, Runware +**Pricing:** ~$0.06/image (API), ~$0.009/image (subscription) + +**Strengths:** +- Best-in-class text rendering (~90% accuracy vs ~30% for most tools) +- Style reference system (upload up to 3 reference images) +- 4.3 billion style presets for consistent brand aesthetics +- Strong at logos and branded typography + +**Ad creative use cases:** +- Generate ad banners with headline text directly in the image +- Create social media graphics with branded text overlays +- Produce multiple design variations with consistent typography +- Generate promotional materials without needing a designer for each iteration + +**Docs:** [Ideogram API](https://developer.ideogram.ai/), [Ideogram](https://ideogram.ai/) + +--- + +### Other Image Tools + +| Tool | Best For | API Status | Notes | +|------|----------|------------|-------| +| **DALL-E 3** (OpenAI) | General image generation | Official API | Integrated with ChatGPT, good text rendering | +| **Midjourney** | Artistic, high-aesthetic images | No official public API | Discord-based; unofficial APIs exist but risk bans | +| **Stable Diffusion** | Self-hosted, customizable | Open source | Best for teams with GPU infrastructure | + +--- + +## Video Generation + +### Google Veo + +Google DeepMind's video generation model, available through the Gemini API and Vertex AI. + +**Best for:** High-quality video ads with native audio, vertical video for social +**API:** Gemini API, Vertex AI +**Pricing:** ~$0.15/sec (Veo 3.1 Fast), ~$0.40/sec (Veo 3.1 Standard) + +**Capabilities:** +- Up to 60 seconds at 1080p +- Native audio generation (dialogue, sound effects, ambient) +- Vertical 9:16 output for Stories/Reels/Shorts +- Upscale to 4K +- Text-to-video and image-to-video + +**Ad creative use cases:** +- Generate short video ads (15-30 sec) from text descriptions +- Create vertical video ads for TikTok, Reels, Shorts +- Produce product demos with voiceover +- Generate multiple video variations from the same prompt with different styles + +**Docs:** [Veo on Vertex AI](https://cloud.google.com/vertex-ai/generative-ai/docs/video/overview) + +--- + +### Kling (Kuaishou) + +Video generation with simultaneous audio-visual generation and camera controls. + +**Best for:** Cinematic video ads, longer-form content, audio-synced video +**API:** Kling API, PiAPI, fal.ai +**Pricing:** ~$0.09/sec (via fal.ai third-party) + +**Capabilities:** +- Up to 3 minutes at 1080p/30-48fps +- Simultaneous audio-visual generation (Kling 2.6) +- Text-to-video and image-to-video +- Motion and camera controls + +**Ad creative use cases:** +- Longer product explainer videos +- Cinematic brand videos with synchronized audio +- Animate product images into video ads + +**Docs:** [Kling AI Developer](https://klingai.com/global/dev/model/video) + +--- + +### Runway + +Video generation and editing platform with strong controllability. + +**Best for:** Controlled video generation, style-consistent content, editing existing footage +**API:** Runway Developer Portal + +**Capabilities:** +- Gen-4: Character/scene consistency across shots +- Motion brush and camera controls +- Image-to-video with reference images +- Video-to-video style transfer + +**Ad creative use cases:** +- Generate video ads with consistent characters/products across scenes +- Style-transfer existing footage to match brand aesthetics +- Extend or remix existing video content + +**Docs:** [Runway API](https://docs.dev.runwayml.com/) + +--- + +### Sora 2 (OpenAI) + +OpenAI's video generation model with synchronized audio. + +**Best for:** High-fidelity video with dialogue and sound +**API:** OpenAI API +**Pricing:** Free tier available; Pro from $0.10-0.50/sec depending on resolution + +**Capabilities:** +- Up to 60 seconds with synchronized audio +- Dialogue, sound effects, and ambient audio +- sora-2 (fast) and sora-2-pro (quality) variants +- Text-to-video and image-to-video + +**Ad creative use cases:** +- Video testimonials and talking-head style ads +- Product demo videos with narration +- Narrative brand videos + +**Docs:** [OpenAI Video Generation](https://platform.openai.com/docs/guides/video-generation) + +--- + +### Seedance 2.0 (ByteDance) + +ByteDance's video generation model with simultaneous audio-visual generation and multimodal inputs. + +**Best for:** Fast, affordable video ads with native audio, multimodal reference inputs +**API:** BytePlus (official), Replicate, WaveSpeedAI, fal.ai (third-party); OpenAI-compatible API format +**Pricing:** ~$0.10-0.80/min depending on resolution (estimated 10-100x cheaper than Sora 2 per clip) + +**Capabilities:** +- Up to 20 seconds at up to 2K resolution +- Simultaneous audio-visual generation (Dual-Branch Diffusion Transformer) +- Text-to-video and image-to-video +- Up to 12 reference files for multimodal input +- OpenAI-compatible API structure + +**Ad creative use cases:** +- High-volume short video ad production at low cost +- Video ads with synchronized voiceover and sound effects in one pass +- Multi-reference generation (feed product images, brand assets, style references) +- Rapid iteration on video ad concepts + +**Docs:** [Seedance](https://seed.bytedance.com/en/seedance2_0) + +--- + +### Higgsfield + +Full-stack video creation platform with cinematic camera controls. + +**Best for:** Social video ads, cinematic style, mobile-first content +**Platform:** [higgsfield.ai](https://higgsfield.ai/) + +**Capabilities:** +- 50+ professional camera movements (zooms, pans, FPV drone shots) +- Image-to-video animation +- Built-in editing, transitions, and keyframing +- All-in-one workflow: image gen, animation, editing + +**Ad creative use cases:** +- Social media video ads with cinematic feel +- Animate product images into dynamic video +- Create multiple video variations with different camera styles +- Quick-turn video content for social campaigns + +--- + +### Video Tool Comparison + +| Tool | Max Length | Audio | Resolution | API | Best For | +|------|-----------|-------|------------|-----|----------| +| **Veo 3.1** | 60 sec | Native | 1080p/4K | Gemini | Vertical social video | +| **Kling 2.6** | 3 min | Native | 1080p | Third-party | Longer cinematic | +| **Runway Gen-4** | 10 sec | No | 1080p | Official | Controlled, consistent | +| **Sora 2** | 60 sec | Native | 1080p | Official | Dialogue-heavy | +| **Seedance 2.0** | 20 sec | Native | 2K | Official + third-party | Affordable high-volume | +| **Higgsfield** | Varies | Yes | 1080p | Web-based | Social, mobile-first | + +--- + +## Voice & Audio Generation + +For layering realistic voiceovers onto video ads, adding narration to product demos, or generating audio for Remotion-rendered videos. These tools turn ad scripts into natural-sounding voice tracks. + +### When to Use Voice Tools + +Many video generators (Veo, Kling, Sora, Seedance) now include native audio. Use standalone voice tools when you need: + +- **Voiceover on silent video** — Runway Gen-4 and Remotion produce silent output +- **Brand voice consistency** — Clone a specific voice for all ads +- **Multi-language versions** — Same ad script in 20+ languages +- **Script iteration** — Re-record voiceover without reshooting video +- **Precise control** — Exact timing, emotion, and pacing + +--- + +### ElevenLabs + +The market leader in realistic voice generation and voice cloning. + +**Best for:** Most natural-sounding voiceovers, brand voice cloning, multilingual +**API:** REST API with streaming support +**Pricing:** ~$0.12-0.30 per 1,000 characters depending on plan; starts at $5/month + +**Capabilities:** +- 29+ languages with natural accent and intonation +- Voice cloning from short audio clips (instant) or longer recordings (professional) +- Emotion and style control +- Streaming for real-time generation +- Voice library with hundreds of pre-built voices + +**Ad creative use cases:** +- Generate voiceover tracks for video ads +- Clone your brand spokesperson's voice for all ad variations +- Produce the same ad in 10+ languages from one script +- A/B test different voice styles (authoritative vs. friendly vs. urgent) + +**API example:** +```bash +curl -X POST "https://api.elevenlabs.io/v1/text-to-speech/{voice_id}" \ + -H "xi-api-key: $ELEVENLABS_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "text": "Stop wasting hours on manual reporting. Try DataFlow free for 14 days.", + "model_id": "eleven_multilingual_v2", + "voice_settings": {"stability": 0.5, "similarity_boost": 0.75} + }' --output voiceover.mp3 +``` + +**Docs:** [ElevenLabs API](https://elevenlabs.io/docs/api-reference/text-to-speech) + +--- + +### OpenAI TTS + +Simple, affordable text-to-speech built into the OpenAI API. + +**Best for:** Quick voiceovers, cost-effective at scale, simple integration +**API:** OpenAI API (same SDK as GPT/DALL-E) +**Pricing:** $15/million chars (standard), $30/million chars (HD); ~$0.015/min with gpt-4o-mini-tts + +**Capabilities:** +- 13 built-in voices (no custom cloning) +- Multiple languages +- Real-time streaming +- HD quality option +- Simple API — same SDK you already use for GPT + +**Ad creative use cases:** +- Fast, cheap voiceover for draft/test ad versions +- High-volume narration at low cost +- Prototype ad audio before investing in premium voice + +**Docs:** [OpenAI TTS](https://platform.openai.com/docs/guides/text-to-speech) + +--- + +### Cartesia Sonic + +Ultra-low latency voice generation built for real-time applications. + +**Best for:** Real-time voice, lowest latency, emotional expressiveness +**API:** REST + WebSocket streaming +**Pricing:** Starts at $5/month; pay-as-you-go from $0.03/min + +**Capabilities:** +- 40ms time-to-first-audio (fastest in class) +- 15+ languages +- Nonverbal expressiveness: laughter, breathing, emotional inflections +- Sonic Turbo for even lower latency +- Streaming API for real-time generation + +**Ad creative use cases:** +- Real-time ad preview during creative iteration +- Interactive demo videos with dynamic narration +- Ads requiring natural laughter, sighs, or emotional reactions + +**Docs:** [Cartesia Sonic](https://docs.cartesia.ai/build-with-cartesia/tts-models/latest) + +--- + +### Voicebox (Open Source) + +Free, local-first voice synthesis studio powered by Qwen3-TTS. The open-source alternative to ElevenLabs. + +**Best for:** Free voice cloning, local/private generation, zero-cost batch production +**API:** Local REST API at `http://localhost:8000` +**Pricing:** Free (MIT license). Runs entirely on your machine. +**Stack:** Tauri (Rust) + React + FastAPI (Python) + +**Capabilities:** +- Voice cloning from short audio samples via Qwen3-TTS +- Multi-language support (English, Chinese, more planned) +- Multi-track timeline editor for composing conversations +- 4-5x faster inference on Apple Silicon via MLX Metal acceleration +- Local REST API for programmatic generation +- No cloud dependency — all processing on-device + +**Ad creative use cases:** +- Free voice cloning for brand spokesperson across all ad variations +- Batch generate voiceovers without per-character costs +- Private/local generation when ad content is sensitive or pre-launch +- Prototype voice variations before committing to a paid service + +**API example:** +```bash +curl -X POST http://localhost:8000/generate \ + -H "Content-Type: application/json" \ + -d '{"text": "Stop wasting hours on manual reporting.", "profile_id": "abc123", "language": "en"}' +``` + +**Install:** Desktop apps for macOS and Windows at [voicebox.sh](https://voicebox.sh), or build from source: +```bash +git clone https://github.com/jamiepine/voicebox.git +cd voicebox && make setup && make dev +``` + +**Docs:** [GitHub](https://github.com/jamiepine/voicebox) + +--- + +### Other Voice Tools + +| Tool | Best For | Differentiator | API | +|------|----------|---------------|-----| +| **PlayHT** | Large voice library, low latency | 900+ voices, <300ms latency, ultra-realistic | [play.ht](https://play.ht/) | +| **Resemble AI** | Enterprise voice cloning | On-premise deployment, real-time speech-to-speech | [resemble.ai](https://www.resemble.ai/) | +| **WellSaid Labs** | Ethical, commercial-safe voices | Voices from compensated actors, safe for commercial use | [wellsaid.io](https://www.wellsaid.io/) | +| **Fish Audio** | Budget-friendly, emotion control | ~50-70% cheaper than ElevenLabs, emotion tags | [fish.audio](https://fish.audio/) | +| **Murf AI** | Non-technical teams | Browser-based studio, 200+ voices | [murf.ai](https://murf.ai/) | +| **Google Cloud TTS** | Google ecosystem, scale | 220+ voices, 40+ languages, enterprise SLAs | [Google TTS](https://cloud.google.com/text-to-speech) | +| **Amazon Polly** | AWS ecosystem, cost | Neural voices, SSML control, cheap at volume | [Amazon Polly](https://aws.amazon.com/polly/) | + +--- + +### Voice Tool Comparison + +| Tool | Quality | Cloning | Languages | Latency | Price/1K chars | +|------|---------|---------|-----------|---------|----------------| +| **ElevenLabs** | Best | Yes (instant + pro) | 29+ | ~200ms | $0.12-0.30 | +| **OpenAI TTS** | Good | No | 13+ | ~300ms | $0.015-0.030 | +| **Cartesia Sonic** | Very good | No | 15+ | ~40ms | ~$0.03/min | +| **PlayHT** | Very good | Yes | 140+ | <300ms | ~$0.10-0.20 | +| **Fish Audio** | Good | Yes | 13+ | ~200ms | ~$0.05-0.10 | +| **WellSaid** | Very good | No (actor voices) | English | ~300ms | Custom pricing | +| **Voicebox** | Good | Yes (local) | 2+ | Local | Free (open source) | + +### Choosing a Voice Tool + +``` +Need voiceover for ads? +├── Need to clone a specific brand voice? +│ ├── Best quality → ElevenLabs +│ ├── Enterprise/on-premise → Resemble AI +│ └── Budget-friendly → Fish Audio, PlayHT +├── Need multilingual (same ad, many languages)? +│ ├── Most languages → PlayHT (140+) +│ └── Best quality → ElevenLabs (29+) +├── Need free / open source / local? +│ └── Voicebox (MIT, runs on your machine) +├── Need cheap, fast, good-enough? +│ └── OpenAI TTS ($0.015/min) +├── Need commercially-safe licensing? +│ └── WellSaid Labs (actor-compensated voices) +└── Need real-time/interactive? + └── Cartesia Sonic (40ms TTFA) +``` + +### Workflow: Voice + Video + +``` +1. Write ad script (use ad-creative skill for copy) +2. Generate voiceover with ElevenLabs/OpenAI TTS +3. Generate or render video: + a. Silent video from Runway/Remotion → layer voice track + b. Or use Veo/Sora/Seedance with native audio (skip separate VO) +4. Combine with ffmpeg if layering separately: + ffmpeg -i video.mp4 -i voiceover.mp3 -c:v copy -c:a aac output.mp4 +5. Generate variations (different scripts, voices, or languages) +``` + +--- + +## Code-Based Video: Remotion + +For templated, data-driven video ads at scale, Remotion is the best option. Unlike AI video generators that produce unique video from prompts, Remotion uses React code to render deterministic, brand-perfect video from templates and data. + +**Best for:** Templated ad variations, personalized video, brand-consistent production +**Stack:** React + TypeScript +**Pricing:** Free for individuals/small teams; commercial license required for 4+ employees +**Docs:** [remotion.dev](https://www.remotion.dev/) + +### Why Remotion for Ads + +| AI Video Generators | Remotion | +|---------------------|----------| +| Unique output each time | Deterministic, pixel-perfect | +| Prompt-based, less control | Full code control over every frame | +| Hard to match brand exactly | Exact brand colors, fonts, spacing | +| One-at-a-time generation | Batch render hundreds from data | +| No dynamic data insertion | Personalize with names, prices, stats | + +### Ad Creative Use Cases + +**1. Dynamic product ads** +Feed a JSON array of products and render a unique video ad for each: +```tsx +// Simplified Remotion component for product ads +export const ProductAd: React.FC<{ + productName: string; + price: string; + imageUrl: string; + tagline: string; +}> = ({productName, price, imageUrl, tagline}) => { + return ( + + +

{productName}

+

{tagline}

+
{price}
+
Shop Now
+ + ); +}; +``` + +**2. A/B test video variations** +Render the same template with different headlines, CTAs, or color schemes: +```tsx +const variations = [ + {headline: "Save 50% Today", cta: "Get the Deal", theme: "urgent"}, + {headline: "Join 10K+ Teams", cta: "Start Free", theme: "social-proof"}, + {headline: "Built for Speed", cta: "Try It Now", theme: "benefit"}, +]; +// Render all variations programmatically +``` + +**3. Personalized outreach videos** +Generate videos addressing prospects by name for cold outreach or sales. + +**4. Social ad batch production** +Render the same content across different aspect ratios: +- 1:1 for feed +- 9:16 for Stories/Reels +- 16:9 for YouTube + +### Remotion Workflow for Ad Creative + +``` +1. Design template in React (or use AI to generate the component) +2. Define data schema (products, headlines, CTAs, images) +3. Feed data array into template +4. Batch render all variations +5. Upload to ad platform +``` + +### Getting Started + +```bash +# Create a new Remotion project +npx create-video@latest + +# Render a single video +npx remotion render src/index.ts MyComposition out/video.mp4 + +# Batch render from data +npx remotion render src/index.ts MyComposition --props='{"data": [...]}' +``` + +--- + +## Choosing the Right Tool + +### Decision Tree + +``` +Need video ads? +├── Templated, data-driven (same structure, different data) +│ └── Use Remotion +├── Unique creative from prompts (exploratory) +│ ├── Need dialogue/voiceover? → Sora 2, Veo 3.1, Kling 2.6, Seedance 2.0 +│ ├── Need consistency across scenes? → Runway Gen-4 +│ ├── Need vertical social video? → Veo 3.1 (native 9:16) +│ ├── Need high volume at low cost? → Seedance 2.0 +│ └── Need cinematic camera work? → Higgsfield, Kling +└── Both → Use AI gen for hero creative, Remotion for variations + +Need image ads? +├── Need text/headlines in image? → Ideogram +├── Need product consistency across variations? → Flux (multi-ref) +├── Need quick iterations on existing images? → Nano Banana Pro +├── Need highest visual quality? → Flux Pro, Midjourney +└── Need high volume at low cost? → Flux Klein, Nano Banana +``` + +### Cost Comparison for 100 Ad Variations + +| Approach | Tool | Approximate Cost | +|----------|------|-----------------| +| 100 static images | Nano Banana Pro | ~$4-24 | +| 100 static images | Flux Dev | ~$1-2 | +| 100 static images | Ideogram API | ~$6 | +| 100 × 15-sec videos | Veo 3.1 Fast | ~$225 | +| 100 × 15-sec videos | Remotion (templated) | ~$0 (self-hosted render) | +| 10 hero videos + 90 templated | Veo + Remotion | ~$22 + render time | + +### Recommended Workflow for Scaled Ad Production + +1. **Generate hero creative** with AI (Nano Banana, Flux, Veo) — high-quality, exploratory +2. **Build templates** in Remotion based on winning creative patterns +3. **Batch produce variations** with Remotion using data (products, headlines, CTAs) +4. **Iterate** — use AI tools for new angles, Remotion for scale + +This hybrid approach gives you the creative exploration of AI generators and the consistency and scale of code-based rendering. + +--- + +## Platform-Specific Image Specs + +When generating images for ads, request the correct dimensions: + +| Platform | Placement | Aspect Ratio | Recommended Size | +|----------|-----------|-------------|-----------------| +| Meta Feed | Single image | 1:1 | 1080x1080 | +| Meta Stories/Reels | Vertical | 9:16 | 1080x1920 | +| Meta Carousel | Square | 1:1 | 1080x1080 | +| Google Display | Landscape | 1.91:1 | 1200x628 | +| Google Display | Square | 1:1 | 1200x1200 | +| LinkedIn Feed | Landscape | 1.91:1 | 1200x627 | +| LinkedIn Feed | Square | 1:1 | 1200x1200 | +| TikTok Feed | Vertical | 9:16 | 1080x1920 | +| Twitter/X Feed | Landscape | 16:9 | 1200x675 | +| Twitter/X Card | Landscape | 1.91:1 | 800x418 | + +Include these dimensions in your generation prompts to avoid needing to crop or resize. diff --git a/src/adclaw/agents/skills/marketing-ad-creative/references/platform-specs.md b/src/adclaw/agents/skills/marketing-ad-creative/references/platform-specs.md new file mode 100644 index 0000000..c9a3c4a --- /dev/null +++ b/src/adclaw/agents/skills/marketing-ad-creative/references/platform-specs.md @@ -0,0 +1,213 @@ +# Platform Specs Reference + +Complete character limits, format requirements, and best practices for each ad platform. + +--- + +## Google Ads + +### Responsive Search Ads (RSAs) + +| Element | Character Limit | Required | Notes | +|---------|----------------|----------|-------| +| Headline | 30 chars | 3 minimum, 15 max | Any 3 may be shown together | +| Description | 90 chars | 2 minimum, 4 max | Any 2 may be shown together | +| Display path 1 | 15 chars | Optional | Appears after domain in URL | +| Display path 2 | 15 chars | Optional | Appears after path 1 | +| Final URL | No limit | Required | Landing page URL | + +**Combination rules:** +- Google selects up to 3 headlines and 2 descriptions to show +- Headlines appear separated by " | " or stacked +- Any headline can appear in any position unless pinned +- Pinning reduces Google's ability to optimize — use sparingly + +**Pinning strategy:** +- Pin your brand name to position 1 if brand guidelines require it +- Pin your strongest CTA to position 2 or 3 +- Leave most headlines unpinned for machine learning + +**Headline mix recommendation (15 headlines):** +- 3-4 keyword-focused (match search intent) +- 3-4 benefit-focused (what they get) +- 2-3 social proof (numbers, awards, customers) +- 2-3 CTA-focused (action to take) +- 1-2 differentiators (why you over competitors) +- 1 brand name headline + +**Description mix recommendation (4 descriptions):** +- 1 benefit + proof point +- 1 feature + outcome +- 1 social proof + CTA +- 1 urgency/offer + CTA (if applicable) + +### Performance Max + +| Element | Character Limit | Notes | +|---------|----------------|-------| +| Headline | 30 chars (5 required) | Short headlines for various placements | +| Long headline | 90 chars (5 required) | Used in display, video, discover | +| Description | 90 chars (1 required, 5 max) | Accompany various ad formats | +| Business name | 25 chars | Required | + +### Display Ads + +| Element | Character Limit | +|---------|----------------| +| Headline | 30 chars | +| Long headline | 90 chars | +| Description | 90 chars | +| Business name | 25 chars | + +--- + +## Meta Ads (Facebook & Instagram) + +### Single Image / Video / Carousel + +| Element | Recommended | Maximum | Notes | +|---------|-------------|---------|-------| +| Primary text | 125 chars | 2,200 chars | Text above image; truncated after ~125 | +| Headline | 40 chars | 255 chars | Below image; truncated after ~40 | +| Description | 30 chars | 255 chars | Below headline; may not show | +| URL display link | 40 chars | N/A | Optional custom display URL | + +**Placement-specific notes:** +- **Feed**: All elements show; primary text most visible +- **Stories/Reels**: Primary text overlaid; keep under 72 chars +- **Right column**: Only headline visible; skip description +- **Audience Network**: Varies by publisher + +**Best practices:** +- Front-load the hook in primary text (first 125 chars) +- Use line breaks for readability in longer primary text +- Emojis: test, but don't overuse — 1-2 per ad max +- Questions in primary text increase engagement +- Headline should be a clear CTA or value statement + +### Lead Ads (Instant Form) + +| Element | Limit | +|---------|-------| +| Greeting headline | 60 chars | +| Greeting description | 360 chars | +| Privacy policy text | 200 chars | + +--- + +## LinkedIn Ads + +### Single Image Ad + +| Element | Recommended | Maximum | Notes | +|---------|-------------|---------|-------| +| Intro text | 150 chars | 600 chars | Above the image; truncated after ~150 | +| Headline | 70 chars | 200 chars | Below the image | +| Description | 100 chars | 300 chars | Only shows on Audience Network | + +### Carousel Ad + +| Element | Limit | +|---------|-------| +| Intro text | 255 chars | +| Card headline | 45 chars | +| Card count | 2-10 cards | + +### Message Ad (InMail) + +| Element | Limit | +|---------|-------| +| Subject line | 60 chars | +| Message body | 1,500 chars | +| CTA button | 20 chars | + +### Text Ad + +| Element | Limit | +|---------|-------| +| Headline | 25 chars | +| Description | 75 chars | + +**LinkedIn-specific guidelines:** +- Professional tone, but not boring +- Use job-specific language the audience recognizes +- Statistics and data points perform well +- Avoid consumer-style hype ("Amazing!" "Incredible!") +- First-person testimonials from peers resonate + +--- + +## TikTok Ads + +### In-Feed Ads + +| Element | Recommended | Maximum | Notes | +|---------|-------------|---------|-------| +| Ad text | 80 chars | 100 chars | Above the video | +| Display name | N/A | 40 chars | Brand name | +| CTA button | Platform options | Predefined | Select from TikTok's options | + +### Spark Ads (Boosted Organic) + +| Element | Notes | +|---------|-------| +| Caption | Uses original post caption | +| CTA button | Added by advertiser | +| Display name | Original creator's handle | + +**TikTok-specific guidelines:** +- Native content outperforms polished ads +- First 2 seconds determine if they watch +- Use trending sounds and formats +- Text overlay is essential (most watch with sound off) +- Vertical video only (9:16) + +--- + +## Twitter/X Ads + +### Promoted Tweets + +| Element | Limit | Notes | +|---------|-------|-------| +| Tweet text | 280 chars | Full tweet with image/video | +| Card headline | 70 chars | Website card | +| Card description | 200 chars | Website card | + +### Website Cards + +| Element | Limit | +|---------|-------| +| Headline | 70 chars | +| Description | 200 chars | + +**Twitter/X-specific guidelines:** +- Conversational, casual tone +- Short sentences work best +- One clear message per tweet +- Hashtags: 1-2 max (0 is often better for ads) +- Threads can work for consideration-stage content + +--- + +## Character Counting Tips + +- **Spaces count** as characters on all platforms +- **Emojis** count as 1-2 characters depending on platform +- **Special characters** (|, &, etc.) count as 1 character +- **URLs** in body text count against limits +- **Dynamic keyword insertion** (`{KeyWord:default}`) can exceed limits — set safe defaults +- Always verify in the platform's ad preview before launching + +--- + +## Multi-Platform Creative Adaptation + +When creating for multiple platforms simultaneously, start with the most restrictive format: + +1. **Google Search headlines** (30 chars) — forces the tightest messaging +2. **Expand to Meta headlines** (40 chars) — add a word or two +3. **Expand to LinkedIn intro text** (150 chars) — add context and proof +4. **Expand to Meta primary text** (125+ chars) — full hook and value prop + +This cascading approach ensures your core message works everywhere, then gets enriched for platforms that allow more space. diff --git a/src/adclaw/agents/skills/marketing-ai-seo/SKILL.md b/src/adclaw/agents/skills/marketing-ai-seo/SKILL.md new file mode 100644 index 0000000..320a51c --- /dev/null +++ b/src/adclaw/agents/skills/marketing-ai-seo/SKILL.md @@ -0,0 +1,485 @@ +--- +name: ai-seo +description: "When the user wants to optimize content for AI search engines, get cited by LLMs, or appear in AI-generated answers. Also use when the user mentions 'AI SEO,' 'AEO,' 'GEO,' 'LLMO,' 'answer engine optimization,' 'generative engine optimization,' 'LLM optimization,' 'AI Overviews,' 'optimize for ChatGPT,' 'optimize for Perplexity,' 'AI citations,' 'AI visibility,' 'zero-click search,' 'how do I show up in AI answers,' 'LLM mentions,' or 'optimize for Claude/Gemini.' Use this whenever someone wants their content to be cited or surfaced by AI assistants and AI search engines. For traditional technical and on-page SEO audits, see seo-audit. For structured data implementation, see schema." +metadata: + version: 2.0.1 +--- + +# AI SEO + +You are an expert in AI search optimization — the practice of making content discoverable, extractable, and citable by AI systems including Google AI Overviews, ChatGPT, Perplexity, Claude, Gemini, and Copilot. Your goal is to help users get their content cited as a source in AI-generated answers. + +## Before Starting + +**Check for product marketing context first:** +If `.agents/product-marketing.md` exists (or `.claude/product-marketing.md`, or the legacy `product-marketing-context.md` filename, in older setups), read it before asking questions. Use that context and only ask for information not already covered or specific to this task. + +Gather this context (ask if not provided): + +### 1. Current AI Visibility +- Do you know if your brand appears in AI-generated answers today? +- Have you checked ChatGPT, Perplexity, or Google AI Overviews for your key queries? +- What queries matter most to your business? + +### 2. Content & Domain +- What type of content do you produce? (Blog, docs, comparisons, product pages) +- What's your domain authority / traditional SEO strength? +- Do you have existing structured data (schema markup)? + +### 3. Goals +- Get cited as a source in AI answers? +- Appear in Google AI Overviews for specific queries? +- Compete with specific brands already getting cited? +- Optimize existing content or create new AI-optimized content? + +### 4. Competitive Landscape +- Who are your top competitors in AI search results? +- Are they being cited where you're not? + +--- + +## How AI Search Works + +### The AI Search Landscape + +| Platform | How It Works | Source Selection | +|----------|-------------|----------------| +| **Google AI Overviews** | Summarizes top-ranking pages | Strong correlation with traditional rankings | +| **ChatGPT (with search)** | Searches web, cites sources | Draws from wider range, not just top-ranked | +| **Perplexity** | Always cites sources with links | Favors authoritative, recent, well-structured content | +| **Gemini** | Google's AI assistant | Pulls from Google index + Knowledge Graph | +| **Copilot** | Bing-powered AI search | Bing index + authoritative sources | +| **Claude** | Brave Search (when enabled) | Training data + Brave search results | + +For a deep dive on how each platform selects sources and what to optimize per platform, see [references/platform-ranking-factors.md](references/platform-ranking-factors.md). + +### Key Difference from Traditional SEO + +Traditional SEO gets you ranked. AI SEO gets you **cited**. + +In traditional search, you need to rank on page 1. In AI search, a well-structured page can get cited even if it ranks on page 2 or 3 — AI systems select sources based on content quality, structure, and relevance, not just rank position. + +**Critical stats:** +- AI Overviews appear in ~45% of Google searches +- AI Overviews reduce clicks to websites by up to 58% +- Brands are 6.5x more likely to be cited via third-party sources than their own domains +- Optimized content gets cited 3x more often than non-optimized +- Statistics and citations boost visibility by 40%+ across queries + +### Google's Official Stance vs. Multi-Platform Reality + +This is important to read once before doing anything else. + +**Google's position** ([AI features optimization guide](https://developers.google.com/search/docs/fundamentals/ai-optimization-guide)): +> "The best practices for SEO continue to be relevant because our generative AI features on Google Search are rooted in our core Search ranking and quality systems." + +Google explicitly says: +- **No special markup or files are required** for AI Overviews or AI Mode +- **Don't chunk content for AI** — write for people, organize with normal headings and paragraphs +- **Don't write separate content for AI** — that risks "scaled content abuse" spam policy +- **Helpful, reliable, people-first content** wins — same E-E-A-T standards as regular Search +- **No AI-specific Search Console reporting** — use standard SEO metrics + +**Other AI engines (ChatGPT, Claude, Perplexity, Copilot) behave differently:** +- They actively reward extractable structure — passages, FAQs, comparison tables, definition blocks +- They parse `llms.txt`, structured pricing pages, and machine-readable files when present +- They cite third-party sources (Reddit, Wikipedia, review sites) more heavily than top-ranked pages + +**What this means for the work:** +- The structural patterns in this skill (40–60 word answer blocks, FAQ schema, comparison tables) help **non-Google AI engines** materially. They also don't hurt Google — they're just normal good content organization. +- For Google AI Overviews / AI Mode specifically: optimize for people and core Search, full stop. Strong E-E-A-T, original information, semantic HTML, clean indexability. +- For ChatGPT/Claude/Perplexity: layer on the extractable structure + llms.txt + machine-readable files. + +When in doubt, default to "write for people, organize for clarity" — that satisfies both camps. + +### Query Fan-Out (Google AI Search) + +Google's AI features don't just answer the one query a user typed — they generate **concurrent, related queries** under the hood and retrieve results for each. + +Google's own example: a user asking "how to fix lawns" triggers fan-out queries about herbicides, chemical-free removal, weed prevention, etc. The AI synthesizes across all of them. + +**Implications:** +- Single-page-per-keyword targeting is less effective. Cover the **full topical cluster** so you're retrievable for the fan-out variants too. +- Long-tail intent matters less than topical authority — Google's AI systems understand synonyms and semantic equivalence. +- A page that comprehensively answers a parent topic (with sub-questions covered) will be retrieved more often than narrow per-query pages. + +**Action**: when planning content, brainstorm the 5–10 related queries the AI is likely to fan out to and make sure your content (or your site as a whole) covers them. + +--- + +## AI Visibility Audit + +Before optimizing, assess your current AI search presence. + +### Step 1: Check AI Answers for Your Key Queries + +Test 10-20 of your most important queries across platforms: + +| Query | Google AI Overview | ChatGPT | Perplexity | You Cited? | Competitors Cited? | +|-------|:-----------------:|:-------:|:----------:|:----------:|:-----------------:| +| [query 1] | Yes/No | Yes/No | Yes/No | Yes/No | [who] | +| [query 2] | Yes/No | Yes/No | Yes/No | Yes/No | [who] | + +**Query types to test:** +- "What is [your product category]?" +- "Best [product category] for [use case]" +- "[Your brand] vs [competitor]" +- "How to [problem your product solves]" +- "[Your product category] pricing" + +### Step 2: Analyze Citation Patterns + +When your competitors get cited and you don't, examine: +- **Content structure** — Is their content more extractable? +- **Authority signals** — Do they have more citations, stats, expert quotes? +- **Freshness** — Is their content more recently updated? +- **Schema markup** — Do they have structured data you're missing? +- **Third-party presence** — Are they cited via Wikipedia, Reddit, review sites? + +### Step 3: Content Extractability Check + +For each priority page, verify: + +| Check | Pass/Fail | +|-------|-----------| +| Clear definition in first paragraph? | | +| Self-contained answer blocks (work without surrounding context)? | | +| Statistics with sources cited? | | +| Comparison tables for "[X] vs [Y]" queries? | | +| FAQ section with natural-language questions? | | +| Schema markup (FAQ, HowTo, Article, Product)? | | +| Expert attribution (author name, credentials)? | | +| Recently updated (within 6 months)? | | +| Heading structure matches query patterns? | | +| AI bots allowed in robots.txt? | | + +### Step 4: AI Bot Access Check + +Verify your robots.txt allows AI crawlers. Each AI platform has its own bot, and blocking it means that platform can't cite you: + +- **GPTBot** and **ChatGPT-User** — OpenAI (ChatGPT) +- **PerplexityBot** — Perplexity +- **ClaudeBot** and **anthropic-ai** — Anthropic (Claude) +- **Google-Extended** — Google Gemini and AI Overviews +- **Bingbot** — Microsoft Copilot (via Bing) + +Check your robots.txt for `Disallow` rules targeting any of these. If you find them blocked, you have a business decision to make: blocking prevents AI training on your content but also prevents citation. One middle ground is blocking training-only crawlers (like **CCBot** from Common Crawl) while allowing the search bots listed above. + +See [references/platform-ranking-factors.md](references/platform-ranking-factors.md) for the full robots.txt configuration. + +--- + +## Optimization Strategy + +### The Three Pillars + +``` +1. Structure (make it extractable) +2. Authority (make it citable) +3. Presence (be where AI looks) +``` + +### Pillar 1: Structure — Make Content Extractable + +AI systems extract passages, not pages. Every key claim should work as a standalone statement. + +**Content block patterns:** +- **Definition blocks** for "What is X?" queries +- **Step-by-step blocks** for "How to X" queries +- **Comparison tables** for "X vs Y" queries +- **Pros/cons blocks** for evaluation queries +- **FAQ blocks** for common questions +- **Statistic blocks** with cited sources + +For detailed templates for each block type, see [references/content-patterns.md](references/content-patterns.md). + +**Structural rules:** +- Lead every section with a direct answer (don't bury it) +- Keep key answer passages to 40-60 words (optimal for snippet extraction) +- Use H2/H3 headings that match how people phrase queries +- Tables beat prose for comparison content +- Numbered lists beat paragraphs for process content +- Each paragraph should convey one clear idea + +### Pillar 2: Authority — Make Content Citable + +AI systems prefer sources they can trust. Build citation-worthiness. + +**The Princeton GEO research** (KDD 2024, studied across Perplexity.ai) ranked 9 optimization methods: + +| Method | Visibility Boost | How to Apply | +|--------|:---------------:|--------------| +| **Cite sources** | +40% | Add authoritative references with links | +| **Add statistics** | +37% | Include specific numbers with sources | +| **Add quotations** | +30% | Expert quotes with name and title | +| **Authoritative tone** | +25% | Write with demonstrated expertise | +| **Improve clarity** | +20% | Simplify complex concepts | +| **Technical terms** | +18% | Use domain-specific terminology | +| **Unique vocabulary** | +15% | Increase word diversity | +| **Fluency optimization** | +15-30% | Improve readability and flow | +| ~~Keyword stuffing~~ | **-10%** | **Actively hurts AI visibility** | + +**Best combination:** Fluency + Statistics = maximum boost. Low-ranking sites benefit even more — up to 115% visibility increase with citations. + +**Statistics and data** (+37-40% citation boost) +- Include specific numbers with sources +- Cite original research, not summaries of research +- Add dates to all statistics +- Original data beats aggregated data + +**Expert attribution** (+25-30% citation boost) +- Named authors with credentials +- Expert quotes with titles and organizations +- "According to [Source]" framing for claims +- Author bios with relevant expertise + +**Freshness signals** +- "Last updated: [date]" prominently displayed +- Regular content refreshes (quarterly minimum for competitive topics) +- Current year references and recent statistics +- Remove or update outdated information + +**E-E-A-T alignment** +- First-hand experience demonstrated +- Specific, detailed information (not generic) +- Transparent sourcing and methodology +- Clear author expertise for the topic + +### Pillar 3: Presence — Be Where AI Looks + +AI systems don't just cite your website — they cite where you appear. + +**Third-party sources matter more than your own site:** +- Wikipedia mentions (7.8% of all ChatGPT citations) +- Reddit discussions (1.8% of ChatGPT citations) +- Industry publications and guest posts +- Review sites (G2, Capterra, TrustRadius for B2B SaaS) +- YouTube (frequently cited by Google AI Overviews) +- Quora answers + +**Actions:** +- Ensure your Wikipedia page is accurate and current +- Participate authentically in Reddit communities +- Get featured in industry roundups and comparison articles +- Maintain updated profiles on relevant review platforms +- Create YouTube content for key how-to queries +- Answer relevant Quora questions with depth + +### Machine-Readable Files for AI Agents + +> **Google's stance**: not required for AI Overviews or AI Mode. Their guide explicitly says you don't need new markup, AI files, or markdown to appear in generative AI search. +> +> **Why include them anyway**: non-Google AI engines (ChatGPT, Claude, Perplexity) and autonomous buying agents do reward extractable structure. The files below help with those engines without harming Google. + +AI agents aren't just answering questions — they're becoming buyers. When an AI agent evaluates tools on behalf of a user, it needs structured, parseable information. If your pricing is locked in a JavaScript-rendered page or a "contact sales" wall, agents will skip you and recommend competitors whose information they can actually read. + +Add these machine-readable files to your site root: + +**`/pricing.md` or `/pricing.txt`** — Structured pricing data for AI agents + +```markdown +# Pricing — [Your Product Name] + +## Free +- Price: $0/month +- Limits: 100 emails/month, 1 user +- Features: Basic templates, API access + +## Pro +- Price: $29/month (billed annually) | $35/month (billed monthly) +- Limits: 10,000 emails/month, 5 users +- Features: Custom domains, analytics, priority support + +## Enterprise +- Price: Custom — contact sales@example.com +- Limits: Unlimited emails, unlimited users +- Features: SSO, SLA, dedicated account manager +``` + +**Why this matters now:** +- AI agents increasingly compare products programmatically before a human ever visits your site +- Opaque pricing gets filtered out of AI-mediated buying journeys +- A simple markdown file is trivially parseable by any LLM — no rendering, no JavaScript, no login walls +- Same principle as `robots.txt` (for crawlers), `llms.txt` (for AI context), and `AGENTS.md` (for agent capabilities) + +**Best practices:** +- Use consistent units (monthly vs. annual, per-seat vs. flat) +- Include specific limits and thresholds, not just feature names +- List what's included at each tier, not just what's different +- Keep it updated — stale pricing is worse than no file +- Link to it from your sitemap and main pricing page + +**`/llms.txt`** — Context file for AI systems (see [llmstxt.org](https://llmstxt.org)) + +If you don't have one yet, add an `llms.txt` that gives AI systems a quick overview of what your product does, who it's for, and links to key pages (including your pricing). + +### Schema Markup for AI + +Structured data helps AI systems understand your content. Key schemas: + +| Content Type | Schema | Why It Helps | +|-------------|--------|-------------| +| Articles/Blog posts | `Article`, `BlogPosting` | Author, date, topic identification | +| How-to content | `HowTo` | Step extraction for process queries | +| FAQs | `FAQPage` | Direct Q&A extraction | +| Products | `Product` | Pricing, features, reviews | +| Comparisons | `ItemList` | Structured comparison data | +| Reviews | `Review`, `AggregateRating` | Trust signals | +| Organization | `Organization` | Entity recognition | + +Content with proper schema shows 30-40% higher AI visibility on non-Google AI engines. **Google's note**: structured data is "not required for generative AI search" but is recommended for overall SEO strategy. For implementation, use the **schema** skill. + +--- + +## Agentic Experiences + +Beyond AI search engines summarizing content, autonomous agents are starting to access sites directly — clicking, reading, comparing, even buying on behalf of users. Google's guide flags this as an emerging category to plan for. + +**How agents access your site:** +- **Visual rendering** — they screenshot/read the page like a user would +- **DOM inspection** — they parse the page's HTML structure +- **Accessibility tree** — they rely on the same semantic information assistive tech uses (labels, roles, landmarks, headings) + +**What to do:** +- **Render meaningful content without heavy JS gymnastics** — if the page is blank until 4 frameworks finish loading, agents see blank +- **Semantic HTML** — use `
`, `