commit b14c6b508a861ea1ad7c63c0dd40aba0ebf9c599 Author: saquib Date: Tue Sep 8 11:00:05 2026 +0530 latest code diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..8a11324 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,54 @@ +.git +.gitignore +.gitattributes +.env +.env.* +!.env.example +__pycache__/ +*.py[cod] +*$py.class +*.so +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.coverage +.coverage.* +htmlcov/ +.pytype/ +.pyre/ +*.egg-info/ +dist/ +build/ +.eggs/ +.venv/ +venv/ +env/ +ENV/ +.vscode/ +.idea/ +*.swp +*.swo +*~ +.DS_Store +Thumbs.db +storage_drive/ +storage/ +uploads/ +downloads/ +temp/ +tmp/ +Dockerfile.* +docker-compose*.yml +.docker/ +*.log +logs/ +.pytest_cache/ +htmlcov/ +coverage.xml +redis_data/ +qdrant_data/ +clamav_data/ +*.md +requirements*.txt +openapi.json +.pre-commit-config.yaml diff --git a/.env.development b/.env.development new file mode 100644 index 0000000..da7bce5 --- /dev/null +++ b/.env.development @@ -0,0 +1,68 @@ +APP_ENV=development +HOST=0.0.0.0 +PORT=20001 +DEBUG=False +DB_HOST=106.51.105.22 +DB_PORT=5432 +DB_NAME=docqube_development +DB_USER=docqube_user +DB_PASSWORD=d20kMfypQE8Z +DB_SSLMODE=disable +PGBOUNCER_PORT=6432 +SAAS_TRUST_SECRET=docqube_trust_secret_2026 +SAAS_BASE_URL=http://localhost:8000 +SAAS_PUBLIC_KEY="" +APP_SECRET=Pk7rcnYeSE-9vgTuevzfvKJ5YAs6Bj0QjgFxyJW00oKMnmPlRxUfU8lAm33GAgxL +ALGORITHM=HS256 +ACCESS_TOKEN_EXPIRE_MINUTES=60 +REDIS_HOST=redis +REDIS_PORT=6379 +REDIS_PASSWORD=JVCzB612F573hYwqZZ +CLAMAV_HOST=clamav +CLAMAV_PORT=3310 +CLAMAV_TIMEOUT_SECONDS=20 +CLAMAV_CHUNK_SIZE=1048576 +STORAGE_DRIVE_DIR=storage_drive +API_BASE_URL=https://docqube.maskantech.in +GOOGLE_CLIENT_ID=989963514121-nh9ibk3e4u1dh6jp4ql17tue7f9suk38.apps.googleusercontent.com +GOOGLE_CLIENT_SECRET=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 +DEEPSEEK_API_KEY=sk-95235bd6d3b440d6b6e25bd722813f3c +DEEPSEEK_MODEL=deepseek-chat +DEEPSEEK_BASE_URL=https://api.deepseek.com/v1 +LLM_TEMPERATURE=0.35 +MAX_RESPONSE_TOKENS=1500 +HF_HUB_OFFLINE=0 +TRANSFORMERS_OFFLINE=0 +VECTOR_DB=qdrant +QDRANT_HOST=qdrant +QDRANT_PORT=6333 +QDRANT_COLLECTION=docqube_collection +RERANKER_MODEL=cross-encoder/ms-marco-MiniLM-L-12-v2 +EMBEDDING_MODEL=BAAI/bge-base-en-v1.5 +CHUNK_SIZE=850 +CHUNK_OVERLAP=180 +TOP_K_QA=3 +TOP_K_SUMMARY=4 +MAX_CONTEXT_CHARS=4000 +MIN_SIMILARITY_SCORE=0.18 +LOG_LEVEL=WARNING +CORS_ORIGINS=https://docqube.maskantech.in +SMTP_HOST=smtp.gmail.com +SMTP_PORT=465 +SMTP_USER=info.maskantech@gmail.com +SMTP_PASSWORD=tuthpljtkrwchgxd +MAIL_FROM=info.maskantech@gmail.com +SMTP_SECURE=True +FRONTEND_URL=https://docqube.maskantech.in +CHAT_REDIS_MAX_MESSAGES=10 +CHAT_REDIS_TTL_SECONDS=86400 +CHAT_DAILY_CREDITS_LIMIT=1000 +DOCUSEAL_API_URL=https://api.docuseal.com +DOCUSEAL_API_KEY=prUReaHxSnChGoeeL7xnf5kxsXVLAzfCbn1RvbhnoMz +DOCUSEAL_ACCOUNT_EMAIL=info.maskantech+test@gmail.com +ZOHO_CLIENT_ID="1000.EX179O66BC7FKAWMR17Q4V4ZV4OW9K" +ZOHO_CLIENT_SECRET="9c8e6414aebdbed66b5bc5f9c2a866fb4b4b159a93" +ZOHO_REFRESH_TOKEN="1000.58b1696404de858cd887a29005879ed5.487b427449d87b3fd0f1cb883f4d1b7d" +ZOHO_WEBHOOK_SECRET="0iWuLqg4z41RMWmgI6ql5qbPE3fCE4tc" +ZOHO_DOMAIN="in" +MAX_ACTIVE_DEVICES=5 \ No newline at end of file diff --git a/.env.localdev b/.env.localdev new file mode 100644 index 0000000..38ff103 --- /dev/null +++ b/.env.localdev @@ -0,0 +1,132 @@ +# Environment Configuration +APP_ENV=localdev + +# Server Configuration +HOST=127.0.0.1 +PORT=20001 +DEBUG=True + +# DB_HOST=localhost +# DB_PORT=5432 +# DB_NAME=new-base +# DB_USER=postgres +# DB_PASSWORD=ribai123 +# DB_SSLMODE=disable +# PGBOUNCER_PORT=5432 +DB_HOST=106.51.105.22 +DB_PORT=5432 +DB_NAME=docqube_local +DB_USER=docqube_user +DB_PASSWORD=d20kMfypQE8Z +DB_SSLMODE=disable +PGBOUNCER_PORT=6432 + +# Security Configuration +APP_SECRET=VBN20DIyTbuqRJKKpxB2C4aDVYRUENa1rdzFXPLXwxA +ALGORITHM=HS256 +ACCESS_TOKEN_EXPIRE_MINUTES=60 + +# Redis Configuration +REDIS_HOST=106.51.105.22 +REDIS_PORT=6388 +REDIS_PASSWORD=JVCzB612F573hYwqZZ + +CLAMAV_HOST=106.51.105.22 +CLAMAV_PORT=3310 +CLAMAV_TIMEOUT_SECONDS=20 +CLAMAV_CHUNK_SIZE=1048576 +STORAGE_DRIVE_DIR=storage_drive +API_BASE_URL=http://localhost:20001 + + +#Saas +SAAS_TRUST_SECRET=VBN20DIyTbuqRJKKpxB2C4aDVYRUENa1rdzFXPLXwxA +SAAS_BASE_URL=http://localhost:8000 +SAAS_PUBLIC_KEY="" + +# Google Auth Configuration +GOOGLE_CLIENT_ID=989963514121-nh9ibk3e4u1dh6jp4ql17tue7f9suk38.apps.googleusercontent.com +GOOGLE_CLIENT_SECRET=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 + +# AI & Models Configuration +# ========================= +# AI (DeepSeek) +# ========================= +DEEPSEEK_API_KEY=sk-95235bd6d3b440d6b6e25bd722813f3c +DEEPSEEK_MODEL=deepseek-chat +DEEPSEEK_BASE_URL=https://api.deepseek.com/v1 +LLM_TEMPERATURE=0.35 +MAX_RESPONSE_TOKENS=1500 + +# Extraction / NVIDIA AI Configuration +EXTRACTION_API_KEY="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" +EXTRACTION_BASE_URL=https://integrate.api.nvidia.com/v1 +EXTRACTION_MODEL=mistralai/mistral-medium-3-instruct +EXTRACTION_VISION_MODEL=meta/llama-3.2-90b-vision-instruct + + +# Force Offline Mode for local models (Prevents startup delays/HF checks) +HF_HUB_OFFLINE=0 +TRANSFORMERS_OFFLINE=0 + +# Vector DB Configuration +VECTOR_DB=qdrant +QDRANT_HOST=localhost +QDRANT_PORT=6333 +QDRANT_COLLECTION=docqube_collection +RERANKER_MODEL=cross-encoder/ms-marco-MiniLM-L-12-v2 + +# ========================= +# EMBEDDING +# ========================= +EMBEDDING_MODEL=BAAI/bge-base-en-v1.5 +# RAG SETTINGS +# ========================= +CHUNK_SIZE=850 +CHUNK_OVERLAP=180 +TOP_K_QA=3 +TOP_K_SUMMARY=4 +MAX_CONTEXT_CHARS=4000 +MIN_SIMILARITY_SCORE=0.18 + +# Logging Configuration +LOG_LEVEL=INFO + +# CORS Configuration +CORS_ORIGINS=* + +# Email Configuration +SMTP_HOST=smtp.gmail.com +SMTP_PORT=465 +SMTP_USER=info.maskantech@gmail.com +SMTP_PASSWORD=ftqjghdbthlwpnsh +MAIL_FROM=info.maskantech@gmail.com +SMTP_SECURE=True + + +# Frontend Configuration +FRONTEND_URL=http://localhost:5173 + +# whisper Model Configuration +WHISPER_MODEL=small + + +# Chat Redis Settings +CHAT_REDIS_MAX_MESSAGES=10 +CHAT_REDIS_TTL_SECONDS=86400 +CHAT_DAILY_CREDITS_LIMIT=1000 + +# Docuseal API Settings +# DOCUSEAL_API_URL=https://api.docuseal.com +# DOCUSEAL_API_KEY=prUReaHxSnChGoeeL7xnf5kxsXVLAzfCbn1RvbhnoMz +# DOCUSEAL_ACCOUNT_EMAIL=info.maskantech+test@gmail.com + +#ZOHO API Settings +# ZOHO_CLIENT_ID="1000.EX179O66BC7FKAWMR17Q4V4ZV4OW9K" +# ZOHO_CLIENT_SECRET="9c8e6414aebdbed66b5bc5f9c2a866fb4b4b159a93" +# ZOHO_REFRESH_TOKEN="1000.58b1696404de858cd887a29005879ed5.487b427449d87b3fd0f1cb883f4d1b7d" +ZOHO_WEBHOOK_SECRET="0iWuLqg4z41RMWmgI6ql5qbPE3fCE4tc" +ZOHO_DOMAIN="in" +MAX_ACTIVE_DEVICES=10 + + diff --git a/.env.production b/.env.production new file mode 100644 index 0000000..c78fa98 --- /dev/null +++ b/.env.production @@ -0,0 +1,97 @@ +# Environment Configuration +APP_ENV=production + +# Server Configuration +HOST=127.0.0.1 +PORT=20001 +DEBUG=True + +DB_HOST=127.0.0.1 +DB_PORT=5432 +DB_NAME=prod_docqube +DB_USER=prod_user +DB_PASSWORD='GO5cA8hK3@47Uq9im44' +DB_SSLMODE=disable +PGBOUNCER_PORT=6432 + + +#saas +SAAS_TRUST_SECRET=docqube_trust_secret_2026 +SAAS_BASE_URL=http://localhost:8000 +SAAS_PUBLIC_KEY="" + +# Security Configuration +APP_SECRET=VBN20DIyTbuqRJKKpxB2C4aDVYRUENa1rdzFXPLXwxA +ALGORITHM=HS256 +ACCESS_TOKEN_EXPIRE_MINUTES=60 + +# Redis Configuration +REDIS_HOST=31.57.228.186 +REDIS_PORT=6379 +REDIS_PASSWORD=TP0TmTqyWzBIuvrE + +CLAMAV_HOST=31.57.228.186 +CLAMAV_PORT=3310 +CLAMAV_TIMEOUT_SECONDS=20 +CLAMAV_CHUNK_SIZE=1048576 +STORAGE_DRIVE_DIR=storage_drive +API_BASE_URL=https://api.docqube.com + +# Google Auth Configuration +GOOGLE_CLIENT_ID=989963514121-nh9ibk3e4u1dh6jp4ql17tue7f9suk38.apps.googleusercontent.com +GOOGLE_CLIENT_SECRET=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 + + +# ========================= +# AI (DeepSeek) +# ========================= +DEEPSEEK_API_KEY=sk-f51cbc9eca6b466d949bfdd60a892a97 +DEEPSEEK_MODEL=deepseek-chat +DEEPSEEK_BASE_URL=https://api.deepseek.com/v1 + +# ========================= +# RAG +# ========================= +EMBEDDING_MODEL=BAAI/bge-base-en-v1.5 +CHUNK_SIZE=300 +CHUNK_OVERLAP=80 +TOP_K=3 +# Logging Configuration +LOG_LEVEL=INFO + +# CORS Configuration +CORS_ORIGINS=https://docqube.com,https://api.docqube.com +COOKIE_DOMAIN=.docqube.com +DISABLE_CSRF=True + +# Email Configuration +SMTP_HOST=smtp.hostinger.com +SMTP_PORT=465 +SMTP_USER=support@docqube.com +SMTP_PASSWORD=DQsupport@765$ +MAIL_FROM=support@docqube.com +SMTP_SECURE=True + +# Frontend Configuration +FRONTEND_URL=https://docqube.com + +# whisper Model Configuration +WHISPER_MODEL=small + +# Chat Redis Settings +CHAT_REDIS_MAX_MESSAGES=10 +CHAT_REDIS_TTL_SECONDS=86400 +CHAT_DAILY_CREDITS_LIMIT=1000 + + +# Docuseal API Settings +DOCUSEAL_API_URL=https://api.docuseal.com +DOCUSEAL_API_KEY=prUReaHxSnChGoeeL7xnf5kxsXVLAzfCbn1RvbhnoMz +DOCUSEAL_ACCOUNT_EMAIL=info.maskantech+test@gmail.com + +#ZOHO API Settings +ZOHO_CLIENT_ID="1000.EX179O66BC7FKAWMR17Q4V4ZV4OW9K" +ZOHO_CLIENT_SECRET="9c8e6414aebdbed66b5bc5f9c2a866fb4b4b159a93" +ZOHO_REFRESH_TOKEN="1000.58b1696404de858cd887a29005879ed5.487b427449d87b3fd0f1cb883f4d1b7d" +ZOHO_WEBHOOK_SECRET="0iWuLqg4z41RMWmgI6ql5qbPE3fCE4tc" +ZOHO_DOMAIN="in" \ No newline at end of file diff --git a/.env.test b/.env.test new file mode 100644 index 0000000..0bda686 --- /dev/null +++ b/.env.test @@ -0,0 +1,68 @@ +APP_ENV=test +HOST=0.0.0.0 +PORT=20002 +DEBUG=False +DB_HOST=106.51.105.22 +DB_PORT=5432 +DB_NAME=docqube_test +DB_USER=docqube_user +DB_PASSWORD=d20kMfypQE8Z +DB_SSLMODE=disable +PGBOUNCER_PORT=6432 +SAAS_TRUST_SECRET=docqube_trust_secret_2026 +SAAS_BASE_URL=http://localhost:8000 +SAAS_PUBLIC_KEY="" +APP_SECRET=VBN20DIyTbuqRJKKpxB2C4aDVYRUENa1rdzFXPLXwxA +ALGORITHM=HS256 +ACCESS_TOKEN_EXPIRE_MINUTES=60 +REDIS_HOST=redis +REDIS_PORT=6379 +REDIS_PASSWORD=JVCzB612F573hYwqZZ +CLAMAV_HOST=clamav +CLAMAV_PORT=3310 +CLAMAV_TIMEOUT_SECONDS=20 +CLAMAV_CHUNK_SIZE=1048576 +STORAGE_DRIVE_DIR=storage_drive +API_BASE_URL=https://docqubeapi-test.maskantech.in +GOOGLE_CLIENT_ID=989963514121-nh9ibk3e4u1dh6jp4ql17tue7f9suk38.apps.googleusercontent.com +GOOGLE_CLIENT_SECRET=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 +DEEPSEEK_API_KEY=sk-43e2e5e6d450447687b3b664f3a91f33 +DEEPSEEK_MODEL=deepseek-chat +DEEPSEEK_BASE_URL=https://api.deepseek.com/v1 +LLM_TEMPERATURE=0.35 +MAX_RESPONSE_TOKENS=1500 +HF_HUB_OFFLINE=0 +TRANSFORMERS_OFFLINE=0 +VECTOR_DB=qdrant +QDRANT_HOST=qdrant +QDRANT_PORT=6333 +QDRANT_COLLECTION=docqube_collection +RERANKER_MODEL=cross-encoder/ms-marco-MiniLM-L-12-v2 +EMBEDDING_MODEL=BAAI/bge-base-en-v1.5 +CHUNK_SIZE=850 +CHUNK_OVERLAP=180 +TOP_K_QA=3 +TOP_K_SUMMARY=4 +MAX_CONTEXT_CHARS=4000 +MIN_SIMILARITY_SCORE=0.18 +LOG_LEVEL=INFO +CORS_ORIGINS=https://docqube-test.maskantech.in,https://docqubeapp-test.maskantech.in,https://docqubeapi-test.maskantech.in +SMTP_HOST=smtp.gmail.com +SMTP_PORT=465 +SMTP_USER=info.maskantech@gmail.com +SMTP_PASSWORD=tuthpljtkrwchgxd +MAIL_FROM=info.maskantech@gmail.com +SMTP_SECURE=True +FRONTEND_URL=https://docqube-test.maskantech.in,https://docqubeapp-test.maskantech.in,https://docqubeapi-test.maskantech.in +CHAT_REDIS_MAX_MESSAGES=10 +CHAT_REDIS_TTL_SECONDS=86400 +CHAT_DAILY_CREDITS_LIMIT=1000 +DOCUSEAL_API_URL=https://api.docuseal.com +DOCUSEAL_API_KEY=prUReaHxSnChGoeeL7xnf5kxsXVLAzfCbn1RvbhnoMz +DOCUSEAL_ACCOUNT_EMAIL=info.maskantech+test@gmail.com +ZOHO_CLIENT_ID="1000.EX179O66BC7FKAWMR17Q4V4ZV4OW9K" +ZOHO_CLIENT_SECRET="9c8e6414aebdbed66b5bc5f9c2a866fb4b4b159a93" +ZOHO_REFRESH_TOKEN="1000.58b1696404de858cd887a29005879ed5.487b427449d87b3fd0f1cb883f4d1b7d" +ZOHO_WEBHOOK_SECRET="0iWuLqg4z41RMWmgI6ql5qbPE3fCE4tc" +ZOHO_DOMAIN="in" +MAX_ACTIVE_DEVICES=5 \ No newline at end of file diff --git a/.env.uat b/.env.uat new file mode 100644 index 0000000..ca5430d --- /dev/null +++ b/.env.uat @@ -0,0 +1,68 @@ +# Environment Configuration +APP_ENV=uat + +# Server Configuration +HOST=127.0.0.1 +PORT=20001 +DEBUG=True + +DB_HOST=106.51.105.22 +DB_PORT=5432 +DB_NAME=docqube_uat +DB_USER=docqube_user +DB_PASSWORD=d20kMfypQE8Z +DB_SSLMODE=disable +PGBOUNCER_PORT=6432 + +# Security Configuration +APP_SECRET=VBN20DIyTbuqRJKKpxB2C4aDVYRUENa1rdzFXPLXwxA +ALGORITHM=HS256 +ACCESS_TOKEN_EXPIRE_MINUTES=60 + +# Redis Configuration +REDIS_HOST=106.51.105.22 +REDIS_PORT=6391 +REDIS_PASSWORD=JVCzB612F573hYwqZZ + +CLAMAV_HOST=106.51.105.22 +CLAMAV_PORT=3310 +CLAMAV_TIMEOUT_SECONDS=20 +CLAMAV_CHUNK_SIZE=1048576 +STORAGE_DRIVE_DIR=storage_drive +API_BASE_URL=http://localhost:20001 + +# Google Auth Configuration +GOOGLE_CLIENT_ID=989963514121-nh9ibk3e4u1dh6jp4ql17tue7f9suk38.apps.googleusercontent.com +GOOGLE_CLIENT_SECRET=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 + +# AI & Models Configuration +OPENAI_API_KEY="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" +OPENAI_BASE_URL=https://integrate.api.nvidia.com/v1 +OPENAI_MODEL=mistralai/mistral-medium-3-instruct +OPENAI_VISION_MODEL=meta/llama-3.2-90b-vision-instruct +MODEL_SERVER_URL=http://localhost:9000 + +# Logging Configuration +LOG_LEVEL=INFO + +# CORS Configuration +CORS_ORIGINS=http://localhost:5173,http://127.0.0.1:5173 + +# Email Configuration +SMTP_HOST=smtp.hostinger.com +SMTP_PORT=465 +SMTP_USER=info@maskantech.in +SMTP_PASSWORD=Infomaskan@123 +MAIL_FROM=info@maskantech.in +SMTP_SECURE=True + +# Frontend Configuration +FRONTEND_URL=http://localhost:5173 + +# whisper Model Configuration +WHISPER_MODEL=small + +# Chat Redis Settings +CHAT_REDIS_MAX_MESSAGES=10 +CHAT_REDIS_TTL_SECONDS=86400 +CHAT_DAILY_CREDITS_LIMIT=1000 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..67179d7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,79 @@ +# Python bytecode and caches +__pycache__/ +*.py[cod] +*$py.class +*.so + +# Virtual environments +.venv/ +venv/ +env/ +ENV/ +env1/ + +# Environment files and local secrets +!.env.example + +# Test, coverage, and static-analysis output +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.pytype/ +.pyre/ +.coverage +.coverage.* +coverage.xml +htmlcov/ +tests/ + +# Build and packaging output +build/ +dist/ +*.egg-info/ +.eggs/ +pip-wheel-metadata/ + +# Application runtime data +app/temp_uploads/* +storage_drive/ +storage/ +uploads/ +downloads/ +temp/ +tmp/ +logs/ +*.log + +# Local ML and Hugging Face caches +.cache/ +cache/ +model_cache/ +hub/ +!app/modules/**/models/ +!app/**/models/ + +# Local Docker/service data +.docker/ +redis_data/ +qdrant_data/ +clamav_data/ + +# IDE and editor files +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS files +.DS_Store +Thumbs.db +Desktop.ini +# Local rework environment — never commit +.env.rework +.env.rework_test +app/temp_uploads/* +!app/temp_uploads/.gitkeep +.env.rework_rls +.gitea/workflows/ +.github/workflows/ \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..81ce884 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,58 @@ +# B4.3 — the fast gates, before the commit rather than after the push. +# +# Only checks that run in about a second belong here. The characterization +# suite, the cross-tenant probes and the query budgets need a database and take +# a minute; they stay in CI. A pre-commit hook that takes a minute gets +# `--no-verify`d, and then none of it runs. +# +# pip install pre-commit && pre-commit install +# +# Everything here is also enforced in CI, so an uninstalled hook is a slower +# feedback loop rather than a hole. + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-json + - id: check-toml + - id: check-merge-conflict + - id: check-added-large-files + args: [--maxkb=512] + - id: detect-private-key + + - repo: local + hooks: + # The three rules that are cheap to check and expensive to discover late. + - id: no-exception-text-in-responses + name: No exception text in HTTP responses + entry: python -m pytest tests/probes/test_error_disclosure.py::test_no_exception_text_is_returned_to_callers -q + language: system + pass_filenames: false + files: ^app/.*\.py$ + description: > + `detail=f"...{e}"` sends internal errors to the caller. It reached an + unauthenticated endpoint once already. + + - id: transaction-discipline + name: Route handlers do not commit + entry: python -m pytest tests/probes/test_transaction_discipline.py -q + language: system + pass_filenames: false + files: ^app/.*\.py$ + description: > + The request owns the transaction. A handler that commits splits one + request into several and leaves partial state behind on failure. + + - id: layering + name: Import layering contracts + entry: lint-imports + language: system + pass_filenames: false + files: ^app/.*\.py$ + description: > + routes → controllers → services → repositories → models, one + direction only. diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..e028118 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.11.8 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8b0ce6e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,80 @@ +ARG PYTHON_VERSION=3.11.8 + +FROM python:${PYTHON_VERSION}-slim-bookworm AS builder + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + UV_COMPILE_BYTECODE=1 \ + UV_LINK_MODE=copy \ + UV_PYTHON_DOWNLOADS=0 + +COPY --from=ghcr.io/astral-sh/uv:0.12.8 /uv /uvx /bin/ + +WORKDIR /app + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + pkg-config \ + libcairo2-dev \ + libpango1.0-dev \ + libgdk-pixbuf-2.0-dev \ + libffi-dev \ + libglib2.0-dev \ + libjpeg-dev \ + libopenjp2-7-dev \ + && rm -rf /var/lib/apt/lists/* + +COPY pyproject.toml uv.lock .python-version ./ + +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync \ + --locked \ + --no-install-project \ + --no-editable + +FROM python:${PYTHON_VERSION}-slim-bookworm AS runtime + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app \ + PATH="/app/.venv/bin:$PATH" + +WORKDIR /app + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + libmagic1 \ + poppler-utils \ + libcairo2 \ + libpango-1.0-0 \ + libpangocairo-1.0-0 \ + libgdk-pixbuf-2.0-0 \ + libffi8 \ + libglib2.0-0 \ + libgl1 \ + libjpeg62-turbo \ + libopenjp2-7 \ + fonts-dejavu-core \ + fonts-liberation \ + shared-mime-info \ + && rm -rf /var/lib/apt/lists/* + +RUN addgroup --system app && \ + adduser --system \ + --ingroup app \ + --home /home/app \ + app + +COPY --from=builder --chown=app:app /app/.venv /app/.venv + +COPY --chown=app:app app ./app +COPY --chown=app:app manage.py . +COPY --chown=app:app alembic ./alembic +COPY --chown=app:app alembic.ini . +COPY --chown=app:app scripts ./scripts + +USER app + +CMD ["python", "manage.py", "run"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..b523c1f --- /dev/null +++ b/README.md @@ -0,0 +1,426 @@ +# Docqube Backend + +**FastAPI backend application for Docqube** + +A modern, container-ready backend built with FastAPI, managed by **uv**, and orchestrated with **Docker Compose**. + +--- + +## Table of Contents + +- [Requirements](#requirements) +- [Python Version](#python-version) +- [Installing uv](#installing-uv) +- [Getting Started](#getting-started) +- [Local Development](#local-development) +- [Dependency Management](#dependency-management) +- [Docker Compose](#docker-compose) +- [Running Celery Worker](#running-celery-worker) +- [Useful uv Commands](#useful-uv-commands) +- [Development Workflow](#development-workflow) +- [Source of Truth](#source-of-truth) + +--- + +## Requirements + +| Tool | Version / Notes | +|-------------------|--------------------------| +| **Python** | `3.11.8` (strict) | +| **uv** | Latest | +| **Docker** | Docker Desktop / Engine | +| **Docker Compose**| v2+ | + +--- + +## Python Version + +This project is **locked to Python 3.11**. + +```text +Python 3.11.8 +``` + +| File | Setting | +|-------------------|----------------------------------| +| `pyproject.toml` | `requires-python = ">=3.11,<3.12"` | +| `.python-version` | `3.11.8` | + +**Verify:** + +```bash +uv run python --version +``` + +> **Expected:** `Python 3.11.8` +> Do **not** use Python 3.12+ unless compatibility has been fully verified and the project configuration intentionally updated. + +--- + +## Installing uv + +### Windows + +```powershell +powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex" +``` + +Close and reopen PowerShell, then verify: + +```powershell +uv --version +``` + +### macOS / Linux + +```bash +curl -LsSf https://astral.sh/uv/install.sh | sh +``` + +Restart the terminal and verify: + +```bash +uv --version +``` + +--- + +## Getting Started + +```bash +# 1. Clone the repository +git clone +cd docqube_backend + +# 2. Install all dependencies +uv sync +``` + +`uv sync` will: + +- Create `.venv` automatically +- Use Python 3.11 +- Resolve dependencies from `pyproject.toml` +- Install exact versions from `uv.lock` +- Install Windows-specific packages when on Windows + +> You do **not** need to manually create or activate a virtual environment. + +--- + +## Local Development + +### Install dependencies + +```bash +uv sync +``` + +### Run the application + +```bash +uv run python manage.py run --env local +``` + +The application starts with the local environment configuration +(example: `http://127.0.0.1:20001`). +Host and port are controlled by the application settings. + +--- + +## Database Migrations & Seeding + +All database migrations and seeds are executed through `uv`: + +### 1. Apply Migrations (By Environment) + +**Local Environment:** +```bash +uv run python manage.py migrate --env local +``` + +**Test Environment:** +```bash +uv run python manage.py migrate --env test +``` + +**Production Environment:** +```bash +uv run python manage.py migrate --env production +``` + +*Or directly via Alembic (uses current environment variables):* +```bash +uv run alembic upgrade head +``` + +### 2. Create a New Migration (Autogenerate from Models) + +```bash +uv run alembic revision --autogenerate -m "your_migration_name" +``` + +### 3. Rollback the Last Migration + +```bash +uv run alembic downgrade -1 +``` + +### 4. Seed Database (By Environment) + +**Local:** +```bash +uv run python manage.py seed --env local +``` + +**Production:** +```bash +uv run python manage.py seed --env production +``` + +--- + +## Dependency Management + +All Python dependencies are managed exclusively through: + +- `pyproject.toml` +- `uv.lock` + +> **Do not** maintain a separate `requirements.txt`. + +### Add a production dependency + +```bash +uv add +``` + +Examples: + +```bash +uv add httpx +uv add "httpx==0.28.1" +uv add "pikepdf>=8.0.0,<11.0.0" +``` + +### Add a development dependency + +```bash +uv add --dev pytest +uv add --dev ruff +uv sync +``` + +### Remove a dependency + +```bash +uv remove +uv sync +``` + +### Upgrade dependencies + +**Single package:** + +```bash +uv lock --upgrade-package +uv sync +``` + +Example: + +```bash +uv lock --upgrade-package fastapi +uv sync +``` + +**Specific version:** + +```bash +uv add "fastapi==0.116.1" +``` + +**All packages** (use only for deliberate maintenance): + +```bash +uv lock --upgrade +uv sync +``` + +### Lockfile + +`uv.lock` **must** be committed to Git. +It guarantees identical dependency resolution across developer machines, CI, Docker, staging, and production. + +> **Never** edit `uv.lock` manually. + +### Inspection commands + +```bash +# Full dependency tree +uv tree + +# Find a package (Windows PowerShell) +uv tree | Select-String pikepdf + +# Find a package (Linux / macOS) +uv tree | grep pikepdf + +# Inspect an installed package +uv pip show pikepdf +uv pip show torch +``` + +--- + +## Docker Compose + +### Development environment + +```bash +# Start (build + run) +docker compose -p docqube-development -f docker-compose.development.yml up -d --build + +# Follow logs +docker compose -p docqube-development -f docker-compose.development.yml logs -f + +# Stop +docker compose -p docqube-development -f docker-compose.development.yml down + +# Normal rebuild +docker compose -p docqube-development -f docker-compose.development.yml up -d --build + +# Completely fresh build (no cache) +docker compose -p docqube-development -f docker-compose.development.yml build --no-cache +docker compose -p docqube-development -f docker-compose.development.yml up -d +``` + +--- + +## Running Celery Worker + +### Locally + +```bash +uv run celery -A app.tasks.celery_app:celery_app worker --loglevel=info +``` + +> The worker requires the correct Redis and environment configuration. + +### Recreate the virtual environment + +If `.venv` becomes corrupted: + +**Windows (PowerShell)** + +```powershell +Remove-Item -Recurse -Force .venv +uv sync +``` + +**Linux / macOS** + +```bash +rm -rf .venv +uv sync +``` + +**Force reinstall** + +```bash +uv sync --reinstall +``` + +--- + +## Useful uv Commands + +| Command | Description | +|----------------------------------------------|-------------------------------------------| +| `uv --version` | Check uv version | +| `uv run python --version` | Check Python version | +| `uv python find` | Locate the Python interpreter | +| `uv sync` | Install / synchronize dependencies | +| `uv add ` | Add a production dependency | +| `uv add --dev ` | Add a development dependency | +| `uv remove ` | Remove a dependency | +| `uv lock` | Update the lockfile | +| `uv lock --upgrade-package ` | Upgrade one package | +| `uv lock --upgrade` | Upgrade all packages | +| `uv tree` | View dependency tree | +| `uv pip show ` | Inspect an installed package | +| `uv run python` | Run Python inside the project environment | +| `uv run python manage.py run --env local` | Run the application | + +--- + +## Development Workflow + +### New developer quick start + +1. Clone the repository +2. Install `uv` +3. Run `uv sync` +4. Create the required environment file (e.g. `.env.development`) +5. Verify Python → `uv run python --version` → `Python 3.11.8` +6. Run locally **or** start the full stack with Docker Compose +7. Check services and view logs + +### Day-to-day workflow + +```bash +# Normal local development +uv sync +uv run python manage.py run --env local + +# Add a package +uv add + +# Remove a package +uv remove + +# Upgrade one package +uv lock --upgrade-package +uv sync +``` + +### Before creating a pull request + +```bash +uv sync +uv run python --version +uv run pytest +docker compose -p docqube-development -f docker-compose.development.yml build +docker compose -p docqube-development -f docker-compose.development.yml up -d +``` + +### Important rules + +**Do** + +- Use `uv sync`, `uv add`, `uv remove`, `uv run` +- Commit `pyproject.toml`, `uv.lock`, and `.python-version` +- Use Docker Compose for the full containerized environment + +**Do not** + +- Use `pip install ...` +- Manually create `env/` or `venv/` +- Manually edit `uv.lock` +- Commit `.venv/` or real `.env` files +- Maintain a manually synchronized `requirements.txt` + +--- + +## Source of Truth + +| Concern | Source of Truth | +|--------------------------------|----------------------------| +| Python version | `.python-version` | +| Python dependencies | `pyproject.toml` | +| Locked dependencies | `uv.lock` | +| Python / Docker environment | `Dockerfile` | +| Environment orchestration | `docker-compose.*.yml` | +| Environment configuration | `.env.*` | + + diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..0e0fb5b --- /dev/null +++ b/alembic.ini @@ -0,0 +1,116 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = alembic + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python-dateutil library that can be +# installed by adding `alembic[tz]` to the pip requirements +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = postgresql+psycopg2://dummy + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the exec runner, execute a binary +# hooks = ruff +# ruff.type = exec +# ruff.executable = %(here)s/.venv/bin/ruff +# ruff.options = --fix REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/README b/alembic/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..a918a1b --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,43 @@ +import sys +import os +from logging.config import fileConfig +from sqlalchemy import engine_from_config, pool +from alembic import context + +# Add project root to path +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from app.core.settings import DATABASE_URL +from app.db.database import Base + +# Import all models here so Alembic can detect them. +# +# `app.db.all_models` is the one that matters. Importing `app.main` reaches +# most models by accident — routes import what they use — but it missed +# `user_roles`, `access_groups` and `user_access_groups`, whose every import is +# function-local. Autogenerate reported all three as removed tables and offered +# to drop the SBAC schema. See tests/probes/test_model_registry.py. +import app.db.all_models # noqa: F401 +import app.main +from app.modules.tenant.models.tenant_contact_model import TenantContact + +target_metadata = Base.metadata + +config = context.config +config.set_main_option("sqlalchemy.url", DATABASE_URL.replace('%', '%%')) + +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +def run_migrations_online(): + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + +run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..fbc4b07 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/06001599321d_add_collaboration_chat.py b/alembic/versions/06001599321d_add_collaboration_chat.py new file mode 100644 index 0000000..b0202ae --- /dev/null +++ b/alembic/versions/06001599321d_add_collaboration_chat.py @@ -0,0 +1,26 @@ +"""add collaboration chat + +Revision ID: 06001599321d +Revises: 22dee9a1d631 +Create Date: 2026-02-26 16:13:54.998247 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '06001599321d' +down_revision: Union[str, None] = '22dee9a1d631' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/alembic/versions/0bffea34ee5f_disable_rls_temporarily.py b/alembic/versions/0bffea34ee5f_disable_rls_temporarily.py new file mode 100644 index 0000000..d656678 --- /dev/null +++ b/alembic/versions/0bffea34ee5f_disable_rls_temporarily.py @@ -0,0 +1,35 @@ +"""disable_rls_temporarily + +Revision ID: 0bffea34ee5f +Revises: 14b4aa5f0f5d +Create Date: 2026-09-05 12:01:30.634229 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '0bffea34ee5f' +down_revision: Union[str, None] = '14b4aa5f0f5d' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +RLS_TABLES = [ + "user_org_units", "org_unit_leads", "user_roles", "access_groups", "user_access_groups", + "org_units", "tenant_subscriptions", "tenant_limit_overrides", "chatbot_documents", + "drive_activities", "drive_files", "drive_folders", "notifications", "tenant_smtp_configs", + "user_files", "projects", "roles", "activity_logs", "tenant_storage_configs", + "chat_sessions", "signing_requests", "tenant_signature_configs", "users" +] + +def upgrade() -> None: + for table in RLS_TABLES: + op.execute(f"ALTER TABLE {table} DISABLE ROW LEVEL SECURITY") + +def downgrade() -> None: + for table in RLS_TABLES: + op.execute(f"ALTER TABLE {table} ENABLE ROW LEVEL SECURITY") diff --git a/alembic/versions/0e1a5f2cdea9_add_max_bytes_quota_to_user_storage_.py b/alembic/versions/0e1a5f2cdea9_add_max_bytes_quota_to_user_storage_.py new file mode 100644 index 0000000..f4b7857 --- /dev/null +++ b/alembic/versions/0e1a5f2cdea9_add_max_bytes_quota_to_user_storage_.py @@ -0,0 +1,38 @@ +"""Add max_bytes_quota to user_storage_usage + +Revision ID: 0e1a5f2cdea9 +Revises: 9a6db38b86a6 +Create Date: 2026-03-02 10:48:38.335359 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '0e1a5f2cdea9' +down_revision: Union[str, None] = '9a6db38b86a6' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('project_versions', sa.Column('drive_file_version_id', sa.Integer(), nullable=True)) + op.create_foreign_key(None, 'project_versions', 'drive_file_versions', ['drive_file_version_id'], ['id']) + op.add_column('user_files', sa.Column('is_deleted', sa.Boolean(), nullable=True)) + op.add_column('user_files', sa.Column('deleted_at', sa.TIMESTAMP(), nullable=True)) + op.add_column('user_storage_usage', sa.Column('max_bytes_quota', sa.BigInteger(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('user_storage_usage', 'max_bytes_quota') + op.drop_column('user_files', 'deleted_at') + op.drop_column('user_files', 'is_deleted') + op.drop_constraint(None, 'project_versions', type_='foreignkey') + op.drop_column('project_versions', 'drive_file_version_id') + # ### end Alembic commands ### diff --git a/alembic/versions/14b4aa5f0f5d_merge_multiple_heads.py b/alembic/versions/14b4aa5f0f5d_merge_multiple_heads.py new file mode 100644 index 0000000..fc6d6e1 --- /dev/null +++ b/alembic/versions/14b4aa5f0f5d_merge_multiple_heads.py @@ -0,0 +1,26 @@ +"""Merge multiple heads + +Revision ID: 14b4aa5f0f5d +Revises: e1_0_add_saas_plan_id, z005_add_virus_scan +Create Date: 2026-09-02 19:15:56.213495 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '14b4aa5f0f5d' +down_revision: Union[str, None] = ('e1_0_add_saas_plan_id', 'z005_add_virus_scan') +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/alembic/versions/15f935213b22_add_tenant_storage_settings.py b/alembic/versions/15f935213b22_add_tenant_storage_settings.py new file mode 100644 index 0000000..ea2fbb3 --- /dev/null +++ b/alembic/versions/15f935213b22_add_tenant_storage_settings.py @@ -0,0 +1,66 @@ +"""add tenant storage settings + +Revision ID: 15f935213b22 +Revises: k002_add_docs_tutorial +Create Date: 2026-04-17 15:49:28.721223 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '15f935213b22' +down_revision: Union[str, None] = 'k002_add_docs_tutorial' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _index_names(bind, table_name: str) -> set[str]: + inspector = sa.inspect(bind) + return {idx["name"] for idx in inspector.get_indexes(table_name)} + + +def upgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + + if not inspector.has_table('tenant_storage_configs'): + op.create_table('tenant_storage_configs', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('tenant_id', sa.UUID(), nullable=False), + sa.Column('b2_key_id', sa.String(length=255), nullable=False), + sa.Column('encrypted_b2_application_key', sa.String(length=512), nullable=False), + sa.Column('b2_quarantine_bucket', sa.String(length=255), nullable=False), + sa.Column('b2_clean_bucket', sa.String(length=255), nullable=False), + sa.Column('b2_endpoint', sa.String(length=512), nullable=True), + sa.Column('is_active', sa.Boolean(), nullable=False), + sa.Column('created_by_id', sa.Integer(), nullable=True), + sa.Column('updated_by_id', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['created_by_id'], ['users.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['updated_by_id'], ['users.id'], ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id') + ) + + existing_indexes = _index_names(bind, 'tenant_storage_configs') + if 'ix_tenant_storage_configs_tenant_id' not in existing_indexes: + with op.batch_alter_table('tenant_storage_configs', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_tenant_storage_configs_tenant_id'), ['tenant_id'], unique=True) + + +def downgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + + if inspector.has_table('tenant_storage_configs'): + existing_indexes = _index_names(bind, 'tenant_storage_configs') + if 'ix_tenant_storage_configs_tenant_id' in existing_indexes: + with op.batch_alter_table('tenant_storage_configs', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_tenant_storage_configs_tenant_id')) + + op.drop_table('tenant_storage_configs') diff --git a/alembic/versions/1cb413ffaaa0_add_saas_mapping_tables.py b/alembic/versions/1cb413ffaaa0_add_saas_mapping_tables.py new file mode 100644 index 0000000..a5db058 --- /dev/null +++ b/alembic/versions/1cb413ffaaa0_add_saas_mapping_tables.py @@ -0,0 +1,59 @@ +"""Add SaaS mapping tables + +Revision ID: 1cb413ffaaa0 +Revises: n002_add_signature_imprints +Create Date: 2026-04-23 23:16:32.782986 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '1cb413ffaaa0' +down_revision: Union[str, None] = 'n002_add_signature_imprints' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # --- SaaSTenantMapping --- + op.create_table( + 'saas_tenant_mappings', + sa.Column('id', sa.dialects.postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('saas_tenant_id', sa.String(), nullable=False), + sa.Column('docqube_tenant_id', sa.dialects.postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('metadata', sa.JSON(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), + sa.ForeignKeyConstraint(['docqube_tenant_id'], ['tenants.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_saas_tenant_mappings_id'), 'saas_tenant_mappings', ['id'], unique=False) + op.create_index(op.f('ix_saas_tenant_mappings_saas_tenant_id'), 'saas_tenant_mappings', ['saas_tenant_id'], unique=True) + + # --- SaaSUserMapping --- + op.create_table( + 'saas_user_mappings', + sa.Column('id', sa.dialects.postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('saas_user_id', sa.String(), nullable=False), + sa.Column('docqube_user_id', sa.Integer(), nullable=False), + sa.Column('metadata', sa.JSON(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), + sa.ForeignKeyConstraint(['docqube_user_id'], ['users.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_saas_user_mappings_id'), 'saas_user_mappings', ['id'], unique=False) + op.create_index(op.f('ix_saas_user_mappings_saas_user_id'), 'saas_user_mappings', ['saas_user_id'], unique=True) + + +def downgrade() -> None: + op.drop_index(op.f('ix_saas_user_mappings_saas_user_id'), table_name='saas_user_mappings') + op.drop_index(op.f('ix_saas_user_mappings_id'), table_name='saas_user_mappings') + op.drop_table('saas_user_mappings') + op.drop_index(op.f('ix_saas_tenant_mappings_saas_tenant_id'), table_name='saas_tenant_mappings') + op.drop_index(op.f('ix_saas_tenant_mappings_id'), table_name='saas_tenant_mappings') + op.drop_table('saas_tenant_mappings') diff --git a/alembic/versions/1f4d2a8c9b77_add_missing_vector_columns.py b/alembic/versions/1f4d2a8c9b77_add_missing_vector_columns.py new file mode 100644 index 0000000..ce4314e --- /dev/null +++ b/alembic/versions/1f4d2a8c9b77_add_missing_vector_columns.py @@ -0,0 +1,67 @@ +"""add missing vector columns + +Revision ID: 1f4d2a8c9b77 +Revises: fb6810c36ffb +Create Date: 2026-04-08 22:05:00 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "1f4d2a8c9b77" +down_revision: Union[str, None] = "fb6810c36ffb" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _table_exists(bind, table_name: str) -> bool: + return sa.inspect(bind).has_table(table_name) + + +def _column_exists(bind, table_name: str, column_name: str) -> bool: + inspector = sa.inspect(bind) + return column_name in {col["name"] for col in inspector.get_columns(table_name)} + + +def upgrade() -> None: + bind = op.get_bind() + + if _table_exists(bind, "vector_indices"): + if not _column_exists(bind, "vector_indices", "index_name"): + op.add_column( + "vector_indices", sa.Column("index_name", sa.String(length=255), nullable=True) + ) + if not _column_exists(bind, "vector_indices", "checksum"): + op.add_column( + "vector_indices", sa.Column("checksum", sa.String(length=255), nullable=True) + ) + if not _column_exists(bind, "vector_indices", "metadata_info"): + op.add_column( + "vector_indices", sa.Column("metadata_info", sa.JSON(), nullable=True) + ) + + if _table_exists(bind, "vector_chunks"): + if not _column_exists(bind, "vector_chunks", "embedding_json"): + op.add_column( + "vector_chunks", sa.Column("embedding_json", sa.JSON(), nullable=True) + ) + + +def downgrade() -> None: + bind = op.get_bind() + + if _table_exists(bind, "vector_chunks"): + if _column_exists(bind, "vector_chunks", "embedding_json"): + op.drop_column("vector_chunks", "embedding_json") + + if _table_exists(bind, "vector_indices"): + if _column_exists(bind, "vector_indices", "metadata_info"): + op.drop_column("vector_indices", "metadata_info") + if _column_exists(bind, "vector_indices", "checksum"): + op.drop_column("vector_indices", "checksum") + if _column_exists(bind, "vector_indices", "index_name"): + op.drop_column("vector_indices", "index_name") diff --git a/alembic/versions/229359b0d1b2_add_terms_accepted.py b/alembic/versions/229359b0d1b2_add_terms_accepted.py new file mode 100644 index 0000000..9dfe144 --- /dev/null +++ b/alembic/versions/229359b0d1b2_add_terms_accepted.py @@ -0,0 +1,64 @@ +"""add_terms_accepted + +Revision ID: 229359b0d1b2 +Revises: 823a486fd287 +Create Date: 2026-04-01 11:28:44.953518 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '229359b0d1b2' +down_revision: Union[str, None] = '823a486fd287' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('drive_comments', schema=None) as batch_op: + batch_op.alter_column('rejected_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + type_=sa.DateTime(), + existing_nullable=True) + + with op.batch_alter_table('drive_stars', schema=None) as batch_op: + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + type_=sa.TIMESTAMP(), + nullable=True, + existing_server_default=sa.text('now()')) + + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.add_column(sa.Column('terms_accepted', sa.Boolean(), nullable=False, server_default=sa.text('false'))) + batch_op.drop_index(batch_op.f('ix_users_email')) + batch_op.create_index(batch_op.f('ix_users_email'), ['email'], unique=False) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_users_email')) + batch_op.create_index(batch_op.f('ix_users_email'), ['email'], unique=True) + batch_op.drop_column('terms_accepted') + + with op.batch_alter_table('drive_stars', schema=None) as batch_op: + batch_op.alter_column('created_at', + existing_type=sa.TIMESTAMP(), + type_=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + + with op.batch_alter_table('drive_comments', schema=None) as batch_op: + batch_op.alter_column('rejected_at', + existing_type=sa.DateTime(), + type_=postgresql.TIMESTAMP(timezone=True), + existing_nullable=True) + + # ### end Alembic commands ### diff --git a/alembic/versions/22dee9a1d631_add_shared_by_user_id_to_projectshare.py b/alembic/versions/22dee9a1d631_add_shared_by_user_id_to_projectshare.py new file mode 100644 index 0000000..6ccef18 --- /dev/null +++ b/alembic/versions/22dee9a1d631_add_shared_by_user_id_to_projectshare.py @@ -0,0 +1,32 @@ +"""Add shared_by_user_id to ProjectShare + +Revision ID: 22dee9a1d631 +Revises: dbaa8ef522c0 +Create Date: 2026-02-26 15:33:00.617693 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '22dee9a1d631' +down_revision: Union[str, None] = 'dbaa8ef522c0' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('project_shares', sa.Column('shared_by_user_id', sa.Integer(), nullable=True)) + op.create_foreign_key(None, 'project_shares', 'users', ['shared_by_user_id'], ['id'], ondelete='CASCADE') + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, 'project_shares', type_='foreignkey') + op.drop_column('project_shares', 'shared_by_user_id') + # ### end Alembic commands ### diff --git a/alembic/versions/29c02cb50737_initial_migration.py b/alembic/versions/29c02cb50737_initial_migration.py new file mode 100644 index 0000000..782db29 --- /dev/null +++ b/alembic/versions/29c02cb50737_initial_migration.py @@ -0,0 +1,258 @@ +"""initial migration + +Revision ID: 29c02cb50737 +Revises: +Create Date: 2026-02-16 19:28:02.116622 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '29c02cb50737' +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('drive_acl', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('resource_type', sa.String(length=10), nullable=False), + sa.Column('resource_id', sa.Integer(), nullable=False), + sa.Column('subject_type', sa.String(length=10), nullable=False), + sa.Column('subject_id', sa.Integer(), nullable=True), + sa.Column('role', sa.String(length=20), nullable=False), + sa.Column('inherited_from', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('users', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('email', sa.String(length=255), nullable=False), + sa.Column('password_hash', sa.String(length=255), nullable=False), + sa.Column('subscription', sa.String(length=50), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_users_email'), 'users', ['email'], unique=True) + op.create_table('drive_activities', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('resource_type', sa.String(length=10), nullable=False), + sa.Column('resource_id', sa.Integer(), nullable=False), + sa.Column('actor_id', sa.Integer(), nullable=False), + sa.Column('activity_type', sa.String(length=50), nullable=False), + sa.Column('activity_metadata', sa.Text(), nullable=True), + sa.Column('ip_address', sa.String(length=64), nullable=True), + sa.Column('device_info', sa.Text(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['actor_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('drive_folders', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('owner_id', sa.Integer(), nullable=False), + sa.Column('parent_id', sa.Integer(), nullable=True), + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('color', sa.String(length=20), nullable=True), + sa.Column('is_starred', sa.Boolean(), nullable=False), + sa.Column('is_trashed', sa.Boolean(), nullable=False), + sa.Column('trashed_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['owner_id'], ['users.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['parent_id'], ['drive_folders.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('drive_shares', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('resource_type', sa.String(length=10), nullable=False), + sa.Column('resource_id', sa.Integer(), nullable=False), + sa.Column('invited_email', sa.String(length=255), nullable=True), + sa.Column('link_token', sa.String(length=255), nullable=True), + sa.Column('role', sa.String(length=20), nullable=False), + sa.Column('expires_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('created_by_id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['created_by_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('link_token') + ) + op.create_table('drive_stars', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('resource_type', sa.String(length=10), nullable=False), + sa.Column('resource_id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('resource_type', 'resource_id', 'user_id', name='uq_drive_star') + ) + op.create_table('projects', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('status', sa.String(length=50), nullable=False), + sa.Column('file_type', sa.String(length=50), nullable=False), + sa.Column('output_format', sa.String(length=50), nullable=False), + sa.Column('progress', sa.Integer(), nullable=False), + sa.Column('current_step', sa.String(length=50), nullable=False), + sa.Column('session_id', sa.String(length=255), nullable=True), + sa.Column('html_content', sa.Text(), nullable=True), + sa.Column('markdown_content', sa.Text(), nullable=True), + sa.Column('xml_content', sa.Text(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('user_files', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('file_name', sa.String(length=512), nullable=False), + sa.Column('s3_key', sa.String(length=1024), nullable=False), + sa.Column('file_size', sa.BigInteger(), nullable=False), + sa.Column('mime_type', sa.String(length=255), nullable=True), + sa.Column('is_deleted', sa.Boolean(), nullable=False), + sa.Column('deleted_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('s3_key') + ) + op.create_table('user_storage_usage', + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('total_bytes', sa.BigInteger(), nullable=False), + sa.Column('file_count', sa.Integer(), nullable=False), + sa.Column('last_updated', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('user_id') + ) + op.create_table('drive_files', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('folder_id', sa.Integer(), nullable=False), + sa.Column('owner_id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=512), nullable=False), + sa.Column('mime_type', sa.String(length=255), nullable=True), + sa.Column('size', sa.BigInteger(), nullable=False), + sa.Column('hash', sa.String(length=128), nullable=True), + sa.Column('latest_version_id', sa.Integer(), nullable=True), + sa.Column('is_starred', sa.Boolean(), nullable=False), + sa.Column('is_trashed', sa.Boolean(), nullable=False), + sa.Column('trashed_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['folder_id'], ['drive_folders.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['owner_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('project_activities', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('project_id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('activity_type', sa.String(length=50), nullable=False), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('activity_metadata', sa.Text(), nullable=True), + sa.Column('ip_address', sa.String(length=64), nullable=True), + sa.Column('device_info', sa.Text(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['project_id'], ['projects.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('project_comments', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('project_id', sa.Integer(), nullable=False), + sa.Column('author_id', sa.Integer(), nullable=False), + sa.Column('body', sa.Text(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('resolved_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('anchor', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.ForeignKeyConstraint(['author_id'], ['users.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['project_id'], ['projects.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('project_shares', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('project_id', sa.Integer(), nullable=False), + sa.Column('shared_with_user_id', sa.Integer(), nullable=False), + sa.Column('shared_by_user_id', sa.Integer(), nullable=False), + sa.Column('permission', sa.String(length=50), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['project_id'], ['projects.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['shared_by_user_id'], ['users.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['shared_with_user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('project_id', 'shared_with_user_id', name='unique_project_user_share') + ) + op.create_table('project_versions', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('project_id', sa.Integer(), nullable=False), + sa.Column('created_by_id', sa.Integer(), nullable=False), + sa.Column('version_number', sa.Integer(), nullable=False), + sa.Column('changes_description', sa.String(length=500), nullable=True), + sa.Column('markdown_content', sa.Text(), nullable=True), + sa.Column('xml_content', sa.Text(), nullable=True), + sa.Column('html_content', sa.Text(), nullable=True), + sa.Column('session_id', sa.String(length=255), nullable=True), + sa.Column('is_current', sa.Boolean(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['created_by_id'], ['users.id'], ), + sa.ForeignKeyConstraint(['project_id'], ['projects.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('drive_comments', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('file_id', sa.Integer(), nullable=False), + sa.Column('author_id', sa.Integer(), nullable=False), + sa.Column('body', sa.Text(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('resolved_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('anchor', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.ForeignKeyConstraint(['author_id'], ['users.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['file_id'], ['drive_files.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('drive_file_versions', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('file_id', sa.Integer(), nullable=False), + sa.Column('version_number', sa.Integer(), nullable=False), + sa.Column('storage_key', sa.Text(), nullable=False), + sa.Column('mime_type', sa.String(length=255), nullable=True), + sa.Column('size', sa.BigInteger(), nullable=False), + sa.Column('checksum', sa.String(length=128), nullable=True), + sa.Column('created_by_id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['created_by_id'], ['users.id'], ), + sa.ForeignKeyConstraint(['file_id'], ['drive_files.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('drive_file_versions') + op.drop_table('drive_comments') + op.drop_table('project_versions') + op.drop_table('project_shares') + op.drop_table('project_comments') + op.drop_table('project_activities') + op.drop_table('drive_files') + op.drop_table('user_storage_usage') + op.drop_table('user_files') + op.drop_table('projects') + op.drop_table('drive_stars') + op.drop_table('drive_shares') + op.drop_table('drive_folders') + op.drop_table('drive_activities') + op.drop_index(op.f('ix_users_email'), table_name='users') + op.drop_table('users') + op.drop_table('drive_acl') + # ### end Alembic commands ### diff --git a/alembic/versions/2b4c62c1cf35_add_subscription_fields_and_.py b/alembic/versions/2b4c62c1cf35_add_subscription_fields_and_.py new file mode 100644 index 0000000..7cf296a --- /dev/null +++ b/alembic/versions/2b4c62c1cf35_add_subscription_fields_and_.py @@ -0,0 +1,51 @@ +"""Add subscription fields and PlanPriceHistory + +Revision ID: 2b4c62c1cf35 +Revises: d2_0_credit_limit_precision +Create Date: 2026-08-19 18:04:44.734002 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '2b4c62c1cf35' +down_revision: Union[str, None] = 'd2_0_credit_limit_precision' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('plan_price_history', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('plan_id', sa.UUID(), nullable=False), + sa.Column('old_price', sa.Numeric(precision=10, scale=2), nullable=True), + sa.Column('new_price', sa.Numeric(precision=10, scale=2), nullable=True), + sa.Column('currency', sa.String(length=10), nullable=True), + sa.Column('changed_by', sa.Integer(), nullable=True), + sa.Column('changed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['plan_id'], ['plans.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_plan_price_history_plan_id'), 'plan_price_history', ['plan_id'], unique=False) + + op.add_column('plans', sa.Column('price_amount', sa.Numeric(precision=10, scale=2), nullable=True)) + op.add_column('plans', sa.Column('interval', sa.String(length=20), nullable=True)) + op.add_column('plans', sa.Column('grace_period_days', sa.Integer(), server_default='0', nullable=False)) + op.add_column('plans', sa.Column('notify_days_before_expiry', sa.Integer(), server_default='7', nullable=False)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('plans', 'notify_days_before_expiry') + op.drop_column('plans', 'grace_period_days') + op.drop_column('plans', 'interval') + op.drop_column('plans', 'price_amount') + op.drop_index(op.f('ix_plan_price_history_plan_id'), table_name='plan_price_history') + op.drop_table('plan_price_history') + # ### end Alembic commands ### diff --git a/alembic/versions/39d5fdfb5386_add_signature_id_to_signature_imprints.py b/alembic/versions/39d5fdfb5386_add_signature_id_to_signature_imprints.py new file mode 100644 index 0000000..d293a28 --- /dev/null +++ b/alembic/versions/39d5fdfb5386_add_signature_id_to_signature_imprints.py @@ -0,0 +1,128 @@ +"""add signature_id to signature_imprints + +Revision ID: 39d5fdfb5386 +Revises: 1cb413ffaaa0 +Create Date: 2026-04-25 15:33:32.819817 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '39d5fdfb5386' +down_revision: Union[str, None] = '1cb413ffaaa0' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Legacy tables/indexes — drop only if present (some envs already lack them). + op.execute("DROP TABLE IF EXISTS vector_chunks CASCADE") + op.execute("DROP TABLE IF EXISTS vector_indices CASCADE") + op.execute("DROP TABLE IF EXISTS chat_token_usage CASCADE") + + op.execute("DROP INDEX IF EXISTS ix_chatbot_documents_drive_file_id") + op.execute("DROP INDEX IF EXISTS ix_chatbot_documents_id") + + with op.batch_alter_table('extraction_data', schema=None) as batch_op: + batch_op.alter_column('confidence_score', + existing_type=sa.DOUBLE_PRECISION(precision=53), + nullable=False) + batch_op.alter_column('extraction_status', + existing_type=sa.VARCHAR(length=50), + nullable=False) + op.execute("DROP INDEX IF EXISTS idx_extraction_data_document_id") + op.create_index(op.f('ix_extraction_data_document_id'), 'extraction_data', ['document_id'], unique=False) + + op.execute("DROP INDEX IF EXISTS ix_signature_imprints_file_page") + op.execute("DROP INDEX IF EXISTS ix_signature_imprints_signing_request") + with op.batch_alter_table('signature_imprints', schema=None) as batch_op: + batch_op.add_column(sa.Column('signature_id', sa.String(length=15), nullable=True)) + batch_op.create_index(batch_op.f('ix_signature_imprints_signing_request_id'), ['signing_request_id'], unique=False) + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('signature_imprints', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_signature_imprints_signing_request_id')) + batch_op.create_index(batch_op.f('ix_signature_imprints_signing_request'), ['signing_request_id'], unique=False) + batch_op.create_index(batch_op.f('ix_signature_imprints_file_page'), ['drive_file_id', 'page_number'], unique=False) + batch_op.drop_column('signature_id') + + with op.batch_alter_table('extraction_data', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_extraction_data_document_id')) + batch_op.create_index(batch_op.f('idx_extraction_data_document_id'), ['document_id'], unique=False) + batch_op.alter_column('extraction_status', + existing_type=sa.VARCHAR(length=50), + nullable=True) + batch_op.alter_column('confidence_score', + existing_type=sa.DOUBLE_PRECISION(precision=53), + nullable=True) + + with op.batch_alter_table('chatbot_documents', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_chatbot_documents_id'), ['id'], unique=False) + batch_op.create_index(batch_op.f('ix_chatbot_documents_drive_file_id'), ['drive_file_id'], unique=False) + + op.create_table('chat_token_usage', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('user_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('chat_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('document_id', sa.VARCHAR(length=255), autoincrement=False, nullable=True), + sa.Column('prompt_tokens', sa.BIGINT(), autoincrement=False, nullable=False), + sa.Column('completion_tokens', sa.BIGINT(), autoincrement=False, nullable=False), + sa.Column('total_tokens', sa.BIGINT(), autoincrement=False, nullable=False), + sa.Column('model_name', sa.VARCHAR(length=100), autoincrement=False, nullable=False), + sa.Column('cost_usd', sa.NUMERIC(precision=12, scale=8), autoincrement=False, nullable=False), + sa.Column('timestamp', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), autoincrement=False, nullable=False), + sa.Column('phase', sa.VARCHAR(length=50), autoincrement=False, nullable=True), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('chat_token_usage_user_id_fkey'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('chat_token_usage_pkey')) + ) + with op.batch_alter_table('chat_token_usage', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_chat_token_usage_user_time'), ['user_id', 'timestamp'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_token_usage_user_id'), ['user_id'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_token_usage_timestamp'), ['timestamp'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_token_usage_phase'), ['phase'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_token_usage_model_time'), ['model_name', 'timestamp'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_token_usage_model_name'), ['model_name'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_token_usage_id'), ['id'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_token_usage_document_id'), ['document_id'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_token_usage_chat_id'), ['chat_id'], unique=False) + + op.create_table('vector_indices', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('user_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('index_data', postgresql.BYTEA(), autoincrement=False, nullable=True), + sa.Column('dimension', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('total_vectors', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('updated_at', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False), + sa.Column('index_name', sa.VARCHAR(length=255), autoincrement=False, nullable=True), + sa.Column('checksum', sa.VARCHAR(length=255), autoincrement=False, nullable=True), + sa.Column('metadata_info', postgresql.JSON(astext_type=sa.Text()), autoincrement=False, nullable=True), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('vector_indices_user_id_fkey'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('vector_indices_pkey')) + ) + with op.batch_alter_table('vector_indices', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_vector_indices_user_id'), ['user_id'], unique=True) + batch_op.create_index(batch_op.f('ix_vector_indices_id'), ['id'], unique=False) + + op.create_table('vector_chunks', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('user_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('document_id', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('chunk_text', sa.TEXT(), autoincrement=False, nullable=False), + sa.Column('chunk_index', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False), + sa.Column('embedding_json', postgresql.JSON(astext_type=sa.Text()), autoincrement=False, nullable=True), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('vector_chunks_user_id_fkey'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('vector_chunks_pkey')) + ) + with op.batch_alter_table('vector_chunks', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_vector_chunks_user_id'), ['user_id'], unique=False) + batch_op.create_index(batch_op.f('ix_vector_chunks_id'), ['id'], unique=False) + batch_op.create_index(batch_op.f('ix_vector_chunks_document_id'), ['document_id'], unique=False) + + # ### end Alembic commands ### diff --git a/alembic/versions/3aa7d970b8c2_enforce_enum_constraint_on_driveacl_role.py b/alembic/versions/3aa7d970b8c2_enforce_enum_constraint_on_driveacl_role.py new file mode 100644 index 0000000..fd70e8a --- /dev/null +++ b/alembic/versions/3aa7d970b8c2_enforce_enum_constraint_on_driveacl_role.py @@ -0,0 +1,51 @@ +"""Enforce enum constraint on DriveACL role + +Revision ID: 3aa7d970b8c2 +Revises: 89508c176437 +Create Date: 2026-03-03 22:06:24.190823 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '3aa7d970b8c2' +down_revision: Union[str, None] = '89508c176437' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +# The four valid drive roles +VALID_ROLES = ('viewer', 'commenter', 'editor', 'owner') + + +def upgrade() -> None: + # 1. Create the PostgreSQL enum type (idempotent via IF NOT EXISTS) + op.execute("CREATE TYPE drive_role AS ENUM ('viewer', 'commenter', 'editor', 'owner')") + + # 2. Sanitise any NULL or invalid rows before adding NOT NULL constraint. + # Default unknown/null values to 'viewer' (least-privileged), then log them. + op.execute( + "UPDATE drive_acls SET role = 'viewer' " + "WHERE role IS NULL OR role NOT IN ('viewer', 'commenter', 'editor', 'owner')" + ) + + # 3. Alter the column: cast existing VARCHAR values to the new enum type + op.execute( + "ALTER TABLE drive_acls " + "ALTER COLUMN role TYPE drive_role USING role::drive_role, " + "ALTER COLUMN role SET NOT NULL" + ) + + +def downgrade() -> None: + # Revert column to plain VARCHAR(50), nullable + op.execute( + "ALTER TABLE drive_acls " + "ALTER COLUMN role TYPE VARCHAR(50) USING role::VARCHAR, " + "ALTER COLUMN role DROP NOT NULL" + ) + # Drop the enum type (only if no other table references it) + op.execute("DROP TYPE IF EXISTS drive_role") diff --git a/alembic/versions/41bed7eb6ed9_refactor_documents_and_add_missing_.py b/alembic/versions/41bed7eb6ed9_refactor_documents_and_add_missing_.py new file mode 100644 index 0000000..cf85d17 --- /dev/null +++ b/alembic/versions/41bed7eb6ed9_refactor_documents_and_add_missing_.py @@ -0,0 +1,100 @@ +"""Refactor documents and add missing fields + +Revision ID: 41bed7eb6ed9 +Revises: 06001599321d +Create Date: 2026-02-27 14:21:01.746676 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '41bed7eb6ed9' +down_revision: Union[str, None] = '06001599321d' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('project_activities') + op.execute("DROP INDEX IF EXISTS ix_collaboration_chat_resource_id") + op.execute("DROP INDEX IF EXISTS ix_collaboration_chat_resource_type") + op.execute("DROP TABLE IF EXISTS collaboration_chat CASCADE") + op.drop_table('project_comments') + op.drop_table('project_shares') + op.add_column('project_versions', sa.Column('changes_description', sa.Text(), nullable=True)) + op.add_column('project_versions', sa.Column('session_id', sa.String(length=255), nullable=True)) + op.drop_column('project_versions', 'description') + op.add_column('projects', sa.Column('drive_file_id', sa.Integer(), nullable=True)) + op.create_index('idx_project_session', 'projects', ['session_id'], unique=False) + op.create_index('idx_project_user_status', 'projects', ['user_id', 'status'], unique=False) + op.create_index(op.f('ix_projects_drive_file_id'), 'projects', ['drive_file_id'], unique=False) + op.create_foreign_key(None, 'projects', 'drive_files', ['drive_file_id'], ['id'], ondelete='CASCADE') + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, 'projects', type_='foreignkey') + op.drop_index(op.f('ix_projects_drive_file_id'), table_name='projects') + op.drop_index('idx_project_user_status', table_name='projects') + op.drop_index('idx_project_session', table_name='projects') + op.drop_column('projects', 'drive_file_id') + op.add_column('project_versions', sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True)) + op.drop_column('project_versions', 'session_id') + op.drop_column('project_versions', 'changes_description') + op.create_table('project_shares', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('project_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('shared_with_user_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('permission', sa.VARCHAR(length=20), autoincrement=False, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), autoincrement=False, nullable=False), + sa.Column('shared_by_user_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.ForeignKeyConstraint(['project_id'], ['projects.id'], name='project_shares_project_id_fkey', ondelete='CASCADE'), + sa.ForeignKeyConstraint(['shared_by_user_id'], ['users.id'], name='project_shares_shared_by_user_id_fkey', ondelete='CASCADE'), + sa.ForeignKeyConstraint(['shared_with_user_id'], ['users.id'], name='project_shares_shared_with_user_id_fkey', ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name='project_shares_pkey') + ) + op.create_table('project_comments', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('project_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('author_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('body', sa.TEXT(), autoincrement=False, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), autoincrement=False, nullable=False), + sa.Column('resolved_at', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=True), + sa.Column('anchor', sa.TEXT(), autoincrement=False, nullable=True), + sa.ForeignKeyConstraint(['author_id'], ['users.id'], name='project_comments_author_id_fkey', ondelete='CASCADE'), + sa.ForeignKeyConstraint(['project_id'], ['projects.id'], name='project_comments_project_id_fkey', ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name='project_comments_pkey') + ) + op.create_table('collaboration_chat', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('resource_type', sa.VARCHAR(length=20), autoincrement=False, nullable=False), + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('user_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('message', sa.TEXT(), autoincrement=False, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), autoincrement=False, nullable=True), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], name='collaboration_chat_user_id_fkey', ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name='collaboration_chat_pkey') + ) + op.create_index('ix_collaboration_chat_resource_type', 'collaboration_chat', ['resource_type'], unique=False) + op.create_index('ix_collaboration_chat_resource_id', 'collaboration_chat', ['resource_id'], unique=False) + op.create_table('project_activities', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('project_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('user_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('activity_type', sa.VARCHAR(length=50), autoincrement=False, nullable=False), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('activity_metadata', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('ip_address', sa.VARCHAR(length=45), autoincrement=False, nullable=True), + sa.Column('device_info', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint(['project_id'], ['projects.id'], name='project_activities_project_id_fkey', ondelete='CASCADE'), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], name='project_activities_user_id_fkey', ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name='project_activities_pkey') + ) + # ### end Alembic commands ### diff --git a/alembic/versions/45c94ee15294_limit_notification_message_to_2000_chars.py b/alembic/versions/45c94ee15294_limit_notification_message_to_2000_chars.py new file mode 100644 index 0000000..c78a1cb --- /dev/null +++ b/alembic/versions/45c94ee15294_limit_notification_message_to_2000_chars.py @@ -0,0 +1,36 @@ +"""Limit Notification message to 2000 chars + +Revision ID: 45c94ee15294 +Revises: d2e2c55290ee +Create Date: 2026-03-03 22:24:55.795394 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '45c94ee15294' +down_revision: Union[str, None] = 'd2e2c55290ee' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('notifications', 'message', + existing_type=sa.TEXT(), + type_=sa.String(length=2000), + existing_nullable=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('notifications', 'message', + existing_type=sa.String(length=2000), + type_=sa.TEXT(), + existing_nullable=False) + # ### end Alembic commands ### diff --git a/alembic/versions/49c19df2c1d4_sync_models.py b/alembic/versions/49c19df2c1d4_sync_models.py new file mode 100644 index 0000000..6889daa --- /dev/null +++ b/alembic/versions/49c19df2c1d4_sync_models.py @@ -0,0 +1,371 @@ +"""sync_models + +Revision ID: 49c19df2c1d4 +Revises: 29c02cb50737 +Create Date: 2026-02-19 11:21:56.538569 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '49c19df2c1d4' +down_revision: Union[str, None] = '29c02cb50737' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('drive_acls', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('resource_type', sa.String(length=20), nullable=False), + sa.Column('resource_id', sa.Integer(), nullable=False), + sa.Column('subject_id', sa.Integer(), nullable=False), + sa.Column('role', sa.String(length=50), nullable=True), + sa.Column('actor_id', sa.Integer(), nullable=True), + sa.Column('created_at', sa.TIMESTAMP(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('chatbot_documents', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('document_id', sa.UUID(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('drive_file_id', sa.Integer(), nullable=True), + sa.Column('filename', sa.String(), nullable=False), + sa.Column('file_path', sa.String(), nullable=True), + sa.Column('cloud_url', sa.String(), nullable=True), + sa.Column('chunks_count', sa.Integer(), nullable=True), + sa.Column('created_at', sa.TIMESTAMP(), nullable=True), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_chatbot_documents_document_id'), 'chatbot_documents', ['document_id'], unique=True) + op.create_index(op.f('ix_chatbot_documents_drive_file_id'), 'chatbot_documents', ['drive_file_id'], unique=False) + op.create_index(op.f('ix_chatbot_documents_id'), 'chatbot_documents', ['id'], unique=False) + op.create_index(op.f('ix_chatbot_documents_user_id'), 'chatbot_documents', ['user_id'], unique=False) + op.create_table('vector_chunks', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('document_id', sa.String(), nullable=True), + sa.Column('chunk_text', sa.String(), nullable=False), + sa.Column('chunk_index', sa.Integer(), nullable=True), + sa.Column('created_at', sa.TIMESTAMP(), nullable=True), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_vector_chunks_document_id'), 'vector_chunks', ['document_id'], unique=False) + op.create_index(op.f('ix_vector_chunks_id'), 'vector_chunks', ['id'], unique=False) + op.create_index(op.f('ix_vector_chunks_user_id'), 'vector_chunks', ['user_id'], unique=False) + op.create_table('vector_indices', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('index_data', sa.LargeBinary(), nullable=True), + sa.Column('dimension', sa.Integer(), nullable=True), + sa.Column('total_vectors', sa.Integer(), nullable=True), + sa.Column('updated_at', sa.TIMESTAMP(), nullable=True), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_vector_indices_id'), 'vector_indices', ['id'], unique=False) + op.create_index(op.f('ix_vector_indices_user_id'), 'vector_indices', ['user_id'], unique=True) + op.drop_table('drive_acl') + op.alter_column('drive_activities', 'resource_type', + existing_type=sa.VARCHAR(length=10), + type_=sa.String(length=20), + nullable=True) + op.alter_column('drive_activities', 'resource_id', + existing_type=sa.INTEGER(), + nullable=True) + op.alter_column('drive_activities', 'actor_id', + existing_type=sa.INTEGER(), + nullable=True) + op.alter_column('drive_activities', 'activity_type', + existing_type=sa.VARCHAR(length=50), + nullable=True) + op.alter_column('drive_activities', 'created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + type_=sa.TIMESTAMP(), + nullable=True, + existing_server_default=sa.text('now()')) + op.drop_constraint(op.f('drive_activities_actor_id_fkey'), 'drive_activities', type_='foreignkey') + op.create_foreign_key(None, 'drive_activities', 'users', ['actor_id'], ['id']) + op.drop_column('drive_activities', 'device_info') + op.drop_column('drive_activities', 'ip_address') + op.add_column('drive_comments', sa.Column('user_id', sa.Integer(), nullable=False)) + op.add_column('drive_comments', sa.Column('content', sa.Text(), nullable=False)) + op.alter_column('drive_comments', 'created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + type_=sa.TIMESTAMP(), + nullable=True, + existing_server_default=sa.text('now()')) + op.drop_constraint(op.f('drive_comments_file_id_fkey'), 'drive_comments', type_='foreignkey') + op.drop_constraint(op.f('drive_comments_author_id_fkey'), 'drive_comments', type_='foreignkey') + op.create_foreign_key(None, 'drive_comments', 'users', ['user_id'], ['id']) + op.create_foreign_key(None, 'drive_comments', 'drive_files', ['file_id'], ['id']) + op.drop_column('drive_comments', 'anchor') + op.drop_column('drive_comments', 'body') + op.drop_column('drive_comments', 'resolved_at') + op.drop_column('drive_comments', 'author_id') + op.add_column('drive_file_versions', sa.Column('s3_key', sa.String(), nullable=True)) + op.add_column('drive_file_versions', sa.Column('content_type', sa.String(), nullable=True)) + op.add_column('drive_file_versions', sa.Column('uploader_id', sa.Integer(), nullable=True)) + op.alter_column('drive_file_versions', 'file_id', + existing_type=sa.INTEGER(), + nullable=True) + op.alter_column('drive_file_versions', 'version_number', + existing_type=sa.INTEGER(), + nullable=True) + op.alter_column('drive_file_versions', 'size', + existing_type=sa.BIGINT(), + nullable=True) + op.alter_column('drive_file_versions', 'created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + type_=sa.TIMESTAMP(), + nullable=True, + existing_server_default=sa.text('now()')) + op.drop_constraint(op.f('drive_file_versions_file_id_fkey'), 'drive_file_versions', type_='foreignkey') + op.drop_constraint(op.f('drive_file_versions_created_by_id_fkey'), 'drive_file_versions', type_='foreignkey') + op.create_foreign_key(None, 'drive_file_versions', 'drive_files', ['file_id'], ['id']) + op.create_foreign_key(None, 'drive_file_versions', 'users', ['uploader_id'], ['id']) + op.drop_column('drive_file_versions', 'checksum') + op.drop_column('drive_file_versions', 'created_by_id') + op.drop_column('drive_file_versions', 'storage_key') + op.drop_column('drive_file_versions', 'mime_type') + op.alter_column('drive_stars', 'user_id', + existing_type=sa.INTEGER(), + nullable=True) + op.alter_column('drive_stars', 'resource_type', + existing_type=sa.VARCHAR(length=10), + nullable=True) + op.alter_column('drive_stars', 'resource_id', + existing_type=sa.INTEGER(), + nullable=True) + op.drop_constraint(op.f('uq_drive_star'), 'drive_stars', type_='unique') + op.drop_constraint(op.f('drive_stars_user_id_fkey'), 'drive_stars', type_='foreignkey') + op.create_foreign_key(None, 'drive_stars', 'users', ['user_id'], ['id']) + op.drop_column('drive_stars', 'created_at') + op.alter_column('project_activities', 'ip_address', + existing_type=sa.VARCHAR(length=64), + type_=sa.String(length=45), + existing_nullable=True) + op.alter_column('project_comments', 'anchor', + existing_type=postgresql.JSONB(astext_type=sa.Text()), + type_=sa.Text(), + existing_nullable=True) + op.alter_column('project_shares', 'permission', + existing_type=sa.VARCHAR(length=50), + type_=sa.String(length=20), + existing_nullable=False) + op.drop_constraint(op.f('unique_project_user_share'), 'project_shares', type_='unique') + op.drop_constraint(op.f('project_shares_shared_by_user_id_fkey'), 'project_shares', type_='foreignkey') + op.drop_column('project_shares', 'updated_at') + op.drop_column('project_shares', 'shared_by_user_id') + op.add_column('project_versions', sa.Column('version_label', sa.String(length=50), nullable=False)) + op.add_column('project_versions', sa.Column('description', sa.Text(), nullable=True)) + op.alter_column('project_versions', 'created_by_id', + existing_type=sa.INTEGER(), + nullable=True) + op.drop_constraint(op.f('project_versions_created_by_id_fkey'), 'project_versions', type_='foreignkey') + op.create_foreign_key(None, 'project_versions', 'users', ['created_by_id'], ['id'], ondelete='SET NULL') + op.drop_column('project_versions', 'changes_description') + op.drop_column('project_versions', 'session_id') + op.add_column('user_files', sa.Column('filename', sa.String(), nullable=False)) + op.add_column('user_files', sa.Column('size', sa.BigInteger(), nullable=True)) + op.add_column('user_files', sa.Column('content_type', sa.String(), nullable=True)) + op.add_column('user_files', sa.Column('status', sa.String(), nullable=True)) + op.alter_column('user_files', 'created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + type_=sa.TIMESTAMP(), + nullable=True, + existing_server_default=sa.text('now()')) + op.drop_constraint(op.f('user_files_s3_key_key'), 'user_files', type_='unique') + op.create_index(op.f('ix_user_files_id'), 'user_files', ['id'], unique=False) + op.create_index(op.f('ix_user_files_user_id'), 'user_files', ['user_id'], unique=False) + op.drop_constraint(op.f('user_files_user_id_fkey'), 'user_files', type_='foreignkey') + op.create_foreign_key(None, 'user_files', 'users', ['user_id'], ['id']) + op.drop_column('user_files', 'file_name') + op.drop_column('user_files', 'deleted_at') + op.drop_column('user_files', 'file_size') + op.drop_column('user_files', 'is_deleted') + op.drop_column('user_files', 'mime_type') + op.add_column('user_storage_usage', sa.Column('id', sa.Integer(), nullable=False)) + op.add_column('user_storage_usage', sa.Column('total_bytes_used', sa.BigInteger(), nullable=True)) + op.add_column('user_storage_usage', sa.Column('total_files_count', sa.Integer(), nullable=True)) + op.alter_column('user_storage_usage', 'last_updated', + existing_type=postgresql.TIMESTAMP(timezone=True), + type_=sa.TIMESTAMP(), + nullable=True, + existing_server_default=sa.text('now()')) + op.create_index(op.f('ix_user_storage_usage_id'), 'user_storage_usage', ['id'], unique=False) + op.create_unique_constraint(None, 'user_storage_usage', ['user_id']) + op.drop_constraint(op.f('user_storage_usage_user_id_fkey'), 'user_storage_usage', type_='foreignkey') + op.create_foreign_key(None, 'user_storage_usage', 'users', ['user_id'], ['id']) + op.drop_column('user_storage_usage', 'total_bytes') + op.drop_column('user_storage_usage', 'file_count') + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('user_storage_usage', sa.Column('file_count', sa.INTEGER(), autoincrement=False, nullable=False)) + op.add_column('user_storage_usage', sa.Column('total_bytes', sa.BIGINT(), autoincrement=False, nullable=False)) + op.drop_constraint(None, 'user_storage_usage', type_='foreignkey') + op.create_foreign_key(op.f('user_storage_usage_user_id_fkey'), 'user_storage_usage', 'users', ['user_id'], ['id'], ondelete='CASCADE') + op.drop_constraint(None, 'user_storage_usage', type_='unique') + op.drop_index(op.f('ix_user_storage_usage_id'), table_name='user_storage_usage') + op.alter_column('user_storage_usage', 'last_updated', + existing_type=sa.TIMESTAMP(), + type_=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + op.drop_column('user_storage_usage', 'total_files_count') + op.drop_column('user_storage_usage', 'total_bytes_used') + op.drop_column('user_storage_usage', 'id') + op.add_column('user_files', sa.Column('mime_type', sa.VARCHAR(length=255), autoincrement=False, nullable=True)) + op.add_column('user_files', sa.Column('is_deleted', sa.BOOLEAN(), autoincrement=False, nullable=False)) + op.add_column('user_files', sa.Column('file_size', sa.BIGINT(), autoincrement=False, nullable=False)) + op.add_column('user_files', sa.Column('deleted_at', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=True)) + op.add_column('user_files', sa.Column('file_name', sa.VARCHAR(length=512), autoincrement=False, nullable=False)) + op.drop_constraint(None, 'user_files', type_='foreignkey') + op.create_foreign_key(op.f('user_files_user_id_fkey'), 'user_files', 'users', ['user_id'], ['id'], ondelete='CASCADE') + op.drop_index(op.f('ix_user_files_user_id'), table_name='user_files') + op.drop_index(op.f('ix_user_files_id'), table_name='user_files') + op.create_unique_constraint(op.f('user_files_s3_key_key'), 'user_files', ['s3_key']) + op.alter_column('user_files', 'created_at', + existing_type=sa.TIMESTAMP(), + type_=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + op.drop_column('user_files', 'status') + op.drop_column('user_files', 'content_type') + op.drop_column('user_files', 'size') + op.drop_column('user_files', 'filename') + op.add_column('project_versions', sa.Column('session_id', sa.VARCHAR(length=255), autoincrement=False, nullable=True)) + op.add_column('project_versions', sa.Column('changes_description', sa.VARCHAR(length=500), autoincrement=False, nullable=True)) + op.drop_constraint(None, 'project_versions', type_='foreignkey') + op.create_foreign_key(op.f('project_versions_created_by_id_fkey'), 'project_versions', 'users', ['created_by_id'], ['id']) + op.alter_column('project_versions', 'created_by_id', + existing_type=sa.INTEGER(), + nullable=False) + op.drop_column('project_versions', 'description') + op.drop_column('project_versions', 'version_label') + op.add_column('project_shares', sa.Column('shared_by_user_id', sa.INTEGER(), autoincrement=False, nullable=False)) + op.add_column('project_shares', sa.Column('updated_at', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False)) + op.create_foreign_key(op.f('project_shares_shared_by_user_id_fkey'), 'project_shares', 'users', ['shared_by_user_id'], ['id'], ondelete='CASCADE') + op.create_unique_constraint(op.f('unique_project_user_share'), 'project_shares', ['project_id', 'shared_with_user_id']) + op.alter_column('project_shares', 'permission', + existing_type=sa.String(length=20), + type_=sa.VARCHAR(length=50), + existing_nullable=False) + op.alter_column('project_comments', 'anchor', + existing_type=sa.Text(), + type_=postgresql.JSONB(astext_type=sa.Text()), + existing_nullable=True) + op.alter_column('project_activities', 'ip_address', + existing_type=sa.String(length=45), + type_=sa.VARCHAR(length=64), + existing_nullable=True) + op.add_column('drive_stars', sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), autoincrement=False, nullable=False)) + op.drop_constraint(None, 'drive_stars', type_='foreignkey') + op.create_foreign_key(op.f('drive_stars_user_id_fkey'), 'drive_stars', 'users', ['user_id'], ['id'], ondelete='CASCADE') + op.create_unique_constraint(op.f('uq_drive_star'), 'drive_stars', ['resource_type', 'resource_id', 'user_id']) + op.alter_column('drive_stars', 'resource_id', + existing_type=sa.INTEGER(), + nullable=False) + op.alter_column('drive_stars', 'resource_type', + existing_type=sa.VARCHAR(length=10), + nullable=False) + op.alter_column('drive_stars', 'user_id', + existing_type=sa.INTEGER(), + nullable=False) + op.add_column('drive_file_versions', sa.Column('mime_type', sa.VARCHAR(length=255), autoincrement=False, nullable=True)) + op.add_column('drive_file_versions', sa.Column('storage_key', sa.TEXT(), autoincrement=False, nullable=False)) + op.add_column('drive_file_versions', sa.Column('created_by_id', sa.INTEGER(), autoincrement=False, nullable=False)) + op.add_column('drive_file_versions', sa.Column('checksum', sa.VARCHAR(length=128), autoincrement=False, nullable=True)) + op.drop_constraint(None, 'drive_file_versions', type_='foreignkey') + op.drop_constraint(None, 'drive_file_versions', type_='foreignkey') + op.create_foreign_key(op.f('drive_file_versions_created_by_id_fkey'), 'drive_file_versions', 'users', ['created_by_id'], ['id']) + op.create_foreign_key(op.f('drive_file_versions_file_id_fkey'), 'drive_file_versions', 'drive_files', ['file_id'], ['id'], ondelete='CASCADE') + op.alter_column('drive_file_versions', 'created_at', + existing_type=sa.TIMESTAMP(), + type_=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + op.alter_column('drive_file_versions', 'size', + existing_type=sa.BIGINT(), + nullable=False) + op.alter_column('drive_file_versions', 'version_number', + existing_type=sa.INTEGER(), + nullable=False) + op.alter_column('drive_file_versions', 'file_id', + existing_type=sa.INTEGER(), + nullable=False) + op.drop_column('drive_file_versions', 'uploader_id') + op.drop_column('drive_file_versions', 'content_type') + op.drop_column('drive_file_versions', 's3_key') + op.add_column('drive_comments', sa.Column('author_id', sa.INTEGER(), autoincrement=False, nullable=False)) + op.add_column('drive_comments', sa.Column('resolved_at', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=True)) + op.add_column('drive_comments', sa.Column('body', sa.TEXT(), autoincrement=False, nullable=False)) + op.add_column('drive_comments', sa.Column('anchor', postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=True)) + op.drop_constraint(None, 'drive_comments', type_='foreignkey') + op.drop_constraint(None, 'drive_comments', type_='foreignkey') + op.create_foreign_key(op.f('drive_comments_author_id_fkey'), 'drive_comments', 'users', ['author_id'], ['id'], ondelete='CASCADE') + op.create_foreign_key(op.f('drive_comments_file_id_fkey'), 'drive_comments', 'drive_files', ['file_id'], ['id'], ondelete='CASCADE') + op.alter_column('drive_comments', 'created_at', + existing_type=sa.TIMESTAMP(), + type_=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + op.drop_column('drive_comments', 'content') + op.drop_column('drive_comments', 'user_id') + op.add_column('drive_activities', sa.Column('ip_address', sa.VARCHAR(length=64), autoincrement=False, nullable=True)) + op.add_column('drive_activities', sa.Column('device_info', sa.TEXT(), autoincrement=False, nullable=True)) + op.drop_constraint(None, 'drive_activities', type_='foreignkey') + op.create_foreign_key(op.f('drive_activities_actor_id_fkey'), 'drive_activities', 'users', ['actor_id'], ['id'], ondelete='CASCADE') + op.alter_column('drive_activities', 'created_at', + existing_type=sa.TIMESTAMP(), + type_=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + op.alter_column('drive_activities', 'activity_type', + existing_type=sa.VARCHAR(length=50), + nullable=False) + op.alter_column('drive_activities', 'actor_id', + existing_type=sa.INTEGER(), + nullable=False) + op.alter_column('drive_activities', 'resource_id', + existing_type=sa.INTEGER(), + nullable=False) + op.alter_column('drive_activities', 'resource_type', + existing_type=sa.String(length=20), + type_=sa.VARCHAR(length=10), + nullable=False) + op.create_table('drive_acl', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('resource_type', sa.VARCHAR(length=10), autoincrement=False, nullable=False), + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('subject_type', sa.VARCHAR(length=10), autoincrement=False, nullable=False), + sa.Column('subject_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('role', sa.VARCHAR(length=20), autoincrement=False, nullable=False), + sa.Column('inherited_from', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('drive_acl_pkey')) + ) + op.drop_index(op.f('ix_vector_indices_user_id'), table_name='vector_indices') + op.drop_index(op.f('ix_vector_indices_id'), table_name='vector_indices') + op.drop_table('vector_indices') + op.drop_index(op.f('ix_vector_chunks_user_id'), table_name='vector_chunks') + op.drop_index(op.f('ix_vector_chunks_id'), table_name='vector_chunks') + op.drop_index(op.f('ix_vector_chunks_document_id'), table_name='vector_chunks') + op.drop_table('vector_chunks') + op.drop_index(op.f('ix_chatbot_documents_user_id'), table_name='chatbot_documents') + op.drop_index(op.f('ix_chatbot_documents_id'), table_name='chatbot_documents') + op.drop_index(op.f('ix_chatbot_documents_drive_file_id'), table_name='chatbot_documents') + op.drop_index(op.f('ix_chatbot_documents_document_id'), table_name='chatbot_documents') + op.drop_table('chatbot_documents') + op.drop_table('drive_acls') + # ### end Alembic commands ### diff --git a/alembic/versions/4a9dc50ce4af_drop_price_display.py b/alembic/versions/4a9dc50ce4af_drop_price_display.py new file mode 100644 index 0000000..e0d7b97 --- /dev/null +++ b/alembic/versions/4a9dc50ce4af_drop_price_display.py @@ -0,0 +1,33 @@ +"""drop_price_display + +Revision ID: 4a9dc50ce4af +Revises: 2b4c62c1cf35 +Create Date: 2026-08-20 11:40:33.072200 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '4a9dc50ce4af' +down_revision: Union[str, None] = '2b4c62c1cf35' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.execute("UPDATE plans SET price_amount = CAST(SUBSTRING(price_display FROM '[\d.]+') AS NUMERIC) WHERE price_display IS NOT NULL AND price_display != 'Contact Us' AND price_amount IS NULL") + op.drop_column('plans', 'price_display') + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('plans', sa.Column('price_display', sa.VARCHAR(length=50), autoincrement=False, nullable=True)) + op.execute("UPDATE plans SET price_display = '$' || price_amount WHERE price_amount IS NOT NULL") + op.execute("UPDATE plans SET price_display = 'Contact Us' WHERE price_amount IS NULL") + # ### end Alembic commands ### diff --git a/alembic/versions/522ee6d91431_add_drive_share_link_fields.py b/alembic/versions/522ee6d91431_add_drive_share_link_fields.py new file mode 100644 index 0000000..558950b --- /dev/null +++ b/alembic/versions/522ee6d91431_add_drive_share_link_fields.py @@ -0,0 +1,30 @@ +"""add_drive_share_link_fields + +Revision ID: 522ee6d91431 +Revises: 8d4dced54966 +Create Date: 2026-02-24 14:35:17.542820 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '522ee6d91431' +down_revision: Union[str, None] = '8d4dced54966' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('drive_shares', sa.Column('is_active', sa.Boolean(), nullable=False)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('drive_shares', 'is_active') + # ### end Alembic commands ### diff --git a/alembic/versions/540c970ff890_add_tenant_chat_token_quota.py b/alembic/versions/540c970ff890_add_tenant_chat_token_quota.py new file mode 100644 index 0000000..ad07c63 --- /dev/null +++ b/alembic/versions/540c970ff890_add_tenant_chat_token_quota.py @@ -0,0 +1,42 @@ +"""add_tenant_chat_token_quota + +Revision ID: 540c970ff890 +Revises: 7fc8dd61bb95 +Create Date: 2026-05-13 11:21:14.192033 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '540c970ff890' +down_revision: Union[str, None] = '7fc8dd61bb95' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Get current connection and inspector + conn = op.get_bind() + inspector = sa.inspect(conn) + + # Check if columns already exist (defensive migration) + columns = [c['name'] for c in inspector.get_columns('tenants')] + + if 'chat_token_daily_limit' not in columns: + op.add_column('tenants', sa.Column('chat_token_daily_limit', sa.BigInteger(), nullable=False, server_default='1000000')) + + if 'chat_tokens_used_today' not in columns: + op.add_column('tenants', sa.Column('chat_tokens_used_today', sa.BigInteger(), nullable=False, server_default='0')) + + if 'chat_tokens_last_reset_at' not in columns: + op.add_column('tenants', sa.Column('chat_tokens_last_reset_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False)) + + +def downgrade() -> None: + op.drop_column('tenants', 'chat_tokens_last_reset_at') + op.drop_column('tenants', 'chat_tokens_used_today') + op.drop_column('tenants', 'chat_token_daily_limit') diff --git a/alembic/versions/56d090047e85_increase_file_size_constraints.py b/alembic/versions/56d090047e85_increase_file_size_constraints.py new file mode 100644 index 0000000..6247c9b --- /dev/null +++ b/alembic/versions/56d090047e85_increase_file_size_constraints.py @@ -0,0 +1,74 @@ +"""increase_file_size_constraints + +Revision ID: 56d090047e85 +Revises: j002_unlimited_tenant_quota +Create Date: 2026-03-26 15:14:47.276886 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '56d090047e85' +down_revision: Union[str, None] = 'j002_unlimited_tenant_quota' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('projects', schema=None) as batch_op: + # Drop old 10MB constraints + batch_op.drop_constraint('ck_project_markdown_max', type_='check') + batch_op.drop_constraint('ck_project_xml_max', type_='check') + batch_op.drop_constraint('ck_project_html_max', type_='check') + + # Add new 30MB constraints + batch_op.create_check_constraint('ck_project_markdown_max', 'octet_length(markdown_content) <= 31457280') + batch_op.create_check_constraint('ck_project_xml_max', 'octet_length(xml_content) <= 31457280') + batch_op.create_check_constraint('ck_project_html_max', 'octet_length(html_content) <= 31457280') + + + + with op.batch_alter_table('project_versions', schema=None) as batch_op: + # Drop old 10MB constraints + batch_op.drop_constraint('ck_version_markdown_max', type_='check') + batch_op.drop_constraint('ck_version_xml_max', type_='check') + batch_op.drop_constraint('ck_version_html_max', type_='check') + + # Add new 30MB constraints + batch_op.create_check_constraint('ck_version_markdown_max', 'octet_length(markdown_content) <= 31457280') + batch_op.create_check_constraint('ck_version_xml_max', 'octet_length(xml_content) <= 31457280') + batch_op.create_check_constraint('ck_version_html_max', 'octet_length(html_content) <= 31457280') + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('project_versions', schema=None) as batch_op: + # Drop new 30MB constraints + batch_op.drop_constraint('ck_version_markdown_max', type_='check') + batch_op.drop_constraint('ck_version_xml_max', type_='check') + batch_op.drop_constraint('ck_version_html_max', type_='check') + + # Re-add old 10MB constraints + batch_op.create_check_constraint('ck_version_markdown_max', 'octet_length(markdown_content) <= 10485760') + batch_op.create_check_constraint('ck_version_xml_max', 'octet_length(xml_content) <= 10485760') + batch_op.create_check_constraint('ck_version_html_max', 'octet_length(html_content) <= 10485760') + + with op.batch_alter_table('projects', schema=None) as batch_op: + # Drop new 30MB constraints + batch_op.drop_constraint('ck_project_markdown_max', type_='check') + batch_op.drop_constraint('ck_project_xml_max', type_='check') + batch_op.drop_constraint('ck_project_html_max', type_='check') + + # Re-add old 10MB constraints + batch_op.create_check_constraint('ck_project_markdown_max', 'octet_length(markdown_content) <= 10485760') + batch_op.create_check_constraint('ck_project_xml_max', 'octet_length(xml_content) <= 10485760') + batch_op.create_check_constraint('ck_project_html_max', 'octet_length(html_content) <= 10485760') + + + # ### end Alembic commands ### diff --git a/alembic/versions/611d6b99ad11_add_multiple_files_to_signing_request.py b/alembic/versions/611d6b99ad11_add_multiple_files_to_signing_request.py new file mode 100644 index 0000000..3f81ded --- /dev/null +++ b/alembic/versions/611d6b99ad11_add_multiple_files_to_signing_request.py @@ -0,0 +1,34 @@ +"""add_multiple_files_to_signing_request + +Revision ID: 611d6b99ad11 +Revises: d551c03861f4 +Create Date: 2026-07-14 18:00:05.273738 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '611d6b99ad11' +down_revision: Union[str, None] = 'd551c03861f4' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column('signing_requests', sa.Column('drive_file_ids', sa.JSON(), nullable=True)) + op.add_column('signing_requests', sa.Column('signed_file_ids', sa.JSON(), nullable=True)) + op.alter_column('signing_requests', 'drive_file_id', + existing_type=sa.INTEGER(), + nullable=True) + + +def downgrade() -> None: + op.alter_column('signing_requests', 'drive_file_id', + existing_type=sa.INTEGER(), + nullable=False) + op.drop_column('signing_requests', 'signed_file_ids') + op.drop_column('signing_requests', 'drive_file_ids') diff --git a/alembic/versions/6c9cc3567866_merge_multiple_heads.py b/alembic/versions/6c9cc3567866_merge_multiple_heads.py new file mode 100644 index 0000000..309e12b --- /dev/null +++ b/alembic/versions/6c9cc3567866_merge_multiple_heads.py @@ -0,0 +1,26 @@ +"""merge_multiple_heads + +Revision ID: 6c9cc3567866 +Revises: 45c94ee15294, ff9052a97835 +Create Date: 2026-03-05 07:53:09.436096 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '6c9cc3567866' +down_revision: Union[str, None] = ('45c94ee15294', 'ff9052a97835') +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/alembic/versions/73dff81eed72_add_tenant_contacts.py b/alembic/versions/73dff81eed72_add_tenant_contacts.py new file mode 100644 index 0000000..8da7cf2 --- /dev/null +++ b/alembic/versions/73dff81eed72_add_tenant_contacts.py @@ -0,0 +1,43 @@ +"""Add tenant contacts + +Revision ID: 73dff81eed72 +Revises: dfae38e4ede3 +Create Date: 2026-08-12 10:56:21.668158 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '73dff81eed72' +down_revision: Union[str, None] = 'dfae38e4ede3' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('tenant_contacts', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('tenant_id', sa.UUID(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('email', sa.String(length=255), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('tenant_id', 'email', name='uq_tenant_contacts_tenant_id_email'), + ) + op.create_index(op.f('ix_tenant_contacts_email'), 'tenant_contacts', ['email'], unique=False) + op.create_index(op.f('ix_tenant_contacts_tenant_id'), 'tenant_contacts', ['tenant_id'], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_tenant_contacts_tenant_id'), table_name='tenant_contacts') + op.drop_index(op.f('ix_tenant_contacts_email'), table_name='tenant_contacts') + op.drop_table('tenant_contacts') + # ### end Alembic commands ### diff --git a/alembic/versions/765b053e3dcd_add_vector_index_checksum.py b/alembic/versions/765b053e3dcd_add_vector_index_checksum.py new file mode 100644 index 0000000..64fdc02 --- /dev/null +++ b/alembic/versions/765b053e3dcd_add_vector_index_checksum.py @@ -0,0 +1,30 @@ +"""add_vector_index_checksum + +Revision ID: 765b053e3dcd +Revises: be67927ab00d +Create Date: 2026-03-03 20:25:42.849543 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '765b053e3dcd' +down_revision: Union[str, None] = 'be67927ab00d' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/alembic/versions/7d7641cc1bb2_refactor_global_signature_config.py b/alembic/versions/7d7641cc1bb2_refactor_global_signature_config.py new file mode 100644 index 0000000..2e618d7 --- /dev/null +++ b/alembic/versions/7d7641cc1bb2_refactor_global_signature_config.py @@ -0,0 +1,32 @@ +"""refactor_global_signature_config + +Revision ID: 7d7641cc1bb2 +Revises: b3a7c1e9f042 +Create Date: 2026-07-21 07:19:00.000000 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '7d7641cc1bb2' +down_revision: Union[str, None] = 'af57c3d9f1c2' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column('global_signature_configs', sa.Column('global_credentials_json', postgresql.JSON(astext_type=sa.Text()), nullable=True)) + op.add_column('global_signature_configs', sa.Column('enabled_providers', postgresql.JSON(astext_type=sa.Text()), server_default='["docuseal", "zoho_sign"]', nullable=False)) + op.drop_column('global_signature_configs', 'docuseal_credentials') + op.drop_column('global_signature_configs', 'zoho_credentials') + + +def downgrade() -> None: + op.add_column('global_signature_configs', sa.Column('zoho_credentials', postgresql.JSON(astext_type=sa.Text()), autoincrement=False, nullable=True)) + op.add_column('global_signature_configs', sa.Column('docuseal_credentials', postgresql.JSON(astext_type=sa.Text()), autoincrement=False, nullable=True)) + op.drop_column('global_signature_configs', 'enabled_providers') + op.drop_column('global_signature_configs', 'global_credentials_json') diff --git a/alembic/versions/7fc8dd61bb95_add_chat_token_usage_and_quota_tables.py b/alembic/versions/7fc8dd61bb95_add_chat_token_usage_and_quota_tables.py new file mode 100644 index 0000000..974cd55 --- /dev/null +++ b/alembic/versions/7fc8dd61bb95_add_chat_token_usage_and_quota_tables.py @@ -0,0 +1,70 @@ +"""Add chat token usage and quota tables + +Revision ID: 7fc8dd61bb95 +Revises: z002_pref_lang +Create Date: 2026-05-12 18:38:23.935092 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '7fc8dd61bb95' +down_revision: Union[str, None] = 'z002_pref_lang' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Get current connection and inspector + conn = op.get_bind() + inspector = sa.inspect(conn) + + # 1. Create chat_token_usage if missing + if not inspector.has_table('chat_token_usage'): + op.create_table('chat_token_usage', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('session_id', sa.Integer(), nullable=True), + sa.Column('prompt_tokens', sa.BigInteger(), nullable=False, server_default='0'), + sa.Column('completion_tokens', sa.BigInteger(), nullable=False, server_default='0'), + sa.Column('total_tokens', sa.BigInteger(), nullable=False, server_default='0'), + sa.Column('model_name', sa.String(length=100), nullable=False), + sa.Column('cost_usd', sa.DECIMAL(precision=12, scale=8), nullable=False, server_default='0.0'), + sa.Column('timestamp', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('phase', sa.String(length=50), nullable=True), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['session_id'], ['chat_sessions.id'], ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('chat_token_usage', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_chat_token_usage_id'), ['id'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_token_usage_timestamp'), ['timestamp'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_token_usage_user_id'), ['user_id'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_token_usage_session_id'), ['session_id'], unique=False) + + # 2. Create chat_token_quota if missing + if not inspector.has_table('chat_token_quota'): + op.create_table('chat_token_quota', + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('daily_limit', sa.BigInteger(), nullable=False), + sa.Column('tokens_used_today', sa.BigInteger(), nullable=False, server_default='0'), + sa.Column('last_reset_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('total_tokens_ever', sa.BigInteger(), nullable=False, server_default='0'), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('user_id') + ) + + +def downgrade() -> None: + conn = op.get_bind() + inspector = sa.inspect(conn) + + if inspector.has_table('chat_token_quota'): + op.drop_table('chat_token_quota') + + if inspector.has_table('chat_token_usage'): + op.drop_table('chat_token_usage') diff --git a/alembic/versions/80be2c61e7fc_add_is_popular_to_plans.py b/alembic/versions/80be2c61e7fc_add_is_popular_to_plans.py new file mode 100644 index 0000000..31a39ef --- /dev/null +++ b/alembic/versions/80be2c61e7fc_add_is_popular_to_plans.py @@ -0,0 +1,30 @@ +"""add_is_popular_to_plans + +Revision ID: 80be2c61e7fc +Revises: 4a9dc50ce4af +Create Date: 2026-08-20 12:47:53.790237 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '80be2c61e7fc' +down_revision: Union[str, None] = '4a9dc50ce4af' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('plans', sa.Column('is_popular', sa.Boolean(), server_default=sa.text('false'), nullable=False)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('plans', 'is_popular') + # ### end Alembic commands ### diff --git a/alembic/versions/823a486fd287_add_created_at_to_drive_stars.py b/alembic/versions/823a486fd287_add_created_at_to_drive_stars.py new file mode 100644 index 0000000..94fd0a3 --- /dev/null +++ b/alembic/versions/823a486fd287_add_created_at_to_drive_stars.py @@ -0,0 +1,36 @@ +"""add created_at to drive_stars + +Revision ID: 823a486fd287 +Revises: 88717d544229 +Create Date: 2026-03-27 23:24:47.417678 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '823a486fd287' +down_revision: Union[str, None] = '88717d544229' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Add column as nullable first + op.add_column('drive_stars', sa.Column('created_at', sa.DateTime(timezone=True), nullable=True)) + + # Update existing stars to current time so they aren't null + op.execute("UPDATE drive_stars SET created_at = NOW() WHERE created_at IS NULL") + + # Make column non-nullable with server default + op.alter_column('drive_stars', 'created_at', + existing_type=sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text('now()')) + + +def downgrade() -> None: + op.drop_column('drive_stars', 'created_at') diff --git a/alembic/versions/8736ab6bd5af_add_notifications_table.py b/alembic/versions/8736ab6bd5af_add_notifications_table.py new file mode 100644 index 0000000..9b782fb --- /dev/null +++ b/alembic/versions/8736ab6bd5af_add_notifications_table.py @@ -0,0 +1,41 @@ +"""add_notifications_table + +Revision ID: 8736ab6bd5af +Revises: 522ee6d91431 +Create Date: 2026-02-24 15:25:34.045912 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '8736ab6bd5af' +down_revision: Union[str, None] = '522ee6d91431' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + 'notifications', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('notif_type', sa.String(length=50), nullable=False), + sa.Column('title', sa.String(length=255), nullable=False), + sa.Column('message', sa.Text(), nullable=False), + sa.Column('resource_type', sa.String(length=20), nullable=True), + sa.Column('resource_id', sa.Integer(), nullable=True), + sa.Column('is_read', sa.Boolean(), nullable=False, server_default=sa.text('false')), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()')), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + ) + op.create_index('ix_notifications_user_id', 'notifications', ['user_id'], unique=False) + + +def downgrade() -> None: + op.drop_index('ix_notifications_user_id', table_name='notifications') + op.drop_table('notifications') diff --git a/alembic/versions/88717d544229_add_is_default_to_roles.py b/alembic/versions/88717d544229_add_is_default_to_roles.py new file mode 100644 index 0000000..32306f8 --- /dev/null +++ b/alembic/versions/88717d544229_add_is_default_to_roles.py @@ -0,0 +1,34 @@ +"""add is_default to roles + +Revision ID: 88717d544229 +Revises: 56d090047e85 +Create Date: 2026-03-27 14:08:16.581384 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '88717d544229' +down_revision: Union[str, None] = '56d090047e85' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('roles', schema=None) as batch_op: + batch_op.add_column(sa.Column('is_default', sa.Boolean(), nullable=False, server_default=sa.text('false'))) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('roles', schema=None) as batch_op: + batch_op.drop_column('is_default') + + # ### end Alembic commands ### diff --git a/alembic/versions/89508c176437_limit_drivecomment_body_to_10000_chars.py b/alembic/versions/89508c176437_limit_drivecomment_body_to_10000_chars.py new file mode 100644 index 0000000..7c47a5f --- /dev/null +++ b/alembic/versions/89508c176437_limit_drivecomment_body_to_10000_chars.py @@ -0,0 +1,36 @@ +"""Limit DriveComment body to 10000 chars + +Revision ID: 89508c176437 +Revises: ff8052a97834 +Create Date: 2026-03-03 22:02:03.251490 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '89508c176437' +down_revision: Union[str, None] = 'ff8052a97834' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('drive_comments', 'body', + existing_type=sa.TEXT(), + type_=sa.String(length=10000), + existing_nullable=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('drive_comments', 'body', + existing_type=sa.String(length=10000), + type_=sa.TEXT(), + existing_nullable=False) + # ### end Alembic commands ### diff --git a/alembic/versions/8d4dced54966_add_is_root_column.py b/alembic/versions/8d4dced54966_add_is_root_column.py new file mode 100644 index 0000000..d9bcdd7 --- /dev/null +++ b/alembic/versions/8d4dced54966_add_is_root_column.py @@ -0,0 +1,59 @@ +"""add is_root column + +Revision ID: 8d4dced54966 +Revises: cfbee4f91fc2 +Create Date: 2026-02-24 11:15:51.402384 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '8d4dced54966' +down_revision: Union[str, None] = 'cfbee4f91fc2' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('drive_comments', sa.Column('author_id', sa.Integer(), nullable=True)) + op.add_column('drive_comments', sa.Column('body', sa.Text(), nullable=True)) + + # Seeding existing comments if any (though usually clean) + op.execute("UPDATE drive_comments SET author_id = user_id WHERE author_id IS NULL") + op.execute("UPDATE drive_comments SET body = content WHERE body IS NULL") + + op.alter_column('drive_comments', 'author_id', nullable=False) + op.alter_column('drive_comments', 'body', nullable=False) + + op.add_column('drive_comments', sa.Column('anchor', sa.Text(), nullable=True)) + op.add_column('drive_comments', sa.Column('resolved_at', sa.DateTime(), nullable=True)) + op.drop_constraint(op.f('drive_comments_user_id_fkey'), 'drive_comments', type_='foreignkey') + op.create_foreign_key(None, 'drive_comments', 'users', ['author_id'], ['id']) + op.drop_column('drive_comments', 'user_id') + op.drop_column('drive_comments', 'content') + + # 🆕 Add is_root safely + op.add_column('drive_folders', sa.Column('is_root', sa.Boolean(), nullable=True)) + op.execute("UPDATE drive_folders SET is_root = false WHERE is_root IS NULL") + op.execute("UPDATE drive_folders SET is_root = true WHERE parent_id IS NULL") + op.alter_column('drive_folders', 'is_root', nullable=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('drive_folders', 'is_root') + op.add_column('drive_comments', sa.Column('content', sa.TEXT(), autoincrement=False, nullable=False)) + op.add_column('drive_comments', sa.Column('user_id', sa.INTEGER(), autoincrement=False, nullable=False)) + op.drop_constraint(None, 'drive_comments', type_='foreignkey') + op.create_foreign_key(op.f('drive_comments_user_id_fkey'), 'drive_comments', 'users', ['user_id'], ['id']) + op.drop_column('drive_comments', 'resolved_at') + op.drop_column('drive_comments', 'anchor') + op.drop_column('drive_comments', 'body') + op.drop_column('drive_comments', 'author_id') + # ### end Alembic commands ### diff --git a/alembic/versions/905726af73c8_add_plan_roles_and_is_system_to_roles.py b/alembic/versions/905726af73c8_add_plan_roles_and_is_system_to_roles.py new file mode 100644 index 0000000..c51bd43 --- /dev/null +++ b/alembic/versions/905726af73c8_add_plan_roles_and_is_system_to_roles.py @@ -0,0 +1,34 @@ +"""Add plan_roles and is_system to roles + +Revision ID: 905726af73c8 +Revises: 80be2c61e7fc +Create Date: 2026-08-20 19:17:20.939383 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '905726af73c8' +down_revision: Union[str, None] = '80be2c61e7fc' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table('plan_roles', + sa.Column('plan_id', sa.UUID(), nullable=False), + sa.Column('role_id', sa.UUID(), nullable=False), + sa.ForeignKeyConstraint(['plan_id'], ['plans.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['role_id'], ['roles.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('plan_id', 'role_id') + ) + op.add_column('roles', sa.Column('is_system', sa.Boolean(), nullable=False, server_default='false')) + + +def downgrade() -> None: + op.drop_column('roles', 'is_system') + op.drop_table('plan_roles') diff --git a/alembic/versions/9557c7307575_tenant_storage_config_and_provider.py b/alembic/versions/9557c7307575_tenant_storage_config_and_provider.py new file mode 100644 index 0000000..6469cd6 --- /dev/null +++ b/alembic/versions/9557c7307575_tenant_storage_config_and_provider.py @@ -0,0 +1,415 @@ +"""tenant_storage_config_and_provider + +Revision ID: 9557c7307575 +Revises: h004_signing_forensic_v2 +Create Date: 2026-03-16 13:00:58.602085 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '9557c7307575' +down_revision: Union[str, None] = 'h004_signing_forensic_v2' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.execute("UPDATE drive_files SET size = 0 WHERE size IS NULL") + op.execute("UPDATE drive_files SET is_starred = false WHERE is_starred IS NULL") + op.execute("UPDATE drive_files SET is_trashed = false WHERE is_trashed IS NULL") + op.execute("UPDATE drive_files SET created_at = CURRENT_TIMESTAMP WHERE created_at IS NULL") + op.execute("UPDATE drive_files SET updated_at = CURRENT_TIMESTAMP WHERE updated_at IS NULL") + + op.execute("UPDATE drive_folders SET is_starred = false WHERE is_starred IS NULL") + op.execute("UPDATE drive_folders SET is_trashed = false WHERE is_trashed IS NULL") + op.execute("UPDATE drive_folders SET created_at = CURRENT_TIMESTAMP WHERE created_at IS NULL") + op.execute("UPDATE drive_folders SET updated_at = CURRENT_TIMESTAMP WHERE updated_at IS NULL") + + op.execute("UPDATE drive_shares SET created_at = CURRENT_TIMESTAMP WHERE created_at IS NULL") + + op.execute("UPDATE project_versions SET is_current = false WHERE is_current IS NULL") + op.execute("UPDATE project_versions SET created_at = CURRENT_TIMESTAMP WHERE created_at IS NULL") + + op.execute("UPDATE projects SET status = 'processing' WHERE status IS NULL") + op.execute("UPDATE projects SET file_type = 'PDF' WHERE file_type IS NULL") + op.execute("UPDATE projects SET output_format = 'EPUB' WHERE output_format IS NULL") + op.execute("UPDATE projects SET progress = 0 WHERE progress IS NULL") + op.execute("UPDATE projects SET current_step = 'upload' WHERE current_step IS NULL") + op.execute("UPDATE projects SET created_at = CURRENT_TIMESTAMP WHERE created_at IS NULL") + op.execute("UPDATE projects SET updated_at = CURRENT_TIMESTAMP WHERE updated_at IS NULL") + + op.execute("UPDATE signing_requests SET created_at = CURRENT_TIMESTAMP WHERE created_at IS NULL") + + op.execute("UPDATE tenants SET created_at = CURRENT_TIMESTAMP WHERE created_at IS NULL") + op.execute("UPDATE tenants SET updated_at = CURRENT_TIMESTAMP WHERE updated_at IS NULL") + + op.execute("UPDATE users SET subscription = 'free' WHERE subscription IS NULL") + op.execute("UPDATE users SET created_at = CURRENT_TIMESTAMP WHERE created_at IS NULL") + + op.create_table('tenant_storage_configs', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('tenant_id', sa.UUID(), nullable=False), + sa.Column('storage_provider', sa.String(length=50), nullable=False), + sa.Column('b2_key_id', sa.String(length=255), nullable=True), + sa.Column('b2_application_key', sa.String(length=512), nullable=True), + sa.Column('b2_bucket_name', sa.String(length=255), nullable=True), + sa.Column('is_active', sa.Boolean(), nullable=False), + sa.Column('created_by_id', sa.Integer(), nullable=True), + sa.Column('updated_by_id', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['created_by_id'], ['users.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['updated_by_id'], ['users.id'], ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('tenant_storage_configs', schema=None) as batch_op: + op.execute("CREATE UNIQUE INDEX IF NOT EXISTS ix_tenant_storage_configs_tenant_id ON tenant_storage_configs (tenant_id)") + + with op.batch_alter_table('accesses', schema=None) as batch_op: + batch_op.alter_column('name', + existing_type=sa.VARCHAR(length=100), + type_=sa.String(length=255), + existing_nullable=False) + op.execute("DROP INDEX IF EXISTS ix_accesses_name") + batch_op.drop_constraint(batch_op.f('uq_accesses_access_code'), type_='unique') + op.execute("CREATE UNIQUE INDEX IF NOT EXISTS ix_accesses_access_code ON accesses (access_code)") + op.execute("CREATE INDEX IF NOT EXISTS ix_accesses_category ON accesses (category)") + op.execute("ALTER TABLE accesses DROP COLUMN IF EXISTS description") + + op.execute("DROP INDEX IF EXISTS idx_drive_activities_actor") + op.execute("DROP INDEX IF EXISTS idx_drive_activities_res") + op.execute("ALTER TABLE drive_activities DROP COLUMN IF EXISTS metadata") + + with op.batch_alter_table('drive_file_versions', schema=None) as batch_op: + batch_op.add_column(sa.Column('storage_provider', sa.String(length=50), nullable=False, server_default='minio')) + op.execute("DROP INDEX IF EXISTS idx_drive_versions_file") + + with op.batch_alter_table('drive_files', schema=None) as batch_op: + batch_op.alter_column('size', + existing_type=sa.BIGINT(), + nullable=False, + existing_server_default=sa.text('0')) + batch_op.alter_column('is_starred', + existing_type=sa.BOOLEAN(), + nullable=False, + existing_server_default=sa.text('false')) + batch_op.alter_column('is_trashed', + existing_type=sa.BOOLEAN(), + nullable=False, + existing_server_default=sa.text('false')) + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('CURRENT_TIMESTAMP')) + batch_op.alter_column('updated_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + op.execute("DROP INDEX IF EXISTS idx_drive_files_folder") + op.execute("DROP INDEX IF EXISTS idx_drive_files_owner") + op.execute("DROP INDEX IF EXISTS idx_drive_files_trashed") + + with op.batch_alter_table('drive_folders', schema=None) as batch_op: + batch_op.alter_column('is_starred', + existing_type=sa.BOOLEAN(), + nullable=False, + existing_server_default=sa.text('false')) + batch_op.alter_column('is_trashed', + existing_type=sa.BOOLEAN(), + nullable=False, + existing_server_default=sa.text('false')) + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('CURRENT_TIMESTAMP')) + batch_op.alter_column('updated_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + op.execute("DROP INDEX IF EXISTS idx_drive_folders_owner") + op.execute("DROP INDEX IF EXISTS idx_drive_folders_parent") + op.execute("DROP INDEX IF EXISTS idx_drive_folders_trashed") + + with op.batch_alter_table('drive_shares', schema=None) as batch_op: + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('CURRENT_TIMESTAMP')) + op.execute("DROP INDEX IF EXISTS idx_drive_shares_resource") + op.execute("DROP INDEX IF EXISTS uq_drive_shares_link") + batch_op.create_unique_constraint(None, ['link_token']) + batch_op.drop_constraint(batch_op.f('drive_shares_created_by_id_fkey'), type_='foreignkey') + batch_op.create_foreign_key(None, 'users', ['created_by_id'], ['id'], ondelete='CASCADE') + + with op.batch_alter_table('project_versions', schema=None) as batch_op: + batch_op.alter_column('is_current', + existing_type=sa.BOOLEAN(), + nullable=False, + existing_server_default=sa.text('false')) + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('CURRENT_TIMESTAMP')) + op.execute("DROP INDEX IF EXISTS idx_project_versions_created_by") + op.execute("DROP INDEX IF EXISTS idx_project_versions_project_id") + + with op.batch_alter_table('projects', schema=None) as batch_op: + batch_op.alter_column('status', + existing_type=sa.VARCHAR(length=50), + nullable=False, + existing_server_default=sa.text("'processing'::character varying")) + batch_op.alter_column('file_type', + existing_type=sa.VARCHAR(length=50), + nullable=False, + existing_server_default=sa.text("'PDF'::character varying")) + batch_op.alter_column('output_format', + existing_type=sa.VARCHAR(length=50), + nullable=False, + existing_server_default=sa.text("'EPUB'::character varying")) + batch_op.alter_column('progress', + existing_type=sa.INTEGER(), + nullable=False, + existing_server_default=sa.text('0')) + batch_op.alter_column('current_step', + existing_type=sa.VARCHAR(length=50), + nullable=False, + existing_server_default=sa.text("'upload'::character varying")) + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('CURRENT_TIMESTAMP')) + batch_op.alter_column('updated_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + op.execute("DROP INDEX IF EXISTS idx_projects_session_id") + op.execute("DROP INDEX IF EXISTS idx_projects_user_id") + batch_op.create_foreign_key(None, 'users', ['user_id'], ['id'], ondelete='CASCADE') + + with op.batch_alter_table('signing_requests', schema=None) as batch_op: + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + + with op.batch_alter_table('system_configurations', schema=None) as batch_op: + batch_op.drop_constraint(batch_op.f('uq_system_configurations_config_key'), type_='unique') + + with op.batch_alter_table('tenant_smtp_configs', schema=None) as batch_op: + batch_op.drop_constraint(batch_op.f('uq_tenant_smtp_configs_tenant_id'), type_='unique') + op.execute("DROP INDEX IF EXISTS ix_tenant_smtp_configs_tenant_id") + op.execute("CREATE UNIQUE INDEX IF NOT EXISTS ix_tenant_smtp_configs_tenant_id ON tenant_smtp_configs (tenant_id)") + + with op.batch_alter_table('tenants', schema=None) as batch_op: + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + batch_op.alter_column('updated_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + batch_op.drop_constraint(batch_op.f('tenants_slug_key'), type_='unique') + op.execute("CREATE INDEX IF NOT EXISTS ix_tenants_id ON tenants (id)") + op.execute("CREATE UNIQUE INDEX IF NOT EXISTS ix_tenants_slug ON tenants (slug)") + + with op.batch_alter_table('user_files', schema=None) as batch_op: + batch_op.add_column(sa.Column('storage_provider', sa.String(length=50), nullable=False, server_default='minio')) + op.execute("DROP INDEX IF EXISTS idx_user_files_s3_key") + op.execute("DROP INDEX IF EXISTS idx_user_files_user") + + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.alter_column('subscription', + existing_type=sa.VARCHAR(length=50), + nullable=False, + existing_server_default=sa.text("'free'::character varying")) + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('CURRENT_TIMESTAMP')) + op.execute("DROP INDEX IF EXISTS idx_users_email") + op.execute("CREATE INDEX IF NOT EXISTS ix_users_email ON users (email)") + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('users', schema=None) as batch_op: + op.execute("DROP INDEX IF EXISTS ix_users_email") + op.execute("CREATE INDEX IF NOT EXISTS idx_users_email ON users (email)") + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('CURRENT_TIMESTAMP')) + batch_op.alter_column('subscription', + existing_type=sa.VARCHAR(length=50), + nullable=True, + existing_server_default=sa.text("'free'::character varying")) + + with op.batch_alter_table('user_files', schema=None) as batch_op: + op.execute("CREATE INDEX IF NOT EXISTS idx_user_files_user ON user_files (user_id)") + op.execute("CREATE INDEX IF NOT EXISTS idx_user_files_s3_key ON user_files (s3_key)") + op.execute("ALTER TABLE user_files DROP COLUMN IF EXISTS storage_provider") + + with op.batch_alter_table('tenants', schema=None) as batch_op: + op.execute("DROP INDEX IF EXISTS ix_tenants_slug") + op.execute("DROP INDEX IF EXISTS ix_tenants_id") + batch_op.create_unique_constraint(batch_op.f('tenants_slug_key'), ['slug'], postgresql_nulls_not_distinct=False) + batch_op.alter_column('updated_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('now()')) + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('now()')) + + with op.batch_alter_table('tenant_smtp_configs', schema=None) as batch_op: + op.execute("DROP INDEX IF EXISTS ix_tenant_smtp_configs_tenant_id") + op.execute("CREATE INDEX IF NOT EXISTS ix_tenant_smtp_configs_tenant_id ON tenant_smtp_configs (tenant_id)") + batch_op.create_unique_constraint(batch_op.f('uq_tenant_smtp_configs_tenant_id'), ['tenant_id'], postgresql_nulls_not_distinct=False) + + with op.batch_alter_table('system_configurations', schema=None) as batch_op: + batch_op.create_unique_constraint(batch_op.f('uq_system_configurations_config_key'), ['config_key'], postgresql_nulls_not_distinct=False) + + with op.batch_alter_table('signing_requests', schema=None) as batch_op: + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('now()')) + + with op.batch_alter_table('projects', schema=None) as batch_op: + batch_op.drop_constraint(None, type_='foreignkey') + op.execute("CREATE INDEX IF NOT EXISTS idx_projects_user_id ON projects (user_id)") + op.execute("CREATE INDEX IF NOT EXISTS idx_projects_session_id ON projects (session_id)") + batch_op.alter_column('updated_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('now()')) + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('CURRENT_TIMESTAMP')) + batch_op.alter_column('current_step', + existing_type=sa.VARCHAR(length=50), + nullable=True, + existing_server_default=sa.text("'upload'::character varying")) + batch_op.alter_column('progress', + existing_type=sa.INTEGER(), + nullable=True, + existing_server_default=sa.text('0')) + batch_op.alter_column('output_format', + existing_type=sa.VARCHAR(length=50), + nullable=True, + existing_server_default=sa.text("'EPUB'::character varying")) + batch_op.alter_column('file_type', + existing_type=sa.VARCHAR(length=50), + nullable=True, + existing_server_default=sa.text("'PDF'::character varying")) + batch_op.alter_column('status', + existing_type=sa.VARCHAR(length=50), + nullable=True, + existing_server_default=sa.text("'processing'::character varying")) + + with op.batch_alter_table('project_versions', schema=None) as batch_op: + op.execute("CREATE INDEX IF NOT EXISTS idx_project_versions_project_id ON project_versions (project_id)") + op.execute("CREATE INDEX IF NOT EXISTS idx_project_versions_created_by ON project_versions (created_by_id)") + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('CURRENT_TIMESTAMP')) + batch_op.alter_column('is_current', + existing_type=sa.BOOLEAN(), + nullable=True, + existing_server_default=sa.text('false')) + + with op.batch_alter_table('drive_shares', schema=None) as batch_op: + batch_op.drop_constraint(None, type_='foreignkey') + batch_op.create_foreign_key(batch_op.f('drive_shares_created_by_id_fkey'), 'users', ['created_by_id'], ['id']) + batch_op.drop_constraint(None, type_='unique') + op.execute("CREATE INDEX IF NOT EXISTS uq_drive_shares_link ON drive_shares (link_token)") + op.execute("CREATE INDEX IF NOT EXISTS idx_drive_shares_resource ON drive_shares (resource_type,resource_id)") + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('CURRENT_TIMESTAMP')) + + with op.batch_alter_table('drive_folders', schema=None) as batch_op: + op.execute("CREATE INDEX IF NOT EXISTS idx_drive_folders_trashed ON drive_folders (is_trashed)") + op.execute("CREATE INDEX IF NOT EXISTS idx_drive_folders_parent ON drive_folders (parent_id)") + op.execute("CREATE INDEX IF NOT EXISTS idx_drive_folders_owner ON drive_folders (owner_id)") + batch_op.alter_column('updated_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('now()')) + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('CURRENT_TIMESTAMP')) + batch_op.alter_column('is_trashed', + existing_type=sa.BOOLEAN(), + nullable=True, + existing_server_default=sa.text('false')) + batch_op.alter_column('is_starred', + existing_type=sa.BOOLEAN(), + nullable=True, + existing_server_default=sa.text('false')) + + with op.batch_alter_table('drive_files', schema=None) as batch_op: + op.execute("CREATE INDEX IF NOT EXISTS idx_drive_files_trashed ON drive_files (is_trashed)") + op.execute("CREATE INDEX IF NOT EXISTS idx_drive_files_owner ON drive_files (owner_id)") + op.execute("CREATE INDEX IF NOT EXISTS idx_drive_files_folder ON drive_files (folder_id)") + batch_op.alter_column('updated_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('now()')) + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('CURRENT_TIMESTAMP')) + batch_op.alter_column('is_trashed', + existing_type=sa.BOOLEAN(), + nullable=True, + existing_server_default=sa.text('false')) + batch_op.alter_column('is_starred', + existing_type=sa.BOOLEAN(), + nullable=True, + existing_server_default=sa.text('false')) + batch_op.alter_column('size', + existing_type=sa.BIGINT(), + nullable=True, + existing_server_default=sa.text('0')) + + with op.batch_alter_table('drive_file_versions', schema=None) as batch_op: + op.execute("CREATE INDEX IF NOT EXISTS idx_drive_versions_file ON drive_file_versions (file_id)") + op.execute("ALTER TABLE drive_file_versions DROP COLUMN IF EXISTS storage_provider") + + with op.batch_alter_table('drive_activities', schema=None) as batch_op: + batch_op.add_column(sa.Column('metadata', sa.TEXT(), autoincrement=False, nullable=True)) + op.execute("CREATE INDEX IF NOT EXISTS idx_drive_activities_res ON drive_activities (resource_type,resource_id)") + op.execute("CREATE INDEX IF NOT EXISTS idx_drive_activities_actor ON drive_activities (actor_id)") + + with op.batch_alter_table('accesses', schema=None) as batch_op: + batch_op.add_column(sa.Column('description', sa.VARCHAR(length=255), autoincrement=False, nullable=True)) + op.execute("DROP INDEX IF EXISTS ix_accesses_category") + op.execute("DROP INDEX IF EXISTS ix_accesses_access_code") + batch_op.create_unique_constraint(batch_op.f('uq_accesses_access_code'), ['access_code'], postgresql_nulls_not_distinct=False) + op.execute("CREATE INDEX IF NOT EXISTS ix_accesses_name ON accesses (name)") + batch_op.alter_column('name', + existing_type=sa.String(length=255), + type_=sa.VARCHAR(length=100), + existing_nullable=False) + + with op.batch_alter_table('tenant_storage_configs', schema=None) as batch_op: + op.execute("DROP INDEX IF EXISTS ix_tenant_storage_configs_tenant_id") + + op.execute("DROP TABLE IF EXISTS tenant_storage_configs") + # ### end Alembic commands ### diff --git a/alembic/versions/9a6db38b86a6_make_drivecomment_generic.py b/alembic/versions/9a6db38b86a6_make_drivecomment_generic.py new file mode 100644 index 0000000..6b03ba2 --- /dev/null +++ b/alembic/versions/9a6db38b86a6_make_drivecomment_generic.py @@ -0,0 +1,57 @@ +"""Make DriveComment generic + +Revision ID: 9a6db38b86a6 +Revises: 41bed7eb6ed9 +Create Date: 2026-02-27 14:32:05.071480 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '9a6db38b86a6' +down_revision: Union[str, None] = '41bed7eb6ed9' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # 1. Add columns as nullable first + op.add_column('drive_comments', sa.Column('resource_type', sa.String(length=20), nullable=True)) + op.add_column('drive_comments', sa.Column('resource_id', sa.Integer(), nullable=True)) + + # 2. Populate data from existing file_id safely + # Use WHERE file_id IS NOT NULL to avoid unnecessary or invalid updates + op.execute("UPDATE drive_comments SET resource_type = 'file', resource_id = file_id WHERE file_id IS NOT NULL") + + # 3. Set to NOT NULL now that data is populated + op.alter_column('drive_comments', 'resource_type', nullable=False) + op.alter_column('drive_comments', 'resource_id', nullable=False) + + # 4. Cleanup old column and constraint + with op.batch_alter_table('drive_comments') as batch_op: + batch_op.drop_column('file_id') + + +def downgrade() -> None: + # 1. Add column back as nullable first + op.add_column('drive_comments', sa.Column('file_id', sa.Integer(), nullable=True)) + + # 2. Restore data from generic columns back to file_id + # Only migrate records that were originally or are currently 'file' types + op.execute("UPDATE drive_comments SET file_id = resource_id WHERE resource_type = 'file'") + + # 3. Delete or handle non-file comments that cannot exist in the old schema + # Alternatively, keep file_id as nullable if preferred, but here we restore original state + op.execute("DELETE FROM drive_comments WHERE resource_type != 'file'") + + # 4. Restore constraints + op.alter_column('drive_comments', 'file_id', nullable=False) + op.create_foreign_key('drive_comments_file_id_fkey', 'drive_comments', 'drive_files', ['file_id'], ['id']) + + # 5. Cleanup generic columns + op.drop_column('drive_comments', 'resource_id') + op.drop_column('drive_comments', 'resource_type') diff --git a/alembic/versions/9af99fc7ca0f_merge_heads.py b/alembic/versions/9af99fc7ca0f_merge_heads.py new file mode 100644 index 0000000..d4b7c0f --- /dev/null +++ b/alembic/versions/9af99fc7ca0f_merge_heads.py @@ -0,0 +1,26 @@ +"""merge heads + +Revision ID: 9af99fc7ca0f +Revises: 15f935213b22, cba70ab35793, v3_extraction +Create Date: 2026-04-18 14:22:43.985119 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '9af99fc7ca0f' +down_revision: Union[str, None] = ('15f935213b22', 'cba70ab35793', 'v3_extraction') +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/alembic/versions/9c49fe120ffb_add_device_and_session_tables.py b/alembic/versions/9c49fe120ffb_add_device_and_session_tables.py new file mode 100644 index 0000000..9a4dd60 --- /dev/null +++ b/alembic/versions/9c49fe120ffb_add_device_and_session_tables.py @@ -0,0 +1,79 @@ +"""Add device and session tables + +Revision ID: 9c49fe120ffb +Revises: z004_add_global_workflows +Create Date: 2026-08-06 15:27:12.015009 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '9c49fe120ffb' +down_revision: Union[str, None] = 'z004_add_global_workflows' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('devices', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('fingerprint', sa.String(length=255), nullable=False), + sa.Column('browser', sa.String(length=100), nullable=True), + sa.Column('os', sa.String(length=100), nullable=True), + sa.Column('device_type', sa.String(length=50), nullable=True), + sa.Column('user_agent', sa.String(length=512), nullable=True), + sa.Column('ip', sa.String(length=50), nullable=True), + sa.Column('country', sa.String(length=100), nullable=True), + sa.Column('state', sa.String(length=100), nullable=True), + sa.Column('city', sa.String(length=100), nullable=True), + sa.Column('latitude', sa.Float(), nullable=True), + sa.Column('longitude', sa.Float(), nullable=True), + sa.Column('first_login', sa.DateTime(timezone=True), nullable=False), + sa.Column('last_login', sa.DateTime(timezone=True), nullable=False), + sa.Column('last_ip', sa.String(length=50), nullable=True), + sa.Column('is_blocked', sa.Boolean(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_devices_fingerprint'), 'devices', ['fingerprint'], unique=False) + op.create_index(op.f('ix_devices_user_id'), 'devices', ['user_id'], unique=False) + op.create_table('sessions', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('device_id', sa.Integer(), nullable=True), + sa.Column('refresh_token_hash', sa.String(length=255), nullable=False), + sa.Column('status', sa.String(length=50), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('last_activity', sa.DateTime(timezone=True), nullable=False), + sa.Column('expires_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('revoked_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('revoked_by', sa.String(length=50), nullable=True), + sa.Column('revoked_by_user_id', sa.Integer(), nullable=True), + sa.Column('reason', sa.String(length=255), nullable=True), + sa.ForeignKeyConstraint(['device_id'], ['devices.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['revoked_by_user_id'], ['users.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_sessions_device_id'), 'sessions', ['device_id'], unique=False) + op.create_index(op.f('ix_sessions_user_id'), 'sessions', ['user_id'], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_sessions_user_id'), table_name='sessions') + op.drop_index(op.f('ix_sessions_device_id'), table_name='sessions') + op.drop_table('sessions') + op.drop_index(op.f('ix_devices_user_id'), table_name='devices') + op.drop_index(op.f('ix_devices_fingerprint'), table_name='devices') + op.drop_table('devices') + # ### end Alembic commands ### diff --git a/alembic/versions/9c4a95ac6726_add_saasrolemapping.py b/alembic/versions/9c4a95ac6726_add_saasrolemapping.py new file mode 100644 index 0000000..8365943 --- /dev/null +++ b/alembic/versions/9c4a95ac6726_add_saasrolemapping.py @@ -0,0 +1,45 @@ +"""Add SaaSRoleMapping + +Revision ID: 9c4a95ac6726 +Revises: 905726af73c8 +Create Date: 2026-08-28 15:36:54.084287 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '9c4a95ac6726' +down_revision: Union[str, None] = '905726af73c8' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('saas_role_mappings', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('saas_role_id', sa.String(), nullable=False), + sa.Column('docqube_role_id', sa.UUID(), nullable=False), + sa.Column('metadata', sa.JSON(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), + sa.ForeignKeyConstraint(['docqube_role_id'], ['roles.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_saas_role_mappings_id'), 'saas_role_mappings', ['id'], unique=False) + op.create_index(op.f('ix_saas_role_mappings_saas_role_id'), 'saas_role_mappings', ['saas_role_id'], unique=True) + op.execute("ALTER TABLE tenant_contacts DROP CONSTRAINT IF EXISTS uq_tenant_contacts_tenant_id_email") + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_unique_constraint('uq_tenant_contacts_tenant_id_email', 'tenant_contacts', ['tenant_id', 'email']) + op.drop_index(op.f('ix_saas_role_mappings_saas_role_id'), table_name='saas_role_mappings') + op.drop_index(op.f('ix_saas_role_mappings_id'), table_name='saas_role_mappings') + op.drop_table('saas_role_mappings') + # ### end Alembic commands ### diff --git a/alembic/versions/9f6521a38bf3_limit_chatbotdocument_file_path_and_.py b/alembic/versions/9f6521a38bf3_limit_chatbotdocument_file_path_and_.py new file mode 100644 index 0000000..cc03f5f --- /dev/null +++ b/alembic/versions/9f6521a38bf3_limit_chatbotdocument_file_path_and_.py @@ -0,0 +1,30 @@ +"""Limit ChatbotDocument file_path and filename to 512 chars + +Revision ID: 9f6521a38bf3 +Revises: 3aa7d970b8c2 +Create Date: 2026-03-03 22:12:00.390551 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '9f6521a38bf3' +down_revision: Union[str, None] = '3aa7d970b8c2' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/alembic/versions/a1b2c3d4e5f6_add_rbac_tables.py b/alembic/versions/a1b2c3d4e5f6_add_rbac_tables.py new file mode 100644 index 0000000..f81d5da --- /dev/null +++ b/alembic/versions/a1b2c3d4e5f6_add_rbac_tables.py @@ -0,0 +1,90 @@ +"""Add RBAC tables (accesses, roles, role_accesses) and role_id to users + +Revision ID: a1b2c3d4e5f6 +Revises: ff9052a97835 +Create Date: 2026-03-05 11:15:00.000000 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import UUID + + +# revision identifiers, used by Alembic. +revision: str = "a1b2c3d4e5f6" +down_revision: Union[str, None] = ("6c9cc3567866", "ff9052a97835") +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # 1. Create accesses table + op.execute( + """ + CREATE TABLE accesses ( + id SERIAL PRIMARY KEY, + name VARCHAR(100) NOT NULL UNIQUE, + description VARCHAR(255), + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL + ); + """ + ) + op.execute("CREATE INDEX ix_accesses_name ON accesses (name);") + + # 2. Create roles table (tenant_id nullable = system-wide role) + op.execute( + """ + CREATE TABLE roles ( + id SERIAL PRIMARY KEY, + tenant_id UUID REFERENCES tenants(id) ON DELETE CASCADE, + name VARCHAR(100) NOT NULL, + description VARCHAR(255), + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL, + CONSTRAINT uq_role_tenant_name UNIQUE (tenant_id, name) + ); + """ + ) + op.execute("CREATE INDEX ix_roles_tenant_id ON roles (tenant_id);") + + # 3. Create role_accesses join table (composite PK) + op.execute( + """ + CREATE TABLE role_accesses ( + role_id INTEGER NOT NULL REFERENCES roles(id) ON DELETE CASCADE, + access_id INTEGER NOT NULL REFERENCES accesses(id) ON DELETE CASCADE, + CONSTRAINT pk_role_accesses PRIMARY KEY (role_id, access_id) + ); + """ + ) + + # 4. Add role_id to users (nullable FK, SET NULL on delete) + op.execute( + """ + ALTER TABLE users + ADD COLUMN role_id INTEGER REFERENCES roles(id) ON DELETE SET NULL; + """ + ) + op.execute("CREATE INDEX ix_users_role_id ON users (role_id);") + + + +def downgrade() -> None: + # Reverse in dependency order + + # Remove role_id from users + op.execute("DROP INDEX IF EXISTS ix_users_role_id;") + op.execute("ALTER TABLE users DROP COLUMN IF EXISTS role_id;") + + # Drop join table + op.execute("DROP TABLE IF EXISTS role_accesses;") + + # Drop roles table (seed data goes with it) + op.execute("DROP INDEX IF EXISTS ix_roles_tenant_id;") + op.execute("DROP TABLE IF EXISTS roles;") + + # Drop accesses table + op.execute("DROP INDEX IF EXISTS ix_accesses_name;") + op.execute("DROP TABLE IF EXISTS accesses;") diff --git a/alembic/versions/a2b3c4d5e6f7_update_access_model_with_hierarchy.py b/alembic/versions/a2b3c4d5e6f7_update_access_model_with_hierarchy.py new file mode 100644 index 0000000..5dcf1a5 --- /dev/null +++ b/alembic/versions/a2b3c4d5e6f7_update_access_model_with_hierarchy.py @@ -0,0 +1,75 @@ +"""Update Access model with hierarchical permission support + +Revision ID: a2b3c4d5e6f7 +Revises: a1b2c3d4e5f6 +Create Date: 2026-03-05 10:00:00.000000 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'a2b3c4d5e6f7' +down_revision = 'a1b2c3d4e5f6' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # FIRST: Drop any existing unique constraint on `name` (constraints may be named differently depending on DB) + op.execute("ALTER TABLE accesses DROP CONSTRAINT IF EXISTS uq_accesses_name CASCADE") + op.execute("ALTER TABLE accesses DROP CONSTRAINT IF EXISTS accesses_name_key CASCADE") + # also drop postgresql-generated index if it survived + op.execute("DROP INDEX IF EXISTS accesses_name_key") + + # Add new columns to accesses table (all nullable first) + op.add_column('accesses', sa.Column('access_code', sa.String(255), nullable=True)) + op.add_column('accesses', sa.Column('category', sa.String(100), nullable=True)) + op.add_column('accesses', sa.Column('parent_id', sa.Integer(), nullable=True)) + + # Create indexes on new columns + op.create_index('ix_access_code', 'accesses', ['access_code']) + op.create_index('ix_category', 'accesses', ['category']) + op.create_index('ix_accesses_parent_id', 'accesses', ['parent_id']) + + # Set default values from existing 'name' field + op.execute(""" + UPDATE accesses + SET access_code = LOWER(REPLACE(name, ' ', '_')), + category = 'General' + WHERE access_code IS NULL + """) + + # Now make columns NOT NULL + op.alter_column('accesses', 'access_code', existing_type=sa.String(255), nullable=False) + op.alter_column('accesses', 'category', existing_type=sa.String(100), nullable=False) + + # Create unique constraint on access_code + op.create_unique_constraint('uq_accesses_access_code', 'accesses', ['access_code']) + + # Create foreign key for parent_id + op.create_foreign_key('fk_accesses_parent_id', 'accesses', 'accesses', ['parent_id'], ['id'], ondelete='CASCADE') + + +def downgrade() -> None: + # Drop foreign key + op.drop_constraint('fk_accesses_parent_id', 'accesses', type_='foreignkey') + + # Drop unique constraint + op.drop_constraint('uq_accesses_access_code', 'accesses', type_='unique') + + # Drop indexes + op.drop_index('ix_accesses_parent_id', 'accesses') + op.drop_index('ix_category', 'accesses') + op.drop_index('ix_access_code', 'accesses') + + # Drop columns + op.drop_column('accesses', 'parent_id') + op.drop_column('accesses', 'category') + op.drop_column('accesses', 'access_code') + + # Re-add unique constraint on 'name' + op.create_unique_constraint('uq_accesses_name', 'accesses', ['name']) + + diff --git a/alembic/versions/a902935ea69e_remove_redundant_id_column_from_user_.py b/alembic/versions/a902935ea69e_remove_redundant_id_column_from_user_.py new file mode 100644 index 0000000..ee2d947 --- /dev/null +++ b/alembic/versions/a902935ea69e_remove_redundant_id_column_from_user_.py @@ -0,0 +1,34 @@ +"""Remove redundant id column from user_storage_usage + +Revision ID: a902935ea69e +Revises: 0e1a5f2cdea9 +Create Date: 2026-03-02 10:53:47.556155 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'a902935ea69e' +down_revision: Union[str, None] = '0e1a5f2cdea9' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index('ix_user_storage_usage_id', table_name='user_storage_usage') + op.drop_constraint('user_storage_usage_user_id_key', 'user_storage_usage', type_='unique') + op.drop_column('user_storage_usage', 'id') + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('user_storage_usage', sa.Column('id', sa.INTEGER(), autoincrement=False, nullable=False)) + op.create_unique_constraint('user_storage_usage_user_id_key', 'user_storage_usage', ['user_id'], postgresql_nulls_not_distinct=False) + op.create_index('ix_user_storage_usage_id', 'user_storage_usage', ['id'], unique=False) + # ### end Alembic commands ### diff --git a/alembic/versions/af57c3d9f1c2_add_signature_config_tables.py b/alembic/versions/af57c3d9f1c2_add_signature_config_tables.py new file mode 100644 index 0000000..92e64c6 --- /dev/null +++ b/alembic/versions/af57c3d9f1c2_add_signature_config_tables.py @@ -0,0 +1,64 @@ +"""add signature config tables + +Revision ID: af57c3d9f1c2 +Revises: daf45366d608 +Create Date: 2026-07-20 18:53:11.916379 + +""" +from typing import Sequence, Union +import uuid + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +# revision identifiers, used by Alembic. +revision: str = 'af57c3d9f1c2' +down_revision: Union[str, None] = 'daf45366d608' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + 'global_signature_configs', + sa.Column('id', postgresql.UUID(as_uuid=True), primary_key=True), + sa.Column('active_provider', sa.String(length=50), nullable=False, server_default='zoho_sign'), + sa.Column('docuseal_credentials', postgresql.JSON(astext_type=sa.Text()), nullable=True), + sa.Column('zoho_credentials', postgresql.JSON(astext_type=sa.Text()), nullable=True), + sa.Column('created_by_id', sa.Integer(), nullable=True), + sa.Column('updated_by_id', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['created_by_id'], ['users.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['updated_by_id'], ['users.id'], ondelete='SET NULL') + ) + + op.execute( + "INSERT INTO global_signature_configs (id, active_provider, created_at, updated_at) " + "VALUES (gen_random_uuid(), 'zoho_sign', now(), now())" + ) + + op.create_table( + 'tenant_signature_configs', + sa.Column('id', postgresql.UUID(as_uuid=True), primary_key=True), + sa.Column('tenant_id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('use_custom_credentials', sa.Boolean(), nullable=False, server_default='false'), + sa.Column('custom_provider', sa.String(length=50), nullable=False, server_default='docuseal'), + sa.Column('custom_credentials_json', postgresql.JSON(astext_type=sa.Text()), nullable=True), + sa.Column('created_by_id', sa.Integer(), nullable=True), + sa.Column('updated_by_id', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['created_by_id'], ['users.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['updated_by_id'], ['users.id'], ondelete='SET NULL') + ) + op.create_index(op.f('ix_tenant_signature_configs_tenant_id'), 'tenant_signature_configs', ['tenant_id'], unique=True) + + +def downgrade() -> None: + op.drop_index(op.f('ix_tenant_signature_configs_tenant_id'), table_name='tenant_signature_configs') + op.drop_table('tenant_signature_configs') + op.drop_table('global_signature_configs') diff --git a/alembic/versions/b029d2f5a0c7_add_signature_id.py b/alembic/versions/b029d2f5a0c7_add_signature_id.py new file mode 100644 index 0000000..3ef076b --- /dev/null +++ b/alembic/versions/b029d2f5a0c7_add_signature_id.py @@ -0,0 +1,36 @@ +"""add_signature_id + +Revision ID: b029d2f5a0c7 +Revises: fb6810c36ffb +Create Date: 2026-04-14 14:59:09.399640 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'b029d2f5a0c7' +down_revision: Union[str, None] = 'fb6810c36ffb' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('signing_requests', schema=None) as batch_op: + batch_op.add_column(sa.Column('signature_id', sa.String(length=128), nullable=True)) + batch_op.create_index(batch_op.f('ix_signing_requests_signature_id'), ['signature_id'], unique=False) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('signing_requests', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_signing_requests_signature_id')) + batch_op.drop_column('signature_id') + + # ### end Alembic commands ### diff --git a/alembic/versions/b0a09cd13348_add_persistent_chat_history_tables.py b/alembic/versions/b0a09cd13348_add_persistent_chat_history_tables.py new file mode 100644 index 0000000..50509b2 --- /dev/null +++ b/alembic/versions/b0a09cd13348_add_persistent_chat_history_tables.py @@ -0,0 +1,131 @@ +"""Add persistent chat history tables + +Revision ID: b0a09cd13348 +Revises: w001_versioning_minor +Create Date: 2026-05-06 16:38:12.054824 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = 'b0a09cd13348' +down_revision: Union[str, None] = 'w001_versioning_minor' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + from sqlalchemy.engine.reflection import Inspector + conn = op.get_bind() + inspector = Inspector.from_engine(conn) + + if not inspector.has_table('activity_logs'): + op.create_table('activity_logs', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('tenant_id', sa.UUID(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=True), + sa.Column('user_name', sa.String(length=255), nullable=True), + sa.Column('user_email', sa.String(length=255), nullable=True), + sa.Column('module', sa.String(length=50), nullable=False), + sa.Column('action', sa.String(length=100), nullable=False), + sa.Column('target_id', sa.String(length=255), nullable=False), + sa.Column('target_type', sa.String(length=50), nullable=False), + sa.Column('target_name', sa.String(length=512), nullable=True), + sa.Column('metadata', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('ip_address', sa.String(length=64), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('request_id', sa.String(length=255), nullable=True), + sa.Column('status', sa.String(length=20), nullable=False), + sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('activity_logs', schema=None) as batch_op: + batch_op.create_index('ix_activity_logs_module_action', ['module', 'action'], unique=False) + batch_op.create_index('ix_activity_logs_target_id', ['target_id'], unique=False) + batch_op.create_index('ix_activity_logs_tenant_created_at', ['tenant_id', 'created_at'], unique=False) + batch_op.create_index('ix_activity_logs_user_created_at', ['user_id', 'created_at'], unique=False) + + if not inspector.has_table('chat_sessions'): + op.create_table('chat_sessions', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('session_id', sa.UUID(), nullable=False), + sa.Column('tenant_id', sa.UUID(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('chatbot_document_id', sa.Integer(), nullable=True), + sa.Column('title', sa.String(length=255), nullable=True), + sa.Column('created_at', sa.TIMESTAMP(), nullable=False), + sa.Column('updated_at', sa.TIMESTAMP(), nullable=False), + sa.ForeignKeyConstraint(['chatbot_document_id'], ['chatbot_documents.id'], ), + sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('chat_sessions', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_chat_sessions_chatbot_document_id'), ['chatbot_document_id'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_sessions_session_id'), ['session_id'], unique=True) + batch_op.create_index(batch_op.f('ix_chat_sessions_tenant_id'), ['tenant_id'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_sessions_user_id'), ['user_id'], unique=False) + + if not inspector.has_table('chat_messages'): + op.create_table('chat_messages', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('message_id', sa.Uuid(), nullable=False), + sa.Column('session_id', sa.Integer(), nullable=False), + sa.Column('role', sa.String(length=20), nullable=False), + sa.Column('content', sa.Text(), nullable=False), + sa.Column('token_count', sa.Integer(), nullable=True), + sa.Column('created_at', sa.TIMESTAMP(), nullable=False), + sa.ForeignKeyConstraint(['session_id'], ['chat_sessions.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('message_id') + ) + with op.batch_alter_table('chat_messages', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_chat_messages_session_id'), ['session_id'], unique=False) + + with op.batch_alter_table('drive_file_versions', schema=None) as batch_op: + # Check if index exists before dropping + indexes = [idx['name'] for idx in inspector.get_indexes('drive_file_versions')] + if 'ix_drive_file_versions_file_major_minor' in indexes: + batch_op.drop_index('ix_drive_file_versions_file_major_minor') + + with op.batch_alter_table('signature_imprints', schema=None) as batch_op: + # Check if index exists before dropping + indexes = [idx['name'] for idx in inspector.get_indexes('signature_imprints')] + if 'ix_signature_imprints_version_id' in indexes: + batch_op.drop_index('ix_signature_imprints_version_id') + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('signature_imprints', schema=None) as batch_op: + batch_op.create_index('ix_signature_imprints_version_id', ['drive_file_version_id'], unique=False) + + with op.batch_alter_table('drive_file_versions', schema=None) as batch_op: + batch_op.create_index('ix_drive_file_versions_file_major_minor', ['file_id', 'version_number', 'minor_version'], unique=False) + + with op.batch_alter_table('chat_messages', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_chat_messages_session_id')) + + op.drop_table('chat_messages') + with op.batch_alter_table('chat_sessions', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_chat_sessions_user_id')) + batch_op.drop_index(batch_op.f('ix_chat_sessions_tenant_id')) + batch_op.drop_index(batch_op.f('ix_chat_sessions_session_id')) + batch_op.drop_index(batch_op.f('ix_chat_sessions_chatbot_document_id')) + + op.drop_table('chat_sessions') + with op.batch_alter_table('activity_logs', schema=None) as batch_op: + batch_op.drop_index('ix_activity_logs_user_created_at') + batch_op.drop_index('ix_activity_logs_tenant_created_at') + batch_op.drop_index('ix_activity_logs_target_id') + batch_op.drop_index('ix_activity_logs_module_action') + + op.drop_table('activity_logs') + # ### end Alembic commands ### diff --git a/alembic/versions/b1_0_explicit_superadmin.py b/alembic/versions/b1_0_explicit_superadmin.py new file mode 100644 index 0000000..f17a2d6 --- /dev/null +++ b/alembic/versions/b1_0_explicit_superadmin.py @@ -0,0 +1,62 @@ +"""B1.0 — make superadmin an explicit flag instead of a null tenant + +Adds users.is_superadmin and backfills it from the signal it replaces, so the +operators who are superadmins today stay superadmins after deploy. + +Until now privilege was inferred from `tenant_id IS NULL`. That made the +*absence* of tenant context a grant of authority: a bug that dropped the tenant +escalated instead of denying. The flag separates "has no tenant" from "may do +anything". + +This migration only adds and backfills. It deliberately does not drop or alter +`tenant_id`, and the application honours both signals for one release, so this +can be reverted without locking anyone out. + +Revision ID: b1_0_explicit_superadmin +Revises: dfae38e4ede3 +""" + +import sqlalchemy as sa +from alembic import op + +revision = "b1_0_explicit_superadmin" +down_revision = "73dff81eed72" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "users", + sa.Column( + "is_superadmin", + sa.Boolean(), + nullable=False, + server_default=sa.false(), + ), + ) + + # Backfill in the same migration, not a follow-up script. If this ran later + # there would be a window in which every superadmin was demoted. + op.execute( + """ + UPDATE users + SET is_superadmin = true + WHERE tenant_id IS NULL + AND is_deleted = false + """ + ) + + op.create_index( + "ix_users_is_superadmin", + "users", + ["is_superadmin"], + postgresql_where=sa.text("is_superadmin"), + ) + + +def downgrade() -> None: + # Safe to reverse: `tenant_id IS NULL` still identifies the same people, + # and the application still honours it. + op.drop_index("ix_users_is_superadmin", table_name="users") + op.drop_column("users", "is_superadmin") diff --git a/alembic/versions/b1_3_row_level_security.py b/alembic/versions/b1_3_row_level_security.py new file mode 100644 index 0000000..7c03b78 --- /dev/null +++ b/alembic/versions/b1_3_row_level_security.py @@ -0,0 +1,87 @@ +"""B1.3 — row-level security on every tenant-owned table + +Enables **and forces** RLS on the 15 tables carrying a `tenant_id`, with a policy +that compares the row's tenant against a session variable the application sets +per request. + +Why this exists when B1.1 already filters queries: the ORM listener attaches to +entity loading, so it cannot filter aggregates — `query(X).count()` compiles to +`SELECT count(*) FROM (SELECT ...)` and comes back unfiltered. RLS applies in the +database regardless of how the query was composed, so it closes a hole the +listener structurally cannot. + +FORCE matters. Without it the policy does not apply to the table's owner, and +migrations run as the owner — so the tables would look protected while the +application, if it ever connected as that role, saw everything. + +This migration is inert until the application sets `docqube.tenant_id`, and +takes effect fully only once the app connects as a NOSUPERUSER / NOBYPASSRLS +role (B1.4). A superuser connection bypasses RLS unconditionally. + +Revision ID: b1_3_row_level_security +Revises: b1_0_explicit_superadmin +""" + +from alembic import op + +revision = "b1_3_row_level_security" +down_revision = "b1_0_explicit_superadmin" +branch_labels = None +depends_on = None + +POLICY = "docqube_tenant_isolation" + +# Derived from the schema when this was written; asserted against +# information_schema by tests/probes/test_rls.py so drift is caught rather than +# assumed. +TENANT_TABLES = [ + "activity_logs", + "chat_sessions", + "chatbot_documents", + "drive_activities", + "drive_files", + "drive_folders", + "notifications", + "projects", + "roles", + "signing_requests", + "tenant_signature_configs", + "tenant_smtp_configs", + "tenant_storage_configs", + "user_files", + "users", +] + +# The predicate, in words: +# * `docqube.bypass = 'on'` — an explicit system operation (migrations, +# background sweeps). Only the application can set this; it is not reachable +# from user input. +# * the row has no tenant — shared system records such as global roles. +# * the row's tenant matches the session variable. +# +# With no variable set, `current_setting(..., true)` returns NULL, `NULLIF` +# keeps it NULL, and `tenant_id = NULL` is NULL — so only shared rows are +# visible. Losing the tenant denies rather than reveals. +USING = """ + coalesce(current_setting('docqube.bypass', true), '') = 'on' + OR tenant_id IS NULL + OR tenant_id = nullif(current_setting('docqube.tenant_id', true), '')::uuid +""" + + +def upgrade() -> None: + for table in TENANT_TABLES: + op.execute(f"ALTER TABLE {table} ENABLE ROW LEVEL SECURITY") + op.execute(f"ALTER TABLE {table} FORCE ROW LEVEL SECURITY") + op.execute(f"DROP POLICY IF EXISTS {POLICY} ON {table}") + op.execute( + f"CREATE POLICY {POLICY} ON {table} " + f"USING ({USING}) WITH CHECK ({USING})" + ) + + +def downgrade() -> None: + for table in TENANT_TABLES: + op.execute(f"DROP POLICY IF EXISTS {POLICY} ON {table}") + op.execute(f"ALTER TABLE {table} NO FORCE ROW LEVEL SECURITY") + op.execute(f"ALTER TABLE {table} DISABLE ROW LEVEL SECURITY") diff --git a/alembic/versions/b3a7c1e9f042_add_collaboration_messages_table.py b/alembic/versions/b3a7c1e9f042_add_collaboration_messages_table.py new file mode 100644 index 0000000..6aa2e01 --- /dev/null +++ b/alembic/versions/b3a7c1e9f042_add_collaboration_messages_table.py @@ -0,0 +1,55 @@ +"""add collaboration_messages table + +Revision ID: b3a7c1e9f042 +Revises: f055f9b157cb +Create Date: 2026-03-02 15:17:00.000000 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'b3a7c1e9f042' +down_revision: Union[str, None] = 'f055f9b157cb' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "collaboration_messages", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "file_id", + sa.Integer(), + sa.ForeignKey("drive_files.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "sender_id", + sa.Integer(), + sa.ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("message", sa.Text(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + ), + ) + + # Performance indexes + op.create_index("ix_collab_msg_file_id", "collaboration_messages", ["file_id"]) + op.create_index("ix_collab_msg_sender_id", "collaboration_messages", ["sender_id"]) + op.create_index("ix_collab_msg_created_at", "collaboration_messages", ["created_at"]) + + +def downgrade() -> None: + op.drop_index("ix_collab_msg_created_at", table_name="collaboration_messages") + op.drop_index("ix_collab_msg_sender_id", table_name="collaboration_messages") + op.drop_index("ix_collab_msg_file_id", table_name="collaboration_messages") + op.drop_table("collaboration_messages") diff --git a/alembic/versions/b6_4_index_review.py b/alembic/versions/b6_4_index_review.py new file mode 100644 index 0000000..205496d --- /dev/null +++ b/alembic/versions/b6_4_index_review.py @@ -0,0 +1,65 @@ +"""B6.4 — indexes on the foreign keys that queries actually filter by + +The database has 34 foreign keys with no supporting index. PostgreSQL does not +create one automatically, and the cost is invisible on a seeded database: a +sequential scan over five rows is instant, over five hundred thousand it is not. + +This adds indexes only where a measured query path filters or joins on the +column — the ones surfaced by the B6.3 query-budget work. Indexing all 34 would +add write cost for paths nothing traverses; the remainder are listed in +`tests/probes/test_index_coverage.py` so they stay visible rather than +forgotten. + +**Deployment note.** `CREATE INDEX` takes a lock that blocks writes for the +duration. On DocQube's current row counts that is brief, but on a large +`drive_files` it will not be — run these with `CONCURRENTLY` outside a +transaction if the table has grown. Alembic cannot do that inside its own +transaction, which is why this migration is written to be easy to translate. + +Revision ID: b6_4_index_review +Revises: b1_3_row_level_security +""" + +from alembic import op + +revision = "b6_4_index_review" +down_revision = "b1_3_row_level_security" +branch_labels = None +depends_on = None + +# (index name, table, columns) — each justified by a query path measured in +# tests/characterization/test_query_budget.py. +INDEXES = [ + # Every folder listing filters files by folder. + ("ix_drive_files_folder_id", "drive_files", "folder_id"), + ("ix_drive_files_owner_id", "drive_files", "owner_id"), + # Hierarchy walks in get_effective_role and breadcrumbs. + ("ix_drive_folders_parent_id", "drive_folders", "parent_id"), + ("ix_drive_folders_owner_id", "drive_folders", "owner_id"), + # The batched star lookup added in B6.3 filters on all three. + ( + "ix_drive_stars_user_resource", + "drive_stars", + "user_id, resource_type, resource_id", + ), + # Version history, loaded per file. + ("ix_drive_file_versions_file_id", "drive_file_versions", "file_id"), + ("ix_project_versions_project_id", "project_versions", "project_id"), + # Permission resolution joins accesses on every authenticated request. + ("ix_role_accesses_access_id", "role_accesses", "access_id"), + # Signing looks requests up by the file they belong to. + ("ix_signing_requests_drive_file_id", "signing_requests", "drive_file_id"), + # Comments and activity are read per resource. + ("ix_drive_comments_author_id", "drive_comments", "author_id"), + ("ix_drive_activities_actor_id", "drive_activities", "actor_id"), +] + + +def upgrade() -> None: + for name, table, columns in INDEXES: + op.execute(f"CREATE INDEX IF NOT EXISTS {name} ON {table} ({columns})") + + +def downgrade() -> None: + for name, _table, _columns in INDEXES: + op.execute(f"DROP INDEX IF EXISTS {name}") diff --git a/alembic/versions/bb224e0b642b_remove_tenant_storage_config_and_use_.py b/alembic/versions/bb224e0b642b_remove_tenant_storage_config_and_use_.py new file mode 100644 index 0000000..d764cc5 --- /dev/null +++ b/alembic/versions/bb224e0b642b_remove_tenant_storage_config_and_use_.py @@ -0,0 +1,48 @@ +"""remove_tenant_storage_config_and_use_system_config + +Revision ID: bb224e0b642b +Revises: i001_dual_bucket_storage +Create Date: 2026-03-17 09:09:02.599878 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'bb224e0b642b' +down_revision: Union[str, None] = 'i001_dual_bucket_storage' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Drop the tenant-specific storage config table + op.drop_table('tenant_storage_configs') + + +def downgrade() -> None: + # Recreate the tenant-specific storage config table + op.create_table( + 'tenant_storage_configs', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('tenant_id', sa.UUID(), nullable=False), + sa.Column('storage_provider', sa.String(length=50), nullable=False), + sa.Column('b2_key_id', sa.String(length=255), nullable=True), + sa.Column('b2_application_key', sa.String(length=512), nullable=True), + sa.Column('b2_quarantine_bucket', sa.String(length=255), nullable=True), + sa.Column('b2_clean_bucket', sa.String(length=255), nullable=True), + sa.Column('is_active', sa.Boolean(), nullable=False), + sa.Column('created_by_id', sa.Integer(), nullable=True), + sa.Column('updated_by_id', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['created_by_id'], ['users.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['updated_by_id'], ['users.id'], ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('tenant_id') + ) + op.create_index(op.f('ix_tenant_storage_configs_tenant_id'), 'tenant_storage_configs', ['tenant_id'], unique=True) diff --git a/alembic/versions/be67927ab00d_add_encrypted_ai_api_key_to_user_model.py b/alembic/versions/be67927ab00d_add_encrypted_ai_api_key_to_user_model.py new file mode 100644 index 0000000..a9e3536 --- /dev/null +++ b/alembic/versions/be67927ab00d_add_encrypted_ai_api_key_to_user_model.py @@ -0,0 +1,38 @@ +"""Add encrypted_ai_api_key to user_model + +Revision ID: be67927ab00d +Revises: b3a7c1e9f042 +Create Date: 2026-03-03 12:43:12.848347 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = 'be67927ab00d' +down_revision: Union[str, None] = 'b3a7c1e9f042' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('collaboration_messages', 'created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + op.add_column('users', sa.Column('encrypted_ai_api_key', sa.String(length=512), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('users', 'encrypted_ai_api_key') + op.alter_column('collaboration_messages', 'created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('now()')) + # ### end Alembic commands ### diff --git a/alembic/versions/c1_0_org_units_and_scoped_roles.py b/alembic/versions/c1_0_org_units_and_scoped_roles.py new file mode 100644 index 0000000..d813cb4 --- /dev/null +++ b/alembic/versions/c1_0_org_units_and_scoped_roles.py @@ -0,0 +1,247 @@ +"""C1/C3 — organisation units, membership, leadership, and scoped role grants + +Four tables, and a backfill that is deliberately a no-op in behaviour. + +`user_roles` exists because `users.role_id` is single-valued: one role, held +across the whole tenant. Scoped access has nowhere to live until a user can hold +the same role in two units. The backfill turns each existing `users.role_id` +into one `user_roles` row with `org_unit_id = NULL`, which means tenant-wide — +exactly the authority that user already had. Nobody gains or loses anything on +this migration. + +`users.role_id` is **not** dropped. Both mechanisms are honoured for one +release, the same shape as the `is_superadmin` transition and for the same +reason: reverting this migration must not be able to remove everyone's +permissions. Removal is a later, deliberate step with its own preflight. + +All four tables carry `tenant_id`, so `app/core/tenant_filter.py` scopes them +automatically, and they are added to the RLS policy here because +`tests/probes/test_isolation_ratchet.py` fails if a tenant-owned table is left +unprotected. + +Revision ID: c1_0_org_units_and_scoped_roles +Revises: b6_4_index_review +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.postgresql import UUID + +revision = "c1_0_org_units_and_scoped_roles" +down_revision = "b6_4_index_review" +branch_labels = None +depends_on = None + +# Kept in step with b1_3_row_level_security.py. +# Named TENANT_TABLES so `test_isolation_ratchet.py` finds it: the ratchet +# scans every migration for this constant, which is what lets a later +# migration add tenant-owned tables without the check going stale. +TENANT_TABLES = ["org_units", "user_org_units", "org_unit_leads"] +POLICY = "tenant_isolation" +USING = """ + coalesce(current_setting('docqube.bypass', true), '') = 'on' + OR tenant_id IS NULL + OR tenant_id = nullif(current_setting('docqube.tenant_id', true), '')::uuid +""" + + +def upgrade() -> None: + op.create_table( + "org_units", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "tenant_id", + UUID(as_uuid=True), + sa.ForeignKey("tenants.id", ondelete="CASCADE"), + nullable=False, + ), + # SET NULL, not CASCADE: deleting a department must not silently delete + # every team beneath it. What happens to orphans is an application + # decision, and it should be visible as one. + sa.Column( + "parent_id", + UUID(as_uuid=True), + sa.ForeignKey("org_units.id", ondelete="SET NULL"), + nullable=True, + ), + sa.Column("name", sa.String(255), nullable=False), + sa.Column("code", sa.String(50), nullable=True), + sa.Column("unit_type", sa.String(50), nullable=True), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("sort_order", sa.Integer(), nullable=False, server_default="0"), + sa.Column("depth", sa.Integer(), nullable=False, server_default="0"), + sa.Column("is_deleted", sa.Boolean(), nullable=False, server_default=sa.false()), + sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + ) + op.create_index("ix_org_units_tenant_id", "org_units", ["tenant_id"]) + op.create_index("ix_org_units_parent_id", "org_units", ["parent_id"]) + op.create_index("ix_org_units_tenant_parent", "org_units", ["tenant_id", "parent_id"]) + # Codes are unique per tenant among *live* rows only — a soft-deleted unit + # must not reserve its code forever. + op.create_index( + "uq_org_units_tenant_code_live", + "org_units", + ["tenant_id", "code"], + unique=True, + postgresql_where=sa.text("NOT is_deleted AND code IS NOT NULL"), + ) + + op.create_table( + "user_org_units", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "tenant_id", + UUID(as_uuid=True), + sa.ForeignKey("tenants.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "user_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=False + ), + sa.Column( + "org_unit_id", + UUID(as_uuid=True), + sa.ForeignKey("org_units.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("is_primary", sa.Boolean(), nullable=False, server_default=sa.false()), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.UniqueConstraint("user_id", "org_unit_id", name="uq_user_org_units_user_unit"), + ) + op.create_index("ix_user_org_units_tenant_id", "user_org_units", ["tenant_id"]) + op.create_index("ix_user_org_units_user_id", "user_org_units", ["user_id"]) + op.create_index("ix_user_org_units_org_unit_id", "user_org_units", ["org_unit_id"]) + # "Exactly one primary" as an index, not as application logic: written in + # Python it is a race between two concurrent writes, which is the same + # defect class as the user_storage_usage get-or-create found under load. + op.create_index( + "uq_user_org_units_one_primary", + "user_org_units", + ["user_id"], + unique=True, + postgresql_where=sa.text("is_primary"), + ) + + op.create_table( + "org_unit_leads", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "tenant_id", + UUID(as_uuid=True), + sa.ForeignKey("tenants.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "org_unit_id", + UUID(as_uuid=True), + sa.ForeignKey("org_units.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "user_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=False + ), + sa.Column("start_date", sa.Date(), nullable=False), + sa.Column("end_date", sa.Date(), nullable=True), + sa.Column("note", sa.Text(), nullable=True), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + ) + op.create_index("ix_org_unit_leads_tenant_id", "org_unit_leads", ["tenant_id"]) + op.create_index("ix_org_unit_leads_org_unit_id", "org_unit_leads", ["org_unit_id"]) + op.create_index("ix_org_unit_leads_user_id", "org_unit_leads", ["user_id"]) + # At most one *current* lead per unit; history is unconstrained, which is + # what makes it usable as history. + op.create_index( + "uq_org_unit_leads_current", + "org_unit_leads", + ["org_unit_id"], + unique=True, + postgresql_where=sa.text("end_date IS NULL"), + ) + + op.create_table( + "user_roles", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "user_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=False + ), + sa.Column( + "role_id", + UUID(as_uuid=True), + sa.ForeignKey("roles.id", ondelete="CASCADE"), + nullable=False, + ), + # NULL means tenant-wide, which is what every grant is today. + sa.Column( + "org_unit_id", + UUID(as_uuid=True), + sa.ForeignKey("org_units.id", ondelete="CASCADE"), + nullable=True, + ), + sa.Column( + "assigned_by_id", + sa.Integer(), + sa.ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.UniqueConstraint("user_id", "role_id", "org_unit_id", name="uq_user_roles_scoped"), + ) + op.create_index("ix_user_roles_user_id", "user_roles", ["user_id"]) + op.create_index("ix_user_roles_role_id", "user_roles", ["role_id"]) + op.create_index("ix_user_roles_org_unit_id", "user_roles", ["org_unit_id"]) + op.create_index("ix_user_roles_lookup", "user_roles", ["user_id", "role_id"]) + # Every foreign key needs a supporting index or a delete on the parent + # sequentially scans this table. `test_index_coverage.py` enforces it. + op.create_index("ix_user_roles_assigned_by_id", "user_roles", ["assigned_by_id"]) + # Postgres treats NULLs as distinct in a UNIQUE constraint, so the triple + # above does not stop the same role being held tenant-wide twice. + op.create_index( + "uq_user_roles_tenant_wide", + "user_roles", + ["user_id", "role_id"], + unique=True, + postgresql_where=sa.text("org_unit_id IS NULL"), + ) + + # Backfill. Behaviour-neutral by construction: a null org unit is + # tenant-wide, which is the authority every user already has. + op.execute( + """ + INSERT INTO user_roles (id, user_id, role_id, org_unit_id, created_at) + SELECT gen_random_uuid(), u.id, u.role_id, NULL, now() + FROM users u + WHERE u.role_id IS NOT NULL + AND u.is_deleted = false + ON CONFLICT DO NOTHING + """ + ) + + for table in TENANT_TABLES: + op.execute(f"ALTER TABLE {table} ENABLE ROW LEVEL SECURITY") + op.execute(f"ALTER TABLE {table} FORCE ROW LEVEL SECURITY") + op.execute(f"DROP POLICY IF EXISTS {POLICY} ON {table}") + op.execute(f"CREATE POLICY {POLICY} ON {table} USING ({USING}) WITH CHECK ({USING})") + + +def downgrade() -> None: + # Safe to reverse: `users.role_id` was never dropped, so removing these + # tables returns the application to the grants it still reads today. + for table in TENANT_TABLES: + op.execute(f"DROP POLICY IF EXISTS {POLICY} ON {table}") + + op.drop_table("user_roles") + op.drop_table("org_unit_leads") + op.drop_table("user_org_units") + op.drop_table("org_units") diff --git a/alembic/versions/c1_1_user_roles_tenant_id.py b/alembic/versions/c1_1_user_roles_tenant_id.py new file mode 100644 index 0000000..f1cc814 --- /dev/null +++ b/alembic/versions/c1_1_user_roles_tenant_id.py @@ -0,0 +1,80 @@ +"""C3.1 — give user_roles a tenant_id, so isolation is structural + +`user_roles` was created without one. That is not a cosmetic omission: both +isolation layers *derive* their coverage from the presence of that column — +`app/core/tenant_filter.py` scopes any model that has it, and the RLS policy +compares it against the session variable. A table without it is protected by +neither, no matter how careful the queries are. + +It is the same shape of gap as `tenants`, which is where "any holder of +`superadmin.tenant.delete` can delete any tenant" lived: a table outside the +mechanism, guarded only by whatever each call site remembered to write. Three +places currently remember. Three is a number that only goes up, and the failure +mode when one forgets is a grant resolving across tenants. + +After this the explicit predicates stay — they are cheap and they document the +intent — but they stop being the only thing standing there. + +**Backfill from the user, not the role.** `roles.tenant_id` is nullable: a null +means a shared/global role template, and a grant of a shared role still belongs +to the tenant of the person holding it. Rows for tenant-less users (superadmins) +stay null, which the RLS policy already treats as globally visible — the same +way their `users` row is treated. + +Revision ID: c1_1_user_roles_tenant_id +Revises: c1_0_org_units_and_scoped_roles +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.postgresql import UUID + +revision = "c1_1_user_roles_tenant_id" +down_revision = "c1_0_org_units_and_scoped_roles" +branch_labels = None +depends_on = None + +TENANT_TABLES = ["user_roles"] +POLICY = "tenant_isolation" +USING = """ + coalesce(current_setting('docqube.bypass', true), '') = 'on' + OR tenant_id IS NULL + OR tenant_id = nullif(current_setting('docqube.tenant_id', true), '')::uuid +""" + + +def upgrade() -> None: + op.add_column( + "user_roles", + sa.Column( + "tenant_id", + UUID(as_uuid=True), + sa.ForeignKey("tenants.id", ondelete="CASCADE"), + nullable=True, + ), + ) + op.create_index("ix_user_roles_tenant_id", "user_roles", ["tenant_id"]) + + op.execute( + """ + UPDATE user_roles ur + SET tenant_id = u.tenant_id + FROM users u + WHERE u.id = ur.user_id + """ + ) + + # Left nullable on purpose. A superadmin has no tenant, so their grants have + # none either; forcing NOT NULL would mean inventing one. + op.execute(f"ALTER TABLE user_roles ENABLE ROW LEVEL SECURITY") + op.execute(f"ALTER TABLE user_roles FORCE ROW LEVEL SECURITY") + op.execute(f"DROP POLICY IF EXISTS {POLICY} ON user_roles") + op.execute(f"CREATE POLICY {POLICY} ON user_roles USING ({USING}) WITH CHECK ({USING})") + + +def downgrade() -> None: + op.execute(f"DROP POLICY IF EXISTS {POLICY} ON user_roles") + op.execute("ALTER TABLE user_roles NO FORCE ROW LEVEL SECURITY") + op.execute("ALTER TABLE user_roles DISABLE ROW LEVEL SECURITY") + op.drop_index("ix_user_roles_tenant_id", table_name="user_roles") + op.drop_column("user_roles", "tenant_id") diff --git a/alembic/versions/c3_0_org_unit_path.py b/alembic/versions/c3_0_org_unit_path.py new file mode 100644 index 0000000..8e7b981 --- /dev/null +++ b/alembic/versions/c3_0_org_unit_path.py @@ -0,0 +1,77 @@ +"""C3 — a materialised path, so a subtree is a prefix match + +`ScopeService` resolved authority with a recursive CTE, once per access code per +request. That is correct and it is the thing that gets slower as the tree gets +deeper, in the way that shows up as "the admin page feels slow" rather than as +an error — the query-count budgets cannot catch it, because the *count* does not +change. + +A materialised path turns the walk into a prefix match: + + /a1b2.../c3d4.../e5f6.../ + +The subtree of a unit is every row whose path starts with that unit's path. +Indexed with `text_pattern_ops`, that is an index range scan. + +**Text, not `ltree`.** `ltree` is the tidier type and needs an extension, which +is a deployment prerequisite for a performance improvement — a poor trade. Text +with a prefix index needs nothing and behaves identically for this query. + +**Why the delimiters matter.** Ids are stored between slashes, so +`LIKE '/a/b/%'` cannot match `/a/bc/...`. Without them a prefix match would leak +across sibling branches whose ids share a leading substring — which for UUIDs is +rare, and rare is worse than never. + +The column is nullable and the CTE remains as a fallback for any row that has +not been backfilled, so this migration cannot break resolution even if the +backfill is incomplete. + +Revision ID: c3_0_org_unit_path +Revises: c5_0_access_groups +""" + +import sqlalchemy as sa +from alembic import op + +revision = "c3_0_org_unit_path" +down_revision = "c5_0_access_groups" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("org_units", sa.Column("path", sa.Text(), nullable=True)) + + # Prefix matching needs `text_pattern_ops`; the default opclass supports + # equality and ordering but not `LIKE 'prefix%'` as an index scan. + op.execute( + "CREATE INDEX ix_org_units_path_prefix " + "ON org_units (path text_pattern_ops)" + ) + op.create_index("ix_org_units_tenant_path", "org_units", ["tenant_id", "path"]) + + # Backfill, walking down from the roots. Done once here so the application + # never has to compute a path it did not write. + op.execute( + """ + WITH RECURSIVE tree AS ( + SELECT id, '/' || id::text || '/' AS path + FROM org_units + WHERE parent_id IS NULL + UNION ALL + SELECT c.id, t.path || c.id::text || '/' + FROM org_units c + JOIN tree t ON c.parent_id = t.id + ) + UPDATE org_units u + SET path = tree.path + FROM tree + WHERE u.id = tree.id + """ + ) + + +def downgrade() -> None: + op.drop_index("ix_org_units_tenant_path", table_name="org_units") + op.execute("DROP INDEX IF EXISTS ix_org_units_path_prefix") + op.drop_column("org_units", "path") diff --git a/alembic/versions/c3abf44266de_enforce_b2_storage_provider.py b/alembic/versions/c3abf44266de_enforce_b2_storage_provider.py new file mode 100644 index 0000000..623435f --- /dev/null +++ b/alembic/versions/c3abf44266de_enforce_b2_storage_provider.py @@ -0,0 +1,40 @@ +"""enforce_b2_storage_provider + +Revision ID: c3abf44266de +Revises: bb224e0b642b +Create Date: 2026-03-17 10:54:51.326256 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'c3abf44266de' +down_revision: Union[str, None] = 'bb224e0b642b' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # 1. Update existing records from 'minio' to 'b2' + op.execute("UPDATE user_files SET storage_provider = 'b2' WHERE storage_provider = 'minio'") + op.execute("UPDATE drive_file_versions SET storage_provider = 'b2' WHERE storage_provider = 'minio'") + + # 2. Change server default to 'b2' + with op.batch_alter_table('user_files', schema=None) as batch_op: + batch_op.alter_column('storage_provider', server_default='b2') + + with op.batch_alter_table('drive_file_versions', schema=None) as batch_op: + batch_op.alter_column('storage_provider', server_default='b2') + + +def downgrade() -> None: + # 1. Change server default back to 'minio' + with op.batch_alter_table('user_files', schema=None) as batch_op: + batch_op.alter_column('storage_provider', server_default='minio') + + with op.batch_alter_table('drive_file_versions', schema=None) as batch_op: + batch_op.alter_column('storage_provider', server_default='minio') diff --git a/alembic/versions/c4_0_grant_expiry.py b/alembic/versions/c4_0_grant_expiry.py new file mode 100644 index 0000000..3a7331e --- /dev/null +++ b/alembic/versions/c4_0_grant_expiry.py @@ -0,0 +1,46 @@ +"""C4A — grants can expire + +`user_roles.expires_at`, nullable. Null means permanent, which is every existing +row, so this migration changes nobody's authority. + +Temporary elevation, contractor access and just-in-time grants all need this, and +all three are ordinary enterprise asks. Without it the only way to end a grant is +to remember to revoke it, and "remember to" is not an access control. + +Expired rows are **not** deleted. They stay, and they become history — which is +what C4B reads to answer "who had access in March". A swept-away row answers +nothing. + +Revision ID: c4_0_grant_expiry +Revises: c1_1_user_roles_tenant_id +""" + +import sqlalchemy as sa +from alembic import op + +revision = "c4_0_grant_expiry" +down_revision = "c1_1_user_roles_tenant_id" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "user_roles", + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True), + ) + + # Partial: only live grants are ever queried by expiry, and indexing the + # permanent ones (which is all of them today) would be indexing a column + # that is null for every row. + op.create_index( + "ix_user_roles_expires_at", + "user_roles", + ["expires_at"], + postgresql_where=sa.text("expires_at IS NOT NULL"), + ) + + +def downgrade() -> None: + op.drop_index("ix_user_roles_expires_at", table_name="user_roles") + op.drop_column("user_roles", "expires_at") diff --git a/alembic/versions/c4_1_access_log_retention.py b/alembic/versions/c4_1_access_log_retention.py new file mode 100644 index 0000000..2368c2b --- /dev/null +++ b/alembic/versions/c4_1_access_log_retention.py @@ -0,0 +1,39 @@ +"""C4C — index the access timeline, so retention and history are cheap + +Two reads matter and neither had an index: + +- the history endpoint, filtering one tenant's `access` events by date; +- the retention sweeper, finding events older than the policy. + +Both are `(tenant_id, module, created_at)`. Partial on `module = 'access'`, +because ordinary activity is far more numerous and does not share these +queries — indexing it here would triple the index for no reader. + +Retention itself is a **setting**, not a constant: how long access records are +kept is the sort of thing a customer contract specifies, and it must be +answerable without a deploy. See `ACCESS_LOG_RETENTION_DAYS`. + +Revision ID: c4_1_access_log_retention +Revises: c4_0_grant_expiry +""" + +import sqlalchemy as sa +from alembic import op + +revision = "c4_1_access_log_retention" +down_revision = "c4_0_grant_expiry" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_index( + "ix_activity_logs_access_timeline", + "activity_logs", + ["tenant_id", "created_at"], + postgresql_where=sa.text("module = 'access'"), + ) + + +def downgrade() -> None: + op.drop_index("ix_activity_logs_access_timeline", table_name="activity_logs") diff --git a/alembic/versions/c5_0_access_groups.py b/alembic/versions/c5_0_access_groups.py new file mode 100644 index 0000000..c1b5148 --- /dev/null +++ b/alembic/versions/c5_0_access_groups.py @@ -0,0 +1,161 @@ +"""C5.1 — access groups, and grants that can name one + +Two new tables, and one genuinely risky change: `user_roles.user_id` becomes +nullable, so a grant can name a **group** instead. + +That weakens an invariant every existing query relied on. Four readers exist and +all four were reviewed before this migration was written: + + * `app/core/scope.py` — resolution; gains a group branch + * `app/modules/auth/services/permission_service.py` — the coarse gate; likewise + * `app/modules/org/routes/org_routes.py` — lists a user's own grants; unchanged + behaviour, it simply will not list grants a user holds *through* a group + * `app/modules/org/services/access_maintenance.py` — the expiry sweeper; its + audit entry uses the principal, which may now be a group + +The `CHECK` is what keeps the pair honest. "Exactly one of two nullable columns" +is the kind of invariant that survives in the schema and rots in code — one new +call site that sets both, and the resolver counts the grant twice. + +Existing rows all have `user_id`, so the constraint holds the moment it is +added, and nobody's authority changes. + +Revision ID: c5_0_access_groups +Revises: c4_1_access_log_retention +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.postgresql import UUID + +revision = "c5_0_access_groups" +down_revision = "c4_1_access_log_retention" +branch_labels = None +depends_on = None + +TENANT_TABLES = ["access_groups", "user_access_groups"] +POLICY = "tenant_isolation" +USING = """ + coalesce(current_setting('docqube.bypass', true), '') = 'on' + OR tenant_id IS NULL + OR tenant_id = nullif(current_setting('docqube.tenant_id', true), '')::uuid +""" + + +def upgrade() -> None: + op.create_table( + "access_groups", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "tenant_id", + UUID(as_uuid=True), + sa.ForeignKey("tenants.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("name", sa.String(255), nullable=False), + sa.Column("description", sa.Text(), nullable=True), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.UniqueConstraint("tenant_id", "name", name="uq_access_groups_tenant_name"), + ) + op.create_index("ix_access_groups_tenant_id", "access_groups", ["tenant_id"]) + + op.create_table( + "user_access_groups", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "tenant_id", + UUID(as_uuid=True), + sa.ForeignKey("tenants.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "user_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=False + ), + sa.Column( + "group_id", + UUID(as_uuid=True), + sa.ForeignKey("access_groups.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.UniqueConstraint("user_id", "group_id", name="uq_user_access_groups"), + ) + op.create_index("ix_user_access_groups_tenant_id", "user_access_groups", ["tenant_id"]) + op.create_index("ix_user_access_groups_user_id", "user_access_groups", ["user_id"]) + op.create_index("ix_user_access_groups_group_id", "user_access_groups", ["group_id"]) + op.create_index( + "ix_user_access_groups_lookup", "user_access_groups", ["user_id", "group_id"] + ) + + # The risky half. + op.add_column( + "user_roles", + sa.Column( + "group_id", + UUID(as_uuid=True), + # CASCADE: deleting a group takes its grants with it. A grant whose + # principal no longer exists is a row nobody can see, revoke or + # audit — and it must not keep applying. + sa.ForeignKey("access_groups.id", ondelete="CASCADE"), + nullable=True, + ), + ) + op.alter_column("user_roles", "user_id", existing_type=sa.Integer(), nullable=True) + op.create_index("ix_user_roles_group_id", "user_roles", ["group_id"]) + + op.create_check_constraint( + "ck_user_roles_exactly_one_principal", + "user_roles", + "(user_id IS NOT NULL) <> (group_id IS NOT NULL)", + ) + + # The tenant-wide uniqueness guard was written for user grants. Group grants + # need the same protection, and the original index cannot serve both + # because `user_id` is now null on half the rows. + op.create_index( + "uq_user_roles_group_tenant_wide", + "user_roles", + ["group_id", "role_id"], + unique=True, + postgresql_where=sa.text("org_unit_id IS NULL AND group_id IS NOT NULL"), + ) + op.create_index( + "uq_user_roles_group_scoped", + "user_roles", + ["group_id", "role_id", "org_unit_id"], + unique=True, + postgresql_where=sa.text("group_id IS NOT NULL"), + ) + + for table in TENANT_TABLES: + op.execute(f"ALTER TABLE {table} ENABLE ROW LEVEL SECURITY") + op.execute(f"ALTER TABLE {table} FORCE ROW LEVEL SECURITY") + op.execute(f"DROP POLICY IF EXISTS {POLICY} ON {table}") + op.execute(f"CREATE POLICY {POLICY} ON {table} USING ({USING}) WITH CHECK ({USING})") + + +def downgrade() -> None: + for table in TENANT_TABLES: + op.execute(f"DROP POLICY IF EXISTS {POLICY} ON {table}") + + # Group grants cannot survive the column being dropped, and they cannot be + # converted into user grants either — the whole point is that they name no + # single user. Removing them is the only honest reverse. + op.execute("DELETE FROM user_roles WHERE group_id IS NOT NULL") + + op.drop_index("uq_user_roles_group_scoped", table_name="user_roles") + op.drop_index("uq_user_roles_group_tenant_wide", table_name="user_roles") + op.drop_constraint("ck_user_roles_exactly_one_principal", "user_roles", type_="check") + op.drop_index("ix_user_roles_group_id", table_name="user_roles") + op.alter_column("user_roles", "user_id", existing_type=sa.Integer(), nullable=False) + op.drop_column("user_roles", "group_id") + + op.drop_table("user_access_groups") + op.drop_table("access_groups") diff --git a/alembic/versions/c7d4e6f1a9b0_add_tenant_storage_quota.py b/alembic/versions/c7d4e6f1a9b0_add_tenant_storage_quota.py new file mode 100644 index 0000000..bc740a0 --- /dev/null +++ b/alembic/versions/c7d4e6f1a9b0_add_tenant_storage_quota.py @@ -0,0 +1,66 @@ +"""add tenant storage quota bytes + +Revision ID: c7d4e6f1a9b0 +Revises: e81a9d22ca5f +Create Date: 2026-03-07 09:25:00.000000 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "c7d4e6f1a9b0" +down_revision: Union[str, None] = "e81a9d22ca5f" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "tenants", + sa.Column("storage_quota_bytes", sa.BigInteger(), nullable=True), + ) + + # Preserve existing effective quota behavior by summing each tenant's + # current per-user quotas. For tenants with no users, default to 1 GB. + op.execute( + """ + UPDATE tenants t + SET storage_quota_bytes = COALESCE( + ( + SELECT SUM(COALESCE(usu.max_bytes_quota, 1073741824)) + FROM users u + LEFT JOIN user_storage_usage usu ON usu.user_id = u.id + WHERE u.tenant_id = t.id + ), + 1073741824 + ) + """ + ) + + op.alter_column( + "tenants", + "storage_quota_bytes", + existing_type=sa.BigInteger(), + nullable=False, + server_default=sa.text("1073741824"), + ) + + op.create_check_constraint( + "ck_tenants_storage_quota_bytes_positive", + "tenants", + "storage_quota_bytes > 0", + ) + + +def downgrade() -> None: + op.drop_constraint( + "ck_tenants_storage_quota_bytes_positive", + "tenants", + type_="check", + ) + op.drop_column("tenants", "storage_quota_bytes") diff --git a/alembic/versions/ca14c4ab10d9_merge_multiple_heads.py b/alembic/versions/ca14c4ab10d9_merge_multiple_heads.py new file mode 100644 index 0000000..4059914 --- /dev/null +++ b/alembic/versions/ca14c4ab10d9_merge_multiple_heads.py @@ -0,0 +1,26 @@ +"""Merge multiple heads + +Revision ID: ca14c4ab10d9 +Revises: b0a09cd13348, y001_add_user_mail_from +Create Date: 2026-05-08 13:47:42.707120 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'ca14c4ab10d9' +down_revision: Union[str, None] = ('b0a09cd13348', 'y001_add_user_mail_from') +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/alembic/versions/cba70ab35793_create_chat_token_usage_table.py b/alembic/versions/cba70ab35793_create_chat_token_usage_table.py new file mode 100644 index 0000000..43ff6c4 --- /dev/null +++ b/alembic/versions/cba70ab35793_create_chat_token_usage_table.py @@ -0,0 +1,83 @@ +"""Create chat_token_usage table + +Revision ID: cba70ab35793 +Revises: cd27562afa2e +Create Date: 2026-04-16 13:39:46.842540 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'cba70ab35793' +down_revision: Union[str, None] = 'cd27562afa2e' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +# (index_name, columns, unique, use_naming_convention) +# use_naming_convention=True wraps in batch_op.f(); False uses the raw string +# (matches the original auto-generated/hand-written split). +_INDEXES = [ + ('ix_chat_token_usage_chat_id', ['chat_id'], False, True), + ('ix_chat_token_usage_document_id', ['document_id'], False, True), + ('ix_chat_token_usage_id', ['id'], False, True), + ('ix_chat_token_usage_model_name', ['model_name'], False, True), + ('ix_chat_token_usage_model_time', ['model_name', 'timestamp'], False, False), + ('ix_chat_token_usage_phase', ['phase'], False, True), + ('ix_chat_token_usage_timestamp', ['timestamp'], False, True), + ('ix_chat_token_usage_user_id', ['user_id'], False, True), + ('ix_chat_token_usage_user_time', ['user_id', 'timestamp'], False, False), +] + + +def _index_names(bind, table_name: str) -> set[str]: + inspector = sa.inspect(bind) + return {idx["name"] for idx in inspector.get_indexes(table_name)} + + +def upgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + + if not inspector.has_table('chat_token_usage'): + op.create_table('chat_token_usage', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('chat_id', sa.Integer(), nullable=True), + sa.Column('document_id', sa.String(length=255), nullable=True), + sa.Column('prompt_tokens', sa.BigInteger(), nullable=False), + sa.Column('completion_tokens', sa.BigInteger(), nullable=False), + sa.Column('total_tokens', sa.BigInteger(), nullable=False), + sa.Column('model_name', sa.String(length=100), nullable=False), + sa.Column('cost_usd', sa.DECIMAL(precision=12, scale=8), nullable=False), + sa.Column('timestamp', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('phase', sa.String(length=50), nullable=True), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + + existing = _index_names(bind, 'chat_token_usage') + with op.batch_alter_table('chat_token_usage', schema=None) as batch_op: + for name, cols, unique, use_f in _INDEXES: + if name not in existing: + ref = batch_op.f(name) if use_f else name + batch_op.create_index(ref, cols, unique=unique) + + +def downgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + + if inspector.has_table('chat_token_usage'): + existing = _index_names(bind, 'chat_token_usage') + with op.batch_alter_table('chat_token_usage', schema=None) as batch_op: + for name, _, _, use_f in reversed(_INDEXES): + if name in existing: + ref = batch_op.f(name) if use_f else name + batch_op.drop_index(ref) + + op.drop_table('chat_token_usage') diff --git a/alembic/versions/cd27562afa2e_add_chat_token_usage_tracking.py b/alembic/versions/cd27562afa2e_add_chat_token_usage_tracking.py new file mode 100644 index 0000000..43d28ec --- /dev/null +++ b/alembic/versions/cd27562afa2e_add_chat_token_usage_tracking.py @@ -0,0 +1,116 @@ +"""Add chat token usage tracking + +Revision ID: cd27562afa2e +Revises: 1f4d2a8c9b77 +Create Date: 2026-04-16 13:05:09.572442 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = 'cd27562afa2e' +down_revision: Union[str, None] = '1f4d2a8c9b77' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _table_exists(bind, table_name: str) -> bool: + return sa.inspect(bind).has_table(table_name) + + +def upgrade() -> None: + bind = op.get_bind() + + if _table_exists(bind, 'chatbot_documents'): + with op.batch_alter_table('chatbot_documents', schema=None) as batch_op: + batch_op.alter_column('chunks_count', + existing_type=sa.INTEGER(), + nullable=False) + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(), + nullable=False) + + if _table_exists(bind, 'vector_chunks'): + with op.batch_alter_table('vector_chunks', schema=None) as batch_op: + batch_op.alter_column('document_id', + existing_type=sa.VARCHAR(), + nullable=False) + batch_op.alter_column('chunk_text', + existing_type=sa.VARCHAR(), + type_=sa.Text(), + existing_nullable=False) + batch_op.alter_column('chunk_index', + existing_type=sa.INTEGER(), + nullable=False) + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + nullable=False) + batch_op.drop_constraint('vector_chunks_user_id_fkey', type_='foreignkey') + batch_op.create_foreign_key(None, 'users', ['user_id'], ['id'], ondelete='CASCADE') + + if _table_exists(bind, 'vector_indices'): + with op.batch_alter_table('vector_indices', schema=None) as batch_op: + batch_op.alter_column('total_vectors', + existing_type=sa.INTEGER(), + nullable=False) + batch_op.alter_column('updated_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + nullable=False) + batch_op.drop_constraint('vector_indices_user_id_fkey', type_='foreignkey') + batch_op.create_foreign_key(None, 'users', ['user_id'], ['id'], ondelete='CASCADE') + + +def downgrade() -> None: + bind = op.get_bind() + + if _table_exists(bind, 'vector_indices'): + with op.batch_alter_table('vector_indices', schema=None) as batch_op: + batch_op.drop_constraint(None, type_='foreignkey') + batch_op.create_foreign_key('vector_indices_user_id_fkey', 'users', ['user_id'], ['id']) + batch_op.drop_index(batch_op.f('ix_vector_indices_user_id')) + batch_op.create_index('ix_vector_indices_user_id', ['user_id'], unique=True) + batch_op.create_index('ix_vector_indices_id', ['id'], unique=False) + batch_op.alter_column('updated_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + nullable=True) + batch_op.alter_column('total_vectors', + existing_type=sa.INTEGER(), + nullable=True) + + if _table_exists(bind, 'vector_chunks'): + with op.batch_alter_table('vector_chunks', schema=None) as batch_op: + batch_op.drop_constraint(None, type_='foreignkey') + batch_op.create_foreign_key('vector_chunks_user_id_fkey', 'users', ['user_id'], ['id']) + batch_op.create_index('ix_vector_chunks_id', ['id'], unique=False) + batch_op.alter_column('created_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + nullable=True) + batch_op.alter_column('chunk_index', + existing_type=sa.INTEGER(), + nullable=True) + batch_op.alter_column('chunk_text', + existing_type=sa.Text(), + type_=sa.VARCHAR(), + existing_nullable=False) + batch_op.alter_column('document_id', + existing_type=sa.VARCHAR(), + nullable=True) + + if _table_exists(bind, 'chatbot_documents'): + with op.batch_alter_table('chatbot_documents', schema=None) as batch_op: + batch_op.create_index('ix_chatbot_documents_id', ['id'], unique=False) + batch_op.create_index('ix_chatbot_documents_drive_file_id', ['drive_file_id'], unique=False) + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(), + nullable=True) + batch_op.alter_column('chunks_count', + existing_type=sa.INTEGER(), + nullable=True) diff --git a/alembic/versions/cfbee4f91fc2_add_updated_at_defaults.py b/alembic/versions/cfbee4f91fc2_add_updated_at_defaults.py new file mode 100644 index 0000000..915bffb --- /dev/null +++ b/alembic/versions/cfbee4f91fc2_add_updated_at_defaults.py @@ -0,0 +1,52 @@ +"""add_updated_at_defaults + +Revision ID: cfbee4f91fc2 +Revises: 49c19df2c1d4 +Create Date: 2026-02-19 11:25:54.874662 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'cfbee4f91fc2' +down_revision: Union[str, None] = '49c19df2c1d4' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('drive_folders', 'updated_at', + existing_type=sa.DateTime(timezone=True), + server_default=sa.text('now()'), + existing_nullable=False) + op.alter_column('drive_files', 'updated_at', + existing_type=sa.DateTime(timezone=True), + server_default=sa.text('now()'), + existing_nullable=False) + op.alter_column('projects', 'updated_at', + existing_type=sa.DateTime(timezone=True), + server_default=sa.text('now()'), + existing_nullable=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('projects', 'updated_at', + existing_type=sa.DateTime(timezone=True), + server_default=None, + existing_nullable=False) + op.alter_column('drive_files', 'updated_at', + existing_type=sa.DateTime(timezone=True), + server_default=None, + existing_nullable=False) + op.alter_column('drive_folders', 'updated_at', + existing_type=sa.DateTime(timezone=True), + server_default=None, + existing_nullable=False) + # ### end Alembic commands ### diff --git a/alembic/versions/d1_0_plans_and_entitlements.py b/alembic/versions/d1_0_plans_and_entitlements.py new file mode 100644 index 0000000..e36b65e --- /dev/null +++ b/alembic/versions/d1_0_plans_and_entitlements.py @@ -0,0 +1,219 @@ +"""D1 — plans, limits, subscriptions and overrides + +The landing page advertises four priced plans and the backend had never heard of +a plan. Meanwhile `tenants` carried four limit columns set per tenant by hand, so +two customers on the same advertised plan could silently have different limits. + +This gives those limits something to be derived from. + +**Behaviour-neutral by construction.** Every tenant's four current values become +**overrides**, not plan limits — so whatever they have today is exactly what they +keep, whichever plan they are placed on. The plan is then chosen to match, or +`custom` when nothing does. Nobody's limits move. + +The `tenants` columns are **not** dropped. They stay for one release and remain +the last-resort fallback in `EntitlementService`, the same transition shape as +`users.role_id` and for the same reason: a rollback must not be able to remove +every tenant's quota. + +`plans` and `plan_limits` are deliberately **not** tenant-scoped — a catalogue is +shared, like `accesses`. `tenant_subscriptions` and `tenant_limit_overrides` are, +and are added to the RLS policy here. + +Revision ID: d1_0_plans_and_entitlements +Revises: c3_0_org_unit_path +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.postgresql import UUID + +revision = "d1_0_plans_and_entitlements" +down_revision = "c3_0_org_unit_path" +branch_labels = None +depends_on = None + +TENANT_TABLES = ["tenant_subscriptions", "tenant_limit_overrides"] +POLICY = "tenant_isolation" +USING = """ + coalesce(current_setting('docqube.bypass', true), '') = 'on' + OR tenant_id IS NULL + OR tenant_id = nullif(current_setting('docqube.tenant_id', true), '')::uuid +""" + +# The four plans the pricing page already sells, with the limits it claims. +# Seeded here rather than in `scripts/seed.py` so that a database which has run +# migrations is one a customer could be placed on — the page and the schema +# start out agreeing. +# +# -1 means unlimited, matching the convention `tenants.envelope_limit` already +# uses. +GB = 1024 ** 3 +PLANS = [ + # code, name, price, public, order + ("starter", "Starter plan", "$12", True, 1), + ("professional", "Professional plan", "$22", True, 2), + ("elite", "Elite plan", "$29", True, 3), + ("custom", "Custom plan", "Contact Us", True, 4), +] +LIMITS = { + "starter": {"storage_bytes": 10 * GB, "seats": 5, + "chat_tokens_daily": 500_000, "envelopes": 25}, + "professional": {"storage_bytes": 100 * GB, "seats": 25, + "chat_tokens_daily": 2_000_000, "envelopes": 250}, + "elite": {"storage_bytes": 500 * GB, "seats": 100, + "chat_tokens_daily": 10_000_000, "envelopes": -1}, + "custom": {"storage_bytes": -1, "seats": -1, + "chat_tokens_daily": -1, "envelopes": -1}, +} + + +def upgrade() -> None: + op.create_table( + "plans", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column("code", sa.String(50), nullable=False, unique=True), + sa.Column("name", sa.String(255), nullable=False), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("price_display", sa.String(50), nullable=True), + sa.Column("currency", sa.String(10), nullable=True), + sa.Column("is_public", sa.Boolean(), nullable=False, server_default=sa.true()), + sa.Column("sort_order", sa.Integer(), nullable=False, server_default="0"), + sa.Column("created_at", sa.DateTime(timezone=True), + server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), + server_default=sa.func.now(), nullable=False), + ) + op.create_index("ix_plans_code", "plans", ["code"]) + + op.create_table( + "plan_limits", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column("plan_id", UUID(as_uuid=True), + sa.ForeignKey("plans.id", ondelete="CASCADE"), nullable=False), + sa.Column("key", sa.String(50), nullable=False), + sa.Column("value", sa.BigInteger(), nullable=False), + sa.UniqueConstraint("plan_id", "key", name="uq_plan_limits_plan_key"), + ) + op.create_index("ix_plan_limits_plan_id", "plan_limits", ["plan_id"]) + + op.create_table( + "tenant_subscriptions", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column("tenant_id", UUID(as_uuid=True), + sa.ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False), + # RESTRICT: deleting a plan somebody is on would silently drop them to + # the defaults. Move them first, deliberately. + sa.Column("plan_id", UUID(as_uuid=True), + sa.ForeignKey("plans.id", ondelete="RESTRICT"), nullable=False), + sa.Column("status", sa.String(20), nullable=False, server_default="active"), + sa.Column("started_at", sa.DateTime(timezone=True), + server_default=sa.func.now(), nullable=False), + sa.Column("current_period_end", sa.DateTime(timezone=True), nullable=True), + sa.Column("cancel_at_period_end", sa.Boolean(), nullable=False, + server_default=sa.false()), + sa.Column("external_ref", sa.String(255), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), + server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), + server_default=sa.func.now(), nullable=False), + # One live subscription per tenant: a second "current" row is an + # ambiguity nothing can resolve. + sa.UniqueConstraint("tenant_id", name="uq_tenant_subscriptions_tenant"), + ) + op.create_index("ix_tenant_subscriptions_tenant_id", "tenant_subscriptions", ["tenant_id"]) + op.create_index("ix_tenant_subscriptions_plan_id", "tenant_subscriptions", ["plan_id"]) + op.create_index("ix_tenant_subscriptions_status", "tenant_subscriptions", ["status"]) + op.create_index("ix_tenant_subscriptions_external_ref", "tenant_subscriptions", ["external_ref"]) + + op.create_table( + "tenant_limit_overrides", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column("tenant_id", UUID(as_uuid=True), + sa.ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False), + sa.Column("key", sa.String(50), nullable=False), + sa.Column("value", sa.BigInteger(), nullable=False), + sa.Column("note", sa.Text(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), + server_default=sa.func.now(), nullable=False), + sa.UniqueConstraint("tenant_id", "key", name="uq_tenant_limit_overrides"), + ) + op.create_index("ix_tenant_limit_overrides_tenant_id", "tenant_limit_overrides", ["tenant_id"]) + + # --- seed the catalogue ------------------------------------------------- + for code, name, price, public, order in PLANS: + op.execute( + sa.text( + "INSERT INTO plans (id, code, name, price_display, currency, " + "is_public, sort_order, created_at, updated_at) " + "VALUES (gen_random_uuid(), :code, :name, :price, 'USD', " + ":public, :order, now(), now())" + ).bindparams(code=code, name=name, price=price, public=public, order=order) + ) + for key, value in LIMITS[code].items(): + op.execute( + sa.text( + "INSERT INTO plan_limits (id, plan_id, key, value) " + "SELECT gen_random_uuid(), id, :key, :value FROM plans " + " WHERE code = :code" + ).bindparams(key=key, value=value, code=code) + ) + + # --- backfill, behaviour-neutral ---------------------------------------- + # + # Current values become **overrides**, so every tenant keeps exactly what it + # has regardless of which plan it lands on. The plan is a label until + # somebody removes the override. + op.execute( + """ + INSERT INTO tenant_limit_overrides (id, tenant_id, key, value, note, created_at) + SELECT gen_random_uuid(), t.id, v.key, v.value, + 'Carried over from the tenant row at D1; not a negotiated exception', + now() + FROM tenants t + CROSS JOIN LATERAL (VALUES + ('storage_bytes', t.storage_quota_bytes), + ('seats', COALESCE(t.max_users_allowed, -1)), + ('chat_tokens_daily', t.chat_token_daily_limit), + ('envelopes', t.envelope_limit) + ) AS v(key, value) + WHERE t.is_deleted = false + ON CONFLICT DO NOTHING + """ + ) + + # Every live tenant is placed on `custom`, not on a guessed plan. + # + # Matching limits to a plan would be inference, and inferring a commercial + # fact is how somebody ends up on Starter because their quota happened to + # look like it. `custom` is honest: nobody has said what they are on yet, + # and their overrides mean it changes nothing. + op.execute( + """ + INSERT INTO tenant_subscriptions (id, tenant_id, plan_id, status, + started_at, created_at, updated_at) + SELECT gen_random_uuid(), t.id, p.id, 'active', now(), now(), now() + FROM tenants t + CROSS JOIN (SELECT id FROM plans WHERE code = 'custom') p + WHERE t.is_deleted = false + ON CONFLICT DO NOTHING + """ + ) + + for table in TENANT_TABLES: + op.execute(f"ALTER TABLE {table} ENABLE ROW LEVEL SECURITY") + op.execute(f"ALTER TABLE {table} FORCE ROW LEVEL SECURITY") + op.execute(f"DROP POLICY IF EXISTS {POLICY} ON {table}") + op.execute(f"CREATE POLICY {POLICY} ON {table} USING ({USING}) WITH CHECK ({USING})") + + +def downgrade() -> None: + for table in TENANT_TABLES: + op.execute(f"DROP POLICY IF EXISTS {POLICY} ON {table}") + + # Safe to reverse: the `tenants` columns were never touched, so removing + # these tables returns the application to the limits it still reads. + op.drop_table("tenant_limit_overrides") + op.drop_table("tenant_subscriptions") + op.drop_table("plan_limits") + op.drop_table("plans") diff --git a/alembic/versions/d2_0_credit_limit_precision.py b/alembic/versions/d2_0_credit_limit_precision.py new file mode 100644 index 0000000..7267d2c --- /dev/null +++ b/alembic/versions/d2_0_credit_limit_precision.py @@ -0,0 +1,61 @@ +"""D2 — a credit limit stops being rounded to a whole number + +`chat_credit_quota.daily_credit_limit` is `bigint` in the database and +`DECIMAL(12, 2)` on the model. PostgreSQL resolves that disagreement by +rounding, silently, on write: + + INSERT ... VALUES (0.60::numeric(12,2)) -> stored: 1 + INSERT ... VALUES (10.75::numeric(12,2)) -> stored: 11 + INSERT ... VALUES (10.25::numeric(12,2)) -> stored: 10 + +So a tenant admin setting a limit of 0.60 credits gets 1 — a 67% overshoot — and +nothing anywhere reports it. Whether that has bitten depends on whether anyone +has entered a fraction, which is not a property worth relying on. + +Widening, so no value is at risk: every bigint currently stored is representable +in `numeric(12, 2)`. `credits_used_today` and `total_credits_ever` are left as +`bigint` deliberately — they count tokens, which are whole. + +The reverse casts back to `bigint` and therefore *does* round. That is the +honest reverse of a widening: going back to a narrower type cannot preserve what +the wider one held. It is written that way rather than left unimplemented so the +migration can actually be reversed, but rolling this back after fractional +limits have been set will round them. + +Found by `alembic revision --autogenerate`, which had never been run against +this schema, and which after the model-registry fix reports this as the single +remaining difference between the models and the database. + +Revision ID: d2_0_credit_limit_precision +Revises: d1_0_plans_and_entitlements +""" + +import sqlalchemy as sa +from alembic import op + +revision = "d2_0_credit_limit_precision" +down_revision = "d1_0_plans_and_entitlements" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.alter_column( + "chat_credit_quota", + "daily_credit_limit", + existing_type=sa.BigInteger(), + type_=sa.DECIMAL(precision=12, scale=2), + existing_nullable=False, + postgresql_using="daily_credit_limit::numeric(12,2)", + ) + + +def downgrade() -> None: + op.alter_column( + "chat_credit_quota", + "daily_credit_limit", + existing_type=sa.DECIMAL(precision=12, scale=2), + type_=sa.BigInteger(), + existing_nullable=False, + postgresql_using="round(daily_credit_limit)::bigint", + ) diff --git a/alembic/versions/d2e2c55290ee_add_10mb_check_constraints_on_document_.py b/alembic/versions/d2e2c55290ee_add_10mb_check_constraints_on_document_.py new file mode 100644 index 0000000..9927d5b --- /dev/null +++ b/alembic/versions/d2e2c55290ee_add_10mb_check_constraints_on_document_.py @@ -0,0 +1,60 @@ +"""Add 10MB CHECK constraints on document content fields + +Revision ID: d2e2c55290ee +Revises: 9f6521a38bf3 +Create Date: 2026-03-03 22:17:00.564459 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'd2e2c55290ee' +down_revision: Union[str, None] = '9f6521a38bf3' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +# 10 MB limit in bytes +_MAX = 10 * 1024 * 1024 # 10485760 + + +def upgrade() -> None: + # projects table — three content columns + op.create_check_constraint( + 'ck_project_markdown_max', 'projects', + f'octet_length(markdown_content) <= {_MAX}' + ) + op.create_check_constraint( + 'ck_project_xml_max', 'projects', + f'octet_length(xml_content) <= {_MAX}' + ) + op.create_check_constraint( + 'ck_project_html_max', 'projects', + f'octet_length(html_content) <= {_MAX}' + ) + + # project_versions table — three content columns + op.create_check_constraint( + 'ck_version_markdown_max', 'project_versions', + f'octet_length(markdown_content) <= {_MAX}' + ) + op.create_check_constraint( + 'ck_version_xml_max', 'project_versions', + f'octet_length(xml_content) <= {_MAX}' + ) + op.create_check_constraint( + 'ck_version_html_max', 'project_versions', + f'octet_length(html_content) <= {_MAX}' + ) + + +def downgrade() -> None: + op.drop_constraint('ck_version_html_max', 'project_versions', type_='check') + op.drop_constraint('ck_version_xml_max', 'project_versions', type_='check') + op.drop_constraint('ck_version_markdown_max', 'project_versions', type_='check') + op.drop_constraint('ck_project_html_max', 'projects', type_='check') + op.drop_constraint('ck_project_xml_max', 'projects', type_='check') + op.drop_constraint('ck_project_markdown_max', 'projects', type_='check') diff --git a/alembic/versions/d551c03861f4_rename_quota_to_credits.py b/alembic/versions/d551c03861f4_rename_quota_to_credits.py new file mode 100644 index 0000000..fc18046 --- /dev/null +++ b/alembic/versions/d551c03861f4_rename_quota_to_credits.py @@ -0,0 +1,38 @@ +"""rename_quota_to_credits + +Revision ID: d551c03861f4 +Revises: 540c970ff890 +Create Date: 2026-05-13 13:01:51.510123 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'd551c03861f4' +down_revision: Union[str, None] = '540c970ff890' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Rename Table + op.rename_table('chat_token_quota', 'chat_credit_quota') + + # Rename Columns in chat_credit_quota + op.alter_column('chat_credit_quota', 'daily_limit', new_column_name='daily_credit_limit') + op.alter_column('chat_credit_quota', 'tokens_used_today', new_column_name='credits_used_today') + op.alter_column('chat_credit_quota', 'total_tokens_ever', new_column_name='total_credits_ever') + + +def downgrade() -> None: + # Rename Columns back + op.alter_column('chat_credit_quota', 'daily_credit_limit', new_column_name='daily_limit') + op.alter_column('chat_credit_quota', 'credits_used_today', new_column_name='tokens_used_today') + op.alter_column('chat_credit_quota', 'total_credits_ever', new_column_name='total_tokens_ever') + + # Rename Table back + op.rename_table('chat_credit_quota', 'chat_token_quota') diff --git a/alembic/versions/daf45366d608_add_envelope_limits_to_tenants.py b/alembic/versions/daf45366d608_add_envelope_limits_to_tenants.py new file mode 100644 index 0000000..9ee0286 --- /dev/null +++ b/alembic/versions/daf45366d608_add_envelope_limits_to_tenants.py @@ -0,0 +1,34 @@ +"""Add envelope limits to tenants + +Revision ID: daf45366d608 +Revises: 611d6b99ad11 +Create Date: 2026-07-18 10:34:04.536445 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'daf45366d608' +down_revision: Union[str, None] = '611d6b99ad11' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column('tenants', sa.Column('envelope_limit', sa.BigInteger(), nullable=False, server_default='-1')) + op.add_column('tenants', sa.Column('envelopes_used', sa.BigInteger(), nullable=False, server_default='0')) + op.add_column('tenants', sa.Column('envelope_reset_days', sa.Integer(), nullable=True)) + op.add_column('tenants', sa.Column('envelope_last_reset_at', sa.DateTime(timezone=True), nullable=False, server_default=sa.text('now()'))) + pass + + +def downgrade() -> None: + op.drop_column('tenants', 'envelope_last_reset_at') + op.drop_column('tenants', 'envelope_reset_days') + op.drop_column('tenants', 'envelopes_used') + op.drop_column('tenants', 'envelope_limit') + pass diff --git a/alembic/versions/dbaa8ef522c0_add_color_to_drive_comments.py b/alembic/versions/dbaa8ef522c0_add_color_to_drive_comments.py new file mode 100644 index 0000000..b66df2c --- /dev/null +++ b/alembic/versions/dbaa8ef522c0_add_color_to_drive_comments.py @@ -0,0 +1,38 @@ +"""add color to drive comments + +Revision ID: dbaa8ef522c0 +Revises: 8736ab6bd5af +Create Date: 2026-02-26 14:23:14.999888 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = 'dbaa8ef522c0' +down_revision: Union[str, None] = '8736ab6bd5af' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('drive_comments', sa.Column('color', sa.String(length=20), nullable=True)) + op.alter_column('notifications', 'created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('notifications', 'created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('now()')) + op.drop_column('drive_comments', 'color') + # ### end Alembic commands ### diff --git a/alembic/versions/dfae38e4ede3_device_constraints_and_session_updates.py b/alembic/versions/dfae38e4ede3_device_constraints_and_session_updates.py new file mode 100644 index 0000000..0f745b9 --- /dev/null +++ b/alembic/versions/dfae38e4ede3_device_constraints_and_session_updates.py @@ -0,0 +1,28 @@ +"""device constraints and session updates + +Revision ID: dfae38e4ede3 +Revises: 9c49fe120ffb +Create Date: 2026-08-07 11:03:54.251473 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = 'dfae38e4ede3' +down_revision: Union[str, None] = '9c49fe120ffb' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + + op.create_unique_constraint('uq_device_user_fingerprint', 'devices', ['user_id', 'fingerprint']) + + +def downgrade() -> None: + op.drop_constraint('uq_device_user_fingerprint', 'devices', type_='unique') diff --git a/alembic/versions/e1_0_add_saas_plan_id.py b/alembic/versions/e1_0_add_saas_plan_id.py new file mode 100644 index 0000000..9cf4ff4 --- /dev/null +++ b/alembic/versions/e1_0_add_saas_plan_id.py @@ -0,0 +1,32 @@ +"""add saas_plan_id to plans + +Revision ID: e1_0_add_saas_plan_id +Revises: 9c4a95ac6726 +Create Date: 2026-09-01 + +Adds a nullable `saas_plan_id` column to the `plans` table so that DocQube +can trace each plan back to the SaaS SubscriptionPlan that originated it via +the provision webhook. Plans created directly inside DocQube will have NULL. +""" + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = "e1_0_add_saas_plan_id" +down_revision = "9c4a95ac6726" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "plans", + sa.Column("saas_plan_id", sa.String(255), nullable=True), + ) + op.create_index("ix_plans_saas_plan_id", "plans", ["saas_plan_id"], unique=True) + + +def downgrade() -> None: + op.drop_index("ix_plans_saas_plan_id", table_name="plans") + op.drop_column("plans", "saas_plan_id") diff --git a/alembic/versions/e81a9d22ca5f_migrate_roles_access_to_uuid.py b/alembic/versions/e81a9d22ca5f_migrate_roles_access_to_uuid.py new file mode 100644 index 0000000..dec3a3b --- /dev/null +++ b/alembic/versions/e81a9d22ca5f_migrate_roles_access_to_uuid.py @@ -0,0 +1,116 @@ +"""migrate_roles_access_to_uuid + +Revision ID: e81a9d22ca5f +Revises: a2b3c4d5e6f7 +Create Date: 2026-03-05 14:42:53.769667 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'e81a9d22ca5f' +down_revision: Union[str, None] = 'a2b3c4d5e6f7' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # --- Users Table --- + # Drop FK pointing to roles + op.drop_constraint('users_role_id_fkey', 'users', type_='foreignkey') + + # --- RoleAccess Table --- + # Drop FKs + op.drop_constraint('role_accesses_role_id_fkey', 'role_accesses', type_='foreignkey') + op.drop_constraint('role_accesses_access_id_fkey', 'role_accesses', type_='foreignkey') + op.drop_constraint('pk_role_accesses', 'role_accesses', type_='primary') + + # --- Accesses Table --- + # FK was named 'fk_accesses_parent_id' in migration a2b3c4d5e6f7 + op.drop_constraint('fk_accesses_parent_id', 'accesses', type_='foreignkey') + + # --- Alter Columns to UUID --- + # Drop serial defaults before changing type + op.execute('ALTER TABLE accesses ALTER COLUMN id DROP DEFAULT') + op.execute('ALTER TABLE roles ALTER COLUMN id DROP DEFAULT') + + # Change types to UUID (tables should be empty) + op.execute('ALTER TABLE accesses ALTER COLUMN id TYPE uuid USING (gen_random_uuid())') + op.execute('ALTER TABLE accesses ALTER COLUMN parent_id TYPE uuid USING NULL') + + op.execute('ALTER TABLE roles ALTER COLUMN id TYPE uuid USING (gen_random_uuid())') + + op.execute('ALTER TABLE role_accesses ALTER COLUMN role_id TYPE uuid USING (gen_random_uuid())') + op.execute('ALTER TABLE role_accesses ALTER COLUMN access_id TYPE uuid USING (gen_random_uuid())') + + op.execute('ALTER TABLE users ALTER COLUMN role_id TYPE uuid USING NULL') + + # Set UUID defaults for primary keys + op.execute('ALTER TABLE accesses ALTER COLUMN id SET DEFAULT gen_random_uuid()') + op.execute('ALTER TABLE roles ALTER COLUMN id SET DEFAULT gen_random_uuid()') + + # Drop leftover serial sequences + op.execute('DROP SEQUENCE IF EXISTS accesses_id_seq CASCADE') + op.execute('DROP SEQUENCE IF EXISTS roles_id_seq CASCADE') + + # --- Re-add Constraints --- + # Accesses + op.create_foreign_key('fk_accesses_parent_id', 'accesses', 'accesses', ['parent_id'], ['id'], ondelete='CASCADE') + + # RoleAccess + op.create_primary_key('pk_role_accesses', 'role_accesses', ['role_id', 'access_id']) + op.create_foreign_key('role_accesses_role_id_fkey', 'role_accesses', 'roles', ['role_id'], ['id'], ondelete='CASCADE') + op.create_foreign_key('role_accesses_access_id_fkey', 'role_accesses', 'accesses', ['access_id'], ['id'], ondelete='CASCADE') + + # Users + op.create_foreign_key('users_role_id_fkey', 'users', 'roles', ['role_id'], ['id'], ondelete='SET NULL') + + +def downgrade() -> None: + # --- Users Table --- + op.drop_constraint('users_role_id_fkey', 'users', type_='foreignkey') + + # --- RoleAccess Table --- + op.drop_constraint('role_accesses_role_id_fkey', 'role_accesses', type_='foreignkey') + op.drop_constraint('role_accesses_access_id_fkey', 'role_accesses', type_='foreignkey') + op.drop_constraint('pk_role_accesses', 'role_accesses', type_='primary') + + # --- Accesses Table --- + op.drop_constraint('fk_accesses_parent_id', 'accesses', type_='foreignkey') + + # Truncate all tables (UUID -> INT cast is destructive) + op.execute('TRUNCATE TABLE role_accesses CASCADE') + op.execute('TRUNCATE TABLE users CASCADE') + op.execute('TRUNCATE TABLE accesses CASCADE') + op.execute('TRUNCATE TABLE roles CASCADE') + + # Drop UUID defaults + op.execute('ALTER TABLE accesses ALTER COLUMN id DROP DEFAULT') + op.execute('ALTER TABLE roles ALTER COLUMN id DROP DEFAULT') + + # Change types back to integer + op.execute('ALTER TABLE accesses ALTER COLUMN id TYPE integer USING 0') + op.execute('ALTER TABLE accesses ALTER COLUMN parent_id TYPE integer USING NULL') + op.execute('ALTER TABLE roles ALTER COLUMN id TYPE integer USING 0') + op.execute('ALTER TABLE role_accesses ALTER COLUMN role_id TYPE integer USING 0') + op.execute('ALTER TABLE role_accesses ALTER COLUMN access_id TYPE integer USING 0') + op.execute('ALTER TABLE users ALTER COLUMN role_id TYPE integer USING NULL') + + # Recreate serial sequences + op.execute("CREATE SEQUENCE accesses_id_seq OWNED BY accesses.id") + op.execute("ALTER TABLE accesses ALTER COLUMN id SET DEFAULT nextval('accesses_id_seq')") + op.execute("CREATE SEQUENCE roles_id_seq OWNED BY roles.id") + op.execute("ALTER TABLE roles ALTER COLUMN id SET DEFAULT nextval('roles_id_seq')") + + # --- Re-add Constraints --- + op.create_foreign_key('fk_accesses_parent_id', 'accesses', 'accesses', ['parent_id'], ['id'], ondelete='CASCADE') + + op.create_primary_key('pk_role_accesses', 'role_accesses', ['role_id', 'access_id']) + op.create_foreign_key('role_accesses_role_id_fkey', 'role_accesses', 'roles', ['role_id'], ['id'], ondelete='CASCADE') + op.create_foreign_key('role_accesses_access_id_fkey', 'role_accesses', 'accesses', ['access_id'], ['id'], ondelete='CASCADE') + + op.create_foreign_key('users_role_id_fkey', 'users', 'roles', ['role_id'], ['id'], ondelete='SET NULL') diff --git a/alembic/versions/f055f9b157cb_add_cascade_deletes_to_drive_file_.py b/alembic/versions/f055f9b157cb_add_cascade_deletes_to_drive_file_.py new file mode 100644 index 0000000..128e295 --- /dev/null +++ b/alembic/versions/f055f9b157cb_add_cascade_deletes_to_drive_file_.py @@ -0,0 +1,36 @@ +"""Add cascade deletes to drive_file_versions + +Revision ID: f055f9b157cb +Revises: a902935ea69e +Create Date: 2026-03-02 11:08:38.477933 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'f055f9b157cb' +down_revision: Union[str, None] = 'a902935ea69e' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint('drive_file_versions_file_id_fkey', 'drive_file_versions', type_='foreignkey') + op.create_foreign_key(None, 'drive_file_versions', 'drive_files', ['file_id'], ['id'], ondelete='CASCADE') + op.drop_constraint('drive_stars_user_id_fkey', 'drive_stars', type_='foreignkey') + op.create_foreign_key(None, 'drive_stars', 'users', ['user_id'], ['id'], ondelete='CASCADE') + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, 'drive_stars', type_='foreignkey') + op.create_foreign_key('drive_stars_user_id_fkey', 'drive_stars', 'users', ['user_id'], ['id']) + op.drop_constraint(None, 'drive_file_versions', type_='foreignkey') + op.create_foreign_key('drive_file_versions_file_id_fkey', 'drive_file_versions', 'drive_files', ['file_id'], ['id']) + # ### end Alembic commands ### diff --git a/alembic/versions/fb6810c36ffb_add_has_completed_tutorial_to_user.py b/alembic/versions/fb6810c36ffb_add_has_completed_tutorial_to_user.py new file mode 100644 index 0000000..9073fa3 --- /dev/null +++ b/alembic/versions/fb6810c36ffb_add_has_completed_tutorial_to_user.py @@ -0,0 +1,34 @@ +"""Add has_completed_tutorial to User + +Revision ID: fb6810c36ffb +Revises: 229359b0d1b2 +Create Date: 2026-04-02 11:27:02.719590 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'fb6810c36ffb' +down_revision: Union[str, None] = '229359b0d1b2' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.add_column(sa.Column('has_completed_tutorial', sa.Boolean(), server_default=sa.text('false'), nullable=False)) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.drop_column('has_completed_tutorial') + + # ### end Alembic commands ### diff --git a/alembic/versions/ff8052a97834_add_ip_address_to_drive_activities.py b/alembic/versions/ff8052a97834_add_ip_address_to_drive_activities.py new file mode 100644 index 0000000..b3d8593 --- /dev/null +++ b/alembic/versions/ff8052a97834_add_ip_address_to_drive_activities.py @@ -0,0 +1,30 @@ +"""Add ip_address to drive_activities + +Revision ID: ff8052a97834 +Revises: 765b053e3dcd +Create Date: 2026-03-03 21:48:41.907463 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'ff8052a97834' +down_revision: Union[str, None] = '765b053e3dcd' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('drive_activities', sa.Column('ip_address', sa.String(length=64), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('drive_activities', 'ip_address') + # ### end Alembic commands ### diff --git a/alembic/versions/ff9052a97835_add_multi_tenancy.py b/alembic/versions/ff9052a97835_add_multi_tenancy.py new file mode 100644 index 0000000..01dca5f --- /dev/null +++ b/alembic/versions/ff9052a97835_add_multi_tenancy.py @@ -0,0 +1,104 @@ +"""Add multi_tenancy + +Revision ID: ff9052a97835 +Revises: ff8052a97834 +Create Date: 2026-03-05 10:00:00.000000 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import UUID + + +# revision identifiers, used by Alembic. +revision: str = "ff9052a97835" +down_revision: Union[str, None] = "ff8052a97834" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # 1. Create tenants table + op.execute( + """ + CREATE TABLE tenants ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name VARCHAR(255) NOT NULL, + slug VARCHAR(100) NOT NULL UNIQUE, + is_active BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() + ); + """ + ) + + + # 3. Add tenant_id columns (nullable first) + tables = [ + "users", + "projects", + "drive_folders", + "drive_files", + "drive_activities", + "chatbot_documents", + "notifications", + "user_files", + ] + for table in tables: + op.execute( + f"ALTER TABLE {table} ADD COLUMN tenant_id UUID REFERENCES tenants(id);" + ) + + # 4. Backfill existing data + for table in tables: + op.execute( + f"UPDATE {table} SET tenant_id = '00000000-0000-0000-0000-000000000001';" + ) + + # 5. Set NOT NULL (except users) + not_null_tables = [ + "projects", + "drive_folders", + "drive_files", + "drive_activities", + "chatbot_documents", + "notifications", + "user_files", + ] + for table in not_null_tables: + op.execute(f"ALTER TABLE {table} ALTER COLUMN tenant_id SET NOT NULL;") + + # 6. Create indexes + for table in tables: + op.execute(f"CREATE INDEX ix_{table}_tenant_id ON {table}(tenant_id);") + + # 7. Constraint unique (tenant_id, email) + op.execute("ALTER TABLE users DROP CONSTRAINT IF EXISTS users_email_key;") + op.execute("ALTER TABLE users DROP CONSTRAINT IF EXISTS uq_users_email;") + op.execute( + "ALTER TABLE users ADD CONSTRAINT uq_user_tenant_email UNIQUE (tenant_id, email);" + ) + + +def downgrade() -> None: + op.execute("ALTER TABLE users DROP CONSTRAINT IF EXISTS uq_user_tenant_email;") + op.execute("ALTER TABLE users ADD CONSTRAINT users_email_key UNIQUE (email);") + + tables = [ + "users", + "projects", + "drive_folders", + "drive_files", + "drive_activities", + "chatbot_documents", + "notifications", + "user_files", + ] + for table in tables: + op.execute(f"DROP INDEX IF EXISTS ix_{table}_tenant_id;") + op.execute(f"ALTER TABLE {table} DROP COLUMN IF EXISTS tenant_id;") + + op.execute("DROP TABLE IF EXISTS tenants;") diff --git a/alembic/versions/g001_add_signing_requests_table.py b/alembic/versions/g001_add_signing_requests_table.py new file mode 100644 index 0000000..11afce5 --- /dev/null +++ b/alembic/versions/g001_add_signing_requests_table.py @@ -0,0 +1,38 @@ +"""add signing_requests table + +Revision ID: g001_signing +Revises: c7d4e6f1a9b0 +Create Date: 2026-03-06 +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import UUID, JSON + +# revision identifiers +revision = "g001_signing" +down_revision = "c7d4e6f1a9b0" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "signing_requests", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column("tenant_id", UUID(as_uuid=True), sa.ForeignKey("tenants.id"), nullable=False, index=True), + sa.Column("user_id", sa.Integer, sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True), + sa.Column("drive_file_id", sa.Integer, sa.ForeignKey("drive_files.id", ondelete="CASCADE"), nullable=False), + sa.Column("docuseal_template_id", sa.Integer, nullable=True), + sa.Column("docuseal_submission_id", sa.Integer, nullable=True), + sa.Column("docuseal_slug", sa.String(255), nullable=True), + sa.Column("signing_type", sa.String(20), nullable=False), + sa.Column("status", sa.String(20), nullable=False, server_default="pending"), + sa.Column("submitters_json", JSON, nullable=True), + sa.Column("signed_file_id", sa.Integer, sa.ForeignKey("drive_files.id"), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + ) + + +def downgrade(): + op.drop_table("signing_requests") diff --git a/alembic/versions/g002_replace_docuseal_with_placements.py b/alembic/versions/g002_replace_docuseal_with_placements.py new file mode 100644 index 0000000..065c06f --- /dev/null +++ b/alembic/versions/g002_replace_docuseal_with_placements.py @@ -0,0 +1,41 @@ +"""replace docuseal columns with placements_json + +Revision ID: g002_placements +Revises: g001_signing +Create Date: 2026-03-07 +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import JSON + +# revision identifiers +revision = "g002_placements" +down_revision = "g001_signing" +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_column("signing_requests", "docuseal_template_id") + op.drop_column("signing_requests", "docuseal_submission_id") + op.drop_column("signing_requests", "docuseal_slug") + op.add_column( + "signing_requests", + sa.Column("placements_json", JSON, nullable=True), + ) + + +def downgrade(): + op.drop_column("signing_requests", "placements_json") + op.add_column( + "signing_requests", + sa.Column("docuseal_template_id", sa.Integer, nullable=True), + ) + op.add_column( + "signing_requests", + sa.Column("docuseal_submission_id", sa.Integer, nullable=True), + ) + op.add_column( + "signing_requests", + sa.Column("docuseal_slug", sa.String(255), nullable=True), + ) diff --git a/alembic/versions/h003_add_tenant_smtp_settings.py b/alembic/versions/h003_add_tenant_smtp_settings.py new file mode 100644 index 0000000..650633d --- /dev/null +++ b/alembic/versions/h003_add_tenant_smtp_settings.py @@ -0,0 +1,76 @@ +"""add tenant smtp settings + +Revision ID: h003_tenant_smtp +Revises: g002_placements +Create Date: 2026-03-09 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +# revision identifiers, used by Alembic. +revision: str = "h003_tenant_smtp" +down_revision: Union[str, None] = "g002_placements" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + + if not inspector.has_table("tenant_smtp_configs"): + op.create_table( + "tenant_smtp_configs", + sa.Column( + "id", + postgresql.UUID(as_uuid=True), + nullable=False, + server_default=sa.text("gen_random_uuid()"), + ), + sa.Column("tenant_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("smtp_host", sa.String(length=255), nullable=False), + sa.Column("smtp_port", sa.Integer(), nullable=False, server_default=sa.text("587")), + sa.Column("smtp_user", sa.String(length=255), nullable=True), + sa.Column("encrypted_smtp_password", sa.String(length=512), nullable=True), + sa.Column("smtp_secure", sa.Boolean(), nullable=False, server_default=sa.text("false")), + sa.Column("mail_from", sa.String(length=255), nullable=False), + sa.Column("is_active", sa.Boolean(), nullable=False, server_default=sa.text("true")), + sa.Column("created_by_id", sa.Integer(), nullable=True), + sa.Column("updated_by_id", sa.Integer(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["created_by_id"], ["users.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["updated_by_id"], ["users.id"], ondelete="SET NULL"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("tenant_id", name="uq_tenant_smtp_configs_tenant_id"), + ) + + existing_indexes = {idx["name"] for idx in inspector.get_indexes("tenant_smtp_configs")} + if "ix_tenant_smtp_configs_tenant_id" not in existing_indexes: + op.create_index( + "ix_tenant_smtp_configs_tenant_id", + "tenant_smtp_configs", + ["tenant_id"], + unique=False, + ) + + +def downgrade() -> None: + op.drop_index("ix_tenant_smtp_configs_tenant_id", table_name="tenant_smtp_configs") + op.drop_table("tenant_smtp_configs") diff --git a/alembic/versions/h004_add_signing_forensic_columns.py b/alembic/versions/h004_add_signing_forensic_columns.py new file mode 100644 index 0000000..424a3ac --- /dev/null +++ b/alembic/versions/h004_add_signing_forensic_columns.py @@ -0,0 +1,48 @@ +"""add forensic metadata columns to signing_requests + +Revision ID: h004_signing_forensic_v2 +Revises: h004_limits_soft_delete +Create Date: 2026-03-09 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import JSON + + +# revision identifiers, used by Alembic. +revision: str = "h004_signing_forensic_v2" +down_revision: Union[str, None] = "h004_limits_soft_delete" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Safe: only add if column doesn't already exist (idempotent for existing DBs) + conn = op.get_bind() + inspector = sa.inspect(conn) + existing = {c["name"] for c in inspector.get_columns("signing_requests")} + + if "ip_address" not in existing: + op.add_column( + "signing_requests", + sa.Column("ip_address", sa.String(64), nullable=True), + ) + if "user_agent" not in existing: + op.add_column( + "signing_requests", + sa.Column("user_agent", sa.String(512), nullable=True), + ) + if "signing_metadata" not in existing: + op.add_column( + "signing_requests", + sa.Column("signing_metadata", JSON, nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("signing_requests", "signing_metadata") + op.drop_column("signing_requests", "user_agent") + op.drop_column("signing_requests", "ip_address") diff --git a/alembic/versions/h004_limits_and_soft_delete.py b/alembic/versions/h004_limits_and_soft_delete.py new file mode 100644 index 0000000..57f82f0 --- /dev/null +++ b/alembic/versions/h004_limits_and_soft_delete.py @@ -0,0 +1,108 @@ +"""add limits config and soft delete columns + +Revision ID: h004_limits_soft_delete +Revises: h003_tenant_smtp +Create Date: 2026-03-09 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +# revision identifiers, used by Alembic. +revision: str = "h004_limits_soft_delete" +down_revision: Union[str, None] = "h003_tenant_smtp" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "users", sa.Column("is_active", sa.Boolean(), nullable=False, server_default=sa.text("true")) + ) + op.add_column( + "users", sa.Column("is_deleted", sa.Boolean(), nullable=False, server_default=sa.text("false")) + ) + op.add_column("users", sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True)) + op.add_column("users", sa.Column("deleted_by_id", sa.Integer(), nullable=True)) + op.create_foreign_key( + "fk_users_deleted_by_id_users", + "users", + "users", + ["deleted_by_id"], + ["id"], + ondelete="SET NULL", + ) + + op.add_column( + "tenants", sa.Column("is_deleted", sa.Boolean(), nullable=False, server_default=sa.text("false")) + ) + op.add_column("tenants", sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True)) + op.add_column("tenants", sa.Column("deleted_by_id", sa.Integer(), nullable=True)) + op.add_column("tenants", sa.Column("max_users_allowed", sa.Integer(), nullable=True)) + op.create_foreign_key( + "fk_tenants_deleted_by_id_users", + "tenants", + "users", + ["deleted_by_id"], + ["id"], + ondelete="SET NULL", + ) + + op.create_table( + "system_configurations", + sa.Column( + "id", + postgresql.UUID(as_uuid=True), + nullable=False, + server_default=sa.text("gen_random_uuid()"), + ), + sa.Column("config_key", sa.String(length=100), nullable=False), + sa.Column("int_value", sa.BigInteger(), nullable=True), + sa.Column("bool_value", sa.Boolean(), nullable=True), + sa.Column("text_value", sa.Text(), nullable=True), + sa.Column("created_by_id", sa.Integer(), nullable=True), + sa.Column("updated_by_id", sa.Integer(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.ForeignKeyConstraint(["created_by_id"], ["users.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["updated_by_id"], ["users.id"], ondelete="SET NULL"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("config_key", name="uq_system_configurations_config_key"), + ) + op.create_index( + "ix_system_configurations_config_key", + "system_configurations", + ["config_key"], + unique=True, + ) + + +def downgrade() -> None: + op.drop_index("ix_system_configurations_config_key", table_name="system_configurations") + op.drop_table("system_configurations") + + op.drop_constraint("fk_tenants_deleted_by_id_users", "tenants", type_="foreignkey") + op.drop_column("tenants", "max_users_allowed") + op.drop_column("tenants", "deleted_by_id") + op.drop_column("tenants", "deleted_at") + op.drop_column("tenants", "is_deleted") + + op.drop_constraint("fk_users_deleted_by_id_users", "users", type_="foreignkey") + op.drop_column("users", "deleted_by_id") + op.drop_column("users", "deleted_at") + op.drop_column("users", "is_deleted") + op.drop_column("users", "is_active") diff --git a/alembic/versions/i001_dual_bucket_storage.py b/alembic/versions/i001_dual_bucket_storage.py new file mode 100644 index 0000000..40a92fe --- /dev/null +++ b/alembic/versions/i001_dual_bucket_storage.py @@ -0,0 +1,56 @@ +"""Replace b2_bucket_name with b2_quarantine_bucket and b2_clean_bucket + +Revision ID: i001_dual_bucket_storage +Revises: 9557c7307575 +Create Date: 2026-03-17 08:00:00.000000 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'i001_dual_bucket_storage' +down_revision = '9557c7307575' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # 1. Add the two new bucket columns (nullable at first) + op.add_column( + 'tenant_storage_configs', + sa.Column('b2_quarantine_bucket', sa.String(255), nullable=True) + ) + op.add_column( + 'tenant_storage_configs', + sa.Column('b2_clean_bucket', sa.String(255), nullable=True) + ) + + # 2. Migrate existing b2_bucket_name to quarantine (best-effort) + op.execute(""" + UPDATE tenant_storage_configs + SET b2_quarantine_bucket = b2_bucket_name + WHERE b2_bucket_name IS NOT NULL + """) + + # 3. Drop the old column + op.drop_column('tenant_storage_configs', 'b2_bucket_name') + + +def downgrade() -> None: + # 1. Restore the old column + op.add_column( + 'tenant_storage_configs', + sa.Column('b2_bucket_name', sa.String(255), nullable=True) + ) + + # 2. Restore data from quarantine bucket (best-effort) + op.execute(""" + UPDATE tenant_storage_configs + SET b2_bucket_name = b2_quarantine_bucket + WHERE b2_quarantine_bucket IS NOT NULL + """) + + # 3. Drop the new columns + op.drop_column('tenant_storage_configs', 'b2_clean_bucket') + op.drop_column('tenant_storage_configs', 'b2_quarantine_bucket') diff --git a/alembic/versions/j001_comments_fix.py b/alembic/versions/j001_comments_fix.py new file mode 100644 index 0000000..ef7c144 --- /dev/null +++ b/alembic/versions/j001_comments_fix.py @@ -0,0 +1,31 @@ +"""Add rejected_at column to drive_comments + +Revision ID: j001_comments_fix +Revises: c3abf44266de +Create Date: 2026-03-24 + +""" + +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision: str = "j001_comments_fix" +down_revision: Union[str, tuple, None] = "c3abf44266de" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute(""" + ALTER TABLE drive_comments + ADD COLUMN IF NOT EXISTS rejected_at TIMESTAMP WITH TIME ZONE DEFAULT NULL + """) + + +def downgrade() -> None: + op.execute(""" + ALTER TABLE drive_comments + DROP COLUMN IF EXISTS rejected_at + """) diff --git a/alembic/versions/j002_allow_unlimited_tenant_quota.py b/alembic/versions/j002_allow_unlimited_tenant_quota.py new file mode 100644 index 0000000..745ee9e --- /dev/null +++ b/alembic/versions/j002_allow_unlimited_tenant_quota.py @@ -0,0 +1,38 @@ +"""allow unlimited tenant storage quota sentinel + +Revision ID: j002_unlimited_tenant_quota +Revises: j001_comments_fix +Create Date: 2026-03-25 +""" + +from typing import Sequence, Union + +from alembic import op + + +# revision identifiers, used by Alembic. +revision: str = "j002_unlimited_tenant_quota" +down_revision: Union[str, None] = "j001_comments_fix" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +CONSTRAINT_NAME = "ck_tenants_storage_quota_bytes_positive" + + +def upgrade() -> None: + op.drop_constraint(CONSTRAINT_NAME, "tenants", type_="check") + op.create_check_constraint( + CONSTRAINT_NAME, + "tenants", + "storage_quota_bytes = -1 OR storage_quota_bytes > 0", + ) + + +def downgrade() -> None: + op.drop_constraint(CONSTRAINT_NAME, "tenants", type_="check") + op.create_check_constraint( + CONSTRAINT_NAME, + "tenants", + "storage_quota_bytes > 0", + ) diff --git a/alembic/versions/k001_limit_signature_id_to_15_chars.py b/alembic/versions/k001_limit_signature_id_to_15_chars.py new file mode 100644 index 0000000..695957b --- /dev/null +++ b/alembic/versions/k001_limit_signature_id_to_15_chars.py @@ -0,0 +1,42 @@ +"""limit signature_id to 15 characters + +Revision ID: k001_limit_signature_id_15 +Revises: 56d090047e85 +Create Date: 2026-04-15 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "k001_limit_signature_id_15" +down_revision: Union[str, None] = "b029d2f5a0c7" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Truncate any existing values that exceed 15 chars before altering the column + op.execute( + "UPDATE signing_requests SET signature_id = LEFT(signature_id, 15) WHERE LENGTH(signature_id) > 15" + ) + op.alter_column( + "signing_requests", + "signature_id", + existing_type=sa.String(128), + type_=sa.String(15), + existing_nullable=True, + ) + + +def downgrade() -> None: + op.alter_column( + "signing_requests", + "signature_id", + existing_type=sa.String(15), + type_=sa.String(128), + existing_nullable=True, + ) diff --git a/alembic/versions/k002_add_docs_tutorial_flag.py b/alembic/versions/k002_add_docs_tutorial_flag.py new file mode 100644 index 0000000..a074dbc --- /dev/null +++ b/alembic/versions/k002_add_docs_tutorial_flag.py @@ -0,0 +1,28 @@ +"""Add has_completed_docs_tutorial to User + +Revision ID: k002_add_docs_tutorial +Revises: l001_merge_heads +Create Date: 2026-04-17 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'k002_add_docs_tutorial' +down_revision: Union[str, None] = 'l001_merge_heads' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.add_column(sa.Column('has_completed_docs_tutorial', sa.Boolean(), server_default=sa.text('false'), nullable=False)) + + +def downgrade() -> None: + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.drop_column('has_completed_docs_tutorial') diff --git a/alembic/versions/l001_merge_heads.py b/alembic/versions/l001_merge_heads.py new file mode 100644 index 0000000..7a28fe6 --- /dev/null +++ b/alembic/versions/l001_merge_heads.py @@ -0,0 +1,25 @@ +"""merge all heads into single linear chain + +Revision ID: l001_merge_heads +Revises: b029d2f5a0c7, k001_limit_signature_id_15 +Create Date: 2026-04-15 +""" + +from typing import Sequence, Union + +from alembic import op + + +# revision identifiers, used by Alembic. +revision: str = "l001_merge_heads" +down_revision: Union[str, tuple] = ("56d090047e85", "k001_limit_signature_id_15") +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/alembic/versions/m001_add_activity_logs_table.py b/alembic/versions/m001_add_activity_logs_table.py new file mode 100644 index 0000000..0ca9ede --- /dev/null +++ b/alembic/versions/m001_add_activity_logs_table.py @@ -0,0 +1,87 @@ +"""add activity_logs table + +Revision ID: m001_add_activity_logs +Revises: 9af99fc7ca0f +Create Date: 2026-04-20 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +# revision identifiers, used by Alembic. +revision: str = 'm001_add_activity_logs' +down_revision: Union[str, None] = '9af99fc7ca0f' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +_INDEXES = [ + ("ix_activity_logs_tenant_created_at", ["tenant_id", "created_at"]), + ("ix_activity_logs_user_created_at", ["user_id", "created_at"]), + ("ix_activity_logs_module_action", ["module", "action"]), + ("ix_activity_logs_target_id", ["target_id"]), +] + + +def _index_names(bind, table_name: str) -> set[str]: + inspector = sa.inspect(bind) + return {idx["name"] for idx in inspector.get_indexes(table_name)} + + +def upgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + + if not inspector.has_table("activity_logs"): + op.create_table( + "activity_logs", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("tenant_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("user_id", sa.Integer(), nullable=True), + sa.Column("user_name", sa.String(length=255), nullable=True), + sa.Column("user_email", sa.String(length=255), nullable=True), + sa.Column("module", sa.String(length=50), nullable=False), + sa.Column("action", sa.String(length=100), nullable=False), + sa.Column("target_id", sa.String(length=255), nullable=False), + sa.Column("target_type", sa.String(length=50), nullable=False), + sa.Column("target_name", sa.String(length=512), nullable=True), + sa.Column( + "metadata", + postgresql.JSONB(astext_type=sa.Text()), + nullable=False, + ), + sa.Column("ip_address", sa.String(length=64), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("request_id", sa.String(length=255), nullable=True), + sa.Column("status", sa.String(length=20), nullable=False), + sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"]), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="SET NULL"), + sa.PrimaryKeyConstraint("id"), + ) + + existing = _index_names(bind, "activity_logs") + for name, cols in _INDEXES: + if name not in existing: + op.create_index(name, "activity_logs", cols, unique=False) + + +def downgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + + if inspector.has_table("activity_logs"): + existing = _index_names(bind, "activity_logs") + for name, _ in reversed(_INDEXES): + if name in existing: + op.drop_index(name, table_name="activity_logs") + + op.drop_table("activity_logs") diff --git a/alembic/versions/n001_add_user_designation.py b/alembic/versions/n001_add_user_designation.py new file mode 100644 index 0000000..c5bf857 --- /dev/null +++ b/alembic/versions/n001_add_user_designation.py @@ -0,0 +1,29 @@ +"""add designation column to users + +Revision ID: n001_add_user_designation +Revises: m001_add_activity_logs +Create Date: 2026-04-21 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'n001_add_user_designation' +down_revision: Union[str, None] = 'm001_add_activity_logs' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + 'users', + sa.Column('designation', sa.String(length=100), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column('users', 'designation') diff --git a/alembic/versions/n002_add_signature_imprints.py b/alembic/versions/n002_add_signature_imprints.py new file mode 100644 index 0000000..6c4551c --- /dev/null +++ b/alembic/versions/n002_add_signature_imprints.py @@ -0,0 +1,62 @@ +"""add signature_imprints table + +Revision ID: n002_add_signature_imprints +Revises: n001_add_user_designation +Create Date: 2026-04-21 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +# revision identifiers, used by Alembic. +revision: str = 'n002_add_signature_imprints' +down_revision: Union[str, None] = 'n001_add_user_designation' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + 'signature_imprints', + sa.Column('id', postgresql.UUID(as_uuid=True), primary_key=True), + sa.Column( + 'signing_request_id', + postgresql.UUID(as_uuid=True), + sa.ForeignKey('signing_requests.id', ondelete='CASCADE'), + nullable=False, + ), + sa.Column( + 'drive_file_id', + sa.Integer(), + sa.ForeignKey('drive_files.id', ondelete='CASCADE'), + nullable=False, + ), + sa.Column('page_number', sa.Integer(), nullable=False), + sa.Column('bbox_points', postgresql.JSON(), nullable=False), + sa.Column( + 'created_at', + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + ) + op.create_index( + 'ix_signature_imprints_file_page', + 'signature_imprints', + ['drive_file_id', 'page_number'], + ) + op.create_index( + 'ix_signature_imprints_signing_request', + 'signature_imprints', + ['signing_request_id'], + ) + + +def downgrade() -> None: + op.drop_index('ix_signature_imprints_signing_request', table_name='signature_imprints') + op.drop_index('ix_signature_imprints_file_page', table_name='signature_imprints') + op.drop_table('signature_imprints') diff --git a/alembic/versions/v3_extraction_data.py b/alembic/versions/v3_extraction_data.py new file mode 100644 index 0000000..27cd8a7 --- /dev/null +++ b/alembic/versions/v3_extraction_data.py @@ -0,0 +1,57 @@ +"""Add extraction_data table + +Revision ID: v3_extraction +Revises: l001_merge_heads +Create Date: 2026-04-18 12:00:00.000000 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = 'v3_extraction' +down_revision: Union[str, None] = 'l001_merge_heads' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _index_names(bind, table_name: str) -> set[str]: + inspector = sa.inspect(bind) + return {idx["name"] for idx in inspector.get_indexes(table_name)} + + +def upgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + + if not inspector.has_table('extraction_data'): + op.create_table('extraction_data', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('document_id', sa.Integer(), nullable=False), + sa.Column('document_type', sa.String(length=50), nullable=True), + sa.Column('raw_text', sa.Text(), nullable=True), + sa.Column('cleaned_text', sa.Text(), nullable=True), + sa.Column('extracted_json', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('confidence_score', sa.Float(), nullable=True), + sa.Column('extraction_status', sa.String(length=50), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['document_id'], ['drive_files.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + + if 'idx_extraction_data_document_id' not in _index_names(bind, 'extraction_data'): + op.create_index(op.f('idx_extraction_data_document_id'), 'extraction_data', ['document_id'], unique=False) + + +def downgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + + if inspector.has_table('extraction_data'): + if 'idx_extraction_data_document_id' in _index_names(bind, 'extraction_data'): + op.drop_index(op.f('idx_extraction_data_document_id'), table_name='extraction_data') + op.drop_table('extraction_data') diff --git a/alembic/versions/w001_versioning_minor_and_version_fk.py b/alembic/versions/w001_versioning_minor_and_version_fk.py new file mode 100644 index 0000000..6a9e156 --- /dev/null +++ b/alembic/versions/w001_versioning_minor_and_version_fk.py @@ -0,0 +1,137 @@ +"""add minor_version + version-level FKs for signing artefacts + +Revision ID: w001_versioning_minor +Revises: v3_extraction, n002_add_signature_imprints, 39d5fdfb5386 +Create Date: 2026-04-28 + +Schema changes: + - drive_file_versions.minor_version (String(8), nullable) + - signing_requests.drive_file_version_id (Integer, nullable, FK) + - signature_imprints.drive_file_version_id (Integer, nullable, FK) + - Composite index on drive_file_versions(file_id, version_number, minor_version) + +Also serves as the merge revision for the three existing heads +('v3_extraction', 'n002_add_signature_imprints', '39d5fdfb5386'). +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "w001_versioning_minor" +down_revision: Union[str, tuple] = ( + "v3_extraction", + "n002_add_signature_imprints", + "39d5fdfb5386", +) +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _column_names(bind, table_name: str) -> set: + inspector = sa.inspect(bind) + return {c["name"] for c in inspector.get_columns(table_name)} + + +def _index_names(bind, table_name: str) -> set: + inspector = sa.inspect(bind) + return {idx["name"] for idx in inspector.get_indexes(table_name)} + + +def upgrade() -> None: + bind = op.get_bind() + + # 1) drive_file_versions.minor_version + if "minor_version" not in _column_names(bind, "drive_file_versions"): + op.add_column( + "drive_file_versions", + sa.Column("minor_version", sa.String(length=8), nullable=True), + ) + + if "ix_drive_file_versions_file_major_minor" not in _index_names( + bind, "drive_file_versions" + ): + op.create_index( + "ix_drive_file_versions_file_major_minor", + "drive_file_versions", + ["file_id", "version_number", "minor_version"], + unique=False, + ) + + # 2) signing_requests.drive_file_version_id + if "drive_file_version_id" not in _column_names(bind, "signing_requests"): + op.add_column( + "signing_requests", + sa.Column("drive_file_version_id", sa.Integer(), nullable=True), + ) + op.create_foreign_key( + "fk_signing_requests_drive_file_version_id", + "signing_requests", + "drive_file_versions", + ["drive_file_version_id"], + ["id"], + ondelete="SET NULL", + ) + + # 3) signature_imprints.drive_file_version_id + if "drive_file_version_id" not in _column_names(bind, "signature_imprints"): + op.add_column( + "signature_imprints", + sa.Column("drive_file_version_id", sa.Integer(), nullable=True), + ) + op.create_foreign_key( + "fk_signature_imprints_drive_file_version_id", + "signature_imprints", + "drive_file_versions", + ["drive_file_version_id"], + ["id"], + ondelete="SET NULL", + ) + if "ix_signature_imprints_version_id" not in _index_names( + bind, "signature_imprints" + ): + op.create_index( + "ix_signature_imprints_version_id", + "signature_imprints", + ["drive_file_version_id"], + ) + + +def downgrade() -> None: + bind = op.get_bind() + + if "drive_file_version_id" in _column_names(bind, "signature_imprints"): + if "ix_signature_imprints_version_id" in _index_names( + bind, "signature_imprints" + ): + op.drop_index( + "ix_signature_imprints_version_id", + table_name="signature_imprints", + ) + op.drop_constraint( + "fk_signature_imprints_drive_file_version_id", + "signature_imprints", + type_="foreignkey", + ) + op.drop_column("signature_imprints", "drive_file_version_id") + + if "drive_file_version_id" in _column_names(bind, "signing_requests"): + op.drop_constraint( + "fk_signing_requests_drive_file_version_id", + "signing_requests", + type_="foreignkey", + ) + op.drop_column("signing_requests", "drive_file_version_id") + + if "minor_version" in _column_names(bind, "drive_file_versions"): + if "ix_drive_file_versions_file_major_minor" in _index_names( + bind, "drive_file_versions" + ): + op.drop_index( + "ix_drive_file_versions_file_major_minor", + table_name="drive_file_versions", + ) + op.drop_column("drive_file_versions", "minor_version") diff --git a/alembic/versions/x001_add_user_smtp_credentials.py b/alembic/versions/x001_add_user_smtp_credentials.py new file mode 100644 index 0000000..bca35ca --- /dev/null +++ b/alembic/versions/x001_add_user_smtp_credentials.py @@ -0,0 +1,48 @@ +"""add user smtp credential columns + +Revision ID: x001_user_smtp_credentials +Revises: w001_versioning_minor +Create Date: 2026-04-30 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "x001_user_smtp_credentials" +down_revision: Union[str, None] = "w001_versioning_minor" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _column_names(bind, table_name: str) -> set: + inspector = sa.inspect(bind) + return {column["name"] for column in inspector.get_columns(table_name)} + + +def upgrade() -> None: + bind = op.get_bind() + column_names = _column_names(bind, "users") + + if "smtp_user" not in column_names: + op.add_column("users", sa.Column("smtp_user", sa.String(length=255), nullable=True)) + + if "encrypted_smtp_password" not in column_names: + op.add_column( + "users", + sa.Column("encrypted_smtp_password", sa.String(length=512), nullable=True), + ) + + +def downgrade() -> None: + bind = op.get_bind() + column_names = _column_names(bind, "users") + + if "encrypted_smtp_password" in column_names: + op.drop_column("users", "encrypted_smtp_password") + + if "smtp_user" in column_names: + op.drop_column("users", "smtp_user") diff --git a/alembic/versions/y001_add_user_mail_from.py b/alembic/versions/y001_add_user_mail_from.py new file mode 100644 index 0000000..33f2130 --- /dev/null +++ b/alembic/versions/y001_add_user_mail_from.py @@ -0,0 +1,39 @@ +"""add user mail_from column + +Revision ID: y001_add_user_mail_from +Revises: x001_user_smtp_credentials +Create Date: 2026-04-30 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "y001_add_user_mail_from" +down_revision: Union[str, None] = "x001_user_smtp_credentials" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _column_names(bind, table_name: str) -> set: + inspector = sa.inspect(bind) + return {column["name"] for column in inspector.get_columns(table_name)} + + +def upgrade() -> None: + bind = op.get_bind() + column_names = _column_names(bind, "users") + + if "mail_from" not in column_names: + op.add_column("users", sa.Column("mail_from", sa.String(length=255), nullable=True)) + + +def downgrade() -> None: + bind = op.get_bind() + column_names = _column_names(bind, "users") + + if "mail_from" in column_names: + op.drop_column("users", "mail_from") diff --git a/alembic/versions/z002_pref_lang.py b/alembic/versions/z002_pref_lang.py new file mode 100644 index 0000000..f377765 --- /dev/null +++ b/alembic/versions/z002_pref_lang.py @@ -0,0 +1,45 @@ +"""add preferred_language to users + +Revision ID: z002_pref_lang +Revises: ca14c4ab10d9 +Create Date: 2026-05-08 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "z002_pref_lang" +down_revision: Union[str, None] = "ca14c4ab10d9" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _column_names(bind, table_name: str) -> set[str]: + inspector = sa.inspect(bind) + return {column["name"] for column in inspector.get_columns(table_name)} + + +def upgrade() -> None: + bind = op.get_bind() + column_names = _column_names(bind, "users") + + if "preferred_language" not in column_names: + op.add_column( + "users", + sa.Column("preferred_language", sa.String(length=16), nullable=True), + ) + op.execute( + "UPDATE users SET preferred_language = 'en' WHERE preferred_language IS NULL" + ) + op.alter_column("users", "preferred_language", nullable=False) + + +def downgrade() -> None: + bind = op.get_bind() + column_names = _column_names(bind, "users") + + if "preferred_language" in column_names: + op.drop_column("users", "preferred_language") diff --git a/alembic/versions/z003_add_disable_pdf_compression_to_tenants.py b/alembic/versions/z003_add_disable_pdf_compression_to_tenants.py new file mode 100644 index 0000000..16ede8b --- /dev/null +++ b/alembic/versions/z003_add_disable_pdf_compression_to_tenants.py @@ -0,0 +1,41 @@ +"""add disable_pdf_compression to tenants + +Revision ID: z003_disable_pdf_compression +Revises: 7d7641cc1bb2 +Create Date: 2026-07-23 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "z003_disable_pdf_compression" +down_revision: Union[str, None] = "7d7641cc1bb2" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _column_names(bind, table_name: str) -> set[str]: + inspector = sa.inspect(bind) + return {column["name"] for column in inspector.get_columns(table_name)} + + +def upgrade() -> None: + bind = op.get_bind() + column_names = _column_names(bind, "tenants") + + if "disable_pdf_compression" not in column_names: + op.add_column( + "tenants", + sa.Column("disable_pdf_compression", sa.Boolean(), nullable=False, server_default="false"), + ) + + +def downgrade() -> None: + bind = op.get_bind() + column_names = _column_names(bind, "tenants") + + if "disable_pdf_compression" in column_names: + op.drop_column("tenants", "disable_pdf_compression") diff --git a/alembic/versions/z004_add_global_workflows.py b/alembic/versions/z004_add_global_workflows.py new file mode 100644 index 0000000..bfbe030 --- /dev/null +++ b/alembic/versions/z004_add_global_workflows.py @@ -0,0 +1,20 @@ +"""dummy z004 + +Revision ID: z004_add_global_workflows +Revises: z003_disable_pdf_compression +Create Date: 2026-08-06 + +""" +from alembic import op +import sqlalchemy as sa + +revision = 'z004_add_global_workflows' +down_revision = 'z003_disable_pdf_compression' +branch_labels = None +depends_on = None + +def upgrade(): + pass + +def downgrade(): + pass diff --git a/alembic/versions/z005_add_virus_scanning_and_quarantine_tables.py b/alembic/versions/z005_add_virus_scanning_and_quarantine_tables.py new file mode 100644 index 0000000..83f5841 --- /dev/null +++ b/alembic/versions/z005_add_virus_scanning_and_quarantine_tables.py @@ -0,0 +1,119 @@ +"""add virus scanning and quarantine tables + +Revision ID: z005_add_virus_scan +Revises: 73dff81eed72 +Create Date: 2026-08-31 19:15:00.000000 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = 'z005_add_virus_scan' +down_revision = '73dff81eed72' +branch_labels = None +depends_on = None + + +def upgrade(): + # 1. virus_scan_settings table + op.create_table( + 'virus_scan_settings', + sa.Column('tenant_id', postgresql.UUID(as_uuid=True), sa.ForeignKey('tenants.id', ondelete='CASCADE'), primary_key=True), + sa.Column('enabled', sa.Boolean(), nullable=False, server_default=sa.text('true')), + sa.Column('file_types', postgresql.JSONB(astext_type=sa.Text()), nullable=False, server_default=sa.text("'[]'::jsonb")), + sa.Column('max_file_size_mb', sa.Integer(), nullable=False, server_default=sa.text('100')), + sa.Column('action_on_detect', sa.String(length=50), nullable=False, server_default=sa.text("'quarantine'")), + sa.Column('notify_admin', sa.Boolean(), nullable=False, server_default=sa.text('true')), + sa.Column('notify_uploader', sa.Boolean(), nullable=False, server_default=sa.text('false')), + sa.Column('auto_suspend_uploader', sa.Boolean(), nullable=False, server_default=sa.text('false')), + sa.Column('suspend_threshold', sa.Integer(), nullable=False, server_default=sa.text('3')), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + ) + + # 2. virus_scan_jobs table + op.create_table( + 'virus_scan_jobs', + sa.Column('id', sa.BigInteger(), primary_key=True, autoincrement=True), + sa.Column('task_id', sa.String(length=255), nullable=True), + sa.Column('tenant_id', postgresql.UUID(as_uuid=True), sa.ForeignKey('tenants.id', ondelete='CASCADE'), nullable=True), + sa.Column('file_id', sa.BigInteger(), nullable=True), + sa.Column('version_id', sa.BigInteger(), nullable=True), + sa.Column('uploader_id', sa.Integer(), sa.ForeignKey('users.id', ondelete='SET NULL'), nullable=True), + sa.Column('storage_key', sa.String(length=1000), nullable=False), + sa.Column('file_size', sa.BigInteger(), nullable=False, server_default=sa.text('0')), + sa.Column('status', sa.String(length=50), nullable=False, server_default=sa.text("'pending'")), + sa.Column('retry_count', sa.Integer(), nullable=False, server_default=sa.text('0')), + sa.Column('error_message', sa.Text(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column('completed_at', sa.DateTime(timezone=True), nullable=True), + ) + op.create_index('ix_virus_scan_jobs_tenant_id', 'virus_scan_jobs', ['tenant_id']) + op.create_index('ix_virus_scan_jobs_task_id', 'virus_scan_jobs', ['task_id']) + op.create_index('ix_virus_scan_jobs_status', 'virus_scan_jobs', ['status']) + + # 3. virus_scan_results table + op.create_table( + 'virus_scan_results', + sa.Column('id', sa.BigInteger(), primary_key=True, autoincrement=True), + sa.Column('tenant_id', postgresql.UUID(as_uuid=True), sa.ForeignKey('tenants.id', ondelete='CASCADE'), nullable=True), + sa.Column('file_id', sa.BigInteger(), nullable=True), + sa.Column('version_id', sa.BigInteger(), nullable=True), + sa.Column('uploader_id', sa.Integer(), sa.ForeignKey('users.id', ondelete='SET NULL'), nullable=True), + sa.Column('status', sa.String(length=50), nullable=False), + sa.Column('threat_name', sa.String(length=255), nullable=True), + sa.Column('action_taken', sa.String(length=50), nullable=False, server_default=sa.text("'none'")), + sa.Column('scan_duration_ms', sa.Integer(), nullable=False, server_default=sa.text('0')), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + ) + op.create_index('ix_virus_scan_results_tenant_id', 'virus_scan_results', ['tenant_id']) + op.create_index('ix_virus_scan_results_status', 'virus_scan_results', ['status']) + op.create_index('ix_virus_scan_results_created_at', 'virus_scan_results', ['created_at']) + + # 4. quarantined_files table + op.create_table( + 'quarantined_files', + sa.Column('id', sa.BigInteger(), primary_key=True, autoincrement=True), + sa.Column('tenant_id', postgresql.UUID(as_uuid=True), sa.ForeignKey('tenants.id', ondelete='CASCADE'), nullable=True), + sa.Column('file_id', sa.BigInteger(), nullable=True), + sa.Column('version_id', sa.BigInteger(), nullable=True), + sa.Column('uploader_id', sa.Integer(), sa.ForeignKey('users.id', ondelete='SET NULL'), nullable=True), + sa.Column('filename', sa.String(length=500), nullable=False), + sa.Column('file_size', sa.BigInteger(), nullable=False, server_default=sa.text('0')), + sa.Column('mime_type', sa.String(length=255), nullable=True), + sa.Column('threat_name', sa.String(length=255), nullable=False), + sa.Column('quarantine_storage_key', sa.String(length=1000), nullable=False), + sa.Column('status', sa.String(length=50), nullable=False, server_default=sa.text("'quarantined'")), + sa.Column('quarantined_at', sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column('released_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('released_by_id', sa.Integer(), sa.ForeignKey('users.id', ondelete='SET NULL'), nullable=True), + sa.Column('deleted_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('deleted_by_id', sa.Integer(), sa.ForeignKey('users.id', ondelete='SET NULL'), nullable=True), + ) + op.create_index('ix_quarantined_files_tenant_id', 'quarantined_files', ['tenant_id']) + op.create_index('ix_quarantined_files_status', 'quarantined_files', ['status']) + + # 5. user_malware_counts table + op.create_table( + 'user_malware_counts', + sa.Column('id', sa.BigInteger(), primary_key=True, autoincrement=True), + sa.Column('tenant_id', postgresql.UUID(as_uuid=True), sa.ForeignKey('tenants.id', ondelete='CASCADE'), nullable=True), + sa.Column('user_id', sa.Integer(), sa.ForeignKey('users.id', ondelete='CASCADE'), nullable=False), + sa.Column('malware_count', sa.Integer(), nullable=False, server_default=sa.text('0')), + sa.Column('last_detected_at', sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column('is_suspended', sa.Boolean(), nullable=False, server_default=sa.text('false')), + sa.Column('suspension_reason', sa.Text(), nullable=True), + ) + op.create_index('ix_user_malware_counts_user_id', 'user_malware_counts', ['user_id']) + op.create_index('ix_user_malware_counts_tenant_id', 'user_malware_counts', ['tenant_id']) + + +def downgrade(): + op.drop_table('user_malware_counts') + op.drop_table('quarantined_files') + op.drop_table('virus_scan_results') + op.drop_table('virus_scan_jobs') + op.drop_table('virus_scan_settings') diff --git a/app/ai_runner.py b/app/ai_runner.py new file mode 100644 index 0000000..72d38f0 --- /dev/null +++ b/app/ai_runner.py @@ -0,0 +1,85 @@ +import sys +import json +import logging +import os +import re + +sys.path.append(os.getcwd()) + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("AI_Runner") + +def main(): + if len(sys.argv) < 4: + print(json.dumps({"status": "failed", "error": "Invalid arguments. Usage: python ai_runner.py "})) + sys.exit(1) + + input_path = sys.argv[1] + session_id = sys.argv[2] + user_id = sys.argv[3] + + try: + from app.core.model_manager import ModelManager + from app.modules.documents.processors.pdf.pdf_converter import UniversalContentIntelligence + from app.modules.documents.processors.pdf.image_extractor import ImageExtractor + + logger.info(f"🧠 Step 1: Running Marker AI on {input_path}") + with ModelManager() as ai: + result = ai.process_document(input_path) + + if hasattr(result, "markdown"): + content = result.markdown + elif isinstance(result, dict) and "markdown" in result: + content = result["markdown"] + else: + try: + from marker.output import text_from_rendered + content = text_from_rendered(result) + except: + content = str(result) + + logger.info("🔍 Step 2: Running Universal Content Intelligence") + metadata = UniversalContentIntelligence.extract_universal_metadata(content) + + figure_image_map = {} + pattern1 = r'!\[Image\s+(\d+)\]\((http[s]?://[^)]+)\)\s*\n\s*\*\*Figure\s+(\d+):' + matches1 = re.findall(pattern1, content, re.IGNORECASE | re.MULTILINE) + for _, image_url, figure_num in matches1: + figure_image_map[figure_num] = image_url + + pattern2 = r'!\[Image\s+(\d+)\]\(([^)]+)\)' + matches2 = re.findall(pattern2, content, re.IGNORECASE) + for image_num, image_url in matches2: + if image_num not in figure_image_map: + figure_image_map[image_num] = image_url + + logger.info("🖼️ Step 3: Extracting images") + images = [] + try: + extractor = ImageExtractor() + images = extractor.process_markdown_output_images(result, session_id=session_id, user_id=user_id) + logger.info(f"✅ Processed {len(images)} images") + except Exception as img_err: + logger.error(f"⚠️ Image extraction failed: {img_err}") + + print(json.dumps({ + "status": "success", + "markdown_content": content, + "session_id": session_id, + "images": images, + "metadata": { + "figures": figure_image_map, + "title": metadata.title if hasattr(metadata, 'title') else "" + } + })) + + except Exception as e: + logger.error(f"❌ AI Runner Failed: {e}") + print(json.dumps({ + "status": "failed", + "error": str(e) + })) + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/app/ai_service/__init__.py b/app/ai_service/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/ai_service/model_server.py b/app/ai_service/model_server.py new file mode 100644 index 0000000..2675fe2 --- /dev/null +++ b/app/ai_service/model_server.py @@ -0,0 +1,31 @@ +from fastapi import FastAPI +from pydantic import BaseModel +from app.core.model_manager import ModelManager + +app = FastAPI(title="AI Model Server") + +model = ModelManager() + + +class ProcessRequest(BaseModel): + file_path: str + + +@app.get("/health") +def health(): + return {"status": "ok"} + + +@app.post("/process") +def process_document(req: ProcessRequest): + """ + Heavy AI processing happens here. + This process should be run separately (e.g., on port 9000). + """ + result = model.process_document(req.file_path) + return {"result": result} + + +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="0.0.0.0", port=9000) diff --git a/app/api/ws_router.py b/app/api/ws_router.py new file mode 100644 index 0000000..f6c918a --- /dev/null +++ b/app/api/ws_router.py @@ -0,0 +1,68 @@ +from fastapi import APIRouter, WebSocket, Query, WebSocketDisconnect +from jose import JWTError, jwt +from sqlalchemy.orm import Session +import logging + +from app.infrastructure.realtime.connection_manager import manager +from app.core.settings import settings +from typing import Optional + +from app.core.ws_auth import authenticate_websocket +from app.db.database import SessionLocal +from app.modules.auth.models.user_model import User +from app.modules.documents.models.document_model import Project +from app.core.token_blacklist import TokenBlacklist + +logger = logging.getLogger(__name__) + +router = APIRouter() + +def _authenticate_ws(token: str, db: Session) -> Optional[User]: + """ + Kept as a thin delegation, not deleted. + + Both WebSocket modules carried their own copy of this and drifted apart; + `app/core/ws_auth.py` is now the single implementation. The name stays + because the handshake below calls it and the probes resolve it by name -- + the point was to remove the second set of *rules*, not the local symbol. + """ + return authenticate_websocket(token, db) + +@router.websocket("/ws/{task_id}") +async def websocket_progress(websocket: WebSocket, task_id: str, token: str = Query(None)): + auth_token = token or websocket.cookies.get("docqube_access_token") + + if auth_token and TokenBlacklist.is_token_blacklisted(auth_token): + await websocket.accept() + await websocket.close(code=4001, reason="Token revoked") + return + + db = SessionLocal() + try: + user = _authenticate_ws(auth_token, db) if auth_token else None + if not user: + logger.warning(f"WebSocket progress auth failed for task {task_id}") + await websocket.close(code=4001, reason="Authentication failed") + return + + project = db.query(Project).filter(Project.session_id == task_id, Project.user_id == user.id).first() + if not project: + logger.warning(f"Unauthorized WebSocket progress access: User {user.id} task {task_id}") + await websocket.close(code=4003, reason="Access denied to this task") + return + + user_id = user.id + finally: + db.close() + + await manager.connect_task(task_id, websocket) + + try: + while True: + await websocket.receive_text() + except WebSocketDisconnect: + logger.info(f"WebSocket progress disconnected: user={user_id} task={task_id}") + except Exception as e: + logger.error(f"WebSocket progress error: user={user_id} task={task_id}: {e}") + finally: + manager.disconnect(websocket) \ No newline at end of file diff --git a/app/assets/stylesheet/Stylesheet.deps.json b/app/assets/stylesheet/Stylesheet.deps.json new file mode 100644 index 0000000..2b4c555 --- /dev/null +++ b/app/assets/stylesheet/Stylesheet.deps.json @@ -0,0 +1,23 @@ +{ + "runtimeTarget": { + "name": ".NETCoreApp,Version=v8.0", + "signature": "" + }, + "compilationOptions": {}, + "targets": { + ".NETCoreApp,Version=v8.0": { + "Stylesheet/1.0.0": { + "runtime": { + "Stylesheet.dll": {} + } + } + } + }, + "libraries": { + "Stylesheet/1.0.0": { + "type": "project", + "serviceable": false, + "sha512": "" + } + } +} \ No newline at end of file diff --git a/app/assets/stylesheet/Stylesheet.dll b/app/assets/stylesheet/Stylesheet.dll new file mode 100644 index 0000000..8003c5e Binary files /dev/null and b/app/assets/stylesheet/Stylesheet.dll differ diff --git a/app/assets/stylesheet/Stylesheet.runtimeconfig.json b/app/assets/stylesheet/Stylesheet.runtimeconfig.json new file mode 100644 index 0000000..1dc0145 --- /dev/null +++ b/app/assets/stylesheet/Stylesheet.runtimeconfig.json @@ -0,0 +1,18 @@ +{ + "runtimeOptions": { + "tfm": "net8.0", + "frameworks": [ + { + "name": "Microsoft.NETCore.App", + "version": "8.0.0" + }, + { + "name": "Microsoft.WindowsDesktop.App", + "version": "8.0.0" + } + ], + "configProperties": { + "System.Runtime.Serialization.EnableUnsafeBinaryFormatterSerialization": true + } + } +} \ No newline at end of file diff --git a/app/assets/stylesheet/jats-html.xsl b/app/assets/stylesheet/jats-html.xsl new file mode 100644 index 0000000..02632ed --- /dev/null +++ b/app/assets/stylesheet/jats-html.xsl @@ -0,0 +1,4447 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <xsl:variable name="authors"> + <xsl:call-template name="author-string"/> + </xsl:variable> + <xsl:value-of select="normalize-space(string($authors))"/> + <xsl:if test="normalize-space(string($authors))">: + </xsl:if> + JATS-HTML Viewer + <!--<xsl:value-of select="/article/front/article-meta/title-group/article-title[1]"/>--> + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + +
+ +
+ + + + +
+ +
+ + + + --> + + + +
+ + + Floating objects + + + +
+ + + + + + + + + + + + +
+
+ + + +
+

+ Journal Information +

+
+ + + + + + +
+   +
+ +   +
+
+ + + + + + +
+

+ Article Information +

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ +
+
+ +
+ + + +
+ +
+ + + + +
+
+
+
+
+   +
+
+
+
+ + + + + +
+ + + + + + +
+ + +

+ + + + Translated + Abstract + + +

+ + +
+ +
+
+ + + + + + + +
+ +
+ +
+ + + + + + + + + + +
+
+

+ Article Information (continued) +

+
+   + +   + +   +
+
+ + + + + + + + +
+
+ +
+
+

+ Process warnings +

+

Warnings reported by the processor due to problematic markup follow:

+
+ +
+
+
+
+ + + + + + + + + + + + + + + + Journal ID + + ( + + + + ) + + + + + + + + + + + + + + + ISSN + + + + + + + + + + ISSN-L + + + + + + + + + + ISBN + + + + + + + + Publisher + + + + + + + + + + + + + + + ( + + ) + + + + + + Notes + + + + + + + + + Title + + + + + + + Subtitle + + + + + + + + + + + + Abbreviated Title + + + + + + + + + + Translated Title + + ( + + + + ) + + + + + + + + + + Translated Subtitle + + ( + + + + ) + + + + + + + + + + + + + + External link + + ( + + + + ) + + + + + + + + + + + + Email + + + + + + + + + + URI + + + + + + + + + + + Self URI + + + + + + + + + + + + + + + + + + + + + Product Information + + + + + + + + + + + + + + + + + + + + + + + Copyright + + + + , + + + + + + + + + + + Copyright statement + + + + + + + Copyright + + + + + + + + License + + ( + + + + , + + + + + + + ) + + + + + + + + + + Date + + + accepted + received + revision requested + revision received + + + + + + + + + + + + + + Publication date + + + + + + + + + + + + + + + + + Volume + + + + + + + + + + + + + + + + + ( + + + + + + + ID: + + + ) + + + + + + , + + + + + + + + + Volume + + + + + + + + Volume ID + + ( + + + + ) + + + + + + + + + Series + + + + + + + + + + + + Issue + + + + + + + + + + + + + + + + , + + + + + + + + + Issue + + + + + + + + Issue ID + + ( + + + + ) + + + + + + + + + Issue title + + + + + + + Issue sponsor + + + + + + + Issue part + + + + + + + + + + Page + + + s + + + + + + - + + + + (pp. + + ) + + + + + + + + + + Electronic Location + Identifier + + + + + + + + + Supplement Info + + + + + + + + Related + + object + article + + + ( + + + + ) + + + + + + + + + + + + + + + + + + + + + + + + + + Conference + + + + + ( + + , + + + ) + + + + + + + + + + + + + + Conference date + + + + + + + Conference + + + + + + + Conference + + + + + + + Conference number + + + + + + + Conference location + + + + + + + Conference sponsor + + + + + + + Conference theme + + + + + + + + + + ( + + + + ) + + + + + + + + + + + + + + + + + + + + Accession ID + Coden + DOI + Manuscript ID + Medline ID + Publisher Item ID + PubMed ID + Publisher ID + Serial Item and Contribution ID + DOAJ ID + arXiv.org ID + + Article Id + + ( + + + + ) + + + + + + + + + + + + Contract + + + + + + + Contract Sponsor + + + + + + + Grant Number + + + + + + + Grant Sponsor + + + + + + + + + + + + + Funded by + + + + + + + Award ID + + + + + + + Award Recipient + + + + + + + Principal Investigator + + + + + + + Funding + + + + + + + Open Access + + + + + + + + + +
+ +
+ + + + + “” + + + +

+ + : +

+ + + +

+ +

+ + + + + + + + + + + + + + Alternative title + + ( + + + + ) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+ + + +
+
+ + +
+
+
+ + + + + +
+
+
 
+
+
+ +
+
+
+
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + (Equal contributor) + + + +
+ + + + + Anonymous + + + + + + + + + + + + + + + , + + + + + , + + + + + + + + + + + + + [ + + ] + + + + + +
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Correspondence should be addressed to

  + + + + + + + + Comment + + + + + + + Bio + + + + + + + On behalf of + + + + + + + + + + + + Author notes + + + + + + + + + + + + + Correspondence to + + + + + + + + + + + + + + + + + + + + + Supplementary material + + + + + + + + + + + + Categories + + + + + + + + + + + + + +
+ +
+ + + + + + Subject + + + + + + + Series title + + + + + + + Series description + + + + + + + + + Keywords + + + + + + + + + + + + + + + + Keyword + + + + +
    + +
+ + + +
  • + +
    • +     + + + + + Compound keyword + + + + + + + + + + + Keyword part + + ( + + + + ) + + + + + + + + + + Counts + + + + + + + + + + + + + + + + + + + + + Count + + ( + + + + ) + + + + + Figures + + + Tables + + + Equations + + + References + + + Pages + + + Words + + + + + Custom metadata + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + + + + + +
    +     + + + + + + + + + + + +
    + + + +
    +     + + + +
    + + + + +
    + +
    +     + +
    +     + + + +
    + + +
    +     + + + + + + + + + + + + + + + + + + + + + + + + +

    + +

    +     +     + + + + + + + + +

    +   +

    +     +     + + + + + + + + +

    +   +

    +     +     + + + + + + + + +

    + +

    +     +     + + + + + +

    + +

    +     +     + + + + +
    + +
    +     +     + + + + + + + + + + + + + + +
    + +
    +     +     +     + + + + + + , + + + + + + +

    + +

    +     + + +
    + +
    +     + + +

    + +

    +     + + + + + + + chem-struct-wrap + + + + + +
    + + + display: float; clear: both + + + + + +
    +     + + + +
    + +
    +     + + + +
    + + + +
    +     + + + + +
    + + + + + + Glossary + + + + +
    +     + + + +

    + [Textual form] + +

    +     + + + + + +
    + + +
    +     + + + + + + + + + + + + + + + + +
    + + + +
    +     + + + +
      + +
    +     + + + +
      + +
    +     + + + + + + lower-alpha + upper-alpha + lower-roman + upper-roman + decimal + + +
      + +
    +     + + + +
  • + +
    • +     + + + + + + + + + + + + + + + + + + + + +

    + + first + + + + +

    +     + + + + + + + + +

    + + + + + + + + + +

    +     + + + +

    + + +

    +     + + + +
    + + +

    + Copyright + + + + , + + +

    +     + +
    +     + + + +

    + +

    +     + + + +
    + + +
    + +
    +
    + +
    +
    + +
    +
    +     + +
    + +
    +     + + + +
    + + +
    +     + + + +
    + +

    + +

    +
    +     + + + +
    + + +
    +     + + + +
    + + +
    +     + + + + 
    +      
+
    +     + + + + + + + + + + + + + + + +
    +
    + + +   + + + +
    +
    + +
    +
    +     + + + + +

    + + +

    +     + + + + + +

    + +

    +     +
    + + +
    +     + + + + + + + + + + + + + +
    + + +
    +     + + + + + + +

    + + +

    +     + + + + + + + : + + + + +
    + + + +
    +     + + + + + + + + + + + + + + + + + + + + + + + +
    + + +
    +     + + + +

    + +

    +     + + + + + [ + + ] + + + + + + + + + + + + + + + + +
    + +
    +     + + + + + + + + + + + TableData + + + + + + + + + + + + + +
    +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + [ + + ] + + + + + + + + + + , + + + + + + + + + + + + + + + + + + +
    +     + + + + + + + + + + + [ + + + + + + + + + + + + + + + ] + + + + + + + + + + + +
    +     + + + + + + + + + + + + + +
    + +
    +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + [Private character + + + + + ] + + + + + + (Glyph not rendered) + + + + + [Related article:] + + + + + + [Related object:] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Notes + + + + + + + + + + Acknowledgements + + + + + + + Appendices + + + + + + + Biography + + + + + + + Notes + + + + + + + Glossary + + + + + + + References + + + + + + + + + + Notes + + + + + + + + + +
    + + + + + + + + + + + + + + + + + +
    +     + + + + + + + + + + + + + + + + + + + + + + + + + +
    + + +
    +     + + + +

    + + + + + + + +

    +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + +
    +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Box + + + + + + + + + + + + + + Formula + + + + + + + + + + + + + + Formula (chemical) + + + + + + + + + + + + + + Figure + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + [ + + ] + + + + + + + + + + + + + + + + [ + + ] + + + + + + + Abbreviation + + + + Communicated by + + + + Contributed by + + + + Conflicts of interest + + + + Corresponding author + + + + Current affiliation + + + + Deceased + + + + Edited by + + + + Equal contributor + + + + Financial disclosure + + + + On leave + + + + + + Participating researcher + + + + Current address + + + + Presented at + + + + Presented by + + + + Previously at + + + + Study group member + + + + Supplementary material + + + + Supported by + + + + + + + + + + + + + + + + + + + + + + + + + + Statement + + + + + + + + + + + + + + Supplementary Material + + + + + + + + + + + + + + Table + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +   +   +   +   +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + etal. + + + + + + + + + + + + + ( + + + electronic + print + print and electronic + electronic preprint + print preprint + corrected, electronic + corrected, print + retracted, electronic + retracted, print + + + + + + ) + + + + + + + + + + + + + + + : + + + + + + + + + + + + +

    + +

    +     + + + + + + + +
    + + + + + + + + +
    + +
    +
    +     + + + + + + + + + + + + + + + + + { label + (or @symbol) + needed for + + + [@id=' + + '] + + } + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + named anchor + + + + + + + + + + + + + + + + + + + + + + +
  • + + + +
    • +     +     +     + + +

    Elements are cross-referenced without labels.

    +

    Either the element should be provided a label, or their cross-reference(s) should + have literal text content.

    +
      + +
    +     + + + + + +
  • + In + +
      + +
    • + +
      • +       +
    + +
    • +     +     + + +

    Elements with different 'specific-use' assignments appearing together

    +
      + +
    +     +     + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + January + February + March + April + May + June + July + August + September + October + November + December + + + + + + + + + + + + + + + + + , + + + and + + + + + + + + + + + +
    +
    +

    + This display is generated from + NISO JATS XML with + + + + . The XSLT engine is + + . +

    +
    +     + + + + + + + + + + + + + + + + [@ + + =' + + '] + + + + + + + + + + + + + + + + / + + + [ + + ] + + + + + + /@ + + + + + + /comment() + + [ + + ] + + + + + + /processing-instruction() + + [ + + ] + + + + + + /text() + + [ + + ] + + + + +   + + DOI: + PMID: + Scopus: + + +   + + + + +
    + + +

    Query

    + + + + + + + +
    + + + + + + + + + + + + + + + +
    +     +     + + + + + + + + + + + vol. + + , + + + + + + + + + + + no. + + , + + + + + + + + + + + pp. + + + , + + + + + + + + + + + + + + - + + pp. + + + + , + + + + + + + + + + + + “” + + , + + + + + + + , + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/app/core/cancellation.py b/app/core/cancellation.py new file mode 100644 index 0000000..ea760ba --- /dev/null +++ b/app/core/cancellation.py @@ -0,0 +1,13 @@ +import redis +from app.core.settings import settings + +redis_client = redis.from_url( + settings.REDIS_URL or "redis://localhost:6379/0", + decode_responses=True +) + +def cancel_task(task_id: str): + redis_client.set(f"cancel:{task_id}", "1", ex=86400) + +def is_cancelled(task_id: str): + return redis_client.exists(f"cancel:{task_id}") == 1 diff --git a/app/core/clamav.py b/app/core/clamav.py new file mode 100644 index 0000000..5691f9b --- /dev/null +++ b/app/core/clamav.py @@ -0,0 +1,118 @@ +import io +import socket +import struct +import logging +from typing import BinaryIO, Iterable, Optional + +logger = logging.getLogger(__name__) + +from app.core.settings import settings + +class ClamAVError(Exception): + """Base exception for ClamAV scanning failures.""" + + +class ClamAVConnectionError(ClamAVError): + """Raised when the ClamAV service cannot be reached.""" + + +class MalwareDetectedError(ClamAVError): + """Raised when ClamAV reports malware in a payload.""" + + def __init__(self, virus_name: str, response: str): + self.virus_name = virus_name + self.response = response + super().__init__(f"Malware detected: {virus_name}") + + +def _response_status(response: str) -> tuple[str, Optional[str]]: + text = response.strip().strip("\x00") + + if " FOUND" in text: + payload = text.split(": ", 1)[1] if ": " in text else text + virus_name = payload.rsplit(" FOUND", 1)[0].strip() or "Unknown threat" + return "found", virus_name + + if text.endswith("OK") or " OK" in text: + return "ok", None + + return "error", None + + +def _recv_clamav_response(sock: socket.socket) -> str: + chunks = [] + while True: + data = sock.recv(4096) + if not data: + break + chunks.append(data) + if b"\x00" in data: + break + return b"".join(chunks).decode("utf-8", errors="replace") + + +def _scan_chunks(chunks: Iterable[bytes]) -> None: + host = settings.CLAMAV_HOST + port = settings.CLAMAV_PORT + timeout = float(settings.CLAMAV_TIMEOUT_SECONDS) + + try: + logger.info(f"Connecting to ClamAV for scan at {host}:{port}...") + with socket.create_connection((host, port), timeout=timeout) as sock: + sock.settimeout(timeout) + sock.sendall(b"zINSTREAM\0") + + total_bytes = 0 + for chunk in chunks: + if not chunk: + continue + total_bytes += len(chunk) + sock.sendall(struct.pack("!I", len(chunk))) + sock.sendall(chunk) + + sock.sendall(struct.pack("!I", 0)) + response = _recv_clamav_response(sock) + logger.info(f"ClamAV scan completed. Sent {total_bytes} bytes. Response: {response.strip()}") + except (OSError, socket.timeout) as exc: + raise ClamAVConnectionError( + f"Unable to connect to ClamAV at {host}:{port}" + ) from exc + + status, virus_name = _response_status(response) + if status == "found": + logger.warning(f"SCAN RESULT: MALWARE FOUND - {virus_name}") + raise MalwareDetectedError(virus_name=virus_name or "Unknown threat", response=response) + if status != "ok": + logger.error(f"SCAN RESULT: ERROR - Unexpected response: {response}") + raise ClamAVError(f"Unexpected ClamAV response: {response}") + + logger.info("SCAN RESULT: CLEAN - Document is safe.") + + +def scan_file_for_malware(file_obj: BinaryIO) -> None: + """ + Scan a file-like object using ClamAV INSTREAM and rewind it for later reads. + """ + chunk_size = max(1, int(settings.CLAMAV_CHUNK_SIZE)) + + can_seek = hasattr(file_obj, "seek") + if can_seek: + file_obj.seek(0) + + try: + def iter_chunks(): + while True: + chunk = file_obj.read(chunk_size) + if not chunk: + break + yield chunk + + _scan_chunks(iter_chunks()) + finally: + if can_seek: + file_obj.seek(0) + + +def scan_bytes_for_malware(payload: bytes) -> None: + """Scan an in-memory payload using ClamAV.""" + scan_file_for_malware(io.BytesIO(payload)) diff --git a/app/core/context.py b/app/core/context.py new file mode 100644 index 0000000..8b70f91 --- /dev/null +++ b/app/core/context.py @@ -0,0 +1,4 @@ +from contextvars import ContextVar + +client_ip_ctx_var: ContextVar[str] = ContextVar("client_ip", default=None) +user_agent_ctx_var: ContextVar[str] = ContextVar("user_agent", default=None) diff --git a/app/core/crypto.py b/app/core/crypto.py new file mode 100644 index 0000000..8876caf --- /dev/null +++ b/app/core/crypto.py @@ -0,0 +1,35 @@ +import base64 +import hashlib +import os +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from app.core.settings import settings + +def _get_key() -> bytes: + """Derive a consistent 32-byte key from APP_SECRET for AES-256.""" + return hashlib.sha256(settings.APP_SECRET.encode('utf-8')).digest() + +def encrypt_data(data: str) -> str: + """Encrypts a string using AES-256-GCM and returns a base64 encoded string.""" + if not data: + return data + aesgcm = AESGCM(_get_key()) + nonce = os.urandom(12) + ciphertext = aesgcm.encrypt(nonce, data.encode('utf-8'), None) + combined = nonce + ciphertext + return base64.b64encode(combined).decode('utf-8') + +def decrypt_data(token: str) -> str: + """Decrypts a base64 encoded AES-256-GCM string.""" + if not token: + return token + try: + combined = base64.b64decode(token.encode('utf-8')) + if len(combined) < 12: + return "" + nonce = combined[:12] + ciphertext = combined[12:] + aesgcm = AESGCM(_get_key()) + decrypted = aesgcm.decrypt(nonce, ciphertext, None) + return decrypted.decode('utf-8') + except Exception: + return "" diff --git a/app/core/mail.py b/app/core/mail.py new file mode 100644 index 0000000..803743f --- /dev/null +++ b/app/core/mail.py @@ -0,0 +1,250 @@ +import smtplib +import logging +import os +from email.mime.text import MIMEText +from email.mime.multipart import MIMEMultipart +from email.mime.base import MIMEBase +from email import encoders +from typing import Optional, Dict, Any, List +from uuid import UUID +from jinja2 import Environment, FileSystemLoader +from sqlalchemy.orm import Session + +from app.core.settings import settings +from app.core.crypto import decrypt_data + +logger = logging.getLogger(__name__) + +template_dir = os.path.join( + os.path.dirname(os.path.dirname(__file__)), "templates", "emails" +) +try: + jinja_env = Environment(loader=FileSystemLoader(template_dir)) +except Exception as e: + logger.warning(f"Failed to initialize Jinja template environment: {e}") + jinja_env = None + + +def send_email( + subject: str, + recipient: str, + body: Optional[str] = None, + html_body: Optional[str] = None, + template_name: Optional[str] = None, + template_context: Optional[Dict[str, Any]] = None, + db: Optional[Session] = None, + tenant_id: Optional[UUID] = None, + user_id: Optional[int] = None, + attachments: Optional[List[Dict[str, Any]]] = None, +): + """ + Sends an email using the configured SMTP settings. + In development, if SMTP is not configured, it logs the email. + """ + if template_name and template_context is not None and jinja_env: + try: + template = jinja_env.get_template(template_name) + html_body = template.render(**template_context) + if not body: + body = f"Please view this email in an HTML-compatible client. Subject: {subject}" + except Exception as e: + logger.error(f"Failed to render email template {template_name}: {e}") + return + elif template_name and not jinja_env: + logger.error("Jinja2 environment not initialized. Cannot render template '%s'", template_name) + return + + if not body and not html_body: + logger.error("Email must have either a body or html_body") + return + + smtp_config = _resolve_smtp_config(db=db, tenant_id=tenant_id, user_id=user_id) + logger.info( + "SMTP source selected: %s (tenant_id=%s)", + smtp_config.get("source", "unknown"), + str(tenant_id) if tenant_id else "none", + ) + + if not smtp_config["smtp_host"]: + return + + try: + _send_with_config( + smtp_config=smtp_config, + subject=subject, + recipient=recipient, + body=body, + html_body=html_body, + attachments=attachments, + ) + logger.info( + "Email sent successfully to %s via %s SMTP", + recipient, + smtp_config.get("source", "unknown"), + ) + return + except Exception as e: + logger.error(f"Failed to send email to {recipient} via {smtp_config.get('source', 'unknown')} SMTP: {e}") + + fallback_config = None + if smtp_config.get("source") == "user": + fallback_config = _resolve_smtp_config(db=db, tenant_id=tenant_id, user_id=None) + elif smtp_config.get("source") == "tenant": + fallback_config = _resolve_smtp_config(db=None, tenant_id=None, user_id=None) + + if fallback_config and fallback_config.get("smtp_host"): + logger.warning( + "Retrying email delivery via fallback SMTP after %s SMTP failure (tenant_id=%s)", + smtp_config.get("source", "unknown"), + str(tenant_id) if tenant_id else "none", + ) + try: + _send_with_config( + smtp_config=fallback_config, + subject=subject, + recipient=recipient, + body=body, + html_body=html_body, + attachments=attachments, + ) + logger.info( + "Email sent successfully to %s via fallback SMTP after %s SMTP failure", + recipient, + smtp_config.get("source", "unknown"), + ) + return + except Exception as fallback_error: + logger.error( + "Fallback SMTP also failed for %s: %s", + recipient, + fallback_error, + ) + + +def _send_with_config( + smtp_config: Dict[str, Any], + subject: str, + recipient: str, + body: Optional[str], + html_body: Optional[str], + attachments: Optional[List[Dict[str, Any]]] = None, +) -> None: + msg = MIMEMultipart() + msg["Subject"] = subject + msg["From"] = smtp_config["mail_from"] + msg["To"] = recipient + + if body or html_body: + content_part = MIMEMultipart("alternative") + if body: + content_part.attach(MIMEText(body, "plain")) + if html_body: + content_part.attach(MIMEText(html_body, "html")) + msg.attach(content_part) + + if attachments: + for attachment in attachments: + part = MIMEBase("application", "octet-stream") + part.set_payload(attachment["content"]) + encoders.encode_base64(part) + part.add_header( + "Content-Disposition", + f"attachment; filename={attachment['filename']}", + ) + msg.attach(part) + + if smtp_config["smtp_secure"] or smtp_config["smtp_port"] == 465: + with smtplib.SMTP_SSL(smtp_config["smtp_host"], smtp_config["smtp_port"]) as server: + if smtp_config["smtp_user"] and smtp_config["smtp_password"]: + server.login(smtp_config["smtp_user"], smtp_config["smtp_password"]) + server.send_message(msg) + else: + with smtplib.SMTP(smtp_config["smtp_host"], smtp_config["smtp_port"]) as server: + if smtp_config["smtp_user"] and smtp_config["smtp_password"]: + server.starttls() + server.login(smtp_config["smtp_user"], smtp_config["smtp_password"]) + server.send_message(msg) + + +def _resolve_smtp_config( + db: Optional[Session] = None, + tenant_id: Optional[UUID] = None, + user_id: Optional[int] = None, +) -> Dict[str, Any]: + fallback = { + "source": "fallback", + "smtp_host": settings.SMTP_HOST, + "smtp_port": settings.SMTP_PORT or 587, + "smtp_user": settings.SMTP_USER, + "smtp_password": settings.SMTP_PASSWORD, + "smtp_secure": settings.SMTP_SECURE, + "mail_from": settings.MAIL_FROM, + } + + close_db = False + if db is None and (tenant_id is not None or user_id is not None): + try: + from app.db.database import SessionLocal + db = SessionLocal() + close_db = True + except Exception as err: + logger.error("Failed to create temporary DB session for SMTP resolution: %s", err) + + try: + base_config = fallback + + if db is not None and tenant_id is not None: + try: + from app.modules.tenant.repositories.tenant_smtp_repository import TenantSMTPRepository + + cfg = TenantSMTPRepository(db).get_by_tenant_id(tenant_id) + if cfg and cfg.is_active: + base_config = { + "source": "tenant", + "smtp_host": cfg.smtp_host, + "smtp_port": cfg.smtp_port, + "smtp_user": cfg.smtp_user, + "smtp_password": ( + decrypt_data(cfg.encrypted_smtp_password) + if cfg.encrypted_smtp_password + else None + ), + "smtp_secure": cfg.smtp_secure, + "mail_from": cfg.mail_from, + } + except Exception as e: + logger.error( + "SMTP config lookup failed for tenant_id=%s, using fallback SMTP: %s", + tenant_id, + e, + ) + + if db is None or user_id is None: + return base_config + + try: + from app.modules.auth.models.user_model import User + + user = db.query(User).filter(User.id == user_id).first() + if not user or not user.smtp_user or not user.encrypted_smtp_password: + return base_config + + return { + "source": "user", + "smtp_host": base_config["smtp_host"], + "smtp_port": base_config["smtp_port"], + "smtp_user": user.smtp_user, + "smtp_password": decrypt_data(user.encrypted_smtp_password), + "smtp_secure": base_config["smtp_secure"], + "mail_from": user.mail_from or base_config["mail_from"], + } + except Exception as e: + logger.error( + "User SMTP credential lookup failed for user_id=%s, using base SMTP: %s", + user_id, + e, + ) + return base_config + finally: + if close_db and db is not None: + db.close() diff --git a/app/core/model_manager.py b/app/core/model_manager.py new file mode 100644 index 0000000..ad431f3 --- /dev/null +++ b/app/core/model_manager.py @@ -0,0 +1,70 @@ +import logging +import torch + +logger = logging.getLogger("AI_Model_Manager") + + +class ModelManager: + + _instance = None + _initialized = False + + def __new__(cls): + if cls._instance is None: + cls._instance = super(ModelManager, cls).__new__(cls) + return cls._instance + + def __init__(self): + pass + + def load_models(self): + + if ModelManager._initialized: + return + + logger.info("🧠 Lazy loading AI models...") + + torch.set_num_threads(2) + + from marker.converters.pdf import PdfConverter + from marker.models import create_model_dict + + self.model_cache = create_model_dict() + + config = { + "use_llm": False, + "extract_images": True, + "extract_tables": True, + "extract_equations": True, + "extract_layout": True, + } + + self.converter = PdfConverter( + artifact_dict=self.model_cache, + config=config + ) + + ModelManager._initialized = True + logger.info("✅ Models ready") + + def process_document(self, file_path): + + self.load_models() + + ext = file_path.lower() + + if ext.endswith(".pdf"): + return self.converter(file_path) + + if ext.endswith((".jpg", ".jpeg", ".png")): + from surya.ocr import run_ocr + return run_ocr(file_path) + + if ext.endswith(".docx"): + from docx import Document + doc = Document(file_path) + return {"markdown": "\n".join(p.text for p in doc.paragraphs)} + + raise ValueError("Unsupported file") + +model_manager = ModelManager() diff --git a/app/core/path_validation.py b/app/core/path_validation.py new file mode 100644 index 0000000..2537e6e --- /dev/null +++ b/app/core/path_validation.py @@ -0,0 +1,88 @@ +""" +path_validation.py — server-side file path and filename sanitization utilities. + +These helpers prevent path traversal attacks (C-017 / H-038) by rejecting +dangerous patterns before paths are stored in the DB or used in file I/O. +""" +import re +import logging + +logger = logging.getLogger(__name__) + +_TRAVERSAL_PATTERN = re.compile( + r""" + \.\. # double-dot (parent directory traversal) + | \x00 # null byte + | [<>:"|?*] # Windows-forbidden characters + """, + re.VERBOSE, +) + +_SEPARATOR_PATTERN = re.compile(r"[/\\]") + +MAX_PATH_LENGTH = 512 +MAX_FILENAME_LENGTH = 255 + + +def validate_storage_path(path: str) -> str: + """ + Validate a storage key / relative path before writing it to the DB or + using it in an S3 / B2 operation. + + Accepts: + - B2 / S3 object keys (e.g. "chatbot/42/abc123.pdf") + - Internal sentinel URIs (e.g. "drive://123") + + Raises: + ValueError – with a human-readable reason on rejection. + + Returns: + The original `path` unchanged if it passes all checks. + """ + if not path: + raise ValueError("File path must not be empty.") + + if len(path) > MAX_PATH_LENGTH: + raise ValueError( + f"File path exceeds the maximum allowed length " + f"({len(path)} > {MAX_PATH_LENGTH})." + ) + + if path.startswith("drive://"): + return path + + if _TRAVERSAL_PATTERN.search(path): + logger.warning("Rejected dangerous storage path: %r", path) + raise ValueError("File path contains disallowed characters or sequences.") + + return path + + +def validate_filename(filename: str) -> str: + """ + Validate an upload filename before storing it or using it on disk. + + Raises: + ValueError – with a human-readable reason on rejection. + + Returns: + The original `filename` unchanged if it passes all checks. + """ + if not filename: + raise ValueError("Filename must not be empty.") + + if len(filename) > MAX_FILENAME_LENGTH: + raise ValueError( + f"Filename exceeds the maximum allowed length " + f"({len(filename)} > {MAX_FILENAME_LENGTH})." + ) + + if _SEPARATOR_PATTERN.search(filename): + logger.warning("Rejected filename containing path separator: %r", filename) + raise ValueError("Filename must not contain path separators.") + + if _TRAVERSAL_PATTERN.search(filename): + logger.warning("Rejected dangerous filename: %r", filename) + raise ValueError("Filename contains disallowed characters or sequences.") + + return filename diff --git a/app/core/progress.py b/app/core/progress.py new file mode 100644 index 0000000..0efcd1d --- /dev/null +++ b/app/core/progress.py @@ -0,0 +1,17 @@ +import redis +import json +from app.core.settings import settings + +redis_client = redis.from_url( + settings.REDIS_URL or "redis://localhost:6379/0", + decode_responses=True +) + +def publish_progress(task_id, progress, message): + redis_client.publish( + f"task:{task_id}", + json.dumps({ + "progress": progress, + "message": message + }) + ) diff --git a/app/core/pubsub_listener.py b/app/core/pubsub_listener.py new file mode 100644 index 0000000..9533158 --- /dev/null +++ b/app/core/pubsub_listener.py @@ -0,0 +1,41 @@ +import asyncio +import json +import redis +import logging +from app.infrastructure.realtime.connection_manager import manager +from app.core.settings import settings + +logger = logging.getLogger(__name__) + +async def start_listener(): + redis_client = redis.from_url( + settings.REDIS_URL or "redis://localhost:6379/0", + decode_responses=True + ) + + pubsub = redis_client.pubsub() + pubsub.psubscribe("task:*") + + logger.info("📡 Redis Pub/Sub listener started") + + while True: + try: + message = await asyncio.get_event_loop().run_in_executor(None, pubsub.get_message, False, 1.0) + + if message and message["type"] == "pmessage": + parts = message["channel"].split(":") + if len(parts) != 2: + continue + task_id = parts[1] + try: + data = json.loads(message["data"]) + except (json.JSONDecodeError, TypeError): + logger.error(f"❌ Received malformed JSON on channel {message['channel']}") + continue + + await manager.send_task(task_id, data) + + await asyncio.sleep(0.01) + except Exception as e: + logger.error(f"Error in Pub/Sub listener: {e}") + await asyncio.sleep(1) diff --git a/app/core/redis.py b/app/core/redis.py new file mode 100644 index 0000000..ffeec75 --- /dev/null +++ b/app/core/redis.py @@ -0,0 +1,136 @@ +""" +Async Redis Pub/Sub client for real-time collaboration messaging. + +This module is SEPARATE from app/db/redis.py (sync cache client). +It uses redis.asyncio for non-blocking publish/subscribe operations +intended for WebSocket fan-out across backend workers. + +Usage: + from app.core.redis import redis_pubsub + + # Publish a message to a file's chat channel + await redis_pubsub.publish(file_id=42, message={...}) + + # Subscribe and iterate over incoming messages + async for msg in redis_pubsub.subscribe(file_id=42): + await websocket.send_json(msg) +""" +import json +import logging +from typing import Any, AsyncIterator, Optional + +import redis.asyncio as aioredis + +from app.core.settings import settings + +logger = logging.getLogger(__name__) + +CHANNEL_PREFIX = "chat:file" + + +def _channel(file_id: int) -> str: + """Build the canonical Redis channel name for a file's chat room.""" + return f"{CHANNEL_PREFIX}:{file_id}" + + +class RedisPubSub: + """ + Lightweight async wrapper around redis.asyncio for chat pub/sub. + + • Lazily creates a shared connection pool on first use. + • publish() and subscribe() are the only public API. + • Graceful degradation: if Redis is unreachable, publish is a no-op + and subscribe yields nothing (the app stays up). + """ + + def __init__(self) -> None: + self._pool: Optional[aioredis.ConnectionPool] = None + self._client: Optional[aioredis.Redis] = None + + + async def connect(self) -> None: + """Create the shared async connection pool (idempotent).""" + if self._client is not None: + return + try: + url = settings.REDIS_URL or "redis://localhost:6379/0" + self._pool = aioredis.ConnectionPool.from_url( + url, decode_responses=True + ) + self._client = aioredis.Redis(connection_pool=self._pool) + await self._client.ping() + logger.info("✅ Async Redis Pub/Sub connected") + except Exception as e: + logger.warning(f"Async Redis Pub/Sub unavailable: {e}") + self._client = None + self._pool = None + + async def disconnect(self) -> None: + """Tear down the pool (call on app shutdown).""" + if self._client: + await self._client.aclose() + self._client = None + if self._pool: + await self._pool.disconnect() + self._pool = None + logger.info("Async Redis Pub/Sub disconnected") + + + async def publish(self, file_id: int, message: dict[str, Any]) -> None: + """ + Publish a JSON message to ``chat:file:{file_id}``. + + No-op if Redis is not connected. + """ + if not self._client: + await self.connect() + if not self._client: + return + try: + channel = _channel(file_id) + payload = json.dumps(message, default=str) + await self._client.publish(channel, payload) + except Exception as e: + logger.error(f"Redis publish error: {e}") + + async def subscribe(self, file_id: int) -> AsyncIterator[dict[str, Any]]: + """ + Subscribe to ``chat:file:{file_id}`` and yield parsed messages. + + Intended to be consumed inside a WebSocket handler:: + + async for msg in redis_pubsub.subscribe(file_id): + await ws.send_json(msg) + + Yields nothing (returns immediately) if Redis is not available. + """ + if not self._client: + await self.connect() + if not self._client: + return + + channel = _channel(file_id) + pubsub = self._client.pubsub() + + try: + await pubsub.subscribe(channel) + logger.debug(f"Subscribed to {channel}") + + async for raw in pubsub.listen(): + if raw["type"] != "message": + continue + try: + yield json.loads(raw["data"]) + except (json.JSONDecodeError, TypeError): + logger.warning(f"Bad payload on {channel}: {raw['data']}") + except Exception as e: + logger.error(f"Redis subscribe error on {channel}: {e}") + finally: + try: + await pubsub.unsubscribe(channel) + await pubsub.aclose() + except Exception: + pass + + +redis_pubsub = RedisPubSub() diff --git a/app/core/request_body.py b/app/core/request_body.py new file mode 100644 index 0000000..069d4ae --- /dev/null +++ b/app/core/request_body.py @@ -0,0 +1,51 @@ +""" +Reading a JSON request body without turning the caller's mistake into our fault. + +`await request.json()` raises `JSONDecodeError` on a malformed body. Nothing in +these routes caught it, so it fell through to the generic handler and became a +**500**. That is wrong in three separate ways: + +- it reports a client error as a server fault, so it pages whoever is on call + and buries genuine failures in noise; +- on the webhook endpoints it tells the sender to retry, because 5xx means + "try again later" and 4xx means "your request is wrong" — a malformed + DocuSeal payload would be redelivered on a schedule, forever; +- `POST /api/sso/login` is unauthenticated, so anyone could produce a 500 with + a one-byte body. + +The discovery harness found this by sending a form body to a JSON endpoint — +which is exactly what a misconfigured integration does on its first day. +""" + +import json + +from fastapi import HTTPException, Request, status + + +async def read_json_body(request: Request, *, what: str = "request body") -> dict: + """ + Parse the body as a JSON object, or raise 400. + + Returns a `dict` specifically: every caller immediately does `.get(...)`, so + a bare list or string would fail with `AttributeError` a line later and land + back in the 500 handler this exists to avoid. + """ + raw = await request.body() + if not raw: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Expected a JSON {what}, but it was empty.", + ) + try: + parsed = json.loads(raw) + except (json.JSONDecodeError, UnicodeDecodeError): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"The {what} is not valid JSON.", + ) from None + if not isinstance(parsed, dict): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"The {what} must be a JSON object.", + ) + return parsed diff --git a/app/core/request_cache.py b/app/core/request_cache.py new file mode 100644 index 0000000..3575587 --- /dev/null +++ b/app/core/request_cache.py @@ -0,0 +1,69 @@ +""" +A memo that lives exactly as long as one request. + +Access resolution is asked the same question many times in a single request. +`require_access` runs once per dependency — several times on a route that +declares more than one — `get_current_user` now resolves the caller's codes so +the response can report them, and `/api/admin/users` resolves assignments for a +page of users. Without a memo each of those repeats the same three queries, one +of which reads the whole `accesses` table. + +Modelled on `app/core/scope.py`, deliberately, and for the same reason: a +`ContextVar` reset once per request is the only cache shape that cannot outlive +the authority it caches. A process-wide cache of "what may this user do" is the +bug where revoking a role takes effect after a restart. + +**Default of `None` means "not in a request".** Celery tasks, scripts and any +code path that never passes through `TenantContextMiddleware` compute every +time. That is correct rather than merely safe — a long-running task must not +pin a permission decision made at its start. + +The trap worth naming, because it is the same one `ScopeService` documents: a +grant that expires *mid-request* stays honoured to the end of that request. +Permissions must not change under a half-finished operation, so this is +intended. Do not "fix" it by shortening the cache. +""" + +from __future__ import annotations + +from contextvars import ContextVar +from typing import Any, Callable, Optional + +_cache: ContextVar[Optional[dict]] = ContextVar("request_cache", default=None) + + +def reset_request_cache() -> None: + """Drop everything memoised. Called once per request, before routing.""" + _cache.set({}) + + +def request_cached(key: str, compute: Callable[[], Any]) -> Any: + """ + Return `compute()`, memoised under *key* for the rest of this request. + + Outside a request the memo is absent and `compute` runs every time. + """ + store = _cache.get() + if store is None: + return compute() + if key not in store: + store[key] = compute() + return store[key] + + +def invalidate_request_cache(prefix: str) -> None: + """ + Forget every memoised key starting with *prefix*. + + Needed because a request may **change** the thing it cached. Granting a role + and then returning the user's roles in the same response would otherwise + serve the pre-grant answer, which reads as "the grant did not work". + """ + store = _cache.get() + if not store: + return + for key in [k for k in store if k.startswith(prefix)]: + del store[key] + + +__all__ = ["reset_request_cache", "request_cached", "invalidate_request_cache"] diff --git a/app/core/rls.py b/app/core/rls.py new file mode 100644 index 0000000..35b3f9c --- /dev/null +++ b/app/core/rls.py @@ -0,0 +1,80 @@ +""" +B1.3 — hand the tenant context to PostgreSQL. + +The RLS policy compares each row's `tenant_id` against `docqube.tenant_id`, a +session variable. Something has to set it, and it has to be set on the same +connection the query runs on — which is why this hooks the session rather than +the request. + +`set_config(..., true)` scopes the setting to the current **transaction**, so it +cannot leak to the next request through a pooled connection. That matters more +than it looks: a connection-scoped setting left behind by one tenant would be +inherited by whoever picked the connection up next. + +Off by the same switch as the ORM listener. The policy itself is inert without a +variable, so an unset switch means RLS is enabled in the database and permissive +in practice — which is the safe way round while it is being rolled out. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from sqlalchemy import event, text +from sqlalchemy.orm import Session + +logger = logging.getLogger(__name__) + + +def _enabled() -> bool: + from app.core.settings import settings + + return bool(getattr(settings, "TENANT_FILTER_ENABLED", False)) + + +def apply_tenant_context(session: Session) -> None: + """Push the current tenant context onto the session's connection.""" + from app.core.tenant_context import ( + current_tenant_id, + is_bypassed, + is_super_admin, + ) + + bypass = is_bypassed() or is_super_admin() + tenant_id = current_tenant_id() + + session.execute( + text("SELECT set_config('docqube.bypass', :v, true)"), + {"v": "on" if bypass else "off"}, + ) + session.execute( + text("SELECT set_config('docqube.tenant_id', :v, true)"), + {"v": str(tenant_id) if tenant_id else ""}, + ) + + +@event.listens_for(Session, "after_begin") +def _set_tenant_on_begin(session: Session, transaction: Any, connection: Any) -> None: + if not _enabled(): + return + try: + from app.core.tenant_context import ( + current_tenant_id, + is_bypassed, + is_super_admin, + ) + + bypass = is_bypassed() or is_super_admin() + tenant_id = current_tenant_id() + + connection.execute( + text("SELECT set_config('docqube.bypass', :v, true)"), + {"v": "on" if bypass else "off"}, + ) + connection.execute( + text("SELECT set_config('docqube.tenant_id', :v, true)"), + {"v": str(tenant_id) if tenant_id else ""}, + ) + except Exception: # noqa: BLE001 - never break a request over this + logger.exception("Failed to apply tenant context to the connection") diff --git a/app/core/sanitization.py b/app/core/sanitization.py new file mode 100644 index 0000000..6cb866a --- /dev/null +++ b/app/core/sanitization.py @@ -0,0 +1,78 @@ +from bs4 import BeautifulSoup +import logging + +logger = logging.getLogger(__name__) + +ALLOWED_TAGS = [ + 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', + 'p', 'br', 'hr', 'pre', 'code', 'blockquote', + 'ul', 'ol', 'li', 'dl', 'dt', 'dd', + 'b', 'strong', 'i', 'em', 'u', 's', 'small', 'sub', 'sup', + 'table', 'thead', 'tbody', 'tfoot', 'tr', 'th', 'td', 'caption', 'colgroup', 'col', + 'div', 'span', 'section', 'article', 'aside', 'header', 'footer', + 'img', 'a', 'figure', 'figcaption', + 'math', 'mrow', 'mi', 'mo', 'mn', 'msup', 'msub', 'msubsup', 'mfrac', 'msqrt', 'mroot', 'mover', 'munder', 'munderover', 'mtd', 'mtr', 'mtable', 'mstyle', 'merror', 'mpadded', 'mphantom', 'mfenced', 'menclose' +] + +ALLOWED_ATTRIBUTES = { + '*': ['id', 'class', 'title', 'lang'], + 'a': ['href', 'target', 'rel', 'name'], + 'img': ['src', 'alt', 'width', 'height', 'loading'], + 'table': ['border', 'cellpadding', 'cellspacing', 'frame', 'rules', 'summary', 'width'], + 'th': ['align', 'valign', 'colspan', 'rowspan', 'scope', 'width'], + 'td': ['align', 'valign', 'colspan', 'rowspan', 'width'], + 'col': ['span', 'width'], + 'colgroup': ['span', 'width'], + 'section': ['sec-type'], + 'div': ['id', 'class'], + 'span': ['id', 'class'], + 'math': ['display', 'xmlns'], + 'mtd': ['columnspan', 'rowspan', 'align'], + 'mtr': ['rowalign'], + 'mtable': ['equalrows', 'equalcolumns', 'displaystyle'], +} + +def sanitize_html(html_content: str) -> str: + """ + Sanitize HTML content using BeautifulSoup4. + Removes potentially dangerous tags (script, iframe, object, etc.) and attributes. + """ + if not html_content: + return "" + + try: + soup = BeautifulSoup(html_content, 'lxml') + + for tag in soup.findAll(): + if tag.name.lower() not in ALLOWED_TAGS: + if tag.name.lower() in ['script', 'style', 'iframe', 'object', 'embed', 'applet', 'link']: + tag.decompose() + else: + tag.unwrap() + + for tag in soup.findAll(): + allowed_attrs = ALLOWED_ATTRIBUTES.get(tag.name.lower(), []) + global_attrs = ALLOWED_ATTRIBUTES.get('*', []) + + attrs_to_remove = [attr for attr in tag.attrs if attr.lower() not in allowed_attrs and attr.lower() not in global_attrs] + + for attr in ['href', 'src']: + if attr in tag.attrs: + val = tag[attr].lower().strip() + if val.startswith('javascript:') or val.startswith('data:text/html') or val.startswith('vbscript:'): + attrs_to_remove.append(attr) + + for attr in list(tag.attrs.keys()): + if attr.lower().startswith('on'): + attrs_to_remove.append(attr) + + for attr in attrs_to_remove: + del tag[attr] + + if soup.body: + return "".join([str(c) for c in soup.body.contents]) + return str(soup) + + except Exception as e: + logger.error(f"HTML Sanitization failed: {e}") + return "Sanitization Error - Access Denied" diff --git a/app/core/schemas.py b/app/core/schemas.py new file mode 100644 index 0000000..6bfebb8 --- /dev/null +++ b/app/core/schemas.py @@ -0,0 +1,55 @@ +""" +Shared response envelopes. + +DocQube's mutations return a small number of repeated shapes — `{"message": …}`, +`{"status": …}`, `{"success": …}` and a couple of counters. Declaring them once +means the generated frontend client sees one type per shape instead of a +hundred anonymous inline objects. + +Every model here matches a response observed in +`tests/characterization/discovered_mutation_shapes.json`. None of them is a +guess, and none carries an optional field the endpoint does not return — a +declared field is *added* as null rather than omitted, so a wider model changes +the payload just as surely as a narrower one drops it. +""" + +from typing import Optional + +from pydantic import BaseModel + + +class MessageOut(BaseModel): + """`{"message": "..."}` — the most common acknowledgement in the codebase.""" + + message: str + + +class StatusOut(BaseModel): + """`{"status": "ok"}` / `{"status": "success"}`.""" + + status: str + + +class StatusMessageOut(BaseModel): + """`{"status": "...", "message": "..."}`.""" + + status: str + message: Optional[str] = None + + +class SuccessOut(BaseModel): + """`{"success": true}`.""" + + success: bool + + +class DeletedCountOut(BaseModel): + """`{"deleted": n}` — bulk delete.""" + + deleted: int + + +class MarkedCountOut(BaseModel): + """`{"marked": n}` — bulk mark-as-read.""" + + marked: int diff --git a/app/core/scope.py b/app/core/scope.py new file mode 100644 index 0000000..41515f9 --- /dev/null +++ b/app/core/scope.py @@ -0,0 +1,332 @@ +""" +C4 — where does this user hold this permission? + +Ported from `docqube_platform_backend/app/services/scope_service.py`, adapted to +resolve through DocQube's `RoleAccess`/`Access` rather than the base's +`RolePermission`/`Permission`. + +The whole design rests on one separation, and it is worth stating plainly +because it is what makes the rollout incremental: + +- `require_access("code")` stays exactly as it is — the **coarse** gate, + answering "do you hold this code at all". Every one of the 227 endpoints + keeps working, unchanged, today and after. +- `ScopeService.authority(user, code)` answers the **scope** question + separately, at the data layer, for endpoints that have been converted. + +So this file adds a capability without altering a single existing decision. An +endpoint becomes org-aware when someone converts it, one at a time, with a test. + +`authority()` returns one of two things, and the difference is deliberate: + + TENANT_WIDE a sentinel — the user holds this everywhere in their tenant + set[UUID] the org units they hold it in, *including descendants* + +It never returns None. A caller must not be able to confuse "no authority" with +"authority everywhere" by testing falsiness — an empty set is falsy and +`TENANT_WIDE` is truthy, which is the safe way round, but the sentinel is a +distinct object so `== TENANT_WIDE` is unambiguous. + +**Subtree inheritance is the point.** Grant a role at *Engineering* and it +applies to *Platform* and *Infrastructure* without further assignment. That is a +recursive CTE walking `parent_id` **downward**; a join written the other way +matches ancestors just as happily and silently promotes every team lead to a +department head. `tests/probes/test_org_scope.py` asserts the direction +explicitly for exactly that reason. +""" + +from __future__ import annotations + +import uuid +from contextvars import ContextVar +from typing import Union + +from sqlalchemy import func, or_, select +from sqlalchemy.orm import Session + +TENANT_WIDE = "TENANT_WIDE" + +Authority = Union[str, set] + +_cache: ContextVar[dict] = ContextVar("scope_cache", default=None) + + +def reset_scope_cache() -> None: + """Drop any memoised authority. Called once per request.""" + _cache.set({}) + + +def _cached(key: str, compute): + store = _cache.get() + if store is None: + return compute() + if key not in store: + store[key] = compute() + return store[key] + + +class ScopeService: + def __init__(self, db: Session): + self.db = db + + + def authority(self, user, access_code: str) -> Authority: + """`TENANT_WIDE`, or the set of org units where the user holds *code*.""" + key = f"{user.id}:{access_code}" + return _cached(key, lambda: self._resolve(user, access_code)) + + def can_reach(self, user, access_code: str, org_unit_id) -> bool: + """Does the user hold *code* over this particular unit?""" + auth = self.authority(user, access_code) + if auth == TENANT_WIDE: + return True + if org_unit_id is None: + return False + return org_unit_id in auth + + def can_access_user(self, actor, target_user_id: int, access_code: str) -> bool: + """ + May *actor* act on this user, under this access code? + + Three ways to yes, and the order matters: + + 1. the actor holds the code tenant-wide — the state every user is in + today, so this is the answer that keeps the application unchanged; + 2. the target is the actor themselves. Someone scoped to one team must + still be able to read their own record, or scoping a user removes + their own profile from them; + 3. the target belongs to a unit the actor has authority over. + + A user in **no** unit is reachable only by (1) or (2). That is + deliberate: an unfiled user belongs to the tenant, not to a team, and a + manager scoped to one team should not inherit everyone nobody has + filed yet. + """ + auth = self.authority(actor, access_code) + if auth == TENANT_WIDE: + return True + if int(target_user_id) == int(actor.id): + return True + if not auth: + return False + + from app.modules.org.models.org_model import UserOrgUnit + + return ( + self.db.query(UserOrgUnit.id) + .filter( + UserOrgUnit.user_id == target_user_id, + UserOrgUnit.org_unit_id.in_(auth), + ) + .first() + is not None + ) + + def visible_user_ids(self, actor, access_code: str): + """ + The users *actor* may see, or `None` for "no restriction". + + `None` and `set()` mean opposite things and both are falsy, which is + exactly the confusion that makes scoped access fail open. Callers must + branch on `is None`, never on truthiness — and there is a probe that + fails if a converted endpoint gets it wrong. + """ + auth = self.authority(actor, access_code) + if auth == TENANT_WIDE: + return None + if not auth: + return {int(actor.id)} + + from app.modules.org.models.org_model import UserOrgUnit + + rows = ( + self.db.query(UserOrgUnit.user_id) + .filter(UserOrgUnit.org_unit_id.in_(auth)) + .distinct() + .all() + ) + return {int(r[0]) for r in rows} | {int(actor.id)} + + def can_delegate(self, actor, access_code: str, target_org_unit_id) -> bool: + """ + May *actor* grant a role that applies at `target_org_unit_id`? + + The escalation this closes: without it, a manager scoped to one team can + grant **themselves** a tenant-wide role and become an administrator of + the whole tenant. Every scoped-permission system has this hole unless it + is closed deliberately, because granting is just another write and looks + like one. + + Two rules: + + - **tenant-wide can only be granted by tenant-wide.** `None` means "this + role, everywhere", and nobody may hand out more authority than they + hold; + - a scoped actor may only grant within the subtree they already have + authority over. + + Mirrors `_can_delegate` in the base's `user_service.py`. + """ + auth = self.authority(actor, access_code) + if auth == TENANT_WIDE: + return True + if target_org_unit_id is None: + return False + return target_org_unit_id in auth + + def visible_unit_ids(self, user, access_code: str): + """ + `None` when the user's authority is tenant-wide, else the id set. + + Shaped for query building: `None` means "add no predicate", a set means + `.filter(Model.org_unit_id.in_(ids))`, and an **empty** set means the + user sees nothing — which callers must handle rather than treating as + "no filter". That distinction is the whole failure mode of scoped + access, so it is expressed in the return type. + """ + auth = self.authority(user, access_code) + return None if auth == TENANT_WIDE else auth + + + def _group_ids(self, user) -> list: + """ + The groups this user belongs to, within their own tenant. + + Filtered by tenant on **both** sides. A membership row naming another + tenant's group must confer nothing, and `user_access_groups` carries its + own `tenant_id` precisely so that a forged or stale row cannot reach + across. + """ + from app.modules.org.models.group_model import AccessGroup, UserAccessGroup + + rows = ( + self.db.query(UserAccessGroup.group_id) + .join(AccessGroup, AccessGroup.id == UserAccessGroup.group_id) + .filter( + UserAccessGroup.user_id == user.id, + UserAccessGroup.tenant_id == user.tenant_id, + AccessGroup.tenant_id == user.tenant_id, + ) + .all() + ) + return [r[0] for r in rows] + + def _resolve(self, user, access_code: str) -> Authority: + from app.modules.auth.models.access_model import Access + from app.modules.auth.models.role_access_model import RoleAccess + from app.modules.auth.models.role_model import Role + from app.modules.auth.models.user_role_model import UserRole + + rows = ( + self.db.query(UserRole.org_unit_id) + .join(Role, Role.id == UserRole.role_id) + .join(RoleAccess, RoleAccess.role_id == Role.id) + .join(Access, Access.id == RoleAccess.access_id) + .filter( + or_( + UserRole.user_id == user.id, + UserRole.group_id.in_(self._group_ids(user)), + ), + Access.access_code == access_code, + or_( + Role.tenant_id == user.tenant_id, + Role.tenant_id.is_(None), + ), + or_( + UserRole.expires_at.is_(None), + UserRole.expires_at > func.now(), + ), + ) + .distinct() + .all() + ) + + if user.role_id and self._legacy_role_grants(user, access_code): + return TENANT_WIDE + + if not rows: + return set() + + anchors: set[uuid.UUID] = set() + for (org_unit_id,) in rows: + if org_unit_id is None: + return TENANT_WIDE + anchors.add(org_unit_id) + + return self._subtree_ids(user.tenant_id, anchors) + + def _legacy_role_grants(self, user, access_code: str) -> bool: + from app.modules.auth.models.access_model import Access + from app.modules.auth.models.role_access_model import RoleAccess + + return ( + self.db.query(RoleAccess.role_id) + .join(Access, Access.id == RoleAccess.access_id) + .filter( + RoleAccess.role_id == user.role_id, + Access.access_code == access_code, + ) + .first() + is not None + ) + + def _subtree_ids(self, tenant_id, root_ids: set) -> set: + """ + Every live unit at or beneath *root_ids*, within this tenant. + + Both filters matter. `tenant_id` means a grant naming another tenant's + unit resolves to nothing rather than reaching into it — defence against + a forged or stale `org_unit_id`. `NOT is_deleted` means a removed unit + stops conferring access immediately, including for the branch beneath + it, because the walk cannot pass *through* a deleted node. + """ + from app.modules.org.models.org_model import OrgUnit + + if not root_ids: + return set() + + anchors = ( + self.db.query(OrgUnit.id, OrgUnit.path) + .filter( + OrgUnit.id.in_(root_ids), + OrgUnit.tenant_id == tenant_id, + OrgUnit.is_deleted.is_(False), + ) + .all() + ) + if anchors and all(path for _, path in anchors): + found: set = set() + for _, path in anchors: + rows = ( + self.db.query(OrgUnit.id) + .filter( + OrgUnit.tenant_id == tenant_id, + OrgUnit.is_deleted.is_(False), + OrgUnit.path.like(f"{path}%"), + ) + .all() + ) + found |= {r[0] for r in rows} + return found + + anchor = ( + select(OrgUnit.id.label("id")) + .where( + OrgUnit.id.in_(root_ids), + OrgUnit.tenant_id == tenant_id, + OrgUnit.is_deleted.is_(False), + ) + .cte("org_subtree", recursive=True) + ) + + descendants = select(OrgUnit.id).where( + OrgUnit.parent_id == anchor.c.id, + OrgUnit.tenant_id == tenant_id, + OrgUnit.is_deleted.is_(False), + ) + + tree = anchor.union(descendants) + return set(self.db.execute(select(tree.c.id)).scalars().all()) + + +__all__ = ["ScopeService", "TENANT_WIDE", "reset_scope_cache"] diff --git a/app/core/security.py b/app/core/security.py new file mode 100644 index 0000000..0533a43 --- /dev/null +++ b/app/core/security.py @@ -0,0 +1,39 @@ +import uuid +from datetime import datetime, timedelta +from typing import Optional +from jose import jwt +from passlib.context import CryptContext +from app.core.settings import settings + +pwd_context = CryptContext(schemes=["pbkdf2_sha256"], deprecated="auto") + +def create_access_token(data: dict, expires_delta: Optional[timedelta] = None): + to_encode = data.copy() + expire = datetime.utcnow() + ( + expires_delta or timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES) + ) + to_encode.update({ + "exp": expire, + "jti": str(uuid.uuid4()) + }) + encoded_jwt = jwt.encode(to_encode, settings.APP_SECRET, algorithm=settings.ALGORITHM) + return encoded_jwt + +def create_refresh_token(data: dict, expires_delta: Optional[timedelta] = None): + to_encode = data.copy() + expire = datetime.utcnow() + ( + expires_delta or timedelta(days=7) + ) + to_encode.update({ + "exp": expire, + "jti": str(uuid.uuid4()), + "type": "refresh" + }) + encoded_jwt = jwt.encode(to_encode, settings.APP_SECRET, algorithm=settings.ALGORITHM) + return encoded_jwt + +def get_password_hash(password: str): + return pwd_context.hash(password) + +def verify_password(plain_password: str, hashed_password: str): + return pwd_context.verify(plain_password, hashed_password) diff --git a/app/core/settings.py b/app/core/settings.py new file mode 100644 index 0000000..db3543e --- /dev/null +++ b/app/core/settings.py @@ -0,0 +1,268 @@ +""" +Central Application Settings +Loads environment variables using Pydantic v2 +""" + +import os +from typing import Optional, List +from pydantic_settings import BaseSettings, SettingsConfigDict +from pydantic import ConfigDict, model_validator + + +class Settings(BaseSettings): + + APP_ENV: str + + HOST: str + PORT: int + DEBUG: bool + + DB_HOST: str + ZOHO_CLIENT_ID: str = "" + ZOHO_CLIENT_SECRET: str = "" + ZOHO_REFRESH_TOKEN: str = "" + ZOHO_WEBHOOK_SECRET: str = "" + ZOHO_DOMAIN: str = "in" + DB_PORT: int + DB_NAME: str + DB_USER: str + DB_PASSWORD: str + DB_SSLMODE: str + + PGBOUNCER_PORT: Optional[int] = None + + @property + def DATABASE_URL(self) -> str: + import urllib.parse + user = urllib.parse.quote(self.DB_USER, safe='') + password = urllib.parse.quote(self.DB_PASSWORD, safe='') + return ( + f"postgresql+psycopg2://{user}:" + f"{password}@" + f"{self.DB_HOST}:{self.DB_PORT}/" + f"{self.DB_NAME}?sslmode={self.DB_SSLMODE}" + ) + + @property + def PGBOUNCER_URL(self) -> Optional[str]: + if not self.PGBOUNCER_PORT: + return None + import urllib.parse + user = urllib.parse.quote(self.DB_USER, safe='') + password = urllib.parse.quote(self.DB_PASSWORD, safe='') + return ( + f"postgresql+psycopg2://{user}:" + f"{password}@" + f"{self.DB_HOST}:{self.PGBOUNCER_PORT}/" + f"{self.DB_NAME}?sslmode={self.DB_SSLMODE}" + ) + + APP_SECRET: str + ALGORITHM: str + ACCESS_TOKEN_EXPIRE_MINUTES: int + MAX_ACTIVE_DEVICES: int = 3 + + REDIS_HOST: Optional[str] + REDIS_PORT: Optional[int] + REDIS_PASSWORD: Optional[str] + + @property + def REDIS_URL(self) -> Optional[str]: + if not self.REDIS_HOST: + return None + + import urllib.parse + + encoded_password = ( + urllib.parse.quote( + self.REDIS_PASSWORD) if self.REDIS_PASSWORD else None + ) + + if encoded_password: + return f"redis://:{encoded_password}@{self.REDIS_HOST}:{self.REDIS_PORT}/0" + return f"redis://{self.REDIS_HOST}:{self.REDIS_PORT}/0" + + CHAT_REDIS_MAX_MESSAGES: int + CHAT_REDIS_TTL_SECONDS: int + + STORAGE_DRIVE_DIR: str + + CLAMAV_HOST: str + CLAMAV_PORT: int + CLAMAV_TIMEOUT_SECONDS: float = 20.0 + CLAMAV_CHUNK_SIZE: int = 1024 * 1024 + + API_BASE_URL: str + + GOOGLE_CLIENT_ID: Optional[str] = None + GOOGLE_CLIENT_SECRET: Optional[str] = None + + SAAS_TRUST_SECRET: Optional[str] = None + SAAS_BASE_URL: Optional[str] = None + SAAS_PUBLIC_KEY: Optional[str] = None + + DOCUSEAL_API_URL: str = "http://localhost:3000" + DOCUSEAL_API_KEY: Optional[str] = None + DOCUSEAL_ACCOUNT_EMAIL: Optional[str] = None + + DEEPSEEK_API_KEY: Optional[str] = None + DEEPSEEK_BASE_URL: str = "https://api.deepseek.com/v1" + DEEPSEEK_MODEL: str = "deepseek-chat" + LLM_TEMPERATURE: float = 0.2 + + EMBEDDING_MODEL: str = "BAAI/bge-base-en-v1.5" + + EXTRACTION_API_KEY: Optional[str] = None + EXTRACTION_BASE_URL: str = "https://integrate.api.nvidia.com/v1" + EXTRACTION_MODEL: str = "mistralai/mistral-medium-3-instruct" + EXTRACTION_VISION_MODEL: str = "meta/llama-3.2-90b-vision-instruct" + + HF_HUB_OFFLINE: bool = False + TRANSFORMERS_OFFLINE: bool = False + + CHUNK_SIZE: int = 850 + CHUNK_OVERLAP: int = 180 + TOP_K_QA: int = 3 + TOP_K_RETRIEVAL: int = 12 + TOP_K_RERANK: int = 6 + MAX_CHAT_HISTORY: int = 5 + TOP_K_SUMMARY: int = 4 + MAX_CONTEXT_CHARS: int = 4000 + MAX_RESPONSE_TOKENS: int = 1024 + CHAT_DAILY_CREDITS_LIMIT: int + MIN_SIMILARITY_SCORE: float = 0.18 + RAG_DOMAIN_PROFILE: str = "auto" + RAG_K_SCHEDULE: str = "8,16,24,36,48" + RAG_MAX_CANDIDATES: int = 180 + RAG_STAGNATION_PATIENCE: int = 2 + RAG_RERANK_MAX_DOCS: int = 48 + RAG_SEMANTIC_WEIGHT: float = 1.0 + RAG_KEYWORD_HIT_WEIGHT: float = 0.08 + RAG_KEYWORD_COVERAGE_WEIGHT: float = 0.35 + RAG_COVERAGE_TARGET_RATIO: float = 0.75 + RAG_MAX_SELECTED_DOCS: int = 40 + RAG_CONTEXT_BASE_BUDGET: int = 4200 + RAG_CONTEXT_MAX_BUDGET: int = 9000 + RAG_PER_SOURCE_CHAR_CAP: int = 680 + RAG_ENABLE_EXHAUSTIVE_COVERAGE: bool = True + RAG_COVERAGE_HARD_FAIL: bool = False + RAG_MIN_COVERAGE_TERM_LEN: int = 4 + RAG_COVERAGE_MAX_POINTS: int = 20000 + RAG_COVERAGE_MAX_PASSES: int = 4 + RAG_SCOPE_CACHE_TTL_SECONDS: int = 300 + RAG_SCOPE_CACHE_MAX_DOCS: int = 24 + RAG_MICRO_MAX_SENTENCES: int = 6 + RAG_MICRO_CHUNK_CHAR_CAP: int = 1200 + RAG_STRICT_CONTEXT_MAX_BUDGET: int = 18000 + RAG_STRICT_PER_SOURCE_CHAR_CAP: int = 2000 + RAG_STRICT_EVIDENCE_ONLY_CONTEXT: bool = True + RAG_SECTION_FULL_CHUNK_MAX_CHARS: int = 2600 + RAG_SECTION_FOCUS_ONLY: bool = True + RAG_SECTION_FOCUS_CHUNK_WINDOW: int = 0 + RAG_SECTION_PROBE_NEIGHBOR_WINDOW: int = 2 + RAG_ENABLE_SECTION_FAST_PATH: bool = True + RAG_ENABLE_FAST_PATH_AUTO_DOC: bool = True + RAG_FAST_PATH_AUTO_DOC_TOP_K: int = 10 + RAG_FAST_PATH_AUTO_DOC_MAX_TRY_DOCS: int = 4 + RAG_ENABLE_EARLY_OOD_CHECK: bool = True + RAG_SECTION_FAST_MAX_CHARS: int = 7000 + RAG_SECTION_FAST_MAX_BULLETS: int = 32 + RAG_SECTION_FAST_PER_SUBSECTION_BULLETS: int = 8 + RAG_SECTION_FAST_COMPACT_ROOT: bool = True + RAG_SECTION_FAST_SUBSECTION_CHAR_CAP: int = 900 + RAG_SECTION_FAST_USE_LLM_FOR_ROOT: bool = False + RAG_SECTION_FAST_LLM_MAX_CHARS: int = 7000 + RAG_REQUIRE_DOCUMENT_ID: bool = True + RAG_DOC_OVERVIEW_CONTEXT_CHARS: int = 16000 + RAG_DOC_OVERVIEW_MAX_SECTIONS: int = 14 + RAG_DOC_OVERVIEW_POINTS_PER_SECTION: int = 3 + RAG_RESCUE_CONTEXT_CHAR_CAP: int = 2600 + RAG_RESPONSE_TOKEN_SMALL: int = 180 + RAG_RESPONSE_TOKEN_MEDIUM: int = 280 + RAG_RESPONSE_TOKEN_LARGE: int = 420 + RAG_RESPONSE_TOKEN_HARD_CAP: int = 520 + VECTOR_DB: str = "qdrant" + QDRANT_HOST: str = "localhost" + QDRANT_PORT: int = 6333 + QDRANT_COLLECTION: str = "documents" + RERANKER_MODEL: str = "cross-encoder/ms-marco-MiniLM-L-12-v2" + + SMTP_HOST: Optional[str] = None + SMTP_PORT: Optional[int] = 587 + SMTP_USER: Optional[str] = None + SMTP_PASSWORD: Optional[str] = None + SMTP_SECURE: bool = False + MAIL_FROM: Optional[str] = "noreply@docqube.com" + + FRONTEND_URL: str = "http://localhost:5173" + + LOG_LEVEL: str + + CORS_ORIGINS: str + + @property + def CORS_ORIGINS_LIST(self) -> List[str]: + return [origin.strip() for origin in self.CORS_ORIGINS.split(",") if origin] + + model_config = SettingsConfigDict( + env_file=os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))), + f".env.{os.getenv('APP_ENV', 'development')}", + ), + extra="ignore", + ) + + @model_validator(mode="after") + def validate_production_config(self) -> "Settings": + if self.MAX_ACTIVE_DEVICES < 1: + raise ValueError("MAX_ACTIVE_DEVICES must be at least 1") + + if self.APP_ENV == "production": + if not self.REDIS_HOST: + raise ValueError( + "REDIS_HOST must be explicitly set in production environment " + "to prevent silent fallback to insecure defaults." + ) + # Enforce PgBouncer in production: app/db/database.py silently + # falls back to a direct PostgreSQL connection (DATABASE_URL, + # DB_PORT) whenever PGBOUNCER_PORT is unset — with no error and + # no log warning at the point it matters. That silent fallback + # is what let the API run for a time connected straight to + # Postgres on 5432 instead of through PgBouncer on 6432 during + # a prior incident. Failing fast here, before the DB engine is + # ever created, converts that into a startup error instead. + if not self.PGBOUNCER_PORT: + raise ValueError( + "PGBOUNCER_PORT must be explicitly set in production environment " + "to prevent silently falling back to a direct PostgreSQL connection." + ) + # CORS check + if "*" in self.CORS_ORIGINS_LIST: + raise ValueError( + "CORS_ORIGINS must not contain '*' in production environment " + "to prevent unauthorized cross-origin requests." + ) + if self.ZOHO_CLIENT_ID and not self.ZOHO_WEBHOOK_SECRET: + import logging + logging.getLogger(__name__).warning( + "ZOHO_WEBHOOK_SECRET is empty in production. This silently disables webhook authentication " + "and leaves your Zoho Sign integration vulnerable to spoofed requests." + ) + return self + + COOKIE_DOMAIN: Optional[str] = None + DISABLE_CSRF: bool = False + + TENANT_FILTER_ENABLED: bool = False + + ORG_SCOPE_ENABLED: bool = False + + SUBSCRIPTION_REQUIRED_FOR_MAPPED_TENANTS: bool = False + + SUBSCRIPTION_ENFORCEMENT_ENABLED: bool = False + + ACCESS_LOG_RETENTION_DAYS: int = 365 + +settings = Settings() + +DATABASE_URL = settings.DATABASE_URL diff --git a/app/core/ssrf_protection.py b/app/core/ssrf_protection.py new file mode 100644 index 0000000..e2b8527 --- /dev/null +++ b/app/core/ssrf_protection.py @@ -0,0 +1,79 @@ +import socket +import logging +from urllib.parse import urlparse +from ipaddress import ip_address, ip_network +from fastapi import HTTPException + +logger = logging.getLogger(__name__) + +PRIVATE_IP_RANGES = [ + ip_network('10.0.0.0/8'), + ip_network('172.16.0.0/12'), + ip_network('192.168.0.0/16'), + ip_network('127.0.0.0/8'), + ip_network('169.254.0.0/16'), + ip_network('::1/128'), + ip_network('fc00::/7'), + ip_network('fe80::/10'), +] + +ALLOWED_SCHEMES = ['http', 'https'] +ALLOWED_HOSTS = [] + +def is_safe_url(url: str, allowed_hosts: list = None, allowed_schemes: list = None, require_https: bool = False) -> bool: + """ + Check if a URL is safe from SSRF by validating the scheme, host, and IP. + """ + try: + parsed = urlparse(url) + + schemes = allowed_schemes if allowed_schemes is not None else ALLOWED_SCHEMES + if require_https: + schemes = ['https'] + + if parsed.scheme.lower() not in schemes: + logger.warning(f"SSRF Prevention: Invalid scheme {parsed.scheme} in URL {url}") + return False + + host = parsed.hostname + if not host: + logger.warning(f"SSRF Prevention: No hostname in URL {url}") + return False + + if allowed_hosts and host not in allowed_hosts: + logger.warning(f"SSRF Prevention: Host {host} not in whitelist for URL {url}") + return False + + try: + ips = socket.getaddrinfo(host, parsed.port or (443 if parsed.scheme == 'https' else 80)) + for family, _, _, _, sockaddr in ips: + addr_str = sockaddr[0] + if '%' in addr_str: + addr_str = addr_str.split('%')[0] + + addr = ip_address(addr_str) + for private_range in PRIVATE_IP_RANGES: + if addr in private_range: + logger.warning(f"SSRF Prevention: URL {url} resolved to private IP {addr}") + return False + except (socket.gaierror, ValueError) as e: + logger.warning(f"SSRF Prevention: Could not resolve or parse host {host} for URL {url}: {e}") + return False + + return True + except Exception as e: + logger.error(f"SSRF Prevention: Error validating URL {url}: {e}") + return False + +def get_safe_fetcher(): + """ + Returns a URL fetcher function compatible with WeasyPrint. + """ + from weasyprint import default_url_fetcher + + def safe_fetcher(url, timeout=10, **kwargs): + if not is_safe_url(url): + raise ValueError(f"SSRF Prevention: Blocked request to unsafe URL: {url}") + return default_url_fetcher(url, timeout=timeout, **kwargs) + + return safe_fetcher diff --git a/app/core/tenant_context.py b/app/core/tenant_context.py new file mode 100644 index 0000000..bdcdb9a --- /dev/null +++ b/app/core/tenant_context.py @@ -0,0 +1,114 @@ +""" +B1.1 — tenant context. + +The request's tenant, held in context variables so the query listener can reach +it without every call site passing it down. + +The important state is not the tenant id but **whether the context was set at +all**. DocQube's defect is that a lost tenant currently escalates: `tenant_id IS +NULL` means super admin, so code that forgets to scope a query sees everything. +Distinguishing "no tenant, because nobody set one" from "tenant is None, which +means superuser" is what lets the listener deny rather than grant. +""" + +from __future__ import annotations + +import functools +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from contextvars import ContextVar, Token +from typing import Any, TypeVar +from uuid import UUID + +_tenant_id: ContextVar[UUID | None] = ContextVar("tenant_id", default=None) +_is_super_admin: ContextVar[bool] = ContextVar("is_super_admin", default=False) +_bypass: ContextVar[bool] = ContextVar("tenant_bypass", default=False) +_context_set: ContextVar[bool] = ContextVar("tenant_context_set", default=False) + +F = TypeVar("F", bound=Callable[..., Any]) + + +def current_tenant_id() -> UUID | None: + return _tenant_id.get() + + +def is_super_admin() -> bool: + return _is_super_admin.get() + + +def is_bypassed() -> bool: + return _bypass.get() + + +def context_was_set() -> bool: + """True when something deliberately established a tenant context.""" + return _context_set.get() + + +def set_context( + tenant_id: UUID | None, is_super: bool = False, bypass: bool = False +) -> tuple[Token, Token, Token, Token]: + return ( + _tenant_id.set(tenant_id), + _is_super_admin.set(is_super), + _bypass.set(bypass), + _context_set.set(True), + ) + + +def reset_context(tokens: tuple[Token, Token, Token, Token]) -> None: + t_tid, t_super, t_bypass, t_set = tokens + _tenant_id.reset(t_tid) + _is_super_admin.reset(t_super) + _bypass.reset(t_bypass) + _context_set.reset(t_set) + + +@contextmanager +def scoped_to(tenant_id: UUID | None, is_super: bool = False) -> Iterator[None]: + """Run a block scoped to one tenant.""" + tokens = set_context(tenant_id, is_super=is_super, bypass=False) + try: + yield + finally: + reset_context(tokens) + + +@contextmanager +def unscoped() -> Iterator[None]: + """ + Run a block with tenant filtering switched off. + + For genuine system work — migrations, background sweeps, the seed script. + Every use is a place where the guarantee does not apply, so they should be + few and obvious. + """ + tokens = (_bypass.set(True), _context_set.set(True)) + try: + yield + finally: + _bypass.reset(tokens[0]) + _context_set.reset(tokens[1]) + + +def system_operation(fn: F) -> F: + """Decorator form of `unscoped()`.""" + + @functools.wraps(fn) + def wrapper(*args: Any, **kwargs: Any) -> Any: + with unscoped(): + return fn(*args, **kwargs) + + return wrapper + + +class TenantUnscoped: + """ + Opt a model *out* of tenant filtering. + + Scoping is derived from the presence of a `tenant_id` column rather than + from a marker mixin, so a new tenant-owned model is protected the day it is + written and nobody has to remember anything. This marker exists for the rare + model that carries a `tenant_id` but must be readable across tenants — and + applying it should require an argument in review. + """ diff --git a/app/core/tenant_filter.py b/app/core/tenant_filter.py new file mode 100644 index 0000000..90aaf6b --- /dev/null +++ b/app/core/tenant_filter.py @@ -0,0 +1,180 @@ +""" +B1.1 — the fail-closed query listener. + +Every SELECT against a tenant-owned table gets a tenant predicate added. With no +tenant context it matches **nothing**, which is the whole point: a bug that loses +the tenant used to see everything, because `tenant_id IS NULL` doubled as super +admin. Now the same bug returns an empty page. + +**Off by default, but no longer blocked.** The ordering constraint is satisfied: +the `tenant_id IS NULL` fallback in `is_superadmin()` is gone, privilege is an +explicit flag, and `TenantContextMiddleware` (B1.4) sets both the tenant and the +flag on every request. The whole suite passes with `TENANT_FILTER_ENABLED=true`, +and CI runs it that way. + +It stays off by default because the switch is the rollback — turning it off takes +effect without a deploy — and because enabling it needs the *data* to be ready, +not just the code. `scripts/b1_preflight.py` is the check. + +Which models are scoped is **derived from the schema**, not from a marker list. +A model with a `tenant_id` column is protected the day it is written. Opting out +requires inheriting `TenantUnscoped`, which is visible in review. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from sqlalchemy import event, false, or_ +from sqlalchemy.orm import Session, with_loader_criteria + +logger = logging.getLogger(__name__) + +_scoped_cache: dict[str, Any] | None = None +_scoped_cache_mappers: int = -1 + + +def scoped_models() -> dict[str, Any]: + """ + Mapped classes carrying a `tenant_id` column, by table name. + + Cached, but **invalidated when the registry grows** — and that second half + is load-bearing, not tidiness. + + The cache used to be computed once and kept forever. The first call happens + on the first query, which in a running application is early: before every + model module has necessarily been imported. Anything mapped after that point + was absent from the set and therefore **silently never filtered** — a + tenant-owned table with no tenant predicate, and nothing anywhere to say so. + + Found when `user_roles` gained a `tenant_id` and the isolation ratchet + reported it unscoped with the filter enabled and scoped with it disabled. + The difference was import order: enabling the filter makes the listener run + during startup, which populated the cache before the model was loaded. + + Counting mappers is enough. SQLAlchemy's registry only grows, so a changed + count means new classes and a stale set. + """ + global _scoped_cache, _scoped_cache_mappers + + from app.core.tenant_context import TenantUnscoped + from app.db.database import Base + + mapper_count = len(Base.registry.mappers) + if _scoped_cache is not None and _scoped_cache_mappers == mapper_count: + return _scoped_cache + + found: dict[str, Any] = {} + for mapper in Base.registry.mappers: + cls = mapper.class_ + if issubclass(cls, TenantUnscoped): + continue + if "tenant_id" in mapper.columns: + found[mapper.local_table.name] = cls + _scoped_cache = found + _scoped_cache_mappers = mapper_count + return found + + +def reset_scoped_cache() -> None: + """Forget the derived set — used by tests that define models on the fly.""" + global _scoped_cache, _scoped_cache_mappers + _scoped_cache = None + _scoped_cache_mappers = -1 + + +def _table_names(statement: Any) -> set[str]: + """ + Every table a statement touches, including through subqueries. + + Walking `get_final_froms()` by hand is not enough: `query(X).count()` + compiles to SELECT count(*) FROM (SELECT ...) and the outer FROM is an + anonymous subquery, so a naive walk sees only the alias. SQLAlchemy's own + `find_tables` handles the nesting. + """ + from sqlalchemy.sql.util import find_tables + + try: + return { + table.name + for table in find_tables( + statement, + include_aliases=True, + include_joins=True, + include_selects=True, + include_crud=True, + ) + } + except Exception: # noqa: BLE001 - not every construct can be walked + return set() + + +def _enabled() -> bool: + from app.core.settings import settings + + return bool(getattr(settings, "TENANT_FILTER_ENABLED", False)) + + +@event.listens_for(Session, "do_orm_execute") +def _apply_tenant_filter(execute_state: Any) -> None: + if not _enabled(): + return + if not execute_state.is_select: + return + if execute_state.is_column_load or execute_state.is_relationship_load: + return + + from app.core.tenant_context import ( + context_was_set, + current_tenant_id, + is_bypassed, + is_super_admin, + ) + + if is_bypassed() or is_super_admin(): + from app.core.tenant_metrics import metrics + + metrics.record_bypassed() + return + + by_table = scoped_models() + if not by_table: + return + + targets = { + mapper.class_ + for mapper in execute_state.all_mappers + if mapper.local_table is not None + and mapper.local_table.name in by_table + } + + if not targets: + targets = { + by_table[name] + for name in _table_names(execute_state.statement) + if name in by_table + } + if not targets: + return + + fail_closed = not context_was_set() + tenant_id = current_tenant_id() + + from app.core.tenant_metrics import metrics + + if fail_closed: + names = sorted(cls.__name__ for cls in targets) + metrics.record_fail_closed(names) + logger.warning("Tenant filter closed a query with no tenant context: %s", names) + else: + metrics.record_scoped() + + for cls in targets: + if fail_closed: + criteria = false() + else: + criteria = or_(cls.tenant_id == tenant_id, cls.tenant_id.is_(None)) + execute_state.statement = execute_state.statement.options( + with_loader_criteria(cls, criteria, include_aliases=True) + ) diff --git a/app/core/tenant_metrics.py b/app/core/tenant_metrics.py new file mode 100644 index 0000000..c0e495b --- /dev/null +++ b/app/core/tenant_metrics.py @@ -0,0 +1,73 @@ +""" +B1.6 / B4.5 — see what the tenant filter is doing. + +After B1 lands, the failure mode is not a crash. It is a listener quietly +returning nothing for a legitimate query, and the first report arrives from a +customer saying a page is empty. Counters make that visible in minutes instead. + +Three numbers, deliberately few: + + ``fail_closed`` a query ran with no tenant context and was denied. In + steady state this should be **zero**. Anything above zero + is a code path that lost the tenant — a bug that used to + escalate silently and now denies loudly. + ``scoped`` queries filtered normally. The denominator. + ``bypassed`` explicit system operations. Should be small and stable; a + rise means someone is reaching for ``unscoped()`` to make + a problem go away. + +In-process and reset on restart. That is enough to answer "is the rollout +safe?", which is what B1.6 needs. Wiring these to the metrics backend is B4.5. +""" + +from __future__ import annotations + +import threading +from dataclasses import dataclass, field + + +@dataclass +class TenantFilterMetrics: + fail_closed: int = 0 + scoped: int = 0 + bypassed: int = 0 + fail_closed_by_model: dict[str, int] = field(default_factory=dict) + _lock: threading.Lock = field(default_factory=threading.Lock, repr=False) + + def record_fail_closed(self, models: list[str]) -> None: + with self._lock: + self.fail_closed += 1 + for name in models: + self.fail_closed_by_model[name] = ( + self.fail_closed_by_model.get(name, 0) + 1 + ) + + def record_scoped(self) -> None: + with self._lock: + self.scoped += 1 + + def record_bypassed(self) -> None: + with self._lock: + self.bypassed += 1 + + def snapshot(self) -> dict: + with self._lock: + total = self.fail_closed + self.scoped + self.bypassed + return { + "fail_closed": self.fail_closed, + "scoped": self.scoped, + "bypassed": self.bypassed, + "total": total, + "fail_closed_rate": (self.fail_closed / total) if total else 0.0, + "fail_closed_by_model": dict(self.fail_closed_by_model), + } + + def reset(self) -> None: + with self._lock: + self.fail_closed = 0 + self.scoped = 0 + self.bypassed = 0 + self.fail_closed_by_model.clear() + + +metrics = TenantFilterMetrics() diff --git a/app/core/token_blacklist.py b/app/core/token_blacklist.py new file mode 100644 index 0000000..71292a9 --- /dev/null +++ b/app/core/token_blacklist.py @@ -0,0 +1,74 @@ +from app.db.redis import redis_cache +import logging + +logger = logging.getLogger(__name__) + +class TokenBlacklist: + """ + Service for managing revoked JWT tokens using Redis. + Uses the 'jti' (JWT ID) claim to identify tokens for blacklisting. + """ + + @staticmethod + def add(jti: str, expires_in: int = 3600): + """ + Add a token's JTI to the blacklist with an expiration time. + + Args: + jti: The unique JWT ID (jti claim) + expires_in: Time in seconds until the token naturally expires + """ + if not jti: + return + + try: + key = f"blacklist:{jti}" + redis_cache.set(key, {"revoked": True}, ttl=expires_in) + except Exception as e: + logger.warning(f"Failed to blacklist JTI: {e}") + + @staticmethod + def is_token_blacklisted(token: str) -> bool: + """ + Check whether a raw JWT has been revoked, by reading its `jti` claim. + + Callers that hold a raw token must use this rather than passing the + token to `is_blacklisted`: the blacklist is keyed on `jti`, so a raw + token builds the key `blacklist:`, which is never written + and therefore never matches. Both WebSocket endpoints did exactly that, + which silently disabled revocation on the whole real-time surface. + """ + if not token: + return False + try: + from jose import jwt + + from app.core.settings import settings + + payload = jwt.decode( + token, + settings.APP_SECRET, + algorithms=[settings.ALGORITHM], + options={"verify_exp": False}, + ) + except Exception: + return False + return TokenBlacklist.is_blacklisted(payload.get("jti")) + + @staticmethod + def is_blacklisted(jti: str) -> bool: + """ + Check if a token's JTI is in the blacklist. + + Takes a `jti` claim, not a raw JWT. See `is_token_blacklisted`. + """ + if not jti: + return False + + try: + key = f"blacklist:{jti}" + value = redis_cache.get(key) + return value is not None + except Exception as e: + logger.warning(f"Failed to check JTI blacklist: {e}") + return False diff --git a/app/core/transactions.py b/app/core/transactions.py new file mode 100644 index 0000000..0444f30 --- /dev/null +++ b/app/core/transactions.py @@ -0,0 +1,88 @@ +""" +B2.4 — keeping object storage and the database consistent. + +B2 made the request own the database transaction. Object storage has no such +boundary: an upload to Backblaze is durable the moment it returns, and no +rollback undoes it. So making the database atomic on its own moves the problem +rather than solving it — + + * a **write** that succeeds in storage and then fails in the database leaves + an object with no row pointing at it, and now not even a half-written row to + find it by; + * a **delete** that removes the object first and then fails in the database + leaves a row pointing at nothing, which is worse: the application will try + to serve it. + +Two primitives, used at the point where the storage call happens: + + compensate_on_rollback(session, lambda: delete(key)) + The object is already written. If the transaction rolls back, remove it. + + defer_until_commit(session, lambda: delete(key)) + Do not touch storage yet. If the transaction commits, do it then. + +Both are per-session and fire once. A failure inside a callback is logged and +swallowed: compensation is best-effort by nature, and raising from it would +replace a tidy-up problem with a failed request. + +This is the cheap half of the outbox pattern. The full version — durable intent +that survives a process restart — is B5.1, and is worth having if the roadmap +ever needs guaranteed delivery. What is here covers the common case: a crash +between the two systems within a single request. +""" + +from __future__ import annotations + +import logging +from typing import Callable, List + +from sqlalchemy import event +from sqlalchemy.orm import Session + +logger = logging.getLogger(__name__) + +_ON_ROLLBACK = "_docqube_compensate_on_rollback" +_ON_COMMIT = "_docqube_defer_until_commit" + + +def _bucket(session: Session, key: str) -> List[Callable[[], None]]: + actions = session.info.get(key) + if actions is None: + actions = [] + session.info[key] = actions + return actions + + +def compensate_on_rollback(session: Session, action: Callable[[], None]) -> None: + """Run `action` if this session's transaction is rolled back.""" + _bucket(session, _ON_ROLLBACK).append(action) + + +def defer_until_commit(session: Session, action: Callable[[], None]) -> None: + """Run `action` only once this session's transaction has committed.""" + _bucket(session, _ON_COMMIT).append(action) + + +def _drain(session: Session, key: str, what: str) -> None: + actions = session.info.pop(key, None) + if not actions: + return + for action in actions: + try: + action() + except Exception: # noqa: BLE001 - tidy-up must not fail the request + logger.exception("Storage %s action failed", what) + + +@event.listens_for(Session, "after_commit") +def _run_deferred(session: Session) -> None: + _drain(session, _ON_COMMIT, "post-commit") + session.info.pop(_ON_ROLLBACK, None) + + +@event.listens_for(Session, "after_soft_rollback") +def _run_compensation(session: Session, previous_transaction) -> None: + if getattr(previous_transaction, "nested", False): + return + _drain(session, _ON_ROLLBACK, "compensating") + session.info.pop(_ON_COMMIT, None) diff --git a/app/core/ws_auth.py b/app/core/ws_auth.py new file mode 100644 index 0000000..7522aa7 --- /dev/null +++ b/app/core/ws_auth.py @@ -0,0 +1,93 @@ +""" +B0.4.5 — one authenticator for the WebSocket surface. + +There were two `_authenticate_ws` functions, in `app/api/ws_router.py` and +`app/modules/collab/routes/chat_routes.py`, near-identical and independently +maintained. That is how they drifted, and the drift was not cosmetic: + +- both keyed the token blacklist on the raw JWT where a `jti` was expected, so + revocation silently did nothing on the entire real-time surface; +- neither rejected `refresh` tokens, so a long-lived refresh token could open a + socket it could not have opened an HTTP request with; +- neither checked whether the user was deleted or deactivated; +- and neither checked the **tenant**, so suspending a tenant closed no sockets. + A suspended customer kept live document collaboration and chat for as long as + the connection stayed open, which on a WebSocket is indefinitely. + +The first two were fixed in both copies. Fixing the rest in both copies would +have been the third round of the same edit, so this is the shared version the +plan called for: one function, one set of checks, both callers delegating. + +**Deliberately not `get_current_user`.** That is a FastAPI dependency built on +`Request` and `Depends`; a WebSocket handshake has neither. What it *does* have +is the same token, so the checks are what get shared, not the plumbing. Every +rule here mirrors one in `get_current_user`, and +`tests/probes/test_websocket_auth.py` asserts the two surfaces agree. +""" + +import logging +from typing import Optional + +from jose import JWTError, jwt +from sqlalchemy.orm import Session + +from app.core.settings import settings +from app.core.token_blacklist import TokenBlacklist +from app.modules.auth.models.user_model import User + +logger = logging.getLogger(__name__) + + +def authenticate_websocket(token: str, db: Session) -> Optional[User]: + """ + The user this token authorises for a socket, or None. + + Returns None for every failure rather than raising: the caller's only + remedy is to close the handshake, and distinguishing *why* would tell an + unauthenticated peer whether an account exists. + """ + if not token: + return None + + try: + payload = jwt.decode( + token, settings.APP_SECRET, algorithms=[settings.ALGORITHM] + ) + except JWTError: + return None + + if payload.get("type") in ("refresh", "device_management"): + return None + + jti = payload.get("jti") + if jti and TokenBlacklist.is_blacklisted(jti): + return None + + sub = payload.get("sub") + if not sub: + return None + + try: + user = db.get(User, int(sub)) + except (ValueError, TypeError): + user = db.query(User).filter(User.email == str(sub)).first() + + if user is None: + return None + + if getattr(user, "is_deleted", False) or not getattr(user, "is_active", True): + return None + + tenant = getattr(user, "tenant", None) + if tenant is not None and not getattr(tenant, "is_active", True): + logger.info( + "WebSocket auth refused: tenant %s is not active", getattr(tenant, "id", "?") + ) + return None + if tenant is not None and getattr(tenant, "is_deleted", False): + return None + + return user + + +__all__ = ["authenticate_websocket"] diff --git a/app/db/all_models.py b/app/db/all_models.py new file mode 100644 index 0000000..1fffe37 --- /dev/null +++ b/app/db/all_models.py @@ -0,0 +1,95 @@ +""" +Every mapped class, registered — importing this module is the guarantee. + +SQLAlchemy only knows about a model once its module has been imported. Nothing +was making that happen for the whole set: `alembic/env.py` imported `app.main` +and trusted that starting the application would drag every model in behind it. +Mostly it did, because routes import the models they use at module level. + +Three did not. `UserRole`, `AccessGroup` and `UserAccessGroup` are imported +*inside functions* everywhere they appear — a deliberate style elsewhere in the +codebase, and harmless right up until something needs the full registry: + +- **`alembic revision --autogenerate` proposed `DROP TABLE user_roles`, + `access_groups` and `user_access_groups`.** Autogenerate diffs the database + against `Base.metadata`; a table present in one and absent from the other + reads as "deleted". The three tables missing are the three that carry scoped + role grants and group membership — the entire SBAC layer — so the next person + to run autogenerate and apply the result without reading it drops + authorization and keeps the app booting. +- `Base.metadata.create_all()` would not create them. +- Anything deriving behaviour from the registry sees a partial picture. The + tenant filter is safe here only because `scoped_models()` invalidates on + mapper count, which was itself added after this class of bug bit once. + +So the imports are explicit and in one place. `tests/probes/test_model_registry.py` +fails if a model module exists and is not listed here, which is what keeps the +list honest — a new model is registered on the day it is written, not the day +someone notices it missing from a migration. +""" + +from app.modules.activity_logs.models import activity_log as _activity_log +from app.modules.auth.models import access_model as _access +from app.modules.auth.models import device_model as _device +from app.modules.auth.models import role_access_model as _role_access +from app.modules.auth.models import role_model as _role +from app.modules.auth.models import saas_models as _saas +from app.modules.auth.models import session_model as _session +from app.modules.auth.models import user_model as _user +from app.modules.auth.models import user_role_model as _user_role +from app.modules.billing.models import plan_model as _plan +from app.modules.chat.models import chat_message_model as _chat_message +from app.modules.chat.models import chat_model as _chat +from app.modules.chat.models import chat_session_model as _chat_session +from app.modules.chat.models import chat_usage_model as _chat_usage +from app.modules.collab.models import collab_model as _collab +from app.modules.configuration.models import ( + system_configuration_model as _system_configuration, +) +from app.modules.documents.models import document_model as _document +from app.modules.drive.models import drive_model as _drive +from app.modules.extraction.models import extraction_model as _extraction +from app.modules.notifications.models import notification_model as _notification +from app.modules.org.models import group_model as _group +from app.modules.org.models import org_model as _org +from app.modules.signing.models import signature_config_model as _signature_config +from app.modules.signing.models import signature_imprint as _signature_imprint +from app.modules.signing.models import signing_request as _signing_request +from app.modules.storage.models import storage_model as _storage +from app.modules.tenant.models import tenant_contact_model as _tenant_contact +from app.modules.tenant.models import tenant_model as _tenant +from app.modules.tenant.models import tenant_smtp_config_model as _tenant_smtp +from app.modules.tenant.models import tenant_storage_config_model as _tenant_storage + +__all__ = [ + "_access", + "_activity_log", + "_chat", + "_chat_message", + "_chat_session", + "_chat_usage", + "_collab", + "_device", + "_document", + "_drive", + "_extraction", + "_group", + "_notification", + "_org", + "_plan", + "_role", + "_role_access", + "_saas", + "_session", + "_signature_config", + "_signature_imprint", + "_signing_request", + "_storage", + "_system_configuration", + "_tenant", + "_tenant_contact", + "_tenant_smtp", + "_tenant_storage", + "_user", + "_user_role", +] diff --git a/app/db/database.py b/app/db/database.py new file mode 100644 index 0000000..c4b5dec --- /dev/null +++ b/app/db/database.py @@ -0,0 +1,100 @@ +import logging +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker, DeclarativeBase, Session +from contextlib import contextmanager +from app.core.settings import DATABASE_URL, settings + +from sqlalchemy.pool import NullPool, QueuePool + +logger = logging.getLogger(__name__) + +class Base(DeclarativeBase): + pass + +active_url = settings.PGBOUNCER_URL if (settings.PGBOUNCER_URL and settings.PGBOUNCER_PORT) else DATABASE_URL + +if settings.PGBOUNCER_URL and settings.PGBOUNCER_PORT: + engine = create_engine( + active_url, + poolclass=NullPool, + echo=False, + ) +elif settings.APP_ENV == "development": + engine = create_engine( + active_url, + poolclass=NullPool, + echo=False, + ) +else: + engine = create_engine( + active_url, + poolclass=QueuePool, + pool_pre_ping=True, + pool_recycle=900, + pool_size=3, + max_overflow=7, + pool_timeout=30, + echo=False, + ) + +SessionLocal = sessionmaker( + bind=engine, + autocommit=False, + autoflush=False, +) + +def get_db(): + db = SessionLocal() + try: + yield db + db.commit() + except Exception: + db.rollback() + raise + finally: + db.close() + +@contextmanager +def get_db_connection() -> Session: + db = SessionLocal() + try: + yield db + db.commit() + except Exception: + db.rollback() + raise + finally: + db.close() + +def _to_dict(obj): + if not obj: return None + if hasattr(obj, '__table__'): + return {c.name: getattr(obj, c.name) for c in obj.__table__.columns} + if isinstance(obj, dict): + return obj + return None + +def init_db_pool(): + """Log pool configuration on startup.""" + pool = engine.pool + + if settings.PGBOUNCER_URL and settings.PGBOUNCER_PORT: + logger.info(f"✅ PgBouncer connection successfully routed! (Strict Port {settings.PGBOUNCER_PORT})") + + logger.info( + f"DB pool initialized: size={getattr(pool, 'size', 'N/A')}, " + f"overflow={getattr(pool, '_max_overflow', 'N/A')}, " + f"class={pool.__class__.__name__}" + ) + +def close_db_pool(): + engine.dispose() + logger.info("DB pool disposed — all connections closed") + +def init_db(): + pass + + +from app.core import tenant_filter # noqa: E402,F401 + +from app.core import rls # noqa: E402,F401 diff --git a/app/db/redis.py b/app/db/redis.py new file mode 100644 index 0000000..c39cf3c --- /dev/null +++ b/app/db/redis.py @@ -0,0 +1,94 @@ +import redis +import os +import json +import logging +from typing import Optional, Any, Dict +from app.core.settings import settings + +logger = logging.getLogger(__name__) + +REDIS_HOST = settings.REDIS_HOST or "localhost" +REDIS_PORT = settings.REDIS_PORT or 6379 +REDIS_PASSWORD = settings.REDIS_PASSWORD + +class RedisClient: + def __init__(self): + self.client = None + try: + self.client = redis.Redis( + host=REDIS_HOST, + port=REDIS_PORT, + password=REDIS_PASSWORD, + decode_responses=True, + socket_connect_timeout=5 + ) + self.client.ping() + logger.info(f"✅ Redis Connected: {REDIS_HOST}:{REDIS_PORT}") + except Exception as e: + logger.info(f"Redis not available: {e}. Running without caching.") + self.client = None + + def get(self, key: str) -> Optional[Any]: + if not self.client: + return None + try: + val = self.client.get(key) + return json.loads(val) if val else None + except Exception as e: + logger.debug(f"Redis get error: {e}") + return None + + def set(self, key: str, value: Any, ttl: int = 300): + if not self.client: + return + try: + self.client.setex(key, ttl, json.dumps(value, default=str)) + except Exception as e: + logger.debug(f"Redis set error: {e}") + + def delete(self, key: str): + if not self.client: + return + try: + self.client.delete(key) + except Exception: + pass + + def hgetall(self, key: str) -> Dict[str, str]: + if not self.client: + return {} + try: + return self.client.hgetall(key) or {} + except Exception as e: + logger.debug(f"Redis hgetall error: {e}") + return {} + + def hset(self, key: str, field: str, value: str): + if not self.client: + return + try: + self.client.hset(key, field, value) + except Exception as e: + logger.debug(f"Redis hset error: {e}") + + def hdel(self, key: str, *fields: str): + if not self.client or not fields: + return + try: + self.client.hdel(key, *fields) + except Exception as e: + logger.debug(f"Redis hdel error: {e}") + + def invalidate_pattern(self, pattern: str): + """Invalidate keys matching pattern (e.g. 'drive:user:1:*')""" + if not self.client: + return + try: + keys = list(self.client.scan_iter(pattern)) + if keys: + self.client.delete(*keys) + logger.debug(f"Invalidated {len(keys)} keys for pattern {pattern}") + except Exception as e: + logger.error(f"Redis invalidate error: {e}") + +redis_cache = RedisClient() diff --git a/app/infrastructure/realtime/chat_subscriber.py b/app/infrastructure/realtime/chat_subscriber.py new file mode 100644 index 0000000..8ed705d --- /dev/null +++ b/app/infrastructure/realtime/chat_subscriber.py @@ -0,0 +1,119 @@ +""" +Redis Chat Subscriber Worker — single-per-process background task. + +Subscribes to the Redis pattern ``chat:file:*`` via PSUBSCRIBE and +forwards incoming messages to the local ChatConnectionManager rooms. + +Why one global worker instead of one per WebSocket connection? + • One Redis subscription per process (not per connection) — O(1) vs O(N). + • Works across multiple FastAPI / Uvicorn workers: each worker runs its + own instance of this task, so every local room receives messages + published by any other worker. + • The ``_origin_worker`` tag prevents echo: messages that originated on + THIS worker are skipped (they were already broadcast locally). + +Usage (in main.py lifespan): + from app.infrastructure.realtime.chat_subscriber import start_chat_subscriber, stop_chat_subscriber + + # Startup + await start_chat_subscriber() + + # Shutdown + await stop_chat_subscriber() +""" +import asyncio +import json +import logging +from typing import Optional + +import redis.asyncio as aioredis + +from app.core.settings import settings +from app.infrastructure.realtime.connection_manager import manager + +logger = logging.getLogger(__name__) + +_CHANNEL_PATTERN = "chat:file:*" + +_subscriber_task: Optional[asyncio.Task] = None + + +async def _listen_forever() -> None: + """ + Connect to Redis, pattern-subscribe, and loop forever. + + Automatically reconnects on transient errors with exponential backoff. + """ + backoff = 1 + + while True: + client: Optional[aioredis.Redis] = None + try: + url = settings.REDIS_URL or "redis://localhost:6379/0" + client = aioredis.from_url(url, decode_responses=True) + await client.ping() + logger.info(f"📡 Chat subscriber connected (pattern: {_CHANNEL_PATTERN})") + backoff = 1 + + pubsub = client.pubsub() + await pubsub.psubscribe(_CHANNEL_PATTERN) + + async for raw in pubsub.listen(): + if raw["type"] != "pmessage": + continue + + channel: str = raw["channel"] + try: + file_id = int(channel.rsplit(":", 1)[-1]) + except (ValueError, IndexError): + logger.warning(f"Bad channel name: {channel}") + continue + + try: + data: dict = json.loads(raw["data"]) + except (json.JSONDecodeError, TypeError): + logger.warning(f"Bad payload on {channel}") + continue + + await manager.broadcast_chat(file_id, data) + + except asyncio.CancelledError: + logger.info("Chat subscriber task cancelled — shutting down") + break + + except Exception as e: + logger.error( + f"Chat subscriber error: {e} — reconnecting in {backoff}s" + ) + await asyncio.sleep(backoff) + backoff = min(backoff * 2, 30) + + finally: + if client: + try: + await client.aclose() + except Exception: + pass + + +async def start_chat_subscriber() -> None: + """Launch the subscriber as a background task (idempotent).""" + global _subscriber_task + if _subscriber_task is not None and not _subscriber_task.done(): + return + _subscriber_task = asyncio.create_task(_listen_forever()) + logger.info("✅ Chat subscriber worker started") + + +async def stop_chat_subscriber() -> None: + """Cancel the background task gracefully.""" + global _subscriber_task + if _subscriber_task is None: + return + _subscriber_task.cancel() + try: + await _subscriber_task + except asyncio.CancelledError: + pass + _subscriber_task = None + logger.info("Chat subscriber worker stopped") diff --git a/app/infrastructure/realtime/connection_manager.py b/app/infrastructure/realtime/connection_manager.py new file mode 100644 index 0000000..0666649 --- /dev/null +++ b/app/infrastructure/realtime/connection_manager.py @@ -0,0 +1,144 @@ +""" +ConnectionManager — unified WebSocket manager for all real-time features. + +Currently handles two types of connectivity: +1. Room-based (Chat): file-based collaboration where multiple users share a room. +2. Direct/Task-based: 1:1 connections for tracking specific backend task progress. + +This replaces the old infrastructure/websocket/ws_manager.py for a +unified, production-ready implementation. +""" +import logging +from collections import defaultdict +from dataclasses import dataclass +from typing import Any, Optional, Set, Dict + +from fastapi import WebSocket +from starlette.websockets import WebSocketState + +logger = logging.getLogger(__name__) + + +@dataclass(eq=False) +class Connection: + """Metadata associated with a single WebSocket connection.""" + websocket: WebSocket + room_id: str + user_id: Optional[int] = None + conn_type: str = 'chat' + + +class ConnectionManager: + """ + Manages WebSocket connections partitioned by room_id. + """ + + def __init__(self) -> None: + self._rooms: Dict[str, Set[Connection]] = defaultdict(set) + self._ws_index: Dict[int, Connection] = {} + + + async def connect( + self, + room_id: str, + websocket: WebSocket, + user_id: Optional[int] = None, + conn_type: str = 'chat' + ) -> Connection: + """Accept the WebSocket and register it in the specified room.""" + await websocket.accept() + + conn = Connection( + websocket=websocket, + room_id=room_id, + user_id=user_id, + conn_type=conn_type + ) + self._rooms[room_id].add(conn) + self._ws_index[id(websocket)] = conn + + logger.info( + f"WS connected: type={conn_type} room={room_id} user={user_id} " + f"(room size: {len(self._rooms[room_id])})" + ) + return conn + + async def connect_chat(self, user_id: int, file_id: int, websocket: WebSocket) -> Connection: + """Helper to connect to a chat room.""" + return await self.connect( + room_id=f"chat:{file_id}", + websocket=websocket, + user_id=user_id, + conn_type='chat' + ) + + async def connect_task(self, task_id: str, websocket: WebSocket) -> Connection: + """Helper to connect to a task status channel.""" + return await self.connect( + room_id=f"task:{task_id}", + websocket=websocket, + conn_type='task' + ) + + def disconnect(self, websocket: WebSocket) -> None: + """Remove a WebSocket from its room.""" + conn = self._ws_index.pop(id(websocket), None) + if conn is None: + return + + room = self._rooms.get(conn.room_id) + if room: + room.discard(conn) + if not room: + del self._rooms[conn.room_id] + + logger.info( + f"WS disconnected: room={conn.room_id} user={conn.user_id} " + f"(room size: {len(self._rooms.get(conn.room_id, []))})" + ) + + async def broadcast(self, room_id: str, message: Dict[str, Any]) -> None: + """Send a JSON payload to every connection in the room.""" + room = self._rooms.get(room_id) + if not room: + return + + stale: list[Connection] = [] + + for conn in room: + try: + if conn.websocket.client_state == WebSocketState.CONNECTED: + await conn.websocket.send_json(message) + else: + stale.append(conn) + except Exception: + stale.append(conn) + + for conn in stale: + self.disconnect(conn.websocket) + + async def broadcast_chat(self, file_id: int, data: Dict[str, Any]) -> None: + """Legacy helper for broadcast on chat:file:{file_id}.""" + await self.broadcast(f"chat:{file_id}", data) + + async def send_task(self, task_id: str, data: Dict[str, Any]) -> None: + """Send message to a specific task status channel.""" + await self.broadcast(f"task:{task_id}", data) + + + def get_room_users(self, file_id: int) -> list[int]: + """Return a list of user IDs currently in a file's chat room.""" + room_id = f"chat:{file_id}" + return list({conn.user_id for conn in self._rooms.get(room_id, set()) if conn.user_id}) + + def get_room_size(self, room_id: str) -> int: + """Number of active connections in a specific room.""" + return len(self._rooms.get(room_id, set())) + + @property + def total_connections(self) -> int: + """Total number of active WebSocket connections across all rooms.""" + return len(self._ws_index) + + +manager = ConnectionManager() diff --git a/app/infrastructure/vector/__init__.py b/app/infrastructure/vector/__init__.py new file mode 100644 index 0000000..eaa6e2b --- /dev/null +++ b/app/infrastructure/vector/__init__.py @@ -0,0 +1 @@ +from .vector_client import VectorStoreManager, get_model, get_qdrant_client diff --git a/app/infrastructure/vector/vector_client.py b/app/infrastructure/vector/vector_client.py new file mode 100644 index 0000000..b8a1a77 --- /dev/null +++ b/app/infrastructure/vector/vector_client.py @@ -0,0 +1,361 @@ +import logging +import uuid +import threading +import numpy as np +from typing import List, Optional + +from qdrant_client import QdrantClient +from qdrant_client.models import ( + Distance, + VectorParams, + PointStruct, + Filter, + FieldCondition, + MatchValue, + FilterSelector, + PayloadSchemaType, +) + +from app.core.settings import settings + +logger = logging.getLogger(__name__) + +_model = None +_model_lock = threading.Lock() + + +def get_model(): + """ + Get the embedding library (SentenceTransformer) for generating vector representations. + Uses centralized setting settings.EMBEDDING_MODEL. + Thread-safe singleton — the model is loaded only once. + """ + global _model + if _model is None: + with _model_lock: + if _model is None: + try: + from sentence_transformers import SentenceTransformer + logger.info(f"Loading SentenceTransformer: {settings.EMBEDDING_MODEL}") + _model = SentenceTransformer(settings.EMBEDDING_MODEL, device='cpu') + _model.to('cpu') + logger.info(f"SentenceTransformer loaded successfully (dim={_model.get_sentence_embedding_dimension()})") + except Exception as e: + logger.error(f"Failed to load embedding model: {e}") + raise + return _model + + +_qdrant_client = None +_qdrant_lock = threading.Lock() + + +def get_qdrant_client() -> QdrantClient: + """Get or create a singleton Qdrant client.""" + global _qdrant_client + if _qdrant_client is None: + with _qdrant_lock: + if _qdrant_client is None: + _qdrant_client = QdrantClient( + host=settings.QDRANT_HOST, + port=settings.QDRANT_PORT, + ) + logger.info( + f"Connected to Qdrant at {settings.QDRANT_HOST}:{settings.QDRANT_PORT}" + ) + return _qdrant_client + + +_manager_instance = None +_manager_lock = threading.Lock() + + +class VectorStoreManager: + """ + Handles all vector operations via Qdrant. + Uses singleton pattern — only one instance exists per process. + + Architecture: + Upload -> Chunk -> Embed (BGE) -> Qdrant -> Search -> DeepSeek + + Each Qdrant point stores: + - id: unique UUID + - vector: BGE embedding + - payload: { user_id, document_id, chunk_text, chunk_index } + """ + + _collection_verified = False + + def __new__(cls): + """Singleton: reuse the same instance across all callers.""" + global _manager_instance + if _manager_instance is None: + with _manager_lock: + if _manager_instance is None: + instance = super().__new__(cls) + instance._initialized = False + _manager_instance = instance + return _manager_instance + + def __init__(self): + if self._initialized: + return + self.client = get_qdrant_client() + self.collection_name = settings.QDRANT_COLLECTION + self._ensure_collection() + self._initialized = True + + def _ensure_collection(self): + """Create the Qdrant collection if it doesn't exist. Runs only once.""" + if VectorStoreManager._collection_verified: + return + + try: + collections = self.client.get_collections().collections + exists = any(c.name == self.collection_name for c in collections) + + if not exists: + model = get_model() + dimension = model.get_sentence_embedding_dimension() + + self.client.create_collection( + collection_name=self.collection_name, + vectors_config=VectorParams( + size=dimension, + distance=Distance.COSINE, + ), + ) + + self.client.create_payload_index( + collection_name=self.collection_name, + field_name="user_id", + field_schema=PayloadSchemaType.INTEGER, + ) + self.client.create_payload_index( + collection_name=self.collection_name, + field_name="document_id", + field_schema=PayloadSchemaType.KEYWORD, + ) + + logger.info( + f"Created Qdrant collection '{self.collection_name}' " + f"(dim={dimension}, distance=COSINE)" + ) + else: + logger.info(f"Qdrant collection '{self.collection_name}' ready") + + VectorStoreManager._collection_verified = True + + except Exception as e: + logger.error(f"Failed to ensure Qdrant collection: {e}") + raise + + def create_index(self, user_id: int, document_id: str, chunks: List[str]): + """ + Calculates embeddings for chunks and upserts them into Qdrant. + Returns the number of indexed chunks. + """ + if not chunks: + return 0 + + user_id = int(user_id) + model = get_model() + + embeddings = model.encode(chunks, show_progress_bar=False) + embeddings = np.array(embeddings).astype("float32") + + points = [] + for idx, (chunk_text, embedding) in enumerate(zip(chunks, embeddings)): + point_id = str(uuid.uuid4()) + points.append( + PointStruct( + id=point_id, + vector=embedding.tolist(), + payload={ + "user_id": user_id, + "document_id": document_id, + "chunk_text": chunk_text, + "chunk_index": idx, + }, + ) + ) + + try: + batch_size = 100 + for i in range(0, len(points), batch_size): + batch = points[i : i + batch_size] + self.client.upsert( + collection_name=self.collection_name, + points=batch, + ) + + logger.info( + f"Indexed {len(chunks)} chunks in Qdrant " + f"(user={user_id}, doc={document_id})" + ) + return len(chunks) + + except Exception as e: + error_str = str(e) + if "doesn't exist" in error_str or "Not found: Collection" in error_str: + logger.warning(f"Qdrant collection '{self.collection_name}' missing. Re-creating...") + VectorStoreManager._collection_verified = False + self._ensure_collection() + + batch_size = 100 + for i in range(0, len(points), batch_size): + batch = points[i : i + batch_size] + self.client.upsert( + collection_name=self.collection_name, + points=batch, + ) + + logger.info( + f"Indexed {len(chunks)} chunks in Qdrant after recreation " + f"(user={user_id}, doc={document_id})" + ) + return len(chunks) + + logger.error(f"Failed to upsert vectors to Qdrant: {e}") + raise + + def search( + self, + user_id: int, + question: str, + document_id: Optional[str] = None, + top_k: int = 10, + ) -> List[dict]: + """ + Searches Qdrant for chunks most similar to the question. + Returns the top_k most relevant chunks with their scores. + """ + try: + user_id = int(user_id) + model = get_model() + + query_embedding = model.encode([question], show_progress_bar=False) + query_vector = query_embedding[0].tolist() + + must_conditions = [ + FieldCondition(key="user_id", match=MatchValue(value=user_id)) + ] + + if document_id: + must_conditions.append( + FieldCondition( + key="document_id", match=MatchValue(value=document_id) + ) + ) + + query_filter = Filter(must=must_conditions) + + results = self.client.search( + collection_name=self.collection_name, + query_vector=query_vector, + query_filter=query_filter, + limit=top_k, + score_threshold=None, + ) + + min_score = settings.MIN_SIMILARITY_SCORE or 0.2 + filtered = [] + + for hit in results: + print(f"SCORE: {hit.score:.4f} | TEXT: {hit.payload['chunk_text'][:80]}...") + + if hit.score >= min_score: + filtered.append({ + "text": hit.payload["chunk_text"], + "score": hit.score + }) + + logger.info( + f"Qdrant search: {len(filtered)} filtered results (from {len(results)}) for user {user_id}" + ) + return filtered + + except Exception as e: + error_str = str(e) + if "doesn't exist" in error_str or "Not found: Collection" in error_str: + logger.warning(f"Qdrant collection '{self.collection_name}' missing during search. Re-creating...") + VectorStoreManager._collection_verified = False + self._ensure_collection() + return [] + + logger.error(f"Search failed for user {user_id}: {e}") + return [] + + def delete_document(self, user_id: int, document_id: str): + """ + Deletes all vectors associated with a specific document from Qdrant. + """ + try: + self.client.delete( + collection_name=self.collection_name, + points_selector=FilterSelector( + filter=Filter( + must=[ + FieldCondition( + key="user_id", match=MatchValue(value=user_id) + ), + FieldCondition( + key="document_id", + match=MatchValue(value=document_id), + ), + ] + ) + ), + ) + logger.info(f"Deleted vectors for user {user_id}, doc {document_id}") + + except Exception as e: + error_str = str(e) + if "doesn't exist" in error_str or "Not found: Collection" in error_str: + logger.warning(f"Qdrant collection '{self.collection_name}' missing during delete. Ignoring.") + VectorStoreManager._collection_verified = False + self._ensure_collection() + else: + logger.error(f"Delete failed: {e}") + + def delete_user_data(self, user_id: int): + """ + Deletes ALL vectors for a user (used for account cleanup / reset). + """ + try: + self.client.delete( + collection_name=self.collection_name, + points_selector=FilterSelector( + filter=Filter( + must=[ + FieldCondition( + key="user_id", match=MatchValue(value=user_id) + ), + ] + ) + ), + ) + logger.info(f"Deleted all vectors for user {user_id}") + + except Exception as e: + error_str = str(e) + if "doesn't exist" in error_str or "Not found: Collection" in error_str: + logger.warning(f"Qdrant collection '{self.collection_name}' missing during delete_user_data. Ignoring.") + VectorStoreManager._collection_verified = False + self._ensure_collection() + else: + logger.error(f"Delete all failed for user {user_id}: {e}") + + +def preload(): + """ + Preload the embedding model and verify Qdrant connection at startup. + Call this during app initialization to avoid cold-start latency. + """ + try: + logger.info("Preloading embedding model and Qdrant connection...") + get_model() + VectorStoreManager() + logger.info("Preload complete - ready for queries") + except Exception as e: + logger.warning(f"Preload failed (will retry on first request): {e}") diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..b8973a6 --- /dev/null +++ b/app/main.py @@ -0,0 +1,449 @@ +import os +import logging +import asyncio +import tempfile +import shutil +import uuid +from contextlib import asynccontextmanager +from datetime import datetime, timezone + +from fastapi import FastAPI, UploadFile, File, HTTPException, Depends, Request, Response, status +from fastapi.middleware.cors import CORSMiddleware +from fastapi.staticfiles import StaticFiles +from sqlalchemy import text +from sqlalchemy.orm import Session +from app.middleware.csrf import CSRFMiddleware +from app.middleware.ip_middleware import IPMiddleware + +from app.core.settings import settings +from app.db.database import init_db, init_db_pool, close_db_pool, get_db + +import app.db.all_models # noqa: F401 +from app.middleware.auth import get_current_user +from app.modules.auth.models.user_model import User +from app.modules.auth.models.access_model import Access +from app.modules.auth.models.role_model import Role +from app.modules.auth.models.role_access_model import RoleAccess +from app.modules.documents.models.document_model import Project + +from app.modules.auth.routes.auth_routes import router as auth_router +from app.modules.auth.routes.user_routes import router as user_router +from app.modules.auth.routes.me_routes import router as me_router +from app.modules.auth.routes.access_routes import router as access_router +from app.modules.auth.routes.role_routes import router as role_router +from app.modules.auth.routes.admin_dashboard_routes import router as admin_dashboard_router +from app.modules.auth.routes.admin_user_routes import router as admin_user_router +from app.modules.documents.routes.document_routes import router as projects_router +from app.modules.documents.routes.sharing_routes import router as sharing_router +from app.modules.documents.routes.version_routes import router as version_router +from app.modules.documents.routes.comment_routes import ( + router as project_comments_router, +) +from app.modules.documents.routes.details_routes import router as project_details_router +from app.modules.drive.routes.drive_routes import ( + router as drive_router, + public_router as public_drive_router, +) +from app.modules.storage.routes.storage_routes import router as storage_router +from app.modules.chat.routes.chat_routes import router as chatbot_router +from app.tasks.routes import router as task_results_router +from app.modules.notifications.routes.notification_routes import ( + router as notifications_router, +) +from app.modules.documents.routes.export_routes import router as export_router + +from app.modules.speech.routes.speech_routes import router as speech_router +from app.api.ws_router import router as ws_router +from app.core.pubsub_listener import start_listener +from app.core.redis import redis_pubsub +from app.modules.collab.routes.chat_routes import ( + router as collab_chat_router, + rest_router as collab_rest_router, +) +from app.modules.tenant.routes.tenant_routes import router as tenant_router +from app.modules.tenant.routes.tenant_contact_routes import router as tenant_contact_router +from app.modules.signing.routes.signing_routes import router as signing_router +from app.modules.signing.routes.config_routes import router as signing_config_router +from app.modules.configuration.routes.system_configuration_routes import ( + router as system_configuration_router, +) +from app.modules.activity_logs.routes import router as activity_logs_router +from app.infrastructure.realtime.chat_subscriber import ( + start_chat_subscriber, + stop_chat_subscriber, +) +from app.modules.drive.routes.scan_routes import router as scan_router +from app.modules.extraction.routes.extraction_routes import router as extraction_router +from app.modules.editor.pdf_router import router as pdf_editor_router +from app.modules.auth.sso_router import router as sso_router +from app.modules.security.routes.security_routes import router as security_router + +TEMP_DIR = tempfile.mkdtemp(prefix="ai_uploads_") +processing_semaphore = asyncio.Semaphore(1) + +logging.basicConfig( + level=getattr(logging, settings.LOG_LEVEL), + format="%(asctime)s - %(levelname)s - %(message)s", +) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + logging.info(f"Server is starting up in {settings.APP_ENV} mode...") + try: + init_db_pool() + init_db() + + asyncio.create_task(start_listener()) + + await redis_pubsub.connect() + + await start_chat_subscriber() + + from app.infrastructure.vector.vector_client import preload as preload_vector + loop = asyncio.get_running_loop() + loop.run_in_executor(None, preload_vector) + + except Exception as e: + logging.error(f"Critical Startup failure: {e}") + raise e + + yield + + logging.info("Server is shutting down...") + try: + await stop_chat_subscriber() + await redis_pubsub.disconnect() + logging.info("✅ Chat subscriber & Redis Pub/Sub disconnected") + except Exception as e: + logging.error(f"Shutdown error during subscriber/redis stop: {e}") + try: + close_db_pool() + logging.info("✅ Database connection pool closed") + except Exception as e: + logging.error(f"Error closing database pool: {e}") + + try: + shutil.rmtree(TEMP_DIR, ignore_errors=True) + logging.info("🧹 Temp directory cleaned") + except Exception as e: + logging.error(f"Shutdown error during temp cleanup: {e}") + + +app = FastAPI( + lifespan=lifespan, + title="DocQube Document Processing Pipeline API", + description="An interactive API and UI to process PDFs into JATS XML.", + version="2.0.0", +) + +app.add_middleware(CSRFMiddleware) +app.add_middleware(IPMiddleware) + +from app.middleware.tenant_context_middleware import TenantContextMiddleware # noqa: E402 + +app.add_middleware(TenantContextMiddleware) + +app.add_middleware( + CORSMiddleware, + allow_origins=settings.CORS_ORIGINS_LIST, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + # Browsers only expose a small header safelist to JS on cross-origin + # responses by default (Content-Range/Accept-Ranges/Content-Length are + # NOT in it) — pdf.js reads these itself to decide whether it can use + # HTTP Range requests for progressive PDF loading, so without this it + # falls back to downloading the entire file before rendering anything. + expose_headers=["Content-Range", "Accept-Ranges", "Content-Length"], +) + + +# File Serving Internal Helper +async def _get_s3_file_response(key: str, db: Session = None, tenant_id: str = None): + from app.infrastructure.storage.local_storage_handler import ( + get_storage_client, + ) + from fastapi.responses import StreamingResponse + import mimetypes + + storage, quarantine_bucket, clean_bucket = get_storage_client( + db=db, tenant_id=tenant_id) + try: + try: + response = storage.get_object(Bucket=clean_bucket, Key=key) + except Exception: + response = storage.get_object(Bucket=quarantine_bucket, Key=key) + + body = response["Body"] + + content_type, _ = mimetypes.guess_type(key) + if not content_type: + content_type = "application/octet-stream" + + return StreamingResponse(body, media_type=content_type) + except HTTPException: + raise + except Exception as e: + logging.error(f"Storage File Proxy Error: {e}") + raise HTTPException(status_code=404, detail="File not found") + + +@app.get("/api/storage_drive/{key:path}") +async def serve_storage_file(key: str, current_user: User = Depends(get_current_user), db: Session = Depends(get_db)): + if ".." in key: + raise HTTPException(status_code=400, detail="Invalid path") + + tenant_part = f"tenants/{current_user.tenant_id}/" if current_user.tenant_id else "" + user_prefix1 = f"{tenant_part}user_{current_user.id}/" + user_prefix2 = f"{tenant_part}users/{current_user.id}/" + conv_prefix = f"{tenant_part}doc_conversion/{current_user.id}/" + chatbot_prefix = f"{tenant_part}chatbot_documents/{current_user.id}/" + images_prefix = f"extracted_images/{current_user.id}/" + + allowed_prefixes = [ + user_prefix1, + user_prefix2, + conv_prefix, + chatbot_prefix, + images_prefix, + f"user_{current_user.id}/", + f"users/{current_user.id}/", + f"doc_conversion/{current_user.id}/", + f"chatbot_documents/{current_user.id}/", + f"extracted_images/{current_user.id}/", + ] + + if not any(key.startswith(p) for p in allowed_prefixes): + raise HTTPException( + status_code=403, detail="Access denied to this storage key") + + return await _get_s3_file_response(key, db=db, tenant_id=current_user.tenant_id) + + +@app.get("/api/serve-image/{session_id}/{filename}") +async def serve_image( + session_id: str, + filename: str, + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db), +): + """ + Alias for serving images extracted during document processing. + """ + if ".." in session_id or ".." in filename: + raise HTTPException(status_code=400, detail="Invalid parameters") + + project = ( + db.query(Project) + .filter(Project.session_id == session_id, Project.user_id == current_user.id) + .first() + ) + + if not project: + raise HTTPException( + status_code=403, detail="Access denied to this session's resources" + ) + + key = f"extracted_images/{current_user.id}/{session_id}/{filename}" + return await _get_s3_file_response(key, db=db, tenant_id=current_user.tenant_id) + + +def _check_database_ready(db: Session) -> bool: + """Lightweight DB reachability check shared by /api/health/ready and the + legacy /api/health. On failure it rolls back before returning — without + this, the session handed back to Depends(get_db)'s own cleanup (which + unconditionally calls db.commit() after a successful return from this + function) is left in an aborted-transaction state, and that commit() + itself raises. That turns a DB failure this function already caught + into an unhandled 500 from the dependency's own teardown — which is + exactly what took the original /api/health down during the PgBouncer + incident, even though it appeared to catch the error internally. + """ + try: + db.execute(text("SELECT 1")) + return True + except Exception: + db.rollback() + return False + + +@app.get("/api/health/live") +def health_live(): + """Liveness: is the API process able to serve HTTP at all. No database, + no PgBouncer, no external dependency of any kind — this is what the + Docker HEALTHCHECK now uses, so a temporarily unreachable database can + no longer get the whole API container killed as unhealthy. + """ + return { + "status": "ok", + "service": "api", + "timestamp": datetime.now(timezone.utc).isoformat(), + } + + +@app.get("/api/health/ready") +def health_ready(response: Response, db: Session = Depends(get_db)): + """Readiness: is the API ready to serve real traffic, i.e. can it reach + the database through the application's actual configured pool/engine + (app.db.database.engine via get_db) — the same one every request uses, + not a separate health-only connection. + """ + if _check_database_ready(db): + return { + "status": "ready", + "database": "connected", + "timestamp": datetime.now(timezone.utc).isoformat(), + } + response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE + return { + "status": "not_ready", + "database": "disconnected", + "timestamp": datetime.now(timezone.utc).isoformat(), + } + + +@app.get("/api/health") +def health_check(db: Session = Depends(get_db)): + """Backward-compatible general health endpoint — response shape kept + identical to before this refactor. Reuses the same DB-reachability + check as /api/health/ready instead of duplicating it; the Docker + HEALTHCHECK no longer depends on this endpoint (see /api/health/live). + """ + if not _check_database_ready(db): + return {"status": "degraded", "database": "error", "error": "database unreachable"} + + health = {"status": "ok", "database": "connected"} + try: + version = db.execute(text("SELECT version_num FROM alembic_version")).scalar() + health["migration_version"] = version + except Exception: + db.rollback() + health["status"] = "degraded" + health["error"] = "migration version lookup failed" + + return health + + +app.include_router(tenant_router, prefix="/api") +app.include_router(tenant_contact_router, prefix="/api/tenant") +app.include_router(auth_router, prefix="/api") +app.include_router(sso_router, prefix="/api") +app.include_router(me_router, prefix="/api") +app.include_router(access_router) +app.include_router(role_router) +app.include_router(admin_dashboard_router) +app.include_router(admin_user_router) +app.include_router(user_router, prefix="/api") +app.include_router(projects_router, prefix="/api") +app.include_router(sharing_router, prefix="/api") +app.include_router(version_router, prefix="/api") +app.include_router(project_comments_router, prefix="/api") +app.include_router(project_details_router, prefix="/api") +app.include_router(drive_router, prefix="/api") +app.include_router(public_drive_router, prefix="/api") +app.include_router(storage_router, prefix="/api") +app.include_router(task_results_router, prefix="/api") +app.include_router(chatbot_router) +app.include_router(notifications_router, prefix="/api") +app.include_router(system_configuration_router) +app.include_router(activity_logs_router) +app.include_router(export_router, prefix="/api") + +app.include_router(speech_router) +app.include_router(ws_router, prefix="/api") +app.include_router(collab_chat_router, prefix="/api") +app.include_router(collab_rest_router, prefix="/api") +app.include_router(signing_router, prefix="/api") +app.include_router(signing_config_router, prefix="/api") +app.include_router(pdf_editor_router, prefix="/api") +app.include_router(scan_router, prefix="/api") +app.include_router(extraction_router, prefix="/api") + +from app.modules.org.routes.org_routes import router as org_router # noqa: E402 + +app.include_router(org_router, prefix="/api") + +from app.modules.billing.routes.billing_routes import router as billing_router # noqa: E402 + +app.include_router(billing_router, prefix="/api") +app.include_router(security_router, prefix="/api") + + +@app.post("/api/extract") +async def extract( + request: Request, + file: UploadFile = File(...), + current_user: User = Depends(get_current_user), +): + """ + On-demand document extraction endpoint. + Loads models only when called and unloads after. + """ + file_id = str(uuid.uuid4()) + ext = os.path.splitext(file.filename)[1].lower() + temp_path = os.path.join(TEMP_DIR, f"{file_id}{ext}") + + MAX_FILE_SIZE = 50 * 1024 * 1024 + + try: + total_size = 0 + with open(temp_path, "wb") as buffer: + while True: + chunk = await file.read(1024 * 1024) + if not chunk: + break + total_size += len(chunk) + if total_size > MAX_FILE_SIZE: + raise HTTPException( + status_code=413, detail="File too large (Max 50MB)" + ) + buffer.write(chunk) + + # Convert PDF to HTML locally, then derive markdown + async with processing_semaphore: + from app.modules.editor.converters import process_pdf_to_html + + loop = asyncio.get_running_loop() + + def _convert(): + html = process_pdf_to_html(temp_path) + from bs4 import BeautifulSoup + markdown = BeautifulSoup( + html, "html.parser").get_text("\n") + return {"html": html, "markdown": markdown} + + result = await loop.run_in_executor(None, _convert) + + return { + "status": "success", + "file_id": file_id, + "data": result, + } + + except HTTPException: + raise + except Exception as e: + error_id = str(uuid.uuid4()) + logging.error( + f"Extraction failed [Ref: {error_id}]: {e}", exc_info=True) + raise HTTPException( + status_code=500, + detail=f"An internal error occurred during processing. Please contact support and provide reference: {error_id}", + ) + + finally: + if os.path.exists(temp_path): + try: + os.remove(temp_path) + except Exception as e: + logging.error(f"Cleanup error removing {temp_path}: {e}") + + +if __name__ == "__main__": + import uvicorn + + uvicorn.run( + "main:app", host=settings.HOST, port=settings.PORT, reload=settings.DEBUG + ) + diff --git a/app/middleware/__init__.py b/app/middleware/__init__.py new file mode 100644 index 0000000..d369da1 --- /dev/null +++ b/app/middleware/__init__.py @@ -0,0 +1,3 @@ +""" +Middleware module for application-wide processing. +""" diff --git a/app/middleware/auth.py b/app/middleware/auth.py new file mode 100644 index 0000000..f37a0ba --- /dev/null +++ b/app/middleware/auth.py @@ -0,0 +1,329 @@ +from typing import Optional +from fastapi import Depends, HTTPException, Request +from fastapi.security import OAuth2PasswordBearer +from jose import JWTError, jwt +from sqlalchemy.orm import Session, selectinload +import logging +from datetime import datetime, timezone + +from app.db.database import get_db +from app.modules.auth.models.role_model import Role +from app.modules.auth.models.role_access_model import RoleAccess +from app.modules.auth.models.user_model import User +from app.core.settings import settings +from app.core.tenant_context import set_context +from app.core.token_blacklist import TokenBlacklist + +logger = logging.getLogger(__name__) + +oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/auth/login", auto_error=False) + + +def _validate_saas_subscription(subscription_details: Optional[dict]) -> None: + if not subscription_details: + return + + status_value = str(subscription_details.get("status") or "").upper() + is_active = subscription_details.get("is_active") + today = datetime.now(timezone.utc).date() + + start_date_raw = subscription_details.get("start_date") + end_date_raw = subscription_details.get("end_date") + + try: + start_date = ( + datetime.fromisoformat(start_date_raw).date() if start_date_raw else None + ) + except ValueError: + start_date = None + + try: + end_date = datetime.fromisoformat(end_date_raw).date() if end_date_raw else None + except ValueError: + end_date = None + + if is_active is False or status_value in {"INACTIVE", "EXPIRED"}: + raise HTTPException(status_code=403, detail="Tenant subscription is inactive") + + if start_date and today < start_date: + raise HTTPException(status_code=403, detail="Tenant subscription is not active yet") + + if end_date and today > end_date: + raise HTTPException(status_code=403, detail="Tenant subscription has expired") + + +def _check_missing_subscription_claim(db, user, subscription_details) -> None: + """ + A platform-managed tenant arriving without a subscription claim. + + `_validate_saas_subscription` only checks a claim that is present. Direct + login sets none, so subscription state is enforced for SSO and not for + direct logins — which is either deliberate or a hole, depending on how the + product is sold. + + `saas_tenant_mappings` is the evidence that settles it per tenant: a mapped + tenant is billed on the platform, an unmapped one is not. So this only ever + looks at mapped tenants, and it cannot affect a customer who was never on + the platform. + + **Logs always, blocks only when configured.** Leave the setting off, watch + the warnings, and enable it once the logs show mapped tenants always arrive + with a claim. Enforcing a billing rule at the login door is the change that + locks real customers out if the assumption is wrong — so the observation + comes first and the enforcement is a deliberate second step. + + Only runs when the claim is *absent*, which is the exceptional path, so the + extra query is not on the hot path for SSO users. + """ + if subscription_details: + return + if not user.tenant_id: + return + + from app.modules.auth.models.saas_models import SaaSTenantMapping + + mapped = ( + db.query(SaaSTenantMapping.id) + .filter(SaaSTenantMapping.docqube_tenant_id == user.tenant_id) + .first() + is not None + ) + if not mapped: + return + + logger.warning( + "Tenant %s is managed by the SaaS platform but this token carries no " + "subscription claim; subscription state is not being enforced for this " + "request (user %s)", + user.tenant_id, + user.id, + ) + + if getattr(settings, "SUBSCRIPTION_REQUIRED_FOR_MAPPED_TENANTS", False): + raise HTTPException( + status_code=403, + detail="Your subscription could not be verified. Please sign in again.", + ) + + +_WRITE_METHODS = frozenset({"POST", "PUT", "PATCH", "DELETE"}) + + +def _enforce_subscription_access(db, request, user) -> None: + """ + A lapsed subscription degrades to read-only. It does not lock anybody out. + + `EntitlementService.is_active()` existed, was tested, and enforced nothing — + it was read by two API responses and by nothing else, so a cancelled + subscription restricted precisely nothing. That is the same failure as the + tenant context in B1: machinery built, machinery proven, machinery never + connected. + + **Read-only rather than refused**, ported from the base and matching how + mature systems behave. An expired customer keeps access to their own + documents and loses the ability to create more. Locking them out makes + export impossible and is bad for renewals: the customer most likely to come + back is the one who can still see what they would be coming back to. + + Off by default, like every other enforcement added in this work. Turning it + on is a commercial decision, and it should be made after somebody has looked + at how many tenants are currently in a lapsed state. + """ + if not getattr(settings, "SUBSCRIPTION_ENFORCEMENT_ENABLED", False): + return + if request.method not in _WRITE_METHODS: + return + + tenant = getattr(user, "tenant", None) + if tenant is None: + return + + from app.modules.billing.services.entitlement_service import EntitlementService + + level = EntitlementService(db).access_level(tenant) + if level == "full": + return + + raise HTTPException( + status_code=402, + detail=( + "Your subscription has lapsed. You can still read and export your " + "documents; renewing restores the ability to make changes." + ), + ) + + +def _attach_resolved_access(db, user) -> None: + """ + Give the user object a way to answer "what may I do" — **lazily**. + + Before this existed there were two answers to that question. The gate + (`require_access` -> `PermissionService`) unioned the legacy `users.role_id` + with every live `user_roles` grant. The *response* (`UserOut.accesses` -> + `User.access_codes`) read the legacy role alone. So granting somebody a + second role gave them API authority the interface would not render — no menu + entry, no route, no button — and multi-role looked broken while being + enforced correctly. + + **Closures, not results.** Resolving eagerly here cost three queries on every + authenticated request, including the many that never ask — `/api/storage/usage` + neither gates on an access code nor serialises the user, and it went from 20 + queries to 23 for an answer nobody read. Attaching the resolvers instead + means the cost lands only where the question is asked, and + `app/core/request_cache.py` makes the second asker free. Net effect on a + *gated* endpoint is a reduction: `PermissionService` used to issue three + queries per `require_access` dependency with no memo at all. + + **Closures and not the session**, because `User` is a model and + `.importlinter` forbids models from importing services — including inside a + function body, which import-linter reads. Keeping the import here is what + keeps the ORM registry independent of request handling. + + **Runs after `set_context`.** Both `user_roles` and `roles` are tenant-owned, + so with `TENANT_FILTER_ENABLED` resolution must happen under the correct + tenant context or it silently returns nothing — which would present as "the + user lost all permissions" rather than as an error. + """ + from app.modules.auth.services.permission_service import PermissionService + from app.modules.auth.services.user_role_service import UserRoleReader + + setattr( + user, + "_resolve_access_codes", + lambda: PermissionService(db).user_access_codes(user), + ) + setattr( + user, + "_resolve_role_refs", + lambda: UserRoleReader(db).assignments(user), + ) + + +def get_current_user( + request: Request, + db: Session = Depends(get_db), + token: Optional[str] = Depends(oauth2_scheme), +) -> User: + token_candidates = [] + if token: + token_candidates.append(token) + if request.cookies.get("docqube_access_token"): + token_candidates.append(request.cookies.get("docqube_access_token")) + if request.cookies.get("access_token"): + token_candidates.append(request.cookies.get("access_token")) + + if not token_candidates: + raise HTTPException(status_code=401, detail="Authentication required") + + for tok in token_candidates: + try: + payload = jwt.decode( + tok, settings.APP_SECRET, algorithms=[settings.ALGORITHM] + ) + jti = payload.get("jti") + if payload.get("type") in ("refresh", "device_management"): + logger.warning(f"Authentication candidate has invalid token type: {payload.get('type')}") + continue + sub = payload.get("sub") or payload.get("id") or payload.get("user_id") or payload.get("email") + tenant_id = payload.get("tenant_id") + + if sub is None: + logger.warning("Authentication candidate is missing a subject claim") + continue + + if jti and TokenBlacklist.is_blacklisted(jti): + logger.warning("AUTH DEBUG: Token blacklisted") + continue + + session_id = payload.get("session_id") + if session_id: + from app.modules.auth.repositories.session_repository import SessionRepository + session_repo = SessionRepository(db) + session = session_repo.get_by_id(session_id) + if not session or session.status != "ACTIVE": + if session and session.status == "REVOKED": + if session.revoked_by == "reauth": + raise HTTPException(status_code=401, detail="You have logged in from another tab on this device. Please refresh.") + else: + raise HTTPException(status_code=401, detail="Your session was remotely logged out from another device.") + raise HTTPException(status_code=401, detail="Your session has expired.") + + from datetime import timedelta + now = datetime.now(timezone.utc) + last_activity = session.last_activity + if last_activity and last_activity.tzinfo is None: + last_activity = last_activity.replace(tzinfo=timezone.utc) + if not last_activity or now - last_activity > timedelta(minutes=5): + session_repo.update_last_activity(session) + + user = None + try: + user_id = int(sub) + user = ( + db.query(User) + .options( + selectinload(User.role) + .selectinload(Role.role_accesses) + .selectinload(RoleAccess.access) + ) + .filter(User.id == user_id) + .first() + ) + except (ValueError, TypeError): + user = ( + db.query(User) + .options( + selectinload(User.role) + .selectinload(Role.role_accesses) + .selectinload(RoleAccess.access) + ) + .filter(User.email == str(sub)) + .first() + ) + + if not user: + logger.warning("AUTH DEBUG: User sub not found in database!") + continue + + if getattr(user, "is_deleted", False) or not getattr(user, "is_active", True): + logger.warning("AUTH DEBUG: User is deleted or inactive") + continue + + if user.tenant_id and tenant_id and str(user.tenant_id) != str(tenant_id): + logger.warning("AUTH DEBUG: Tenant mismatch user.tenant_id vs token") + continue + + tenant = getattr(user, "tenant", None) + if tenant is not None and ( + not getattr(tenant, "is_active", True) + or getattr(tenant, "is_deleted", False) + ): + logger.warning("Tenant %s is not active", user.tenant_id) + raise HTTPException( + status_code=403, + detail="This workspace has been suspended. Contact support.", + ) + + _enforce_subscription_access(db, request, user) + + saas_permissions = payload.get("saas_permissions", []) + setattr(user, "saas_permissions", set(saas_permissions)) + saas_subscription = payload.get("saas_subscription") + _validate_saas_subscription(saas_subscription) + _check_missing_subscription_claim(db, user, saas_subscription) + setattr(user, "saas_subscription", saas_subscription) + + set_context( + user.tenant_id, + is_super=bool(getattr(user, "is_superadmin", False)), + ) + + _attach_resolved_access(db, user) + return user + + except JWTError as err: + logger.warning("Authentication candidate JWT decoding failed: %s", type(err).__name__) + continue + + logger.warning("AUTH DEBUG: All %d token candidates failed authentication!", len(token_candidates)) + raise HTTPException(status_code=401, detail="Authentication required") diff --git a/app/middleware/csrf.py b/app/middleware/csrf.py new file mode 100644 index 0000000..80bd8d7 --- /dev/null +++ b/app/middleware/csrf.py @@ -0,0 +1,58 @@ +import uuid +from starlette.middleware.base import BaseHTTPMiddleware +from fastapi import Request +from fastapi.responses import JSONResponse +from app.core.settings import settings + +class CSRFMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + csrf_cookie = request.cookies.get("csrf_token") + csrf_header = request.headers.get("x-csrf-token") or request.headers.get("X-CSRF-Token") + + needs_validation = ( + request.method in ["POST", "PUT", "PATCH", "DELETE"] and + not request.url.path.startswith("/api/auth/") and + not request.url.path.startswith("/api/sso/") and + not request.url.path == "/api/me/tutorial/finish" and + not "/signing/ocr" in request.url.path and + not "webhook" in request.url.path + ) + + if needs_validation and not settings.DISABLE_CSRF: + if not csrf_cookie or not csrf_header or csrf_cookie != csrf_header: + import logging + logger = logging.getLogger(__name__) + logger.warning( + f"CSRF Failure on {request.url.path}: " + f"cookie_present={bool(csrf_cookie)}, " + f"header_present={bool(csrf_header)}, " + f"match={csrf_cookie == csrf_header}" + ) + + response = JSONResponse( + status_code=403, + content={"detail": "CSRF verification failed"} + ) + if not csrf_cookie: + self._set_csrf_cookie(response) + return response + + response = await call_next(request) + + if not csrf_cookie: + self._set_csrf_cookie(response) + + return response + + def _set_csrf_cookie(self, response): + """Helper to set the CSRF cookie with appropriate security flags.""" + is_prod = settings.APP_ENV == "production" + response.set_cookie( + key="csrf_token", + value=str(uuid.uuid4()), + httponly=False, + samesite="none" if is_prod else "lax", + secure=is_prod, + domain=settings.COOKIE_DOMAIN if is_prod else None, + path="/" + ) \ No newline at end of file diff --git a/app/middleware/ip_middleware.py b/app/middleware/ip_middleware.py new file mode 100644 index 0000000..ffdf738 --- /dev/null +++ b/app/middleware/ip_middleware.py @@ -0,0 +1,16 @@ +from starlette.middleware.base import BaseHTTPMiddleware +from fastapi import Request +from app.core.context import client_ip_ctx_var, user_agent_ctx_var + +class IPMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + x_forwarded_for = request.headers.get("X-Forwarded-For") + if x_forwarded_for: + ip = x_forwarded_for.split(",")[0].strip() + else: + ip = request.client.host if request.client else None + + client_ip_ctx_var.set(ip) + user_agent_ctx_var.set(request.headers.get("User-Agent")) + response = await call_next(request) + return response \ No newline at end of file diff --git a/app/middleware/rbac.py b/app/middleware/rbac.py new file mode 100644 index 0000000..fe9659f --- /dev/null +++ b/app/middleware/rbac.py @@ -0,0 +1,54 @@ +from fastapi import Depends, HTTPException +from sqlalchemy.orm import Session + +from app.db.database import get_db +from app.middleware.auth import get_current_user +from app.modules.drive.models.drive_model import DriveACL + + +ROLE_HIERARCHY = { + "owner": 4, + "editor": 3, + "commenter": 2, + "viewer": 1, +} + + +def require_role(resource_type: str, minimum_role: str): + def checker( + resource_id: int, + db: Session = Depends(get_db), + user=Depends(get_current_user), + ): + + acl = ( + db.query(DriveACL) + .filter( + DriveACL.resource_type == resource_type, + DriveACL.resource_id == resource_id, + DriveACL.subject_type == "user", + DriveACL.subject_id == user.id, + ) + .first() + ) + + if not acl: + raise HTTPException(status_code=403, detail="Access denied") + + user_level = ROLE_HIERARCHY.get(acl.role) + min_level = ROLE_HIERARCHY.get(minimum_role) + + if user_level is None or min_level is None: + import logging + logging.getLogger(__name__).error( + "rbac: unknown role encountered — acl.role=%r, minimum_role=%r", + acl.role, minimum_role, + ) + raise HTTPException(status_code=403, detail="Access denied") + + if user_level < min_level: + raise HTTPException(status_code=403, detail="Insufficient permission") + + return True + + return checker diff --git a/app/middleware/security_headers.py b/app/middleware/security_headers.py new file mode 100644 index 0000000..d3b1ee6 --- /dev/null +++ b/app/middleware/security_headers.py @@ -0,0 +1,15 @@ +from starlette.middleware.base import BaseHTTPMiddleware +from fastapi import Request + + +class SecurityHeadersMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + response = await call_next(request) + + response.headers["X-Content-Type-Options"] = "nosniff" + response.headers["X-Frame-Options"] = "DENY" + response.headers["X-XSS-Protection"] = "1; mode=block" + response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin" + response.headers["Content-Security-Policy"] = "default-src 'self'" + + return response diff --git a/app/middleware/tenant.py b/app/middleware/tenant.py new file mode 100644 index 0000000..21ad6c8 --- /dev/null +++ b/app/middleware/tenant.py @@ -0,0 +1,111 @@ +from typing import Optional +from uuid import UUID +from fastapi import Request, Depends, HTTPException, status +from sqlalchemy.orm import Session + +from app.db.database import get_db +from app.modules.auth.models.user_model import User +from app.middleware.auth import get_current_user +from app.modules.tenant.models.tenant_model import Tenant + + +def get_tenant_from_header( + request: Request, db: Session = Depends(get_db) +) -> Optional[Tenant]: + """ + Extracts X-Tenant-ID from the request headers and validates it. + Used for public routes like registration where the user is not authenticated yet. + """ + tenant_id_str = request.headers.get("X-Tenant-ID") + if not tenant_id_str: + default_tenant = db.query(Tenant).filter(Tenant.slug == "default").first() + return default_tenant + + try: + tenant_id = UUID(tenant_id_str) + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid X-Tenant-ID format" + ) + + tenant = db.query(Tenant).filter(Tenant.id == tenant_id).first() + if not tenant: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Tenant not found" + ) + + if not tenant.is_active: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, detail="Tenant is currently inactive" + ) + + return tenant + + +def get_tenant_id(user: User = Depends(get_current_user)) -> Optional[UUID]: + """ + Extracts tenant_id from the authenticated user context. + Returns None for superadmins. + """ + return user.tenant_id + + +def is_superadmin(user: User = Depends(get_current_user)) -> bool: + """ + Checks whether the user is a superadmin. + + Reads the explicit `is_superadmin` flag, and nothing else. + + It used to also accept `tenant_id IS NULL`. That made the *absence* of + tenant context a grant of authority: any bug that dropped the tenant did not + deny, it escalated. The fallback was kept for exactly one release so that + reverting the B1.0 migration could not lock the operators out mid-deploy; + that release has now shipped and this is its removal. + + **Deploy order matters.** This change assumes `users.is_superadmin` exists + and has been backfilled — that is migration `b1_0_explicit_superadmin`, + which sets the flag for every active user with no tenant. Ship this without + that migration and every superadmin loses access. `scripts/b1_preflight.py` + reports anyone still relying on the old signal before you find out the hard + way. + + A user who loses tenant context now has no tenant and no privilege, which is + the whole point. + """ + return bool(getattr(user, "is_superadmin", False)) + + +def require_superadmin(user: User = Depends(get_current_user)) -> User: + """ + Dependency that enforces superadmin access. + Returns the user if superadmin, raises HTTP 403 otherwise. + """ + if not is_superadmin(user): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Superadmin access required in the Document Vault", + ) + return user + + +def require_tenant(request: Request, user: User = Depends(get_current_user)) -> UUID: + """ + Dependency that enforces tenant context. + Superadmins can provide X-Tenant-ID to simulate tenant context. + """ + if user.tenant_id is None: + tenant_id_str = request.headers.get("X-Tenant-ID") + if not tenant_id_str: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Superadmins must provide X-Tenant-ID header to perform this operation", + ) + try: + return UUID(tenant_id_str) + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid X-Tenant-ID format", + ) + + return user.tenant_id diff --git a/app/middleware/tenant_context_middleware.py b/app/middleware/tenant_context_middleware.py new file mode 100644 index 0000000..65ebc40 --- /dev/null +++ b/app/middleware/tenant_context_middleware.py @@ -0,0 +1,133 @@ +""" +B1.4 — establish the tenant context for every request. + +This is the wiring the rest of B1 was waiting on, and its absence was the single +thing standing between "isolation is built" and "isolation works". The ORM +listener (`app/core/tenant_filter.py`) and the RLS session variables +(`app/core/rls.py`) both read `current_tenant_id()`. Both were built, tested and +correct. Nothing in the application ever set it — the only callers of +`scoped_to()` in the repository were the tests, which supply their own context +and therefore proved the machinery while never exercising the wiring. + +The consequence was quiet, because the flag defaults off: every request ran with +no context, both layers stayed inert, and the application behaved exactly as it +always had. Turning `TENANT_FILTER_ENABLED` on would have failed closed on every +query in the system. + +**Why middleware and not a dependency.** `get_current_user` is a `def`, so +FastAPI runs it in a threadpool, and a `ContextVar` set inside a worker thread +does not propagate back to the request's task — the write is simply lost at the +thread boundary. Middleware runs in the request's own task, so what it sets is +visible to every dependency and handler that follows, including the sync ones +(the context is *copied into* the threadpool, which is all a reader needs). + +**Why raw ASGI and not `BaseHTTPMiddleware`.** `BaseHTTPMiddleware` runs the +downstream app in a separate task and pumps the response through a queue, which +breaks streaming: this application serves file downloads through +`StreamingResponse` and has a Server-Sent Events endpoint on +`text/event-stream`, and SSE under that middleware stops arriving incrementally. +A plain ASGI callable adds no task, no queue and no buffering — it sets a +context variable and gets out of the way. + +**Why the token and not the database.** Reading the user here would mean a query +per request before any route is chosen, and that query would itself need the +context it is trying to establish. The token is signature-verified before its +claims are read, and `get_current_user` independently checks the tenant claim +against the loaded row, rejecting any mismatch — so a forged claim fails there +rather than being trusted here. +""" + +import logging +import uuid + +from jose import JWTError, jwt +from starlette.datastructures import Headers +from starlette.types import ASGIApp, Message, Receive, Scope, Send + +from app.core.request_cache import reset_request_cache +from app.core.scope import reset_scope_cache +from app.core.settings import settings +from app.core.tenant_context import reset_context, set_context + +logger = logging.getLogger(__name__) + +_COOKIE_NAMES = ("docqube_access_token", "access_token") + + +def _token_from(headers: Headers) -> str | None: + authorization = headers.get("authorization") or "" + if authorization.lower().startswith("bearer "): + candidate = authorization[7:].strip() + if candidate: + return candidate + + cookie_header = headers.get("cookie") + if not cookie_header: + return None + for part in cookie_header.split(";"): + name, _, value = part.strip().partition("=") + if name in _COOKIE_NAMES and value: + return value + return None + + +def _context_from_token(token: str) -> tuple[uuid.UUID | None, bool] | None: + """The tenant and privilege a verified token asserts, or None if unusable.""" + try: + claims = jwt.decode(token, settings.APP_SECRET, algorithms=[settings.ALGORITHM]) + except JWTError: + return None + + if claims.get("type") in ("refresh", "device_management"): + return None + + tenant_id = None + raw = claims.get("tenant_id") + if raw: + try: + tenant_id = uuid.UUID(str(raw)) + except (ValueError, AttributeError, TypeError): + logger.warning("Token carried an unparseable tenant_id") + + is_super = bool(claims.get("is_superadmin", False)) + + if tenant_id is None and not is_super: + return None + return tenant_id, is_super + + +class TenantContextMiddleware: + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] not in ("http", "websocket"): + await self.app(scope, receive, send) + return + + reset_scope_cache() + reset_request_cache() + + tokens = None + raw_token = _token_from(Headers(scope=scope)) + if raw_token: + resolved = _context_from_token(raw_token) + if resolved is not None: + tenant_id, is_super = resolved + tokens = set_context(tenant_id, is_super=is_super) + + try: + await self.app(scope, receive, send) + finally: + if tokens is not None: + try: + reset_context(tokens) + except ValueError: + pass + + +def install(app: ASGIApp) -> None: + app.add_middleware(TenantContextMiddleware) + + +__all__ = ["TenantContextMiddleware", "install"] diff --git a/app/modules/activity_logs/__init__.py b/app/modules/activity_logs/__init__.py new file mode 100644 index 0000000..4ed6c2e --- /dev/null +++ b/app/modules/activity_logs/__init__.py @@ -0,0 +1,3 @@ +from app.modules.activity_logs.service import log_drive_event, log_event, log_user_event + +__all__ = ["log_event", "log_drive_event", "log_user_event"] diff --git a/app/modules/activity_logs/constants.py b/app/modules/activity_logs/constants.py new file mode 100644 index 0000000..4685c9b --- /dev/null +++ b/app/modules/activity_logs/constants.py @@ -0,0 +1,67 @@ +from enum import Enum + + +class ActivityLogModule(str, Enum): + DRIVE = "drive" + EDITOR = "editor" + CONVERSION = "conversion" + SIGNING = "signing" + COLLABORATION = "collaboration" + ACCESS = "access" + TENANT = "tenant" + + +class ActivityLogStatus(str, Enum): + SUCCESS = "success" + FAILURE = "failure" + + +class ActivityLogTargetType(str, Enum): + DOCUMENT = "document" + FOLDER = "folder" + VERSION = "version" + SESSION = "session" + TENANT = "tenant" + SIGNING_REQUEST = "signing_request" + ROLE = "role" + USER = "user" + ORG_UNIT = "org_unit" + GROUP = "group" + + +class ActivityLogAction(str, Enum): + FOLDER_CREATED = "folder_created" + FOLDER_DELETED = "folder_deleted" + DOCUMENT_UPLOADED = "document_uploaded" + DOCUMENT_DELETED = "document_deleted" + DOCUMENT_MOVED = "document_moved" + DOCUMENT_SHARED = "document_shared" + ACCESS_REVOKED = "access_revoked" + DOCUMENT_EDITED = "document_edited" + CONVERSION_COMPLETED = "conversion_completed" + DOCUMENT_SIGNED = "document_signed" + DOCUMENT_EMAILED = "document_emailed" + SIGN_REQUEST_SENT = "sign_request_sent" + CONFIG_SMTP_UPDATED = "config_smtp_updated" + CONFIG_STORAGE_UPDATED = "config_storage_updated" + CONFIG_COMPRESSION_UPDATED = "config_compression_updated" + CONFIG_SIGNING_UPDATED = "config_signing_updated" + CONTACT_ADDED = "contact_added" + CONTACT_UPDATED = "contact_updated" + CONTACT_DELETED = "contact_deleted" + DEVICE_LOGOUT = "device_logout" + DEVICE_LOGIN = "device_login" + ROLE_CREATED = "role_created" + USER_DEACTIVATED = "user_deactivated" + SIGN_REMINDER_SENT = "sign_reminder_sent" + + ROLE_GRANTED = "role_granted" + ROLE_REVOKED = "role_revoked" + ROLE_EXPIRED = "role_expired" + ORG_UNIT_CREATED = "org_unit_created" + ORG_UNIT_RENAMED = "org_unit_renamed" + ORG_UNIT_MOVED = "org_unit_moved" + ORG_UNIT_DELETED = "org_unit_deleted" + GROUP_CREATED = "group_created" + GROUP_DELETED = "group_deleted" + DOCUMENT_SIGNATURE_BROKEN = "document_signature_broken" diff --git a/app/modules/activity_logs/models/activity_log.py b/app/modules/activity_logs/models/activity_log.py new file mode 100644 index 0000000..dd65dbc --- /dev/null +++ b/app/modules/activity_logs/models/activity_log.py @@ -0,0 +1,50 @@ +import uuid +from datetime import datetime +from typing import Optional + +from sqlalchemy import DateTime, ForeignKey, Index, Integer, String, func, text +from sqlalchemy.dialects.postgresql import JSONB, UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.db.database import Base + + +class ActivityLog(Base): + __tablename__ = "activity_logs" + __table_args__ = ( + Index("ix_activity_logs_tenant_created_at", "tenant_id", "created_at"), + Index("ix_activity_logs_user_created_at", "user_id", "created_at"), + Index("ix_activity_logs_module_action", "module", "action"), + Index("ix_activity_logs_target_id", "target_id"), + Index( + "ix_activity_logs_access_timeline", + "tenant_id", + "created_at", + postgresql_where=text("module = 'access'"), + ), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + tenant_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("tenants.id"), nullable=False + ) + user_id: Mapped[Optional[int]] = mapped_column( + ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + user_name: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + user_email: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + module: Mapped[str] = mapped_column(String(50), nullable=False) + action: Mapped[str] = mapped_column(String(100), nullable=False) + target_id: Mapped[str] = mapped_column(String(255), nullable=False) + target_type: Mapped[str] = mapped_column(String(50), nullable=False) + target_name: Mapped[Optional[str]] = mapped_column(String(512), nullable=True) + event_metadata: Mapped[dict] = mapped_column("metadata", JSONB, nullable=False) + ip_address: Mapped[Optional[str]] = mapped_column(String(64), nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + request_id: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + status: Mapped[str] = mapped_column(String(20), nullable=False) + + tenant = relationship("Tenant") + user = relationship("User", foreign_keys=[user_id]) diff --git a/app/modules/activity_logs/routes.py b/app/modules/activity_logs/routes.py new file mode 100644 index 0000000..a775b13 --- /dev/null +++ b/app/modules/activity_logs/routes.py @@ -0,0 +1,112 @@ +from datetime import datetime +from typing import Optional +from uuid import UUID + +from fastapi import APIRouter, Depends, Query +from sqlalchemy.orm import Session + +from app.db.database import get_db +from app.middleware.auth import get_current_user +from app.middleware.tenant import require_tenant +from app.modules.activity_logs.schemas import ActivityLogUserOption, PaginatedActivityLogsOut +from app.modules.activity_logs.service import list_activity_logs, search_activity_log_users +from app.modules.auth.dependencies.access_dependency import require_access +from app.modules.auth.models.user_model import User + +router = APIRouter(prefix="/api/activity-logs", tags=["Activity Logs"]) + + +@router.get( + "/me", + response_model=PaginatedActivityLogsOut, +) +def get_my_activity_logs( + page: int = Query(1, ge=1), + page_size: int = Query(20, ge=1, le=100), + module: Optional[str] = Query(None), + action: Optional[str] = Query(None), + status: Optional[str] = Query(None), + target_id: Optional[str] = Query(None), + target_name: Optional[str] = Query(None), + target_type: Optional[str] = Query(None), + date_from: Optional[datetime] = Query(None), + date_to: Optional[datetime] = Query(None), + search: Optional[str] = Query(None), + tenant_id: UUID = Depends(require_tenant), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + return list_activity_logs( + db=db, + tenant_id=tenant_id, + page=page, + page_size=page_size, + user_id=current_user.id, + module=module, + action=action, + status=status, + target_id=target_id, + target_name=target_name, + target_type=target_type, + date_from=date_from, + date_to=date_to, + search=search, + ) + +@router.get( + "", + response_model=PaginatedActivityLogsOut, + dependencies=[require_access("admin.logs.read")], +) +def get_activity_logs( + page: int = Query(1, ge=1), + page_size: int = Query(20, ge=1, le=100), + user_id: Optional[int] = Query(None), + module: Optional[str] = Query(None), + action: Optional[str] = Query(None), + status: Optional[str] = Query(None), + target_id: Optional[str] = Query(None), + target_name: Optional[str] = Query(None), + target_type: Optional[str] = Query(None), + date_from: Optional[datetime] = Query(None), + date_to: Optional[datetime] = Query(None), + search: Optional[str] = Query(None), + tenant_id: UUID = Depends(require_tenant), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + return list_activity_logs( + db=db, + tenant_id=tenant_id, + page=page, + page_size=page_size, + user_id=user_id, + module=module, + action=action, + status=status, + target_id=target_id, + target_name=target_name, + target_type=target_type, + date_from=date_from, + date_to=date_to, + search=search, + ) + +@router.get( + "/users/search", + response_model=list[ActivityLogUserOption], + dependencies=[require_access("admin.logs.read")], +) +def search_activity_log_users_route( + q: str = Query(..., min_length=1), + limit: int = Query(10, ge=1, le=50), + tenant_id: UUID = Depends(require_tenant), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + return search_activity_log_users( + db=db, + tenant_id=tenant_id, + query=q, + limit=limit, + ) diff --git a/app/modules/activity_logs/schemas.py b/app/modules/activity_logs/schemas.py new file mode 100644 index 0000000..2090092 --- /dev/null +++ b/app/modules/activity_logs/schemas.py @@ -0,0 +1,227 @@ +from datetime import datetime +from typing import Any +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, EmailStr, Field, field_serializer, model_validator + +from app.modules.activity_logs.constants import ( + ActivityLogAction, + ActivityLogModule, + ActivityLogStatus, + ActivityLogTargetType, +) + + +class FolderCreatedDeletedMetadata(BaseModel): + parent_folder_id: int | None + + +class DocumentUploadedMetadata(BaseModel): + file_size: int = Field(ge=0) + file_type: str + folder_id: int + + +class DocumentDeletedMetadata(BaseModel): + folder_id: int + + +class DocumentMovedMetadata(BaseModel): + from_folder_id: int + to_folder_id: int + from_folder: str + to_folder: str + + +class DocumentSharedMetadata(BaseModel): + shared_with: EmailStr + permission: str + entity_type: str + + +class AccessRevokedMetadata(BaseModel): + revoked_from: EmailStr + + +class DocumentEditedMetadata(BaseModel): + version_number: int = Field(ge=1) + user_agent: str | None = None + + + +class ConversionCompletedMetadata(BaseModel): + from_: str = Field(alias="from") + to: str + + model_config = ConfigDict(populate_by_name=True) + + +class DocumentSignedMetadata(BaseModel): + signed_by: EmailStr + signed_at: datetime + + +class DocumentEmailedMetadata(BaseModel): + to_emails: list[str] + cc_emails: list[str] = Field(default_factory=list) + + +class SignRequestSentMetadata(BaseModel): + recipients: list[str] + total_signers: int + provider: str + + +class ConfigUpdatedMetadata(BaseModel): + updated_by: str | None = None + config_type: str + + +class ContactActionMetadata(BaseModel): + contact_email: str + contact_name: str | None = None + + +class DeviceLogoutMetadata(BaseModel): + device_name: str + ip_address: str | None = None + + +class DeviceLoginMetadata(BaseModel): + pass + + +class RoleCreatedMetadata(BaseModel): + role_name: str + + +class UserDeactivatedMetadata(BaseModel): + deactivated_user_id: int + deactivated_user_email: str + + +class SignReminderMetadata(BaseModel): + recipient_email: str + request_id: str + + +ACTION_METADATA_MODELS = { + ActivityLogAction.FOLDER_CREATED: FolderCreatedDeletedMetadata, + ActivityLogAction.FOLDER_DELETED: FolderCreatedDeletedMetadata, + ActivityLogAction.DOCUMENT_UPLOADED: DocumentUploadedMetadata, + ActivityLogAction.DOCUMENT_DELETED: DocumentDeletedMetadata, + ActivityLogAction.DOCUMENT_MOVED: DocumentMovedMetadata, + ActivityLogAction.DOCUMENT_SHARED: DocumentSharedMetadata, + ActivityLogAction.ACCESS_REVOKED: AccessRevokedMetadata, + ActivityLogAction.DOCUMENT_EDITED: DocumentEditedMetadata, + ActivityLogAction.DOCUMENT_SIGNATURE_BROKEN: DocumentEditedMetadata, + ActivityLogAction.CONVERSION_COMPLETED: ConversionCompletedMetadata, + ActivityLogAction.DOCUMENT_SIGNED: DocumentSignedMetadata, + ActivityLogAction.DOCUMENT_EMAILED: DocumentEmailedMetadata, + ActivityLogAction.SIGN_REQUEST_SENT: SignRequestSentMetadata, + ActivityLogAction.CONFIG_SMTP_UPDATED: ConfigUpdatedMetadata, + ActivityLogAction.CONFIG_STORAGE_UPDATED: ConfigUpdatedMetadata, + ActivityLogAction.CONFIG_COMPRESSION_UPDATED: ConfigUpdatedMetadata, + ActivityLogAction.CONFIG_SIGNING_UPDATED: ConfigUpdatedMetadata, + ActivityLogAction.CONTACT_ADDED: ContactActionMetadata, + ActivityLogAction.CONTACT_UPDATED: ContactActionMetadata, + ActivityLogAction.CONTACT_DELETED: ContactActionMetadata, + ActivityLogAction.DEVICE_LOGOUT: DeviceLogoutMetadata, + ActivityLogAction.DEVICE_LOGIN: DeviceLoginMetadata, + ActivityLogAction.ROLE_CREATED: RoleCreatedMetadata, + ActivityLogAction.USER_DEACTIVATED: UserDeactivatedMetadata, + ActivityLogAction.SIGN_REMINDER_SENT: SignReminderMetadata, +} + +ACTION_MODULES = { + ActivityLogAction.FOLDER_CREATED: ActivityLogModule.DRIVE, + ActivityLogAction.FOLDER_DELETED: ActivityLogModule.DRIVE, + ActivityLogAction.DOCUMENT_UPLOADED: ActivityLogModule.DRIVE, + ActivityLogAction.DOCUMENT_DELETED: ActivityLogModule.DRIVE, + ActivityLogAction.DOCUMENT_MOVED: ActivityLogModule.DRIVE, + ActivityLogAction.DOCUMENT_SHARED: ActivityLogModule.DRIVE, + ActivityLogAction.ACCESS_REVOKED: ActivityLogModule.DRIVE, + ActivityLogAction.DOCUMENT_EDITED: ActivityLogModule.EDITOR, + ActivityLogAction.DOCUMENT_SIGNATURE_BROKEN: ActivityLogModule.EDITOR, + ActivityLogAction.CONVERSION_COMPLETED: ActivityLogModule.CONVERSION, + ActivityLogAction.DOCUMENT_SIGNED: ActivityLogModule.SIGNING, + ActivityLogAction.DOCUMENT_EMAILED: ActivityLogModule.DRIVE, + ActivityLogAction.SIGN_REQUEST_SENT: ActivityLogModule.SIGNING, + ActivityLogAction.CONFIG_SMTP_UPDATED: ActivityLogModule.TENANT, + ActivityLogAction.CONFIG_STORAGE_UPDATED: ActivityLogModule.TENANT, + ActivityLogAction.CONFIG_COMPRESSION_UPDATED: ActivityLogModule.TENANT, + ActivityLogAction.CONFIG_SIGNING_UPDATED: ActivityLogModule.TENANT, + ActivityLogAction.CONTACT_ADDED: ActivityLogModule.TENANT, + ActivityLogAction.CONTACT_UPDATED: ActivityLogModule.TENANT, + ActivityLogAction.CONTACT_DELETED: ActivityLogModule.TENANT, + ActivityLogAction.DEVICE_LOGOUT: ActivityLogModule.TENANT, + ActivityLogAction.DEVICE_LOGIN: ActivityLogModule.TENANT, + ActivityLogAction.ROLE_CREATED: ActivityLogModule.TENANT, + ActivityLogAction.USER_DEACTIVATED: ActivityLogModule.TENANT, + ActivityLogAction.SIGN_REMINDER_SENT: ActivityLogModule.SIGNING, +} + + +class ActivityLogCreate(BaseModel): + tenant_id: UUID + user_id: int | None = None + user_name: str | None = None + user_email: EmailStr | None = None + module: ActivityLogModule + action: ActivityLogAction + target_id: str + target_type: ActivityLogTargetType + target_name: str | None = None + metadata: dict[str, Any] + ip_address: str | None = None + request_id: str | None = None + status: ActivityLogStatus + + @model_validator(mode="after") + def validate_action_contract(self) -> "ActivityLogCreate": + expected_module = ACTION_MODULES[self.action] + if self.module != expected_module: + raise ValueError( + f"Action {self.action.value} must use module {expected_module.value}" + ) + + metadata_model = ACTION_METADATA_MODELS[self.action] + validated = metadata_model.model_validate(self.metadata) + self.metadata = validated.model_dump(mode="json", by_alias=True) + return self + + +class ActivityLogOut(BaseModel): + id: int + tenant_id: UUID + user_id: int | None = None + user_name: str | None = None + user_email: str | None = None + module: str + action: str + target_id: str + target_type: str + target_name: str | None = None + metadata: dict[str, Any] + ip_address: str | None = None + created_at: datetime + request_id: str | None = None + status: str + description: str + + @field_serializer("tenant_id") + def serialize_tenant_id(self, tenant_id: UUID) -> str: + return str(tenant_id) + + +class PaginatedActivityLogsOut(BaseModel): + items: list[ActivityLogOut] + total: int + page: int + page_size: int + total_pages: int + +class ActivityLogUserOption(BaseModel): + user_id: int + user_name: str | None = None + user_email: str | None = None \ No newline at end of file diff --git a/app/modules/activity_logs/service.py b/app/modules/activity_logs/service.py new file mode 100644 index 0000000..3d8f5e5 --- /dev/null +++ b/app/modules/activity_logs/service.py @@ -0,0 +1,339 @@ +import logging +from datetime import datetime +from math import ceil +from typing import Any + +from sqlalchemy import func, or_ +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm import Session + +from app.core.context import client_ip_ctx_var +from app.db.database import SessionLocal +from app.modules.activity_logs.constants import ( + ActivityLogAction, + ActivityLogModule, + ActivityLogStatus, + ActivityLogTargetType, +) +from app.modules.activity_logs.models.activity_log import ActivityLog +from app.modules.activity_logs.schemas import ( + ActivityLogCreate, + ActivityLogOut, + ActivityLogUserOption, + PaginatedActivityLogsOut, +) + +logger = logging.getLogger(__name__) + + +def log_event( + *, + tenant_id, + user_id=None, + user_name=None, + user_email=None, + module, + action, + target_id, + target_type, + target_name=None, + metadata: dict[str, Any], + ip_address=None, + request_id=None, + status, +) -> int | None: + try: + payload = ActivityLogCreate( + tenant_id=tenant_id, + user_id=user_id, + user_name=user_name, + user_email=user_email, + module=module, + action=action, + target_id=str(target_id), + target_type=target_type, + target_name=target_name, + metadata=metadata, + ip_address=ip_address, + request_id=request_id, + status=status, + ) + except Exception: + logger.exception("Activity log validation failed") + return None + + session = SessionLocal() + try: + row = ActivityLog( + tenant_id=payload.tenant_id, + user_id=payload.user_id, + user_name=payload.user_name, + user_email=payload.user_email, + module=payload.module.value, + action=payload.action.value, + target_id=payload.target_id, + target_type=payload.target_type.value, + target_name=payload.target_name, + event_metadata=payload.metadata, + ip_address=payload.ip_address, + request_id=payload.request_id, + status=payload.status.value, + ) + session.add(row) + session.commit() + session.refresh(row) + return row.id + except SQLAlchemyError: + session.rollback() + logger.exception("Activity log insert failed") + return None + except Exception: + session.rollback() + logger.exception("Unexpected activity log failure") + return None + finally: + session.close() + + +def log_drive_event( + *, + user, + action: ActivityLogAction, + target_id, + target_type: ActivityLogTargetType, + target_name: str | None, + metadata: dict[str, Any], + status: ActivityLogStatus = ActivityLogStatus.SUCCESS, + request_id: str | None = None, +) -> int | None: + return log_event( + tenant_id=user.tenant_id, + user_id=user.id, + user_name=user.name, + user_email=user.email, + module=ActivityLogModule.DRIVE, + action=action, + target_id=target_id, + target_type=target_type, + target_name=target_name, + metadata=metadata, + ip_address=client_ip_ctx_var.get(), + request_id=request_id, + status=status, + ) + + +def log_user_event( + *, + user, + module: ActivityLogModule, + action: ActivityLogAction, + target_id, + target_type: ActivityLogTargetType, + target_name: str | None, + metadata: dict[str, Any], + status: ActivityLogStatus = ActivityLogStatus.SUCCESS, + request_id: str | None = None, +) -> int | None: + return log_event( + tenant_id=user.tenant_id, + user_id=user.id, + user_name=user.name, + user_email=user.email, + module=module, + action=action, + target_id=target_id, + target_type=target_type, + target_name=target_name, + metadata=metadata, + ip_address=client_ip_ctx_var.get(), + request_id=request_id, + status=status, + ) + + +def build_description(action: str, target_name: str | None, metadata: dict[str, Any]) -> str: + quoted_target = f"'{target_name}'" if target_name else "item" + + if action == ActivityLogAction.DOCUMENT_UPLOADED.value: + return f"Document {quoted_target} uploaded" + if action == ActivityLogAction.DOCUMENT_DELETED.value: + return f"Document {quoted_target} deleted" + if action == ActivityLogAction.FOLDER_CREATED.value: + return f"Folder {quoted_target} created" + if action == ActivityLogAction.FOLDER_DELETED.value: + return f"Folder {quoted_target} deleted" + if action == ActivityLogAction.DOCUMENT_MOVED.value: + return ( + f"Document moved from '{metadata.get('from_folder', '')}' " + f"to '{metadata.get('to_folder', '')}'" + ) + if action == ActivityLogAction.DOCUMENT_SHARED.value: + return f"Document {quoted_target} shared with {metadata.get('shared_with', 'recipient')}" + if action == ActivityLogAction.ACCESS_REVOKED.value: + return f"Access revoked from {metadata.get('revoked_from', 'recipient')} for {quoted_target}" + if action == ActivityLogAction.DOCUMENT_EDITED.value: + return ( + f"Document {quoted_target} edited" + f" (version {metadata.get('version_number', 'unknown')})" + ) + if action == ActivityLogAction.CONVERSION_COMPLETED.value: + return ( + f"Document {quoted_target} converted from " + f"{metadata.get('from', 'unknown')} to {metadata.get('to', 'unknown')}" + ) + if action == ActivityLogAction.DOCUMENT_SIGNED.value: + return f"Document signed by {metadata.get('signed_by', 'unknown signer')}" + if action == ActivityLogAction.DOCUMENT_EMAILED.value: + to_emails = metadata.get('to_emails', []) + return f"Document {quoted_target} emailed to {', '.join(to_emails)}" + if action == ActivityLogAction.SIGN_REQUEST_SENT.value: + return f"Sign request sent for {quoted_target} via {metadata.get('provider', 'unknown')}" + if action == ActivityLogAction.CONFIG_SMTP_UPDATED.value: + return "SMTP configuration updated" + if action == ActivityLogAction.CONFIG_STORAGE_UPDATED.value: + return "Storage configuration updated" + if action == ActivityLogAction.CONFIG_COMPRESSION_UPDATED.value: + return "Compression configuration updated" + if action == ActivityLogAction.CONFIG_SIGNING_UPDATED.value: + return "Signing configuration updated" + if action == ActivityLogAction.CONTACT_ADDED.value: + return f"Contact {metadata.get('contact_email', '')} added to address book" + if action == ActivityLogAction.CONTACT_UPDATED.value: + return f"Contact {metadata.get('contact_email', '')} updated in address book" + if action == ActivityLogAction.CONTACT_DELETED.value: + return f"Contact {metadata.get('contact_email', '')} deleted from address book" + if action == ActivityLogAction.DEVICE_LOGOUT.value: + return f"Device {metadata.get('device_name', 'unknown')} logged out" + if action == ActivityLogAction.ROLE_CREATED.value: + return f"Role {metadata.get('role_name', 'unknown')} created" + if action == ActivityLogAction.USER_DEACTIVATED.value: + return f"User {metadata.get('deactivated_user_email', 'unknown')} deactivated" + if action == ActivityLogAction.SIGN_REMINDER_SENT.value: + return f"Sign reminder sent to {metadata.get('recipient_email', 'unknown')}" + + return f"{action.replace('_', ' ').capitalize()} for {quoted_target}" + + +def list_activity_logs( + *, + db: Session, + tenant_id, + page: int, + page_size: int, + user_id: int | None = None, + module: str | None = None, + action: str | None = None, + status: str | None = None, + target_id: str | None = None, + target_name: str | None = None, + target_type: str | None = None, + date_from: datetime | None = None, + date_to: datetime | None = None, + search: str | None = None, +) -> PaginatedActivityLogsOut: + query = db.query(ActivityLog).filter(ActivityLog.tenant_id == tenant_id) + + if user_id is not None: + query = query.filter(ActivityLog.user_id == user_id) + if module: + query = query.filter(ActivityLog.module == module) + if action: + query = query.filter(ActivityLog.action == action) + if status: + query = query.filter(ActivityLog.status == status) + if target_id: + query = query.filter(ActivityLog.target_id == str(target_id)) + if target_name: + query = query.filter(ActivityLog.target_name.ilike(f"%{target_name}%")) + if target_type: + query = query.filter(ActivityLog.target_type == target_type) + if date_from: + query = query.filter(ActivityLog.created_at >= date_from) + if date_to: + query = query.filter(ActivityLog.created_at <= date_to) + if search and search.strip(): + term = f"%{search.strip()}%" + query = query.filter( + or_( + ActivityLog.user_name.ilike(term), + ActivityLog.target_name.ilike(term), + ) + ) + + total = query.count() + items = ( + query.order_by(ActivityLog.created_at.desc(), ActivityLog.id.desc()) + .offset((page - 1) * page_size) + .limit(page_size) + .all() + ) + + rows = [ + ActivityLogOut( + id=row.id, + tenant_id=row.tenant_id, + user_id=row.user_id, + user_name=row.user_name, + user_email=row.user_email, + module=row.module, + action=row.action, + target_id=row.target_id, + target_type=row.target_type, + target_name=row.target_name, + metadata=row.event_metadata, + ip_address=row.ip_address, + created_at=row.created_at, + request_id=row.request_id, + status=row.status, + description=build_description(row.action, row.target_name, row.event_metadata), + ) + for row in items + ] + + return PaginatedActivityLogsOut( + items=rows, + total=total, + page=page, + page_size=page_size, + total_pages=ceil(total / page_size) if total else 0, + ) + +def search_activity_log_users( + *, + db: Session, + tenant_id, + query: str, + limit: int, +) -> list[ActivityLogUserOption]: + term = query.strip() + if not term: + return [] + + rows = ( + db.query( + ActivityLog.user_id, + func.max(ActivityLog.user_name).label("user_name"), + func.max(ActivityLog.user_email).label("user_email"), + ) + .filter( + ActivityLog.tenant_id == tenant_id, + ActivityLog.user_id.isnot(None), + or_( + ActivityLog.user_name.ilike(f"%{term}%"), + ActivityLog.user_email.ilike(f"%{term}%"), + ), + ) + .group_by(ActivityLog.user_id) + .order_by(func.max(ActivityLog.created_at).desc()) + .limit(limit) + .all() + ) + return [ + ActivityLogUserOption( + user_id=row.user_id, + user_name=row.user_name, + user_email=row.user_email, + ) + for row in rows + ] diff --git a/app/modules/auth/controllers/auth_controller.py b/app/modules/auth/controllers/auth_controller.py new file mode 100644 index 0000000..f02ab72 --- /dev/null +++ b/app/modules/auth/controllers/auth_controller.py @@ -0,0 +1,66 @@ +from sqlalchemy.orm import Session +from app.modules.auth.models.user_model import User +from app.modules.auth.services.auth_service import AuthService, UserService +from typing import Any, Optional + + +class AuthController: + @staticmethod + def register_user(payload: Any, tenant: Any, db: Session): + auth_service = AuthService(db) + return auth_service.register(payload, tenant) + + @staticmethod + def login_user(form_data: Any, db: Session, request: Optional[Any] = None): + auth_service = AuthService(db) + return auth_service.login(form_data.username, form_data.password, request) + + @staticmethod + def google_login(payload: Any, db: Session, request: Optional[Any] = None): + auth_service = AuthService(db) + return auth_service.google_login(payload, request) + + @staticmethod + def forgot_password(payload: Any, db: Session): + auth_service = AuthService(db) + return auth_service.forgot_password(payload) + + @staticmethod + def reset_password(payload: Any, db: Session): + auth_service = AuthService(db) + return auth_service.reset_password(payload) + + +class UserController: + @staticmethod + def update_profile( + user: User, + name: Optional[str], + db: Session, + designation: Optional[str] = None, + preferred_language: Optional[str] = None, + ): + user_service = UserService(db) + return user_service.update_profile( + user, + name=name, + designation=designation, + preferred_language=preferred_language, + ) + + @staticmethod + def change_password( + user: User, current_password: str, new_password: str, db: Session + ): + user_service = UserService(db) + return user_service.change_password(user, current_password, new_password) + + @staticmethod + def get_user_stats(user: User, db: Session): + user_service = UserService(db) + return user_service.get_user_stats(user) + + @staticmethod + def delete_account(user: User, db: Session): + user_service = UserService(db) + return user_service.delete_account(user) diff --git a/app/modules/auth/controllers/role_controller.py b/app/modules/auth/controllers/role_controller.py new file mode 100644 index 0000000..4dee9c2 --- /dev/null +++ b/app/modules/auth/controllers/role_controller.py @@ -0,0 +1,78 @@ +""" +Role Controller - handles role management API logic. +""" + +from sqlalchemy.orm import Session +from typing import List, Optional +from uuid import UUID + +from app.modules.auth.models.role_model import Role +from app.modules.auth.schemas.role_schema import ( + RoleCreate, + RoleUpdate, + RoleOut, + RoleWithAccessesOut, + RolePaginatedOut, +) +from app.modules.auth.services.role_service import RoleService + + +class RoleController: + """Controller for role management endpoints.""" + + @staticmethod + def create_role(db: Session, role_data: RoleCreate) -> RoleOut: + """Create a new role.""" + role = RoleService.create_role(db, role_data) + return RoleOut.model_validate(role) + + @staticmethod + def get_all_roles(db: Session, tenant_id: Optional[UUID] = None) -> List[RoleOut]: + """Get all roles.""" + roles = RoleService.get_all_roles(db, tenant_id) + return [RoleOut.model_validate(role) for role in roles] + + @staticmethod + def get_role_by_id(db: Session, role_id: UUID) -> RoleOut: + """Get a role by ID.""" + role = RoleService.get_role_by_id(db, role_id) + return RoleOut.model_validate(role) + + @staticmethod + def get_role_with_accesses(db: Session, role_id: UUID) -> RoleWithAccessesOut: + """Get a role with all its accesses.""" + role = RoleService.get_role_with_accesses(db, role_id) + return RoleWithAccessesOut.model_validate(role) + + @staticmethod + def update_role(db: Session, role_id: UUID, role_data: RoleUpdate) -> RoleOut: + """Update a role.""" + role = RoleService.update_role(db, role_id, role_data) + return RoleOut.model_validate(role) + + @staticmethod + def delete_role(db: Session, role_id: UUID): + """Delete a role.""" + return RoleService.delete_role(db, role_id) + + @staticmethod + def get_roles_paginated( + db: Session, + tenant_id: Optional[UUID] = None, + page: int = 1, + page_size: int = 10, + search: Optional[str] = None, + ) -> RolePaginatedOut: + """Get paginated list of roles.""" + return RoleService.get_roles_paginated( + db=db, + tenant_id=tenant_id, + page=page, + page_size=page_size, + search=search, + ) + + @staticmethod + def assign_role_to_user(db: Session, user_id: int, role_id: UUID): + """Assign a role to a user.""" + return RoleService.assign_role_to_user(db, user_id, role_id) diff --git a/app/modules/auth/dependencies/__init__.py b/app/modules/auth/dependencies/__init__.py new file mode 100644 index 0000000..1c5a202 --- /dev/null +++ b/app/modules/auth/dependencies/__init__.py @@ -0,0 +1,3 @@ +from .access_dependency import require_access, require_any_access, require_all_access + +__all__ = ["require_access", "require_any_access", "require_all_access"] diff --git a/app/modules/auth/dependencies/access_dependency.py b/app/modules/auth/dependencies/access_dependency.py new file mode 100644 index 0000000..ff44baf --- /dev/null +++ b/app/modules/auth/dependencies/access_dependency.py @@ -0,0 +1,91 @@ +""" +Access control dependencies for FastAPI routes. + +These dependencies are used in route definitions to require specific access codes. + +Example: + @router.post("/projects", dependencies=[require_access("project.create")]) + def create_project(user: User = Depends(get_current_user)): + ... +""" + +from typing import List +from fastapi import Depends, HTTPException +from sqlalchemy.orm import Session + +from app.db.database import get_db +from app.middleware.auth import get_current_user +from app.modules.auth.services.permission_service import PermissionService +from app.modules.auth.models.user_model import User + + +def require_access(code: str): + """ + Dependency to require a specific access code. + + Usage in routes: + @router.post("/path", dependencies=[require_access("some.code")]) + def endpoint(...): + ... + + Args: + code: The required access code (e.g. "project.create") + + Returns: + FastAPI Depends wrapper that checks access + """ + + async def check_access( + user: User = Depends(get_current_user), db: Session = Depends(get_db) + ) -> None: + PermissionService(db).require_access(user, code) + + return Depends(check_access) + + +def require_any_access(*codes: str): + """ + Dependency to require at least one of the provided access codes. + + Usage in routes: + @router.post("/path", dependencies=[require_any_access("code1", "code2")]) + def endpoint(...): + ... + + Args: + *codes: One or more access codes (user needs at least one) + + Returns: + FastAPI Depends wrapper that checks access + """ + + async def check_access( + user: User = Depends(get_current_user), db: Session = Depends(get_db) + ) -> None: + PermissionService(db).require_any_access(user, list(codes)) + + return Depends(check_access) + + +def require_all_access(*codes: str): + """ + Dependency to require all of the provided access codes. + + Usage in routes: + @router.post("/path", dependencies=[require_all_access("code1", "code2")]) + def endpoint(...): + ... + + Args: + *codes: One or more access codes (user must have all) + + Returns: + FastAPI Depends wrapper that checks access + """ + + async def check_access( + user: User = Depends(get_current_user), db: Session = Depends(get_db) + ) -> None: + PermissionService(db).require_all_access(user, list(codes)) + + return Depends(check_access) diff --git a/app/modules/auth/middleware/hmac.py b/app/modules/auth/middleware/hmac.py new file mode 100644 index 0000000..f0f88f3 --- /dev/null +++ b/app/modules/auth/middleware/hmac.py @@ -0,0 +1,33 @@ +import hmac +import hashlib +import logging +from fastapi import Request, HTTPException, status +from app.core.settings import settings + +logger = logging.getLogger(__name__) + +async def verify_saas_hmac(request: Request): + """ + Verify HMAC signature from SaaS. + + The signature is calculated as: + HMAC-SHA256(secret, canonical_string) + + where canonical_string is usually user_id=...&email=...&tenant_id=...×tamp=... + """ + signature = request.headers.get("X-Signature") + if not signature: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Missing HMAC signature" + ) + + if not settings.SAAS_TRUST_SECRET: + logger.error("SAAS_TRUST_SECRET is not configured") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Server configuration error" + ) + + + return True diff --git a/app/modules/auth/models/access_model.py b/app/modules/auth/models/access_model.py new file mode 100644 index 0000000..f67a995 --- /dev/null +++ b/app/modules/auth/models/access_model.py @@ -0,0 +1,42 @@ +import uuid +from sqlalchemy import String, DateTime, func, ForeignKey, Index +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship +from typing import Optional +from app.db.database import Base + + +class Access(Base): + __tablename__ = "accesses" + __table_args__ = ( + Index("ix_access_code", "access_code"), + Index("ix_category", "category"), + ) + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + + access_code: Mapped[str] = mapped_column( + String(255), unique=True, nullable=False, index=True + ) + + category: Mapped[str] = mapped_column(String(100), nullable=False, index=True) + + name: Mapped[str] = mapped_column(String(255), nullable=False) + + parent_id: Mapped[Optional[uuid.UUID]] = mapped_column( + ForeignKey("accesses.id", ondelete="CASCADE"), nullable=True, index=True + ) + + created_at: Mapped[DateTime] = mapped_column( + DateTime(timezone=True), server_default=func.now() + ) + + parent = relationship("Access", remote_side=[id], backref="children") + role_accesses = relationship( + "RoleAccess", back_populates="access", cascade="all, delete-orphan" + ) + + def __repr__(self): + return f"" \ No newline at end of file diff --git a/app/modules/auth/models/device_model.py b/app/modules/auth/models/device_model.py new file mode 100644 index 0000000..a7440cc --- /dev/null +++ b/app/modules/auth/models/device_model.py @@ -0,0 +1,43 @@ +import uuid +from datetime import datetime +from sqlalchemy import String, DateTime, ForeignKey, Integer, Boolean, Float, func, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column, relationship +from typing import Optional +from app.db.database import Base + + +class Device(Base): + __tablename__ = "devices" + __table_args__ = ( + UniqueConstraint("user_id", "fingerprint", name="uq_device_user_fingerprint"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + + user_id: Mapped[int] = mapped_column( + Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True + ) + user = relationship("User", backref="devices") + + fingerprint: Mapped[str] = mapped_column(String(255), index=True, nullable=False) + + browser: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + os: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + device_type: Mapped[Optional[str]] = mapped_column(String(50), nullable=True) + user_agent: Mapped[Optional[str]] = mapped_column(String(512), nullable=True) + + ip: Mapped[Optional[str]] = mapped_column(String(50), nullable=True) + country: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + state: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + city: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + latitude: Mapped[Optional[float]] = mapped_column(Float, nullable=True) + longitude: Mapped[Optional[float]] = mapped_column(Float, nullable=True) + + first_login: Mapped[DateTime] = mapped_column(DateTime(timezone=True), default=func.now()) + last_login: Mapped[DateTime] = mapped_column(DateTime(timezone=True), default=func.now()) + last_ip: Mapped[Optional[str]] = mapped_column(String(50), nullable=True) + + is_blocked: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) + + created_at: Mapped[DateTime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + updated_at: Mapped[DateTime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) diff --git a/app/modules/auth/models/role_access_model.py b/app/modules/auth/models/role_access_model.py new file mode 100644 index 0000000..38dd793 --- /dev/null +++ b/app/modules/auth/models/role_access_model.py @@ -0,0 +1,21 @@ +from sqlalchemy import ForeignKey, PrimaryKeyConstraint +from sqlalchemy.orm import Mapped, mapped_column, relationship +from app.db.database import Base +from sqlalchemy.dialects.postgresql import UUID +import uuid + +class RoleAccess(Base): + __tablename__ = "role_accesses" + __table_args__ = ( + PrimaryKeyConstraint("role_id", "access_id", name="pk_role_accesses"), + ) + + role_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("roles.id", ondelete="CASCADE"), nullable=False + ) + + access_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("accesses.id", ondelete="CASCADE"), nullable=False, index=True) + + role = relationship("Role", back_populates="role_accesses") + access = relationship("Access", back_populates="role_accesses") \ No newline at end of file diff --git a/app/modules/auth/models/role_model.py b/app/modules/auth/models/role_model.py new file mode 100644 index 0000000..3116738 --- /dev/null +++ b/app/modules/auth/models/role_model.py @@ -0,0 +1,45 @@ +import uuid +from sqlalchemy import String, DateTime, ForeignKey, UniqueConstraint, func, Boolean +from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.dialects.postgresql import UUID +from typing import Optional, List +from app.db.database import Base + + +class Role(Base): + __tablename__ = "roles" + __table_args__ = ( + UniqueConstraint("tenant_id", "name", name="uq_role_tenant_name"), + ) + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + + tenant_id: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), + ForeignKey("tenants.id", ondelete="CASCADE"), + nullable=True, + index=True, + ) + + name: Mapped[str] = mapped_column(String(100), nullable=False) + + description: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + is_default: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + is_system: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + + created_at: Mapped[DateTime] = mapped_column( + DateTime(timezone=True), server_default=func.now() + ) + + users = relationship("User", back_populates="role") + role_accesses = relationship( + "RoleAccess", back_populates="role", cascade="all, delete-orphan" + ) + tenant = relationship("Tenant", back_populates="roles") + + @property + def accesses(self): + """Return the list of Access objects for this role (used by Pydantic RoleWithAccessesOut).""" + return [ra.access for ra in self.role_accesses if ra.access] diff --git a/app/modules/auth/models/saas_models.py b/app/modules/auth/models/saas_models.py new file mode 100644 index 0000000..74a7b26 --- /dev/null +++ b/app/modules/auth/models/saas_models.py @@ -0,0 +1,44 @@ +import uuid +from sqlalchemy import Column, String, DateTime, func, ForeignKey, JSON +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import relationship +from app.db.database import Base + +class SaaSUserMapping(Base): + __tablename__ = "saas_user_mappings" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, index=True) + saas_user_id = Column(String, nullable=False, unique=True, index=True) + docqube_user_id = Column(ForeignKey("users.id"), nullable=False) + metadata_ = Column("metadata", JSON, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), onupdate=func.now(), server_default=func.now()) + + user = relationship("User", backref="saas_mapping") + +class SaaSTenantMapping(Base): + __tablename__ = "saas_tenant_mappings" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, index=True) + saas_tenant_id = Column(String, unique=True, nullable=False, index=True) + docqube_tenant_id = Column(UUID(as_uuid=True), ForeignKey("tenants.id"), nullable=False) + metadata_ = Column("metadata", JSON, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + tenant = relationship("Tenant") + +class SaaSRoleMapping(Base): + __tablename__ = "saas_role_mappings" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, index=True) + saas_role_id = Column(String, unique=True, nullable=False, index=True) + docqube_role_id = Column(UUID(as_uuid=True), ForeignKey("roles.id", ondelete="CASCADE"), nullable=False) + metadata_ = Column("metadata", JSON, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + role = relationship("Role", backref="saas_mapping") diff --git a/app/modules/auth/models/session_model.py b/app/modules/auth/models/session_model.py new file mode 100644 index 0000000..a1603a4 --- /dev/null +++ b/app/modules/auth/models/session_model.py @@ -0,0 +1,35 @@ +from datetime import datetime +from sqlalchemy import String, DateTime, ForeignKey, Integer, func +from sqlalchemy.orm import Mapped, mapped_column, relationship +from typing import Optional +from app.db.database import Base +from app.modules.auth.models.device_model import Device + + +class Session(Base): + __tablename__ = "sessions" + + id: Mapped[int] = mapped_column(primary_key=True) + + user_id: Mapped[int] = mapped_column( + Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True + ) + user = relationship("User", backref="auth_sessions", foreign_keys=[user_id]) + + device_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("devices.id", ondelete="SET NULL"), nullable=True, index=True + ) + device = relationship("Device", backref="sessions") + + refresh_token_hash: Mapped[str] = mapped_column(String(255), nullable=False) + + status: Mapped[str] = mapped_column(String(50), default="ACTIVE", nullable=False) + + created_at: Mapped[DateTime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + last_activity: Mapped[DateTime] = mapped_column(DateTime(timezone=True), default=func.now()) + expires_at: Mapped[DateTime] = mapped_column(DateTime(timezone=True), nullable=False) + + revoked_at: Mapped[Optional[DateTime]] = mapped_column(DateTime(timezone=True), nullable=True) + revoked_by: Mapped[Optional[str]] = mapped_column(String(50), nullable=True) + revoked_by_user_id: Mapped[Optional[int]] = mapped_column(Integer, ForeignKey("users.id", ondelete="SET NULL"), nullable=True) + reason: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) diff --git a/app/modules/auth/models/user_model.py b/app/modules/auth/models/user_model.py new file mode 100644 index 0000000..2c3dd44 --- /dev/null +++ b/app/modules/auth/models/user_model.py @@ -0,0 +1,226 @@ +import uuid +from datetime import datetime +from sqlalchemy import String, DateTime, func, ForeignKey, UniqueConstraint, Integer, Boolean, Index, text +from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.dialects.postgresql import UUID +from typing import Optional +from app.db.database import Base + + +class User(Base): + __tablename__ = "users" + __table_args__ = ( + UniqueConstraint("tenant_id", "email", name="uq_user_tenant_email"), + Index( + "ix_users_is_superadmin", + "is_superadmin", + postgresql_where=text("is_superadmin"), + ), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + + tenant_id: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), ForeignKey("tenants.id"), nullable=True, index=True + ) + + tenant = relationship("Tenant", back_populates="users", foreign_keys=[tenant_id]) + + is_superadmin: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=False, server_default="false" + ) + + @property + def tenant_envelope_limit(self) -> Optional[int]: + if self.tenant: + return self.tenant.envelope_limit + return None + + @property + def tenant_envelopes_used(self) -> Optional[int]: + if self.tenant: + return self.tenant.envelopes_used + return None + + role_id: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), ForeignKey("roles.id", ondelete="SET NULL"), nullable=True, index=True + ) + + role = relationship("Role", back_populates="users") + + name: Mapped[str] = mapped_column(String(255), nullable=False) + + designation: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + preferred_language: Mapped[str] = mapped_column(String(16), nullable=False, default="en") + + email: Mapped[str] = mapped_column(String(255), index=True, nullable=False) + + preferred_language: Mapped[str] = mapped_column(String(16), nullable=False, default="en", server_default="en") + + password_hash: Mapped[str] = mapped_column(String(255), nullable=False) + + subscription: Mapped[str] = mapped_column(String(50), default="free") + is_active: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + terms_accepted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + has_completed_tutorial: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + has_completed_docs_tutorial: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + is_deleted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + deleted_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True) + deleted_by_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + + created_at: Mapped[DateTime] = mapped_column( + DateTime(timezone=True), server_default=func.now() + ) + + encrypted_ai_api_key: Mapped[Optional[str]] = mapped_column( + String(512), nullable=True + ) + + smtp_user: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + + mail_from: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + + encrypted_smtp_password: Mapped[Optional[str]] = mapped_column( + String(512), nullable=True + ) + + projects = relationship( + "Project", back_populates="owner", cascade="all, delete-orphan" + ) + + drive_folders = relationship( + "DriveFolder", back_populates="owner", cascade="all, delete-orphan" + ) + + drive_files = relationship( + "DriveFile", back_populates="owner", cascade="all, delete-orphan" + ) + + project_versions = relationship( + "ProjectVersion", back_populates="creator", cascade="all, delete-orphan" + ) + + drive_file_versions = relationship( + "DriveFileVersion", back_populates="creator", cascade="all, delete-orphan" + ) + + drive_comments = relationship( + "DriveComment", back_populates="author", cascade="all, delete-orphan" + ) + + drive_activities = relationship( + "DriveActivity", back_populates="actor", cascade="all, delete-orphan" + ) + + user_files = relationship( + "UserFile", back_populates="user", cascade="all, delete-orphan" + ) + + storage_usage = relationship( + "UserStorageUsage", + back_populates="user", + uselist=False, + cascade="all, delete-orphan", + ) + + notifications = relationship( + "Notification", + back_populates="user", + cascade="all, delete-orphan", + lazy="dynamic", + ) + chat_token_quota = relationship( + "ChatTokenQuota", + back_populates="user", + uselist=False, + cascade="all, delete-orphan", + ) + deleted_by = relationship("User", remote_side=[id], foreign_keys=[deleted_by_id]) + + @property + def role_name(self) -> Optional[str]: + return self.role.name if self.role else None + + @property + def access_codes(self) -> list[str]: + """ + Every access code this user holds. + + **Prefers what `get_current_user` resolved.** This property used to be an + independent second answer to "what may this user do", computed from the + single legacy `users.role_id` and nothing else — so a user holding a role + through `user_roles` had authority that `require_access` honoured and the + product would not show them. `/api/me/profile` reports this list and the + frontend drives every route guard, menu entry and button from it, so an + incomplete answer here is indistinguishable from having no permission. + + `_resolve_access_codes` is attached by `get_current_user` and calls + `PermissionService`, the one correct resolver: legacy role, `user_roles` + grants (live and in-tenant only), group grants, SaaS claims, and + descendant expansion. It is a **closure rather than a computed list** so + that requests which never ask do not pay for the answer; the result is + memoised per request, so asking twice is free. + + The fallback below is the old computation, kept for objects that never + passed through authentication — fixtures, background tasks, a detached + row. It is deliberately not an error: raising here would turn "no session + attached" into a 500 on paths that only wanted a display name. + """ + resolver = getattr(self, "_resolve_access_codes", None) + if resolver is not None: + return list(resolver()) + + codes = set() + if self.role: + codes.update([ra.access.access_code for ra in self.role.role_accesses if ra.access]) + + saas_permissions = getattr(self, "saas_permissions", set()) + codes.update(saas_permissions) + + return list(codes) + + @property + def role_refs(self) -> list: + """ + Every role this user holds, as resolved assignment dicts. + + Named `role_refs` rather than `roles` on purpose: `roles` would sit one + typo away from a relationship name in a class that already has `role`, + and a plain attribute shadowing a mapper attribute fails at import time + in a way that is tedious to diagnose. `UserOut` exposes it as `roles`. + + Resolved through a closure attached by `get_current_user`, for the same + reason as `access_codes`: only the responses that report roles should pay + for reading them. + + Off that path the primary role is synthesised, so a detached object still + describes itself honestly rather than claiming no roles at all. + """ + resolver = getattr(self, "_resolve_role_refs", None) + if resolver is not None: + return list(resolver()) + + if self.role is None: + return [] + return [ + { + "grant_id": None, + "role_id": self.role.id, + "role_name": self.role.name, + "source": "primary", + "org_unit_id": None, + "org_unit_name": None, + "group_id": None, + "group_name": None, + "expires_at": None, + "assigned_by_id": None, + "created_at": None, + "is_expired": False, + } + ] + + @property + def subscription_details(self): + return getattr(self, "saas_subscription", None) diff --git a/app/modules/auth/models/user_role_model.py b/app/modules/auth/models/user_role_model.py new file mode 100644 index 0000000..79b5397 --- /dev/null +++ b/app/modules/auth/models/user_role_model.py @@ -0,0 +1,160 @@ +""" +C3 — a role held *somewhere*. + +`User.role_id` gives every user exactly one role, held across the whole tenant. +That is the thing scope cannot be built on: until a user can hold the same role +in two places, or two roles in one place, "scoped access" has nowhere to live. + +This is the join table the base uses (`docqube_platform_backend/app/models/rbac.py`), +with one difference: it points at DocQube's existing `roles`, so the access-code +vocabulary is unchanged and all 227 endpoints keep working exactly as they do +today. + + org_unit_id IS NULL -> tenant-wide, the grant everyone has today + org_unit_id = X -> this role, within X and everything beneath it + +The null case is what makes the migration a no-op: the backfill turns each +`User.role_id` into one row with a null unit, which is precisely the authority +that user already had. + +**Both mechanisms are honoured for one release.** `User.role_id` is not dropped +here. That is the same transition shape as `users.is_superadmin`, for the same +reason — a rollback must not be able to remove everyone's permissions — and it +ends the same way, with a preflight script and a deliberate removal. +""" + +import uuid +from datetime import datetime, timezone +from typing import Optional + +from sqlalchemy import DateTime, ForeignKey, Index, UniqueConstraint, func +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.db.database import Base + +import app.modules.org.models.group_model # noqa: F401 + + +class UserRole(Base): + __tablename__ = "user_roles" + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + tenant_id: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), + ForeignKey("tenants.id", ondelete="CASCADE"), + nullable=True, + index=True, + ) + """ + The tenant this grant belongs to. + + Present so isolation is **structural** rather than remembered: the ORM + listener scopes any model carrying this column, and the RLS policy compares + it against the session variable. Without it the table sat outside both + layers, guarded only by whatever predicate each call site wrote — the same + shape of gap that let any holder of `superadmin.tenant.delete` delete any + tenant. + + Nullable, because a superadmin has no tenant and neither do their grants. + """ + + user_id: Mapped[Optional[int]] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), nullable=True, index=True + ) + group_id: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), + ForeignKey("access_groups.id", ondelete="CASCADE"), + nullable=True, + index=True, + ) + """ + The principal: exactly one of `user_id` or `group_id`. + + Enforced by a CHECK constraint, not by the application. "Exactly one of two + nullable columns" survives in the schema and rots in code — one new call + site that sets both, and the resolver counts the grant twice. + + `user_id` became nullable in C5.1. That weakened an invariant every prior + query relied on, so all four readers were reviewed before the migration: + `core/scope.py`, `auth/services/permission_service.py`, + `org/routes/org_routes.py` and `org/services/access_maintenance.py`. + """ + role_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("roles.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + org_unit_id: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), + ForeignKey("org_units.id", ondelete="CASCADE"), + nullable=True, + index=True, + ) + assigned_by_id: Mapped[Optional[int]] = mapped_column( + ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + + expires_at: Mapped[Optional[datetime]] = mapped_column( + DateTime(timezone=True), nullable=True + ) + """ + When this grant stops applying. Null means permanent. + + Enforced in the resolution query, not in Python — an expired grant must be + invisible to *every* reader, including any future one that queries this + table directly. Filtering after the fact is how one caller ends up honouring + a dead grant. + + Expired rows are kept rather than swept: they are the history that answers + "who had access in March". + """ + + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + + role = relationship("Role", foreign_keys=[role_id]) + user = relationship("User", foreign_keys=[user_id]) + + __table_args__ = ( + UniqueConstraint( + "user_id", "role_id", "org_unit_id", name="uq_user_roles_scoped" + ), + Index( + "uq_user_roles_tenant_wide", + "user_id", + "role_id", + unique=True, + postgresql_where=(org_unit_id.is_(None)), + ), + Index("ix_user_roles_lookup", "user_id", "role_id"), + Index( + "uq_user_roles_group_scoped", + "group_id", + "role_id", + "org_unit_id", + unique=True, + postgresql_where=(group_id.isnot(None)), + ), + Index( + "uq_user_roles_group_tenant_wide", + "group_id", + "role_id", + unique=True, + postgresql_where=(org_unit_id.is_(None) & group_id.isnot(None)), + ), + Index( + "ix_user_roles_expires_at", + "expires_at", + postgresql_where=(expires_at.isnot(None)), + ), + Index("ix_user_roles_assigned_by_id", "assigned_by_id"), + ) + + def __repr__(self) -> str: # pragma: no cover - debugging aid + where = self.org_unit_id or "tenant-wide" + return f"" diff --git a/app/modules/auth/repositories/device_repository.py b/app/modules/auth/repositories/device_repository.py new file mode 100644 index 0000000..139f9f5 --- /dev/null +++ b/app/modules/auth/repositories/device_repository.py @@ -0,0 +1,45 @@ +from typing import List, Optional +from sqlalchemy.orm import Session +from app.modules.auth.models.device_model import Device +from datetime import datetime, timezone + +class DeviceRepository: + def __init__(self, db: Session): + self.db = db + + def get_by_id(self, device_id: int) -> Optional[Device]: + return self.db.query(Device).filter(Device.id == device_id).first() + + def get_by_fingerprint(self, user_id: int, fingerprint: str) -> Optional[Device]: + return self.db.query(Device).filter( + Device.user_id == user_id, + Device.fingerprint == fingerprint + ).first() + + def get_user_devices(self, user_id: int) -> List[Device]: + from app.modules.auth.models.session_model import Session as AuthSession + return self.db.query(Device).join(AuthSession, Device.id == AuthSession.device_id).filter( + Device.user_id == user_id, + AuthSession.status == "ACTIVE", + AuthSession.expires_at > datetime.now(timezone.utc) + ).order_by(Device.last_login.desc()).distinct().all() + + def create(self, device: Device) -> Device: + self.db.add(device) + self.db.flush() + self.db.refresh(device) + return device + + def update_login_activity(self, device: Device, ip: str) -> Device: + device.last_login = datetime.now(timezone.utc) + if ip: + device.last_ip = ip + self.db.flush() + self.db.refresh(device) + return device + + def block_device(self, device: Device) -> Device: + device.is_blocked = True + self.db.flush() + self.db.refresh(device) + return device diff --git a/app/modules/auth/repositories/session_repository.py b/app/modules/auth/repositories/session_repository.py new file mode 100644 index 0000000..e81b93e --- /dev/null +++ b/app/modules/auth/repositories/session_repository.py @@ -0,0 +1,72 @@ +from typing import List, Optional +from sqlalchemy.orm import Session +from app.modules.auth.models.session_model import Session as AuthSession +from datetime import datetime, timezone + +class SessionRepository: + def __init__(self, db: Session): + self.db = db + + def get_by_id(self, session_id: int) -> Optional[AuthSession]: + return self.db.query(AuthSession).filter(AuthSession.id == session_id).first() + + def get_active_sessions_for_user(self, user_id: int) -> List[AuthSession]: + return self.db.query(AuthSession).filter( + AuthSession.user_id == user_id, + AuthSession.status == "ACTIVE", + AuthSession.expires_at > datetime.now(timezone.utc) + ).all() + + def count_active_sessions(self, user_id: int) -> int: + return self.db.query(AuthSession).filter( + AuthSession.user_id == user_id, + AuthSession.status == "ACTIVE", + AuthSession.expires_at > datetime.now(timezone.utc) + ).count() + + def create(self, session: AuthSession) -> AuthSession: + self.db.add(session) + self.db.flush() + self.db.refresh(session) + return session + + def update_last_activity(self, session: AuthSession) -> AuthSession: + session.last_activity = datetime.now(timezone.utc) + return session + + def revoke(self, session: AuthSession, by: str = "user", revoked_by_user_id: Optional[int] = None, reason: Optional[str] = None) -> AuthSession: + session.status = "REVOKED" + session.revoked_at = datetime.now(timezone.utc) + session.revoked_by = by + if revoked_by_user_id: + session.revoked_by_user_id = revoked_by_user_id + if reason: + session.reason = reason + self.db.flush() + self.db.refresh(session) + + from app.db.redis import redis_cache + redis_cache.set(f"session_revoked:{session.id}", "1", ttl=600) + import json + if redis_cache.client: + redis_cache.client.publish(f"user_events:{session.user_id}", json.dumps({"type": "SESSION_REVOKED", "session_id": session.id})) + + return session + + def revoke_all_for_device(self, device_id: int, by: str = "user") -> None: + self.db.query(AuthSession).filter( + AuthSession.device_id == device_id, + AuthSession.status == "ACTIVE" + ).update({ + "status": "REVOKED", + "revoked_at": datetime.now(timezone.utc), + "revoked_by": by + }) + + from app.modules.auth.models.device_model import Device + device = self.db.query(Device).filter(Device.id == device_id).first() + if device: + from app.db.redis import redis_cache + import json + if redis_cache.client: + redis_cache.client.publish(f"user_events:{device.user_id}", json.dumps({"type": "DEVICE_REVOKED", "device_id": device.id})) diff --git a/app/modules/auth/repositories/user_repository.py b/app/modules/auth/repositories/user_repository.py new file mode 100644 index 0000000..c67b9ec --- /dev/null +++ b/app/modules/auth/repositories/user_repository.py @@ -0,0 +1,43 @@ +from typing import Optional, List +from sqlalchemy.orm import Session +from app.modules.auth.models.user_model import User + +class UserRepository: + def __init__(self, db: Session): + self.db = db + + def get_by_email(self, email: str) -> Optional[User]: + return self.db.query(User).filter(User.email == email.lower().strip()).first() + + def get_by_id(self, user_id: int) -> Optional[User]: + return self.db.query(User).filter(User.id == user_id).first() + + def create(self, user: User) -> User: + self.db.add(user) + self.db.flush() + self.db.refresh(user) + return user + + def search_users( + self, query: str, limit: int = 10, tenant_id=None + ) -> List[User]: + """ + Search users by name or email. + + `tenant_id` is required in practice: without it this returns users from + every tenant, including their email addresses and permission sets. The + parameter defaults to None only so a superadmin caller can search across + tenants deliberately — callers that serve tenant users must always pass + the caller's own tenant. + """ + q = self.db.query(User).filter( + User.is_deleted.is_(False), + (User.name.ilike(f"%{query}%")) | (User.email.ilike(f"%{query}%")), + ) + if tenant_id is not None: + q = q.filter(User.tenant_id == tenant_id) + return q.limit(limit).all() + + def get_users_by_ids(self, ids: List[int]) -> List[User]: + return self.db.query(User).filter(User.id.in_(ids), User.is_deleted.is_(False)).all() + diff --git a/app/modules/auth/routes/access_routes.py b/app/modules/auth/routes/access_routes.py new file mode 100644 index 0000000..b0639c1 --- /dev/null +++ b/app/modules/auth/routes/access_routes.py @@ -0,0 +1,76 @@ +""" +Access control routes. + +Endpoints for retrieving user's access codes and managing permissions. +Used by frontend to determine what UI elements to show. +""" + +from fastapi import APIRouter, Depends +from sqlalchemy.orm import Session + +from app.db.database import get_db +from app.middleware.auth import get_current_user +from app.modules.auth.models.user_model import User +from app.modules.auth.services.permission_service import PermissionService + +from typing import List + +from app.modules.auth.schemas.access_schema import AccessOut, RoleAccessesOut + +router = APIRouter(prefix="/api/access", tags=["Access"]) + + +@router.get("/get", response_model=RoleAccessesOut) +def get_user_accesses( + user: User = Depends(get_current_user), + db: Session = Depends(get_db), +): + """ + Get current user's access codes. + + This endpoint is called by the frontend to determine which UI elements + to show/hide based on the user's permissions. + + Returns: + { + "accesses": ["project.create", "drive.file.upload", ...], + "role": "superadmin" | "tenant_admin" | null, + "roles": ["tenant_admin", "approver"] + } + + `role` is the **primary** role and is unchanged. `roles` is every role the + caller holds — primary, granted, and through a group — because a user may + hold several and a single name cannot say so. + """ + perm_service = PermissionService(db) + accesses = sorted(list(perm_service.user_access_codes(user))) + + seen: list[str] = [] + for assignment in user.role_refs: + name = assignment.get("role_name") + if name and name not in seen: + seen.append(name) + + return { + "accesses": accesses, + "role": user.role.name if user.role else None, + "roles": seen, + "tenant_id": str(user.tenant_id) if user.tenant_id else None, + } + + +@router.get("/all", response_model=List[AccessOut]) +def get_all_system_accesses( + user: User = Depends(get_current_user), + db: Session = Depends(get_db), +): + """ + Get all available access codes in the platform. + Typically used for role management UI. + """ + from app.modules.auth.dependencies.access_dependency import require_access + + perm_service = PermissionService(db) + perm_service.require_any_access(user, ["superadmin.role.read", "admin.role.read"]) + + return perm_service.get_all_system_accesses() diff --git a/app/modules/auth/routes/admin_dashboard_routes.py b/app/modules/auth/routes/admin_dashboard_routes.py new file mode 100644 index 0000000..db2eb89 --- /dev/null +++ b/app/modules/auth/routes/admin_dashboard_routes.py @@ -0,0 +1,43 @@ +from typing import Literal + +from fastapi import APIRouter, Depends, Query +from sqlalchemy.orm import Session + +from app.db.database import get_db +from app.middleware.auth import get_current_user +from app.modules.auth.dependencies.access_dependency import require_access +from app.modules.auth.models.user_model import User +from app.modules.auth.schemas.admin_dashboard_schema import ( + AdminDashboardMetricsOut, + MetricTrendOnlyOut, +) +from app.modules.auth.services.admin_dashboard_service import AdminDashboardService + +router = APIRouter(prefix="/api/admin", tags=["Admin Dashboard"]) + + +@router.get( + "/dashboard-metrics", + response_model=AdminDashboardMetricsOut, + dependencies=[require_access("admin.user.read")], +) +def get_dashboard_metrics( + granularity: Literal["weekly", "monthly", "yearly"] = Query("weekly"), + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db), +): + return AdminDashboardService(db).get_tenant_metrics(current_user, granularity) + + +@router.get( + "/dashboard-metrics/trends/{metric}", + response_model=MetricTrendOnlyOut, + dependencies=[require_access("admin.user.read")], +) +def get_dashboard_metric_trend( + metric: Literal["signatures", "uploads"], + granularity: Literal["weekly", "monthly", "yearly"] = Query("weekly"), + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db), +): + return AdminDashboardService(db).get_trend_data(current_user, metric, granularity) diff --git a/app/modules/auth/routes/admin_user_routes.py b/app/modules/auth/routes/admin_user_routes.py new file mode 100644 index 0000000..474738a --- /dev/null +++ b/app/modules/auth/routes/admin_user_routes.py @@ -0,0 +1,655 @@ +from fastapi import APIRouter, Depends, Query, HTTPException, status +from sqlalchemy import func +from sqlalchemy.orm import Session, joinedload +from typing import Optional, List +from uuid import UUID +from pydantic import BaseModel, Field, field_validator +from datetime import datetime +import re +import logging + +from app.db.database import get_db +from app.core.scope import ScopeService +from app.core.settings import settings +from app.middleware.auth import get_current_user +from app.modules.auth.models.user_model import User +from app.modules.auth.models.role_model import Role +from app.modules.auth.models.role_access_model import RoleAccess +from app.modules.auth.schemas.auth_schema import UserOut +from app.modules.auth.schemas.role_schema import ( + AddUserRoleIn, + EffectiveAccessOut, + RoleAssignmentOut, +) +from app.modules.auth.services.permission_service import PermissionService +from app.modules.auth.services.user_role_service import UserRoleReader, invalidate_for_user +from app.modules.auth.dependencies.access_dependency import require_access +from app.modules.auth.services.privilege import holds_superadmin_access +from app.modules.configuration.services.system_configuration_service import ( + SystemConfigurationService, +) +from app.modules.chat.models.chat_usage_model import ChatTokenQuota + +from app.modules.configuration.schemas.system_configuration_schema import ( + SystemLimitsOut, +) + +from app.core.security import get_password_hash +from app.core.schemas import MessageOut + +router = APIRouter(prefix="/api/admin/users", tags=["Admin Users"]) +logger = logging.getLogger(__name__) + + +def _is_superadmin(user: User) -> bool: + """A user is treated as a superadmin if they hold any `superadmin.*` access.""" + return holds_superadmin_access(user) + + +def _resolve_effective_tenant_id( + current_user: User, requested_tenant_id: Optional[UUID] +) -> Optional[UUID]: + """ + Tenant admins are always scoped to their own tenant. + Superadmins may pass a tenant_id to narrow the scope, or None for all tenants. + """ + if _is_superadmin(current_user): + return requested_tenant_id + return current_user.tenant_id + + +@router.get( + "/summary/limits", + response_model=SystemLimitsOut, + dependencies=[require_access("admin.user.read")], +) +def get_user_management_limits(db: Session = Depends(get_db)): + """Get global user limits snapshot for administration UI.""" + return SystemConfigurationService(db).get_limits_snapshot() + + +class UpdateUserIn(BaseModel): + name: Optional[str] = None + designation: Optional[str] = Field(None, max_length=100) + is_active: Optional[bool] = None + role_id: Optional[UUID] = None + tenant_id: Optional[UUID] = None + password: Optional[str] = Field(None, min_length=12) + + @field_validator("password") + @classmethod + def password_complexity(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + if not re.search(r"[A-Z]", v): + raise ValueError("Password must contain at least one uppercase letter") + if not re.search(r"[0-9]", v): + raise ValueError("Password must contain at least one number") + if not re.search(r"[^A-Za-z0-9]", v): + raise ValueError("Password must contain at least one special character") + return v + + +class UserListOut(BaseModel): + id: int + name: str + designation: Optional[str] = None + email: str + is_active: bool + tenant_id: Optional[UUID] = None + tenant_name: Optional[str] = None + role_id: Optional[UUID] = None + role: Optional[str] = None + subscription: Optional[str] = None + is_deleted: bool = False + deleted_at: Optional[datetime] = None + chat_credit_daily_limit: Optional[float] = None + chat_credits_used_today: Optional[float] = None + tenant_chat_token_limit: Optional[int] = None + tenant_total_credits_allocated: Optional[float] = None + + roles: List[RoleAssignmentOut] = [] + """ + Every role this user holds, primary first. + + **Additive.** `role` and `role_id` above are retained and still mean the + primary role, so every existing consumer of this shape is unaffected. + """ + role_count: int = 0 + + class Config: + from_attributes = True + + +class PaginatedUsersOut(BaseModel): + items: List[UserListOut] + total: int + page: int + page_size: int + total_pages: int + + +from app.modules.chat.repositories.chat_repository import ChatbotRepository + +def _get_single_user_allocated_quota(user: User, db: Session) -> float: + if not user.tenant_id: + return 0.0 + repo = ChatbotRepository(db) + return repo.get_tenant_total_user_quota(user.tenant_id) + + +def _to_user_list_out(user: User, tenant_total_credits_allocated: float, assignments: list) -> UserListOut: + return UserListOut( + id=user.id, + name=user.name, + designation=user.designation, + email=user.email, + is_active=getattr(user, "is_active", True), + tenant_id=user.tenant_id, + tenant_name=user.tenant.name if user.tenant else None, + role_id=user.role_id, + role=user.role.name if user.role else None, + subscription=user.subscription, + is_deleted=getattr(user, "is_deleted", False), + deleted_at=getattr(user, "deleted_at", None), + chat_credit_daily_limit=getattr(user, "chat_token_quota", None).daily_credit_limit if getattr(user, "chat_token_quota", None) else None, + chat_credits_used_today=round((getattr(user, "chat_token_quota", None).credits_used_today / 1000.0), 2) if getattr(user, "chat_token_quota", None) else 0, + tenant_chat_token_limit=user.tenant.chat_token_daily_limit if user.tenant else None, + tenant_total_credits_allocated=tenant_total_credits_allocated, + roles=[RoleAssignmentOut(**a) for a in assignments], + role_count=len(assignments), + ) + + +@router.get( + "", + response_model=PaginatedUsersOut, + dependencies=[require_access("admin.user.read")], +) +def list_all_users( + page: int = Query(1, ge=1), + page_size: int = Query(20, ge=1, le=100), + search: Optional[str] = Query(None), + tenant_id: Optional[UUID] = Query(None), + include_deleted: bool = Query(False), + only_deleted: bool = Query(False), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """List all users with pagination, search, and optional tenant filter.""" + query = db.query(User).options( + joinedload(User.role), + joinedload(User.tenant), + joinedload(User.chat_token_quota), + ) + + if settings.ORG_SCOPE_ENABLED: + visible = ScopeService(db).visible_user_ids(current_user, "admin.user.read") + if visible is not None: + query = query.filter(User.id.in_(visible)) + + effective_tenant_id = _resolve_effective_tenant_id(current_user, tenant_id) + if effective_tenant_id: + query = query.filter(User.tenant_id == effective_tenant_id) + + if only_deleted: + query = query.filter(User.is_deleted == True) + elif not include_deleted: + query = query.filter(User.is_deleted == False) + + if search and search.strip(): + term = f"%{search.strip()}%" + query = query.filter(User.name.ilike(term) | User.email.ilike(term)) + + total = query.count() + offset = (page - 1) * page_size + users = query.offset(offset).limit(page_size).all() + total_pages = (total + page_size - 1) // page_size if total > 0 else 0 + + by_user = UserRoleReader(db).assignments_for_users(users) + + # Batch compute tenant allocated quotas in 1 single SQL query for all returned users + tenant_ids = {u.tenant_id for u in users if u.tenant_id} + tenant_quota_map = {} + if tenant_ids: + rows = ( + db.query(User.tenant_id, func.sum(ChatTokenQuota.daily_credit_limit)) + .join(ChatTokenQuota, User.id == ChatTokenQuota.user_id) + .filter(User.tenant_id.in_(tenant_ids), User.is_deleted == False) + .group_by(User.tenant_id) + .all() + ) + tenant_quota_map = {r[0]: float(r[1] or 0.0) for r in rows} + + return PaginatedUsersOut( + items=[_to_user_list_out(u, tenant_quota_map.get(u.tenant_id, 0.0), by_user.get(u.id, [])) for u in users], + total=total, + page=page, + page_size=page_size, + total_pages=total_pages, + ) + + +@router.get( + "/{user_id}", + response_model=UserListOut, + dependencies=[require_access("admin.user.read")], +) +def get_user( + user_id: int, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user) +): + """Get a single user by ID.""" + query = ( + db.query(User) + .options(joinedload(User.role), joinedload(User.tenant), joinedload(User.chat_token_quota)) + .filter(User.id == user_id) + ) + if not _is_superadmin(current_user) and current_user.tenant_id: + query = query.filter(User.tenant_id == current_user.tenant_id) + user = query.first() + if not user: + raise HTTPException(status_code=404, detail="User not found") + + if settings.ORG_SCOPE_ENABLED and not ScopeService(db).can_access_user( + current_user, user.id, "admin.user.read" + ): + raise HTTPException(status_code=404, detail="User not found") + + return _to_user_list_out( + user, + _get_single_user_allocated_quota(user, db), + UserRoleReader(db).assignments(user) + ) + + +@router.patch( + "/{user_id}", + response_model=UserListOut, + dependencies=[require_access("admin.user.update")], +) +def update_user( + user_id: int, + payload: UpdateUserIn, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """Update a user's name, active status, role, tenant, or password.""" + query = ( + db.query(User) + .options(joinedload(User.role), joinedload(User.tenant), joinedload(User.chat_token_quota)) + .filter(User.id == user_id) + ) + if not _is_superadmin(current_user) and current_user.tenant_id: + query = query.filter(User.tenant_id == current_user.tenant_id) + user = query.first() + if not user: + raise HTTPException(status_code=404, detail="User not found") + if getattr(user, "is_deleted", False): + raise HTTPException(status_code=400, detail="Cannot update a deleted user") + + if payload.name is not None: + user.name = payload.name + if payload.designation is not None: + user.designation = (payload.designation or "").strip() or None + was_active = user.is_active + if payload.is_active is not None: + if payload.is_active == False and user.id == current_user.id: + raise HTTPException(status_code=400, detail="You cannot deactivate your own account") + user.is_active = payload.is_active + previous_role = user.role + role_changed = False + new_role = None + if payload.role_id is not None: + role = db.query(Role).filter(Role.id == payload.role_id).first() + if not role: + raise HTTPException(status_code=404, detail="Role not found") + role_changed = user.role_id != role.id + new_role = role + user.role_id = role.id + if payload.tenant_id is not None: + user.tenant_id = payload.tenant_id + if payload.password is not None: + user.password_hash = get_password_hash(payload.password) + + db.flush() + db.refresh(user) + db.refresh(user) + + if role_changed: + from app.modules.org.services.access_audit import record_primary_role_change + + record_primary_role_change(db, current_user, user, previous_role, new_role) + invalidate_for_user(user.id) + + user_out = ( + db.query(User) + .options(joinedload(User.role), joinedload(User.tenant)) + .filter(User.id == user_id) + .first() + ) + + if was_active and payload.is_active is False: + try: + from app.modules.activity_logs.service import log_event + from app.modules.activity_logs.constants import ActivityLogModule, ActivityLogAction, ActivityLogStatus, ActivityLogTargetType + if user_out.tenant_id: + log_event( + tenant_id=user_out.tenant_id, + user_id=current_user.id, + user_email=current_user.email, + module=ActivityLogModule.TENANT, + action=ActivityLogAction.USER_DEACTIVATED, + target_id=str(user_out.id), + target_type=ActivityLogTargetType.USER, + metadata={ + "deactivated_user_id": user_out.id, + "deactivated_user_email": user_out.email + }, + status=ActivityLogStatus.SUCCESS + ) + except Exception as e: + import logging + logging.getLogger(__name__).error(f"Failed to log USER_DEACTIVATED: {e}") + + return _to_user_list_out( + user_out, + _get_single_user_allocated_quota(user_out, db), + UserRoleReader(db).assignments(user_out) + ) + + +@router.delete( + "/{user_id}", + dependencies=[require_access("admin.user.delete")], + response_model=MessageOut) +def soft_delete_user( + user_id: int, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + query = db.query(User).filter(User.id == user_id) + if not _is_superadmin(current_user) and current_user.tenant_id: + query = query.filter(User.tenant_id == current_user.tenant_id) + user = query.first() + if not user: + raise HTTPException(status_code=404, detail="User not found") + if user.id == current_user.id: + raise HTTPException(status_code=400, detail="You cannot delete your own account") + if user.is_deleted: + return {"message": "User already deleted"} + + user.is_deleted = True + user.is_active = False + user.deleted_at = datetime.utcnow() + user.deleted_by_id = current_user.id + + try: + from app.modules.activity_logs.service import log_event + from app.modules.activity_logs.constants import ActivityLogModule, ActivityLogAction, ActivityLogStatus, ActivityLogTargetType + if user.tenant_id: + log_event( + tenant_id=user.tenant_id, + user_id=current_user.id, + user_email=current_user.email, + module=ActivityLogModule.TENANT, + action=ActivityLogAction.USER_DEACTIVATED, + target_id=str(user.id), + target_type=ActivityLogTargetType.USER, + metadata={ + "deactivated_user_id": user.id, + "deactivated_user_email": user.email + }, + status=ActivityLogStatus.SUCCESS + ) + except Exception as e: + import logging + logging.getLogger(__name__).error(f"Failed to log USER_DEACTIVATED: {e}") + + return {"detail": "User deleted successfully"} + + +@router.post( + "/{user_id}/restore", + response_model=UserListOut, + dependencies=[require_access("admin.user.update")], +) +def restore_user( + user_id: int, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + query = ( + db.query(User) + .options(joinedload(User.role), joinedload(User.tenant)) + .filter(User.id == user_id) + ) + if not _is_superadmin(current_user) and current_user.tenant_id: + query = query.filter(User.tenant_id == current_user.tenant_id) + user = query.first() + if not user: + raise HTTPException(status_code=404, detail="User not found") + if not user.is_deleted: + raise HTTPException(status_code=400, detail="User is not deleted") + + SystemConfigurationService(db).ensure_can_create_user(tenant_id=user.tenant_id) + + user.is_deleted = False + user.is_active = True + user.deleted_at = None + user.deleted_by_id = None + db.flush() + db.refresh(user) + return _to_user_list_out( + user, + _get_single_user_allocated_quota(user, db), + UserRoleReader(db).assignments(user) + ) + + +@router.post( + "/bulk/distribute-credits-equally", + dependencies=[require_access("admin.user.update")], +) +def distribute_credits_equally( + tenant_id: Optional[UUID] = Query(None), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """Distribute the tenant's total credit limit equally among all active users.""" + from app.modules.chat.services.chat_service import ChatbotService + from app.modules.tenant.models.tenant_model import Tenant + + effective_tenant_id = _resolve_effective_tenant_id(current_user, tenant_id) + if not effective_tenant_id: + raise HTTPException(status_code=400, detail="Tenant ID is required for bulk distribution.") + + tenant = db.query(Tenant).filter(Tenant.id == effective_tenant_id).first() + if not tenant: + raise HTTPException(status_code=404, detail="Tenant not found") + + if tenant.chat_token_daily_limit == -1: + raise HTTPException(status_code=400, detail="Cannot distribute equally when Tenant has Unlimited quota.") + + active_users = db.query(User).filter( + User.tenant_id == effective_tenant_id, + User.is_active == True, + User.is_deleted == False + ).all() + + if not active_users: + return {"message": "No active users to distribute credits to."} + + total_tenant_credits = tenant.chat_token_daily_limit / 1000.0 + equal_quota = total_tenant_credits / len(active_users) + + logger.info(f"Bulk distributing {total_tenant_credits} Cr among {len(active_users)} users. Quota per user: {equal_quota}") + + chat_service = ChatbotService(db) + for user in active_users: + logger.info(f"Updating user {user.id} ({user.email}) to {equal_quota} Cr") + chat_service.repo.update_user_quota(user.id, equal_quota) + + return { + "message": f"Successfully distributed {total_tenant_credits} Cr among {len(active_users)} users. Each user now has {equal_quota} Cr.", + "quota_per_user": equal_quota + } + + +def _visible_user_or_404(db: Session, current_user: User, user_id: int) -> User: + """ + The target user, or 404 — never 403. + + A 403 confirms the user exists, which is the leak the cross-tenant probes + were written to catch. Applies the tenant predicate and, when + `ORG_SCOPE_ENABLED`, the org-scope check, exactly as the read endpoints + above do. + """ + query = ( + db.query(User) + .options(joinedload(User.role), joinedload(User.tenant)) + .filter(User.id == user_id) + ) + if not _is_superadmin(current_user) and current_user.tenant_id: + query = query.filter(User.tenant_id == current_user.tenant_id) + user = query.first() + if not user: + raise HTTPException(status_code=404, detail="User not found") + + if settings.ORG_SCOPE_ENABLED and not ScopeService(db).can_access_user( + current_user, user.id, "admin.user.read" + ): + raise HTTPException(status_code=404, detail="User not found") + return user + + +@router.get( + "/{user_id}/roles", + response_model=List[RoleAssignmentOut], + dependencies=[require_access("admin.role.read")], +) +def list_user_roles( + user_id: int, + include_expired: bool = Query( + False, + description=( + "Include grants that have lapsed. They confer nothing; they are " + "shown so an administrator can see that a temporary grant ended " + "rather than wondering where it went." + ), + ), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """Every role this user holds: primary, granted, and through a group.""" + user = _visible_user_or_404(db, current_user, user_id) + assignments = UserRoleReader(db).assignments(user, include_expired=include_expired) + return [RoleAssignmentOut(**a) for a in assignments] + + +@router.post( + "/{user_id}/roles", + response_model=RoleAssignmentOut, + status_code=status.HTTP_201_CREATED, + dependencies=[require_access("admin.role.assign")], +) +def add_user_role( + user_id: int, + payload: AddUserRoleIn, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """ + Grant this user an additional role. + + `org_unit_id` omitted means **tenant-wide**, which is the authority a + primary role already carries and the case the organisation tree could not + express. + + Does **not** touch `users.role_id`. The primary role is left exactly as it + was, so nothing an administrator has already set is silently replaced. + """ + from app.modules.org.services.grant_service import GrantService + + user = _visible_user_or_404(db, current_user, user_id) + + row = GrantService(db).grant( + current_user, + role_id=payload.role_id, + user_id=user.id, + org_unit_id=payload.org_unit_id, + expires_at=payload.expires_at, + ) + + for assignment in UserRoleReader(db).assignments(user): + if assignment["grant_id"] == row.id: + return RoleAssignmentOut(**assignment) + + raise HTTPException( + status_code=500, detail="The grant was written but did not resolve." + ) + + +@router.delete( + "/{user_id}/roles/{grant_id}", + response_model=MessageOut, + dependencies=[require_access("admin.role.assign")], +) +def remove_user_role( + user_id: int, + grant_id: UUID, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """ + Revoke one granted role. + + The **primary** role is not revocable here — it has no `grant_id`, and it is + changed through `PATCH /api/admin/users/{id}`. Offering a Remove button that + silently did nothing to it would be the worst of both. + + A grant reaching this user **through a group** is likewise not revocable + here: it belongs to the group, and removing it from one member would mean + editing the group for everybody. + """ + from app.modules.org.services.grant_service import GrantService + + _visible_user_or_404(db, current_user, user_id) + + row = UserRoleReader(db).grant_row(current_user.tenant_id, grant_id) + if row is None or row.user_id != user_id: + raise HTTPException(status_code=404, detail="Grant not found") + + GrantService(db).revoke(current_user, grant_id) + return {"message": "Role removed"} + + +@router.get( + "/{user_id}/effective-access", + response_model=List[EffectiveAccessOut], + dependencies=[require_access("admin.user.read")], +) +def user_effective_access( + user_id: int, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """ + Every access code this user holds, and which role confers each one. + + "Why can this person do that" answered itself while a user had one role. + With several it does not, and the alternative is reading the database — so + this exists before the interface that makes several roles easy to create. + + Descendant codes are attributed to the role holding their parent, because + that is the row an administrator would edit to take the code away. + """ + user = _visible_user_or_404(db, current_user, user_id) + + attributions = PermissionService(db).granted_by(user) + + return [ + EffectiveAccessOut(access_code=code, granted_by=sources) + for code, sources in sorted(attributions.items()) + ] diff --git a/app/modules/auth/routes/auth_routes.py b/app/modules/auth/routes/auth_routes.py new file mode 100644 index 0000000..0c33ba0 --- /dev/null +++ b/app/modules/auth/routes/auth_routes.py @@ -0,0 +1,499 @@ +from fastapi import APIRouter, Depends, Response, Request, HTTPException +from fastapi.security import OAuth2PasswordRequestForm +from sqlalchemy.orm import Session +from app.db.database import get_db +from app.modules.auth.controllers.auth_controller import AuthController, UserController +from app.modules.auth.repositories.user_repository import UserRepository +from app.modules.auth.schemas.auth_schema import ( + RegisterIn, + Token, + UserOut, + GoogleLoginIn, + UpdateProfileIn, + ChangePasswordIn, + RegisterOut, + ForgotPasswordIn, + ResetPasswordIn, +) +from app.middleware.auth import get_current_user, oauth2_scheme +from app.core.token_blacklist import TokenBlacklist +from app.modules.auth.models.user_model import User +from app.modules.tenant.models.tenant_model import Tenant +from app.middleware.tenant import get_tenant_from_header +from app.core.settings import settings +from jose import jwt +from app.modules.auth.schemas.access_schema import EditorTokenOut +from app.core.schemas import MessageOut, StatusMessageOut + +router = APIRouter(prefix="/auth", tags=["Auth"]) + +@router.post("/register", response_model=RegisterOut) +def register( + request: Request, + payload: RegisterIn, + db: Session = Depends(get_db), + tenant: Tenant = Depends(get_tenant_from_header), +): + return AuthController.register_user(payload, tenant, db) + +@router.post("/login", response_model=Token) +def login( + request: Request, + response: Response, + form_data: OAuth2PasswordRequestForm = Depends(), + db: Session = Depends(get_db), +): + token_data = AuthController.login_user(form_data, db, request) + + response.set_cookie( + key="docqube_access_token", + value=token_data["access_token"], + httponly=True, + secure=settings.APP_ENV == "production", + samesite="none" if settings.APP_ENV == "production" else "lax", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + max_age=3600, + path="/", + ) + + response.set_cookie( + key="docqube_refresh_token", + value=token_data["refresh_token"], + httponly=True, + secure=settings.APP_ENV == "production", + samesite="none" if settings.APP_ENV == "production" else "lax", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + max_age=7 * 24 * 3600, + path="/api/auth/refresh", + ) + + response.set_cookie( + key="docqube_has_session", + value="true", + httponly=False, + secure=settings.APP_ENV == "production", + samesite="none" if settings.APP_ENV == "production" else "lax", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + max_age=7 * 24 * 3600, + path="/", + ) + + if hasattr(request.state, "new_device_id"): + response.set_cookie( + key="docqube_device_id", + value=request.state.new_device_id, + httponly=True, + secure=settings.APP_ENV == "production", + samesite="none" if settings.APP_ENV == "production" else "lax", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + max_age=365 * 24 * 3600, + path="/", + ) + + return token_data + +@router.post("/google", response_model=Token) +def google_login( + request: Request, response: Response, payload: GoogleLoginIn, db: Session = Depends(get_db) +): + token_data = AuthController.google_login(payload, db, request) + + response.set_cookie( + key="docqube_access_token", + value=token_data["access_token"], + httponly=True, + secure=settings.APP_ENV == "production", + samesite="none" if settings.APP_ENV == "production" else "lax", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + max_age=3600, + path="/", + ) + + response.set_cookie( + key="docqube_refresh_token", + value=token_data["refresh_token"], + httponly=True, + secure=settings.APP_ENV == "production", + samesite="none" if settings.APP_ENV == "production" else "lax", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + max_age=7 * 24 * 3600, + path="/api/auth/refresh", + ) + + response.set_cookie( + key="docqube_has_session", + value="true", + httponly=False, + secure=settings.APP_ENV == "production", + samesite="none" if settings.APP_ENV == "production" else "lax", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + max_age=7 * 24 * 3600, + path="/", + ) + + if hasattr(request.state, "new_device_id"): + response.set_cookie( + key="docqube_device_id", + value=request.state.new_device_id, + httponly=True, + secure=settings.APP_ENV == "production", + samesite="none" if settings.APP_ENV == "production" else "lax", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + max_age=365 * 24 * 3600, + path="/", + ) + + return token_data + +@router.post("/forgot-password", response_model=MessageOut) +def forgot_password( + request: Request, payload: ForgotPasswordIn, db: Session = Depends(get_db) +): + return AuthController.forgot_password(payload, db) + +@router.post("/reset-password") +def reset_password( + request: Request, payload: ResetPasswordIn, db: Session = Depends(get_db) +): + return AuthController.reset_password(payload, db) + +@router.get("/me", response_model=UserOut) +@router.get("/verify", response_model=UserOut) +def verify_auth(user: User = Depends(get_current_user)): + return user + +@router.post("/accept-terms", response_model=StatusMessageOut) +def accept_terms(user: User = Depends(get_current_user), db: Session = Depends(get_db)): + if not user.terms_accepted: + user.terms_accepted = True + return {"status": "success", "message": "Terms accepted successfully"} + +@router.post("/refresh", response_model=Token) +def refresh_token(request: Request, response: Response, db: Session = Depends(get_db)): + refresh_token = request.cookies.get("docqube_refresh_token") + if not refresh_token: + raise HTTPException(status_code=401, detail="Refresh token missing") + + try: + payload = jwt.decode( + refresh_token, settings.APP_SECRET, algorithms=[settings.ALGORITHM] + ) + user_id = payload.get("sub") + token_type = payload.get("type") + + if not user_id or token_type != "refresh": + raise HTTPException(status_code=401, detail="Invalid refresh token") + + session_id = payload.get("session_id") + if session_id: + from app.modules.auth.repositories.session_repository import SessionRepository + from app.core.security import verify_password + session_repo = SessionRepository(db) + session = session_repo.get_by_id(session_id) + if not session or session.status != "ACTIVE": + if session and session.status == "REVOKED": + if session.revoked_by == "reauth": + raise HTTPException(status_code=401, detail="You have logged in from another tab on this device. Please refresh.") + else: + raise HTTPException(status_code=401, detail="Your session was remotely logged out from another device.") + raise HTTPException(status_code=401, detail="Your session has expired.") + + if not session.refresh_token_hash or session.refresh_token_hash == "pending" or not verify_password(refresh_token, session.refresh_token_hash): + raise HTTPException(status_code=401, detail="Invalid refresh token hash") + + from datetime import timedelta, datetime, timezone + now = datetime.now(timezone.utc) + last_activity = session.last_activity + if last_activity and last_activity.tzinfo is None: + last_activity = last_activity.replace(tzinfo=timezone.utc) + if not last_activity or now - last_activity > timedelta(minutes=5): + session_repo.update_last_activity(session) + + user_repo = UserRepository(db) + user = user_repo.get_by_id(int(user_id)) + if not user or user.is_deleted or (hasattr(user, "is_active") and not user.is_active): + raise HTTPException(status_code=401, detail="User not found") + + from app.core.security import create_access_token + + payload_access = { + "sub": str(user.id), + "tenant_id": str(user.tenant_id) if user.tenant_id else None, + } + if session_id: + payload_access["session_id"] = session_id + + new_access_token = create_access_token(payload_access) + + response.set_cookie( + key="docqube_access_token", + value=new_access_token, + httponly=True, + secure=settings.APP_ENV == "production", + samesite="none" if settings.APP_ENV == "production" else "lax", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + max_age=3600, + path="/", + ) + + response.set_cookie( + key="docqube_has_session", + value="true", + httponly=False, + secure=settings.APP_ENV == "production", + samesite="none" if settings.APP_ENV == "production" else "lax", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + max_age=7 * 24 * 3600, + path="/", + ) + + return { + "access_token": new_access_token, + "refresh_token": refresh_token, + "token_type": "bearer", + } + + except HTTPException as he: + raise he + except Exception as e: + raise HTTPException(status_code=401, detail="Invalid or expired refresh token") + +@router.get("/editor-token", response_model=EditorTokenOut) +def get_editor_token(current_user: User = Depends(get_current_user)): + from app.core.security import create_access_token + token = create_access_token( + { + "sub": str(current_user.id), + "tenant_id": str(current_user.tenant_id) if current_user.tenant_id else None, + } + ) + return {"token": token} + + +@router.post("/logout") +def logout( + response: Response, + token: str = Depends(oauth2_scheme), + request: Request = None, + db: Session = Depends(get_db) +): + auth_token = token + if not auth_token and request: + auth_token = request.cookies.get("docqube_access_token") + + if auth_token: + try: + unverified_payload = jwt.get_unverified_claims(auth_token) + jti = unverified_payload.get("jti") + exp = unverified_payload.get("exp") + + if jti: + ttl = 3600 + if exp: + from datetime import datetime + + now = datetime.utcnow().timestamp() + ttl = max(1, int(exp - now)) + + TokenBlacklist.add(jti, expires_in=ttl) + + session_id = unverified_payload.get("session_id") + if session_id: + from app.modules.auth.services.auth_service import AuthService + AuthService(db).logout(session_id) + except Exception: + pass + + response.delete_cookie( + "docqube_access_token", + path="/", + samesite="none" if settings.APP_ENV == "production" else "lax", + secure=settings.APP_ENV == "production", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + ) + response.delete_cookie( + "docqube_refresh_token", + path="/api/auth/refresh", + samesite="none" if settings.APP_ENV == "production" else "lax", + secure=settings.APP_ENV == "production", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + ) + response.delete_cookie( + "docqube_has_session", + path="/", + samesite="none" if settings.APP_ENV == "production" else "lax", + secure=settings.APP_ENV == "production", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + ) + return {"status": "success", "message": "Logged out"} + +from pydantic import BaseModel +class ResolveDeviceLimitIn(BaseModel): + logout_device_id: int + +@router.post("/resolve-device-limit") +def resolve_device_limit( + payload: ResolveDeviceLimitIn, + request: Request, + db: Session = Depends(get_db) +): + auth_header = request.headers.get("Authorization") + if not auth_header or not auth_header.startswith("Bearer "): + raise HTTPException(status_code=401, detail="Authentication required") + + token = auth_header.split(" ")[1] + + try: + payload_jwt = jwt.decode(token, settings.APP_SECRET, algorithms=[settings.ALGORITHM]) + jti = payload_jwt.get("jti") + if jti and TokenBlacklist.is_blacklisted(jti): + raise HTTPException(status_code=401, detail="Token blacklisted") + except HTTPException as he: + raise he + except Exception: + raise HTTPException(status_code=401, detail="Invalid token") + + if payload_jwt.get("type") != "device_management": + raise HTTPException(status_code=403, detail="Invalid token type") + + user_id = int(payload_jwt.get("sub")) + + from app.modules.auth.repositories.device_repository import DeviceRepository + from app.modules.auth.repositories.session_repository import SessionRepository + + device_repo = DeviceRepository(db) + device = device_repo.get_by_id(payload.logout_device_id) + + if not device or device.user_id != user_id: + raise HTTPException(status_code=404, detail="Device not found") + + session_repo = SessionRepository(db) + session_repo.revoke_all_for_device(device.id, by="user") + + return {"status": "success", "message": "Device logged out successfully. You can now login."} + +from typing import List +from app.modules.auth.schemas.auth_schema import DeviceOut + +@router.get("/devices", response_model=List[DeviceOut]) +def get_user_devices( + request: Request, + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + from app.modules.auth.repositories.device_repository import DeviceRepository + from app.modules.auth.utils.device_utils import generate_device_fingerprint + + device_repo = DeviceRepository(db) + devices = device_repo.get_user_devices(current_user.id) + + current_fingerprint = generate_device_fingerprint(request) + + results = [] + for d in devices: + results.append({ + "id": d.id, + "browser": d.browser, + "os": d.os, + "device_type": d.device_type, + "city": d.city, + "country": d.country, + "state": d.state, + "last_login": d.last_login, + "is_current_device": d.fingerprint == current_fingerprint + }) + results.sort(key=lambda x: not x["is_current_device"]) + return results + +@router.delete("/devices/{device_id}") +def logout_device( + device_id: int, + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + from app.modules.auth.repositories.device_repository import DeviceRepository + from app.modules.auth.repositories.session_repository import SessionRepository + + device_repo = DeviceRepository(db) + device = device_repo.get_by_id(device_id) + + if not device or device.user_id != current_user.id: + raise HTTPException(status_code=404, detail="Device not found") + + session_repo = SessionRepository(db) + session_repo.revoke_all_for_device(device.id, by="user") + + try: + from app.modules.activity_logs.service import log_event + from app.modules.activity_logs.constants import ActivityLogModule, ActivityLogAction, ActivityLogStatus, ActivityLogTargetType + log_event( + tenant_id=current_user.tenant_id, + user_id=current_user.id, + user_email=current_user.email, + module=ActivityLogModule.TENANT, + action=ActivityLogAction.DEVICE_LOGOUT, + target_id=str(device.id), + target_type=ActivityLogTargetType.SESSION, + metadata={ + "device_name": device.device_name, + "ip_address": device.ip_address, + }, + status=ActivityLogStatus.SUCCESS + ) + except Exception as e: + import logging + logging.getLogger(__name__).error(f"Failed to log device logout: {e}") + + return {"status": "success", "message": "Logged out from device successfully"} + +from fastapi.responses import StreamingResponse +import asyncio + +@router.get("/session/events") +async def session_events( + request: Request, + current_user: User = Depends(get_current_user) +): + """ + SSE Endpoint for real-time session and device revocation events. + Frontend can listen to this to instantly log out when their session is revoked. + """ + from app.core.redis import redis_pubsub + import json + + async def event_generator(): + if not redis_pubsub._client: + await redis_pubsub.connect() + + if not redis_pubsub._client: + return + + pubsub = redis_pubsub._client.pubsub() + channel = f"user_events:{current_user.id}" + await pubsub.subscribe(channel) + + try: + while True: + if await request.is_disconnected(): + break + + message = await pubsub.get_message(ignore_subscribe_messages=True, timeout=1.0) + if message and message["type"] == "message": + data = message["data"] + if isinstance(data, bytes): + data = data.decode('utf-8') + yield f"data: {data}\n\n" + + await asyncio.sleep(0.1) + except asyncio.CancelledError: + pass + except Exception: + pass + finally: + try: + await pubsub.unsubscribe(channel) + await pubsub.aclose() + except Exception: + pass + + return StreamingResponse(event_generator(), media_type="text/event-stream") \ No newline at end of file diff --git a/app/modules/auth/routes/me_routes.py b/app/modules/auth/routes/me_routes.py new file mode 100644 index 0000000..c7da55c --- /dev/null +++ b/app/modules/auth/routes/me_routes.py @@ -0,0 +1,213 @@ +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy.orm import Session +from typing import Optional +import hmac +import hashlib + +from app.db.database import get_db +from app.modules.auth.schemas.access_schema import ( + ApiKeyStatusOut, + MeStatsOut, + StorageNonceOut, +) +from app.modules.auth.controllers.auth_controller import UserController +from app.modules.auth.schemas.auth_schema import ( + UserOut, + UpdateProfileIn, + ChangePasswordIn, + UserSMTPCredentialsOut, + UserSMTPCredentialsPatchIn, +) +from app.middleware.auth import get_current_user +from app.modules.auth.models.user_model import User +from app.core.settings import settings +from app.core.crypto import encrypt_data +from app.core.mail import _resolve_smtp_config +from pydantic import BaseModel +from app.core.schemas import StatusOut + +class TutorialCompleteSchema(BaseModel): + key: str = "dashboard" + +router = APIRouter(prefix="/me", tags=["User Profile"]) + +@router.get("/profile", response_model=UserOut) +def get_profile(user: User = Depends(get_current_user)): + return user + +@router.get("/storage-nonce", response_model=StorageNonceOut) +def get_storage_nonce(user: User = Depends(get_current_user)): + """ + Returns a deterministic, user-specific nonce for client-side encryption. + Combined with APP_SECRET to ensure it's not guessing-prone. + """ + nonce = hmac.new( + settings.APP_SECRET.encode(), + f"user_storage_{user.id}_{user.email}".encode(), + hashlib.sha256 + ).hexdigest() + return {"nonce": nonce} + +@router.patch("/profile", response_model=UserOut) +def update_profile( + payload: UpdateProfileIn, + user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + return UserController.update_profile( + user, + payload.name, + db, + designation=payload.designation, + preferred_language=payload.preferred_language, + ) + +@router.post("/password") +def change_password( + payload: ChangePasswordIn, + user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + return UserController.change_password(user, payload.current_password, payload.new_password, db) + +@router.get("/stats", response_model=MeStatsOut) +def get_stats( + user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + return UserController.get_user_stats(user, db) + +@router.delete("/account") +def delete_account( + user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + return UserController.delete_account(user, db) + +@router.post("/api-key", response_model=StatusOut) +def update_api_key( + payload: dict, + user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + key = payload.get("key") + if key and str(key).strip(): + from app.core.crypto import encrypt_data + user.encrypted_ai_api_key = encrypt_data(str(key).strip()) + else: + user.encrypted_ai_api_key = None + return {"status": "success"} + +@router.get("/api-key/status", response_model=ApiKeyStatusOut) +def get_api_key_status(user: User = Depends(get_current_user)): + return {"has_key": bool(user.encrypted_ai_api_key)} + + +@router.get("/smtp-credentials", response_model=UserSMTPCredentialsOut) +def get_smtp_credentials( + user: User = Depends(get_current_user), + db: Session = Depends(get_db), +): + smtp_config = _resolve_smtp_config( + db=db, + tenant_id=user.tenant_id, + user_id=user.id, + ) + return UserSMTPCredentialsOut( + configured=bool(user.smtp_user and user.encrypted_smtp_password), + source=smtp_config["source"], + smtp_host=smtp_config["smtp_host"], + smtp_port=smtp_config["smtp_port"], + smtp_user=smtp_config["smtp_user"], + mail_from=smtp_config["mail_from"], + has_password=bool(user.encrypted_smtp_password), + ) + + +@router.patch("/smtp-credentials", response_model=UserSMTPCredentialsOut) +def update_smtp_credentials( + payload: UserSMTPCredentialsPatchIn, + user: User = Depends(get_current_user), + db: Session = Depends(get_db), +): + if payload.smtp_user is not None: + cleaned_user = payload.smtp_user.strip() + user.smtp_user = cleaned_user or None + + if payload.mail_from is not None: + cleaned_mail_from = str(payload.mail_from).strip() + user.mail_from = cleaned_mail_from or None + + if payload.smtp_password is not None: + cleaned_password = payload.smtp_password.strip() + user.encrypted_smtp_password = ( + encrypt_data(cleaned_password) if cleaned_password else None + ) + + db.add(user) + db.flush() + db.refresh(user) + + smtp_config = _resolve_smtp_config( + db=db, + tenant_id=user.tenant_id, + user_id=user.id, + ) + return UserSMTPCredentialsOut( + configured=bool(user.smtp_user and user.encrypted_smtp_password), + source=smtp_config["source"], + smtp_host=smtp_config["smtp_host"], + smtp_port=smtp_config["smtp_port"], + smtp_user=smtp_config["smtp_user"], + mail_from=smtp_config["mail_from"], + has_password=bool(user.encrypted_smtp_password), + ) + +@router.post("/tutorial-complete", response_model=UserOut) +def complete_tutorial( + payload: TutorialCompleteSchema, + user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + tutorial_key = payload.key + print(f"DEBUG: Tutorial completion request for user {user.email}, key: {tutorial_key}") + + if tutorial_key == "docs": + user.has_completed_docs_tutorial = True + print(f"DEBUG: Setting has_completed_docs_tutorial = True") + else: + user.has_completed_tutorial = True + print(f"DEBUG: Setting has_completed_tutorial = True") + + db.add(user) +@router.post("/tutorial/finish", response_model=UserOut) +def finish_tutorial( + payload: Optional[TutorialCompleteSchema] = None, + user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + """ + A specific, robust endpoint to mark tutorials as finished. + Automatically handles both dashboard and docs depending on the key. + """ + key = payload.key if payload else "dashboard" + if key == "docs": + user.has_completed_docs_tutorial = True + elif key == "dashboard": + user.has_completed_tutorial = True + elif key == "all": + user.has_completed_tutorial = True + user.has_completed_docs_tutorial = True + + try: + db.add(user) + db.flush() + db.refresh(user) + except HTTPException: + raise + except Exception as e: + db.rollback() + raise HTTPException(status_code=500, detail="Database persistence failed") + raise HTTPException(status_code=500, detail="Database persistence failed") + + return user diff --git a/app/modules/auth/routes/role_routes.py b/app/modules/auth/routes/role_routes.py new file mode 100644 index 0000000..65a63aa --- /dev/null +++ b/app/modules/auth/routes/role_routes.py @@ -0,0 +1,344 @@ +from app.modules.auth.schemas.access_schema import RoleAssignedOut +""" +Role Management Routes - RBAC endpoints for creating, updating, deleting roles. +""" + +from fastapi import APIRouter, Depends, Query, HTTPException, status +from sqlalchemy.orm import Session +from typing import Optional, List +from uuid import UUID +from pydantic import BaseModel + +from app.db.database import get_db +from app.middleware.auth import get_current_user +from app.modules.auth.models.user_model import User +from app.modules.auth.controllers.role_controller import RoleController +from app.modules.auth.schemas.role_schema import ( + RoleCreate, + RoleUpdate, + RoleOut, + RoleWithAccessesOut, + RolePaginatedOut, +) +from app.modules.auth.dependencies.access_dependency import require_access +from app.modules.auth.services.privilege import holds_superadmin_access +from app.modules.auth.services.user_role_service import UserRoleReader +from app.core.schemas import MessageOut + + +class AssignUserRoleIn(BaseModel): + user_id: int + role_id: UUID + + +router = APIRouter(prefix="/api/roles", tags=["Roles"]) + + +def _is_superadmin(current_user: User) -> bool: + """True when the caller holds any superadmin access code.""" + return holds_superadmin_access(current_user) + + +def _assert_role_in_callers_tenant(role, current_user: User, db: Session = None, allow_system_roles: bool = False) -> None: + """ + Refuse to act on a role belonging to another tenant. + + `RoleController.get_role_by_id` looks a role up by primary key alone, so + without this every by-id route was reachable across tenants — a tenant + administrator could read, rename and delete another tenant's roles by + guessing or observing a UUID. Raises 404 rather than 403 so the response + does not confirm that the role exists. + """ + if _is_superadmin(current_user): + return + if role.tenant_id == current_user.tenant_id: + return + + if allow_system_roles and getattr(role, "is_system", False) and current_user.tenant_id and db: + from app.modules.billing.models.plan_model import TenantSubscription, PlanRole + sub = db.query(TenantSubscription).filter( + TenantSubscription.tenant_id == current_user.tenant_id, + TenantSubscription.status == 'active' + ).order_by(TenantSubscription.created_at.desc()).first() + if sub and sub.plan_id: + has_role = db.query(PlanRole).filter( + PlanRole.plan_id == sub.plan_id, + PlanRole.role_id == role.id + ).first() + if has_role: + return + + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Role not found" + ) + + +@router.post( + "/", response_model=RoleOut, dependencies=[require_access("admin.role.create")] +) +def create_role( + payload: RoleCreate, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """ + Create a new role. + + Required access: admin.role.create | superadmin.role.create + + - **role_name**: Name of the role + - **description**: Optional description + - **access_ids**: List of access IDs to assign + - **tenant_id**: Tenant ID (for tenant-specific roles) + """ + if not payload.tenant_id and current_user.tenant_id: + payload.tenant_id = current_user.tenant_id + + result = RoleController.create_role(db, payload) + try: + from app.modules.activity_logs.service import log_event + from app.modules.activity_logs.constants import ActivityLogModule, ActivityLogAction, ActivityLogStatus, ActivityLogTargetType + if payload.tenant_id: + log_event( + tenant_id=payload.tenant_id, + user_id=current_user.id, + user_email=current_user.email, + module=ActivityLogModule.TENANT, + action=ActivityLogAction.ROLE_CREATED, + target_id=str(result.id), + target_type=ActivityLogTargetType.ROLE, + metadata={ + "role_name": payload.role_name + }, + status=ActivityLogStatus.SUCCESS + ) + except Exception as e: + import logging + logging.getLogger(__name__).error(f"Failed to log ROLE_CREATED: {e}") + + return result + + +@router.get( + "/", response_model=List[RoleOut], dependencies=[require_access("admin.role.read")] +) +def get_all_roles( + tenant_id: Optional[UUID] = Query(None), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """ + Get all roles. + + Required access: admin.role.read | superadmin.role.read + + Superadmins see roles across all tenants by default, or a single + tenant when `?tenant_id=` is provided. Tenant admins are + always scoped to their own tenant regardless of the query param. + """ + effective_tenant_id = ( + tenant_id if _is_superadmin(current_user) else current_user.tenant_id + ) + return RoleController.get_all_roles(db, effective_tenant_id) + + +@router.get( + "/{role_id}", + response_model=RoleOut, + dependencies=[require_access("admin.role.read")], +) +def get_role( + role_id: UUID, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """ + Get a specific role by ID. + + Required access: admin.role.read | superadmin.role.read + """ + role = RoleController.get_role_by_id(db, role_id) + _assert_role_in_callers_tenant(role, current_user, db, allow_system_roles=True) + return role + + +@router.get( + "/{role_id}/details", + response_model=RoleWithAccessesOut, + dependencies=[require_access("admin.role.read")], +) +def get_role_with_accesses( + role_id: UUID, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """ + Get a role with all assigned access codes. + + Required access: admin.role.read | superadmin.role.read + + Also reports how many principals hold it, so the editing screen can say what + a change affects. Counted here and not on the list endpoints, which render + up to a hundred roles. + """ + _assert_role_in_callers_tenant( + RoleController.get_role_by_id(db, role_id), current_user, db, allow_system_roles=True + ) + role = RoleController.get_role_with_accesses(db, role_id) + out = RoleWithAccessesOut.model_validate(role) + out.assigned_user_count = UserRoleReader(db).users_holding_role(role_id) + return out + + +@router.patch( + "/{role_id}", + response_model=RoleOut, + dependencies=[require_access("admin.role.update")], +) +def update_role( + role_id: UUID, + payload: RoleUpdate, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """ + Update a role's details and accesses. + + Required access: admin.role.update | superadmin.role.update + + - **role_name**: New role name (optional) + - **description**: New description (optional) + - **access_ids**: New list of access IDs (optional) + """ + role = RoleController.get_role_by_id(db, role_id) + _assert_role_in_callers_tenant(role, current_user) + if role.is_default and current_user.tenant_id: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Default roles cannot be modified by tenant admins." + ) + return RoleController.update_role(db, role_id, payload) + + +@router.delete("/{role_id}", dependencies=[require_access("admin.role.update")], response_model=MessageOut) +def delete_role( + role_id: UUID, + force: bool = Query( + False, + description=( + "Delete even though the role is still held. Required once anybody " + "holds it, so the removal of their authority is a decision rather " + "than a side effect." + ), + ), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """ + Delete a role. + + Required access: admin.role.update | superadmin.role.delete + + **Refuses with 409 while the role is still held**, unless `?force=true`. + + Deleting a role has always taken authority away silently: `user_roles.role_id` + is `ON DELETE CASCADE`, so every grant of it disappears, and `users.role_id` + is `ON DELETE SET NULL`, so every holder loses their primary role. Neither + leaves a trace and neither was ever counted for the operator. With one role + per user that was survivable; with roles granted in several places it is + not, because the operator can no longer hold the affected set in their head. + + The cascade behaviour is unchanged. Only the silence is. + """ + role = RoleController.get_role_by_id(db, role_id) + _assert_role_in_callers_tenant(role, current_user) + if role.is_default and current_user.tenant_id: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Default roles cannot be deleted by tenant admins." + ) + + if not force: + holders = UserRoleReader(db).users_holding_role(role_id) + if holders: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + f"This role cannot be deleted because there are currently {holders} " + f"active {'assignment' if holders == 1 else 'assignments'} of it. " + "Please reassign or remove them first." + ), + ) + + return RoleController.delete_role(db, role_id) + + +@router.get( + "/tenant/{tenant_id}", + response_model=List[RoleOut], + dependencies=[require_access("admin.role.read")], +) +def get_roles_by_tenant( + tenant_id: UUID, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """ + Get all roles for a specific tenant. + + The tenant is named by the caller, so a tenant administrator must not be + able to name somebody else's. + """ + if not _is_superadmin(current_user) and tenant_id != current_user.tenant_id: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Tenant not found" + ) + return RoleController.get_all_roles(db, tenant_id) + + +@router.post( + "/assign-user", + dependencies=[require_access("admin.role.assign")], + response_model=RoleAssignedOut) +def assign_role_to_user( + payload: AssignUserRoleIn, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """Assign a role to a user.""" + return RoleController.assign_role_to_user(db, payload.user_id, payload.role_id) + + +@router.get( + "", + response_model=RolePaginatedOut, + dependencies=[require_access("admin.role.read")], +) +def list_roles_paginated( + page: int = Query(1, ge=1), + page_size: int = Query(10, ge=1, le=100), + search: Optional[str] = Query(None), + tenant_id: Optional[UUID] = Query(None), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """ + Get paginated list of roles with optional search. + + Required access: admin.role.read | superadmin.role.read + + Superadmins see roles across all tenants by default, or a single + tenant when `?tenant_id=` is provided. Tenant admins are + always scoped to their own tenant regardless of the query param. + """ + effective_tenant_id = ( + tenant_id if _is_superadmin(current_user) else current_user.tenant_id + ) + + return RoleController.get_roles_paginated( + db=db, + tenant_id=effective_tenant_id, + page=page, + page_size=page_size, + search=search, + ) diff --git a/app/modules/auth/routes/user_routes.py b/app/modules/auth/routes/user_routes.py new file mode 100644 index 0000000..0bdb13c --- /dev/null +++ b/app/modules/auth/routes/user_routes.py @@ -0,0 +1,33 @@ +from fastapi import APIRouter, Depends, Query +from sqlalchemy.orm import Session +from typing import List + +from app.db.database import get_db +from app.modules.auth.models.user_model import User +from app.modules.auth.repositories.user_repository import UserRepository +from app.modules.auth.schemas.auth_schema import UserOut +from app.modules.auth.services.user_role_service import UserRoleReader, attach_role_refs +from app.middleware.auth import get_current_user + +router = APIRouter(prefix="/users", tags=["Users"]) + +@router.get("/search", response_model=List[UserOut]) +def search_users( + q: str = Query(..., min_length=1), + limit: int = Query(10, ge=1, le=50), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user) +): + """ + Search users by name or email, within the caller's own tenant. + + Previously unscoped: any authenticated user could enumerate every user of + every tenant, complete with email addresses, roles and access codes. A + caller with no tenant is a superadmin under today's model and still + searches across tenants. + """ + user_repo = UserRepository(db) + users = user_repo.search_users(q, limit, tenant_id=current_user.tenant_id) + + attach_role_refs(UserRoleReader(db), users) + return users diff --git a/app/modules/auth/schemas/access_schema.py b/app/modules/auth/schemas/access_schema.py new file mode 100644 index 0000000..a42d09d --- /dev/null +++ b/app/modules/auth/schemas/access_schema.py @@ -0,0 +1,84 @@ +""" +Response schemas for the access (RBAC catalogue) and self-service endpoints. + +Every shape here was taken from a recorded response in +`tests/characterization/snapshots`, not from the ORM models — the drive module +showed how far the hand-written schemas had drifted from what the endpoints +actually return, and a `response_model` that omits a field silently deletes it +from the payload. +""" + +from typing import List, Optional +from uuid import UUID + +from pydantic import BaseModel + + +class AccessOut(BaseModel): + """One row of the access catalogue. 66 exist.""" + + id: UUID + name: str + access_code: str + category: str + parent_id: Optional[UUID] = None + + class Config: + from_attributes = True + + +class RoleAccessesOut(BaseModel): + """`GET /api/access/get` — the caller's roles and the codes they carry.""" + + role: Optional[str] = None + """The **primary** role. Retained and unchanged; `roles` is the full set.""" + + roles: List[str] = [] + """ + Every role the caller holds, primary first. + + Additive, and a plain list of names rather than the richer + `RoleAssignmentOut` used elsewhere: this endpoint exists so the frontend can + decide what to render, and scope and expiry do not change that decision. + `/api/me/profile` carries the detailed shape. + """ + + tenant_id: Optional[str] = None + accesses: List[str] = [] + + +class ApiKeyStatusOut(BaseModel): + has_key: bool + + +class StorageNonceOut(BaseModel): + nonce: str + + +class MeStatsOut(BaseModel): + """ + `GET /api/me/stats`. The camelCase keys are the existing contract — the + frontend reads them as they are, so they are pinned rather than tidied. + Renaming them is a frontend change, not a schema change. + """ + + totalProjects: int = 0 + totalFiles: int = 0 + subscription: Optional[str] = None + joined: Optional[str] = None + + +class EditorTokenOut(BaseModel): + token: str + + +class CeleryHealthOut(BaseModel): + status: str + + +class RoleAssignedOut(BaseModel): + """`POST /api/roles/assign-user`.""" + + message: str + user_id: int + role_id: str diff --git a/app/modules/auth/schemas/admin_dashboard_schema.py b/app/modules/auth/schemas/admin_dashboard_schema.py new file mode 100644 index 0000000..f8b3ab0 --- /dev/null +++ b/app/modules/auth/schemas/admin_dashboard_schema.py @@ -0,0 +1,57 @@ +from pydantic import BaseModel + + +class MetricTrendPointOut(BaseModel): + date: str + value: int + + +class MetricCountOut(BaseModel): + total: int + recent: int + trend: list[MetricTrendPointOut] + + +class TotalMetricOut(BaseModel): + total: int + recent_added: int + recent_deleted: int + + +class ShareMetricOut(BaseModel): + internal: int + external: int + + +class StorageMetricOut(BaseModel): + used_bytes: int + quota_bytes: int + usage_percentage: float + used_formatted: str + quota_formatted: str + + +class ConversionMetricOut(BaseModel): + total: int + completed: int + processing: int + failed: int + completion_rate: float + + +class SimpleMetricOut(BaseModel): + total: int + + +class AdminDashboardMetricsOut(BaseModel): + signatures: MetricCountOut | None = None + uploads: MetricCountOut + users: TotalMetricOut + shares: ShareMetricOut + storage: StorageMetricOut + conversions: ConversionMetricOut | None = None + edits: SimpleMetricOut + + +class MetricTrendOnlyOut(BaseModel): + trend: list[MetricTrendPointOut] diff --git a/app/modules/auth/schemas/auth_schema.py b/app/modules/auth/schemas/auth_schema.py new file mode 100644 index 0000000..10a4188 --- /dev/null +++ b/app/modules/auth/schemas/auth_schema.py @@ -0,0 +1,182 @@ +from pydantic import BaseModel, EmailStr, ConfigDict, Field, field_validator +from typing import Optional +from typing import Literal +from datetime import datetime +import re + + +class Token(BaseModel): + access_token: str + refresh_token: Optional[str] = None + token_type: str = "bearer" + + +class TokenData(BaseModel): + email: Optional[str] = None + + +class RegisterIn(BaseModel): + email: EmailStr + password: str = Field(..., min_length=12) + name: str = Field(..., min_length=2) + designation: Optional[str] = Field(None, max_length=100) + + @field_validator("password") + @classmethod + def password_complexity(cls, v: str) -> str: + if not re.search(r"[A-Z]", v): + raise ValueError("Password must contain at least one uppercase letter") + if not re.search(r"[0-9]", v): + raise ValueError("Password must contain at least one number") + if not re.search(r"[^A-Za-z0-9]", v): + raise ValueError("Password must contain at least one special character") + return v + + +class LoginIn(BaseModel): + email: Optional[EmailStr] = None + username: Optional[str] = None + password: str + + +from uuid import UUID + +from app.modules.auth.schemas.role_schema import RoleAssignmentOut + + +class UserOut(BaseModel): + id: int + email: EmailStr + name: str + designation: Optional[str] = None + preferred_language: str = "en" + subscription: Optional[str] = "free" + created_at: Optional[datetime] = None + tenant_id: Optional[UUID] = None + is_active: Optional[bool] = True + terms_accepted: bool = False + has_completed_tutorial: bool = False + has_completed_docs_tutorial: bool = False + + role: Optional[str] = Field(None, validation_alias="role_name") + role_id: Optional[UUID] = None + roles: list[RoleAssignmentOut] = Field( + default_factory=list, validation_alias="role_refs" + ) + """ + Every role the user holds — primary, granted, and through a group. + + **Additive.** `role` and `role_id` are retained and unchanged, because they + are read by the frontend, by the admin screens and by anything else already + consuming this shape. `role` remains the *primary* role specifically, which + is what the existing dropdown edits. + """ + accesses: list[str] = Field(default_factory=list, validation_alias="access_codes") + subscription_details: Optional[dict] = None + tenant_envelope_limit: Optional[int] = None + tenant_envelopes_used: Optional[int] = None + + model_config = ConfigDict(from_attributes=True, populate_by_name=True) + + +class GoogleLoginIn(BaseModel): + token: Optional[str] = None + credential: Optional[str] = None + tenant_id: Optional[str] = None + + +class UpdateProfileIn(BaseModel): + name: Optional[str] = Field(None, min_length=2, max_length=255) + designation: Optional[str] = Field(None, max_length=100) + preferred_language: Optional[str] = Field(None, min_length=2, max_length=16) + + @field_validator("preferred_language") + @classmethod + def normalize_preferred_language(cls, value: Optional[str]) -> Optional[str]: + if value is None: + return None + normalized = value.strip().lower() + if not normalized: + raise ValueError("Preferred language cannot be empty") + if not re.fullmatch(r"[a-z]{2,3}(?:-[a-z0-9]{2,8})?", normalized): + raise ValueError("Preferred language must be a valid language code") + return normalized + + +class ChangePasswordIn(BaseModel): + current_password: str + new_password: str = Field(..., min_length=12) + + @field_validator("new_password") + @classmethod + def password_complexity(cls, v: str) -> str: + if not re.search(r"[A-Z]", v): + raise ValueError("Password must contain at least one uppercase letter") + if not re.search(r"[0-9]", v): + raise ValueError("Password must contain at least one number") + if not re.search(r"[^A-Za-z0-9]", v): + raise ValueError("Password must contain at least one special character") + return v + + +class UserSMTPCredentialsPatchIn(BaseModel): + smtp_user: Optional[str] = Field(None, max_length=255) + mail_from: Optional[EmailStr] = None + smtp_password: Optional[str] = Field( + None, + description="If omitted during update, existing password is preserved. Send empty string to clear it.", + ) + + +class UserSMTPCredentialsOut(BaseModel): + configured: bool + source: Literal["user", "tenant", "fallback"] + smtp_host: Optional[str] = None + smtp_port: int = 587 + smtp_user: Optional[str] = None + mail_from: Optional[str] = None + has_password: bool = False + + +class UpgradeIn(BaseModel): + plan: str + + +class RegisterOut(BaseModel): + message: str + user_id: Optional[int] = None + email: Optional[EmailStr] = None + tenant_id: Optional[UUID] = None + + +class ForgotPasswordIn(BaseModel): + email: EmailStr + + +class ResetPasswordIn(BaseModel): + token: str + new_password: str = Field(..., min_length=12) + + @field_validator("new_password") + @classmethod + def password_complexity(cls, v: str) -> str: + if not re.search(r"[A-Z]", v): + raise ValueError("Password must contain at least one uppercase letter") + if not re.search(r"[0-9]", v): + raise ValueError("Password must contain at least one number") + if not re.search(r"[^A-Za-z0-9]", v): + raise ValueError("Password must contain at least one special character") + return v + +class DeviceOut(BaseModel): + id: int + browser: Optional[str] = None + os: Optional[str] = None + device_type: Optional[str] = None + city: Optional[str] = None + country: Optional[str] = None + state: Optional[str] = None + last_login: Optional[datetime] = None + is_current_device: bool = False + + model_config = ConfigDict(from_attributes=True) diff --git a/app/modules/auth/schemas/role_schema.py b/app/modules/auth/schemas/role_schema.py new file mode 100644 index 0000000..77cfaad --- /dev/null +++ b/app/modules/auth/schemas/role_schema.py @@ -0,0 +1,141 @@ +from pydantic import BaseModel, Field, ConfigDict +from typing import Literal, Optional, List +from datetime import datetime +from uuid import UUID + + +class RoleAccessOut(BaseModel): + id: UUID + name: str + access_code: str + category: str + parent_id: Optional[UUID] = None + + model_config = ConfigDict(from_attributes=True) + + +class RoleCreate(BaseModel): + role_name: str = Field(..., max_length=100) + description: Optional[str] = None + tenant_id: Optional[UUID] = None + access_ids: Optional[List[UUID]] = [] + is_default: Optional[bool] = False + + +class RoleUpdate(BaseModel): + role_name: Optional[str] = None + description: Optional[str] = None + access_ids: Optional[List[UUID]] = None + is_default: Optional[bool] = None + + +class RoleOut(BaseModel): + id: UUID + role_name: str = Field(..., alias="name") + description: Optional[str] = None + tenant_id: Optional[UUID] = None + is_default: bool = False + is_system: bool = False + created_at: Optional[datetime] = None + + model_config = ConfigDict(from_attributes=True, populate_by_name=True) + + +class RoleWithAccessesOut(RoleOut): + accesses: List[RoleAccessOut] = [] + assigned_user_count: Optional[int] = None + """ + How many principals this role reaches, or null when it was not counted. + + Added to the *details* response and deliberately not to `RoleOut`: the list + endpoints render up to a hundred roles and counting each one would be an + N+1 nobody asked for. It exists so that deleting a role can say what it is + about to do — `user_roles.role_id` is `ON DELETE CASCADE` and + `users.role_id` is `ON DELETE SET NULL`, so a silent delete removes + authority from people the operator was not thinking about. + """ + + +class RoleAssignmentOut(BaseModel): + """ + One role a user holds, and how they hold it. + + `source` is the field that keeps the interface honest: + + "primary" the legacy `users.role_id`. Changed through + `PATCH /api/admin/users/{id}`, **not** revocable as a grant, + and while it is set it cannot be narrowed by anything (see + `ScopeService._resolve`) + "grant" a `user_roles` row naming this user. Added and revoked freely + "group" a `user_roles` row naming a group this user belongs to. It + confers codes; it is removed by editing the group + + Rendering the three identically would imply that removing any of them is the + same action. It is not, and that is exactly where a permissions screen + misleads the person using it. + """ + + grant_id: Optional[UUID] = None + role_id: UUID + role_name: str + source: Literal["primary", "grant", "group"] + org_unit_id: Optional[UUID] = None + org_unit_name: Optional[str] = None + group_id: Optional[UUID] = None + group_name: Optional[str] = None + expires_at: Optional[datetime] = None + assigned_by_id: Optional[int] = None + created_at: Optional[datetime] = None + is_expired: bool = False + + model_config = ConfigDict(from_attributes=True) + + +class AddUserRoleIn(BaseModel): + """ + Add one role to one user. + + `org_unit_id` omitted or null means **tenant-wide** — the authority every + user already has through their primary role. That default is the point of + this endpoint: the organisation tree could only ever grant *scoped to a + unit*, so "Alice is an Editor and an Approver, everywhere" was not + expressible anywhere in the product. + """ + + role_id: UUID + org_unit_id: Optional[UUID] = None + expires_at: Optional[datetime] = None + + +class AccessAttributionOut(BaseModel): + """Which role conferred one access code, and at what scope.""" + + role_id: Optional[UUID] = None + role_name: Optional[str] = None + source: str + org_unit_id: Optional[UUID] = None + org_unit_name: Optional[str] = None + expires_at: Optional[datetime] = None + + +class EffectiveAccessOut(BaseModel): + """ + One access code the user holds, with every reason they hold it. + + Answers "why can this person do that". With one role per user the question + answered itself. With several it does not, and support cannot read the + database. + """ + + access_code: str + granted_by: List[AccessAttributionOut] + + +class RolePaginatedOut(BaseModel): + items: List[RoleOut] + total: int + page: int + page_size: int + total_pages: int + + model_config = ConfigDict(from_attributes=True) diff --git a/app/modules/auth/services/admin_dashboard_service.py b/app/modules/auth/services/admin_dashboard_service.py new file mode 100644 index 0000000..2168cd5 --- /dev/null +++ b/app/modules/auth/services/admin_dashboard_service.py @@ -0,0 +1,557 @@ +from datetime import datetime, timedelta, timezone +from uuid import UUID + +from fastapi import HTTPException +from sqlalchemy import func +from sqlalchemy.orm import Session + +from app.modules.activity_logs.constants import ActivityLogAction, ActivityLogStatus +from app.modules.activity_logs.models.activity_log import ActivityLog +from app.modules.auth.models.user_model import User +from app.modules.auth.services.permission_service import PermissionService +from app.modules.documents.models.document_model import Project +from app.modules.drive.models.drive_model import DriveACL, DriveFile, DriveFolder, DriveShare +from app.modules.signing.models.signing_request import SigningRequest +from app.modules.storage.services.storage_service import StorageService +from app.modules.auth.services.privilege import holds_superadmin_access + +CONVERSION_JOBS_ACCESS_CODE = "metrics:conversion_jobs" +SIGNATURES_TREND_ACCESS_CODE = "metrics:signatures_trend" + + +class AdminDashboardService: + def __init__(self, db: Session): + self.db = db + + def _is_tenant_admin(self, current_user: User) -> bool: + return bool(current_user.tenant_id) and not holds_superadmin_access( + current_user + ) + + def get_tenant_metrics(self, current_user: User, granularity: str = "weekly") -> dict: + tenant_id = current_user.tenant_id + if not tenant_id: + raise HTTPException( + status_code=403, + detail="Tenant-scoped admin metrics are only available for tenant users.", + ) + + # 1. Try Redis cache lookup (tenant + granularity isolated) + cache_key = None + try: + from app.db.redis import redis_cache + from app.modules.drive.constants import CACHE_ADMIN_METRICS, CACHE_ADMIN_METRICS_TTL_SECONDS + cache_key = CACHE_ADMIN_METRICS.format(tenant_id=str(tenant_id), granularity=granularity) + cached = redis_cache.get(cache_key) + if cached: + import json + return json.loads(cached) + except Exception: + pass + + recent_since = datetime.now(timezone.utc) - timedelta(days=7) + trend_config = self._get_trend_config(granularity) + permission_service = PermissionService(self.db) + is_tenant_admin = self._is_tenant_admin(current_user) + can_view_conversions = is_tenant_admin or permission_service.has_access( + current_user, CONVERSION_JOBS_ACCESS_CODE + ) + can_view_signatures = is_tenant_admin or permission_service.has_access( + current_user, SIGNATURES_TREND_ACCESS_CODE + ) + + metrics = { + "signatures": { + "total": self._count_signatures(tenant_id), + "recent": self._count_recent_signatures(tenant_id, recent_since), + "trend": self._get_signature_trend(tenant_id, trend_config), + } + if can_view_signatures + else None, + "uploads": { + "total": self._count_uploads(tenant_id), + "recent": self._count_recent_uploads(tenant_id, recent_since), + "trend": self._get_upload_trend(tenant_id, trend_config), + }, + "users": { + "total": self._count_total_users(tenant_id), + "recent_added": self._count_recent_users_added(tenant_id, recent_since), + "recent_deleted": self._count_recent_users_deleted(tenant_id, recent_since), + }, + "shares": { + "internal": self._count_internal_shares(tenant_id), + "external": self._count_external_shares(tenant_id), + }, + "storage": self._get_storage_metrics(current_user), + "conversions": self._get_conversion_metrics(tenant_id) + if can_view_conversions + else None, + "edits": { + "total": self._count_recent_edits(tenant_id, recent_since), + }, + } + + # 2. Store in Redis + if cache_key: + try: + import json + redis_cache.set(cache_key, json.dumps(metrics, default=str), ex=CACHE_ADMIN_METRICS_TTL_SECONDS) + except Exception: + pass + + return metrics + + def _count_signatures(self, tenant_id: UUID) -> int: + return ( + self.db.query(SigningRequest) + .filter( + SigningRequest.tenant_id == tenant_id, + SigningRequest.status == "completed", + ) + .count() + ) + + def _count_recent_signatures(self, tenant_id: UUID, recent_since: datetime) -> int: + return ( + self.db.query(SigningRequest) + .filter( + SigningRequest.tenant_id == tenant_id, + SigningRequest.status == "completed", + SigningRequest.completed_at >= recent_since, + ) + .count() + ) + + def _count_uploads(self, tenant_id: UUID) -> int: + return self.db.query(DriveFile).filter(DriveFile.tenant_id == tenant_id).count() + + def _count_recent_uploads(self, tenant_id: UUID, recent_since: datetime) -> int: + return ( + self.db.query(DriveFile) + .filter( + DriveFile.tenant_id == tenant_id, + DriveFile.created_at >= recent_since, + ) + .count() + ) + + def _count_total_users(self, tenant_id: UUID) -> int: + return ( + self.db.query(User) + .filter( + User.tenant_id == tenant_id, + User.is_deleted.is_(False), + ) + .count() + ) + + def _count_recent_users_added(self, tenant_id: UUID, recent_since: datetime) -> int: + return ( + self.db.query(User) + .filter( + User.tenant_id == tenant_id, + User.created_at >= recent_since, + ) + .count() + ) + + def _count_recent_users_deleted(self, tenant_id: UUID, recent_since: datetime) -> int: + return ( + self.db.query(User) + .filter( + User.tenant_id == tenant_id, + User.is_deleted.is_(True), + User.deleted_at.isnot(None), + User.deleted_at >= recent_since, + ) + .count() + ) + + def _count_recent_edits(self, tenant_id: UUID, recent_since: datetime) -> int: + return ( + self.db.query(ActivityLog) + .filter( + ActivityLog.tenant_id == tenant_id, + ActivityLog.action == ActivityLogAction.DOCUMENT_EDITED.value, + ActivityLog.status == ActivityLogStatus.SUCCESS.value, + ActivityLog.created_at >= recent_since, + ) + .count() + ) + + def _get_signature_trend(self, tenant_id: UUID, trend_config: dict) -> list[dict]: + if trend_config["mode"] == "rolling_weeks": + return self._build_rolling_week_trend( + self.db.query( + SigningRequest.completed_at.label("event_at"), + ) + .filter( + SigningRequest.tenant_id == tenant_id, + SigningRequest.status == "completed", + SigningRequest.completed_at.isnot(None), + ), + SigningRequest.completed_at, + trend_config["periods"], + ) + + if trend_config["mode"] == "calendar_year_months": + period_bucket = func.date_trunc("month", SigningRequest.completed_at) + return self._build_current_year_month_trend( + self.db.query( + period_bucket.label("period"), + func.count(SigningRequest.id).label("value"), + ) + .filter( + SigningRequest.tenant_id == tenant_id, + SigningRequest.status == "completed", + SigningRequest.completed_at.isnot(None), + ), + SigningRequest.completed_at, + period_bucket, + ) + + period_bucket = func.date_trunc(trend_config["bucket"], SigningRequest.completed_at) + return self._build_period_trend( + self.db.query( + period_bucket.label("period"), + func.count(SigningRequest.id).label("value"), + ) + .filter( + SigningRequest.tenant_id == tenant_id, + SigningRequest.status == "completed", + SigningRequest.completed_at.isnot(None), + ), + SigningRequest.completed_at, + period_bucket, + trend_config, + ) + + def _get_upload_trend(self, tenant_id: UUID, trend_config: dict) -> list[dict]: + if trend_config["mode"] == "rolling_weeks": + return self._build_rolling_week_trend( + self.db.query( + DriveFile.created_at.label("event_at"), + ) + .filter(DriveFile.tenant_id == tenant_id), + DriveFile.created_at, + trend_config["periods"], + ) + + if trend_config["mode"] == "calendar_year_months": + period_bucket = func.date_trunc("month", DriveFile.created_at) + return self._build_current_year_month_trend( + self.db.query( + period_bucket.label("period"), + func.count(DriveFile.id).label("value"), + ) + .filter(DriveFile.tenant_id == tenant_id), + DriveFile.created_at, + period_bucket, + ) + + period_bucket = func.date_trunc(trend_config["bucket"], DriveFile.created_at) + return self._build_period_trend( + self.db.query( + period_bucket.label("period"), + func.count(DriveFile.id).label("value"), + ) + .filter(DriveFile.tenant_id == tenant_id), + DriveFile.created_at, + period_bucket, + trend_config, + ) + + def get_trend_data(self, current_user: User, metric: str, granularity: str = "weekly") -> dict: + tenant_id = current_user.tenant_id + if not tenant_id: + raise HTTPException( + status_code=403, + detail="Tenant-scoped admin metrics are only available for tenant users.", + ) + + trend_config = self._get_trend_config(granularity) + if metric == "signatures": + if not self._is_tenant_admin(current_user): + PermissionService(self.db).require_access( + current_user, SIGNATURES_TREND_ACCESS_CODE + ) + return {"trend": self._get_signature_trend(tenant_id, trend_config)} + if metric == "uploads": + return {"trend": self._get_upload_trend(tenant_id, trend_config)} + + raise HTTPException(status_code=404, detail="Unsupported metric trend requested.") + + def _get_trend_config(self, granularity: str) -> dict: + normalized = granularity.lower() + if normalized == "monthly": + return {"mode": "rolling_weeks", "periods": 5} + if normalized == "yearly": + return {"mode": "calendar_year_months"} + return {"mode": "weekly_days", "bucket": "day", "periods": 7} + + def _build_period_trend(self, query, date_column, period_bucket, trend_config: dict) -> list[dict]: + current_period_start = self._get_period_start(datetime.now(timezone.utc), trend_config["bucket"]) + period_starts = [] + for offset in reversed(range(trend_config["periods"])): + period_starts.append(self._shift_period_start(current_period_start, trend_config["bucket"], -offset)) + + start_datetime = period_starts[0] + end_datetime = self._shift_period_start(current_period_start, trend_config["bucket"], 1) + rows = ( + query.filter( + date_column >= start_datetime, + date_column < end_datetime, + ) + .group_by(period_bucket) + .all() + ) + + counts_by_period = { + row.period.date().isoformat(): row.value + for row in rows + if row.period is not None + } + + return [ + { + "date": period_start.date().isoformat(), + "value": counts_by_period.get(period_start.date().isoformat(), 0), + } + for period_start in period_starts + ] + + def _build_rolling_week_trend(self, query, date_column, periods: int) -> list[dict]: + now = datetime.now(timezone.utc) + today_start = datetime(now.year, now.month, now.day, tzinfo=timezone.utc) + earliest_period_start = today_start - timedelta(days=(periods * 7) - 1) + end_datetime = today_start + timedelta(days=1) + + rows = ( + query.filter( + date_column >= earliest_period_start, + date_column < end_datetime, + ) + .all() + ) + + counts_by_week: dict[int, int] = {} + for row in rows: + if row.event_at is None: + continue + week_index = ((row.event_at.date() - earliest_period_start.date()).days // 7) + 1 + counts_by_week[week_index] = counts_by_week.get(week_index, 0) + 1 + + return [ + { + "date": (earliest_period_start + timedelta(days=(week_index - 1) * 7)).date().isoformat(), + "value": counts_by_week.get(week_index, 0), + } + for week_index in range(1, periods + 1) + ] + + def _build_current_year_month_trend(self, query, date_column, period_bucket) -> list[dict]: + now = datetime.now(timezone.utc) + year_start = datetime(now.year, 1, 1, tzinfo=timezone.utc) + next_year_start = datetime(now.year + 1, 1, 1, tzinfo=timezone.utc) + + rows = ( + query.filter( + date_column >= year_start, + date_column < next_year_start, + ) + .group_by(period_bucket) + .all() + ) + + counts_by_month = { + row.period.date().isoformat(): row.value + for row in rows + if row.period is not None + } + + return [ + { + "date": datetime(now.year, month, 1, tzinfo=timezone.utc).date().isoformat(), + "value": counts_by_month.get( + datetime(now.year, month, 1, tzinfo=timezone.utc).date().isoformat(), + 0, + ), + } + for month in range(1, 13) + ] + + def _get_period_start(self, value: datetime, bucket: str) -> datetime: + if bucket == "year": + return datetime(value.year, 1, 1, tzinfo=timezone.utc) + if bucket == "month": + return datetime(value.year, value.month, 1, tzinfo=timezone.utc) + return datetime(value.year, value.month, value.day, tzinfo=timezone.utc) + + def _shift_period_start(self, value: datetime, bucket: str, step: int) -> datetime: + if bucket == "year": + return datetime(value.year + step, 1, 1, tzinfo=timezone.utc) + + if bucket == "month": + month_index = (value.month - 1) + step + year = value.year + (month_index // 12) + month = (month_index % 12) + 1 + return datetime(year, month, 1, tzinfo=timezone.utc) + + return value + timedelta(days=step) + + def _get_storage_metrics(self, current_user: User) -> dict: + usage = StorageService(self.db).get_storage_usage(current_user) + return { + "used_bytes": int(usage["total_bytes"]), + "quota_bytes": int(usage["quota_bytes"]), + "usage_percentage": float(usage["usage_percentage"]), + "used_formatted": usage["total_formatted"], + "quota_formatted": usage["quota_formatted"], + } + + def _get_conversion_metrics(self, tenant_id: UUID) -> dict: + total = self.db.query(Project).filter(Project.tenant_id == tenant_id).count() + completed = ( + self.db.query(Project) + .filter( + Project.tenant_id == tenant_id, + Project.status == "completed", + ) + .count() + ) + processing = ( + self.db.query(Project) + .filter( + Project.tenant_id == tenant_id, + Project.status == "processing", + ) + .count() + ) + failed = ( + self.db.query(Project) + .filter( + Project.tenant_id == tenant_id, + Project.status == "failed", + ) + .count() + ) + completion_rate = round((completed / total) * 100, 1) if total > 0 else 0.0 + return { + "total": total, + "completed": completed, + "processing": processing, + "failed": failed, + "completion_rate": completion_rate, + } + + def _count_internal_shares(self, tenant_id: UUID) -> int: + return ( + self._count_internal_file_shares(tenant_id) + + self._count_internal_folder_shares(tenant_id) + + self._count_internal_document_shares(tenant_id) + ) + + def _count_external_shares(self, tenant_id: UUID) -> int: + return ( + self._count_external_file_shares(tenant_id) + + self._count_external_folder_shares(tenant_id) + + self._count_external_document_shares(tenant_id) + ) + + def _count_internal_file_shares(self, tenant_id: UUID) -> int: + return ( + self.db.query(DriveACL) + .join( + DriveFile, + (DriveACL.resource_type == "file") & (DriveACL.resource_id == DriveFile.id), + ) + .join(User, DriveACL.subject_id == User.id) + .filter( + DriveFile.tenant_id == tenant_id, + User.tenant_id == tenant_id, + DriveACL.role != "owner", + ) + .count() + ) + + def _count_internal_folder_shares(self, tenant_id: UUID) -> int: + return ( + self.db.query(DriveACL) + .join( + DriveFolder, + (DriveACL.resource_type == "folder") + & (DriveACL.resource_id == DriveFolder.id), + ) + .join(User, DriveACL.subject_id == User.id) + .filter( + DriveFolder.tenant_id == tenant_id, + User.tenant_id == tenant_id, + DriveACL.role != "owner", + ) + .count() + ) + + def _count_internal_document_shares(self, tenant_id: UUID) -> int: + return ( + self.db.query(DriveACL) + .join( + Project, + (DriveACL.resource_type == "document") + & (DriveACL.resource_id == Project.id), + ) + .join(User, DriveACL.subject_id == User.id) + .filter( + Project.tenant_id == tenant_id, + User.tenant_id == tenant_id, + DriveACL.role != "owner", + ) + .count() + ) + + def _count_external_file_shares(self, tenant_id: UUID) -> int: + return ( + self.db.query(DriveShare) + .join( + DriveFile, + (DriveShare.resource_type == "file") + & (DriveShare.resource_id == DriveFile.id), + ) + .filter( + DriveFile.tenant_id == tenant_id, + DriveShare.invited_email.isnot(None), + DriveShare.is_active == True, + ) + .count() + ) + + def _count_external_folder_shares(self, tenant_id: UUID) -> int: + return ( + self.db.query(DriveShare) + .join( + DriveFolder, + (DriveShare.resource_type == "folder") + & (DriveShare.resource_id == DriveFolder.id), + ) + .filter( + DriveFolder.tenant_id == tenant_id, + DriveShare.invited_email.isnot(None), + DriveShare.is_active == True, + ) + .count() + ) + + def _count_external_document_shares(self, tenant_id: UUID) -> int: + return ( + self.db.query(DriveShare) + .join( + Project, + (DriveShare.resource_type == "document") + & (DriveShare.resource_id == Project.id), + ) + .filter( + Project.tenant_id == tenant_id, + DriveShare.invited_email.isnot(None), + DriveShare.is_active == True, + ) + .count() + ) diff --git a/app/modules/auth/services/auth_service.py b/app/modules/auth/services/auth_service.py new file mode 100644 index 0000000..3e4372f --- /dev/null +++ b/app/modules/auth/services/auth_service.py @@ -0,0 +1,528 @@ +import os +import re +import uuid +import logging +from typing import Dict, Any, List, Optional +from fastapi import HTTPException +from sqlalchemy.orm import Session +from google.oauth2 import id_token +from google.auth.transport import requests + +from app.core.settings import settings +from app.db.redis import redis_cache +from app.core.mail import send_email +from app.modules.auth.models.user_model import User +from app.modules.auth.repositories.user_repository import UserRepository +from app.core.security import ( + get_password_hash, + verify_password, + create_access_token, + create_refresh_token, +) +from app.modules.auth.schemas.auth_schema import ( + RegisterIn, + GoogleLoginIn, + UpdateProfileIn, + ChangePasswordIn, + ForgotPasswordIn, + ResetPasswordIn, +) +from app.modules.configuration.services.system_configuration_service import ( + SystemConfigurationService, +) + +logger = logging.getLogger(__name__) + + +class AuthService: + def __init__(self, db: Session): + self.db = db + self.user_repo = UserRepository(db) + + def register(self, payload: RegisterIn, tenant: Any = None) -> dict: + if self.user_repo.get_by_email(payload.email): + raise HTTPException(status_code=400, detail="Email already registered") + + SystemConfigurationService(self.db).ensure_can_create_user( + tenant_id=tenant.id if tenant else None + ) + + user = User( + name=payload.name.strip(), + designation=(payload.designation or "").strip() or None, + email=payload.email.lower().strip(), + password_hash=get_password_hash(payload.password), + subscription="free", + tenant_id=tenant.id if tenant else None, + ) + new_user = self.user_repo.create(user) + + from app.modules.drive.services.drive_service import DriveService + + drive_service = DriveService(self.db) + drive_service.get_or_create_root_folder(new_user) + + from app.modules.storage.models.storage_model import UserStorageUsage + + storage_usage = UserStorageUsage(user_id=new_user.id) + self.db.add(storage_usage) + + try: + from datetime import datetime + + send_email( + subject="Welcome to DocQube!", + recipient=new_user.email, + template_name="welcome.html", + template_context={ + "name": new_user.name, + "login_url": f"{settings.FRONTEND_URL}/login", + "year": datetime.utcnow().year, + }, + db=self.db, + tenant_id=new_user.tenant_id, + ) + except Exception as e: + logger.error(f"Failed to send welcome email for {new_user.email}: {e}") + + return { + "message": "Account created successfully", + "user_id": new_user.id, + "email": new_user.email, + "tenant_id": new_user.tenant_id, + } + + def logout(self, session_id: int): + from app.modules.auth.repositories.session_repository import SessionRepository + session_repo = SessionRepository(self.db) + session = session_repo.get_by_id(session_id) + if session and session.status == "ACTIVE": + session_repo.revoke(session, by="user") + + def _handle_device_login(self, user_id: int, request: Any) -> int: + from app.modules.auth.utils.device_utils import get_client_ip, parse_user_agent, generate_device_fingerprint, get_location_from_ip + from app.modules.auth.repositories.device_repository import DeviceRepository + from app.modules.auth.repositories.session_repository import SessionRepository + from app.modules.auth.models.device_model import Device + from app.modules.auth.models.session_model import Session as AuthSession + from datetime import datetime, timezone, timedelta + + device_repo = DeviceRepository(self.db) + session_repo = SessionRepository(self.db) + + fingerprint = generate_device_fingerprint(request) + ip_addr = get_client_ip(request) + user_agent_str = request.headers.get("User-Agent", "") + browser, os_name, device_type = parse_user_agent(user_agent_str) + + device = device_repo.get_by_fingerprint(user_id, fingerprint) + if device: + if device.is_blocked: + raise HTTPException(status_code=403, detail="Device is blocked") + device_repo.update_login_activity(device, ip_addr) + session_repo.revoke_all_for_device(device.id, by="reauth") + from sqlalchemy import text + self.db.execute(text("SELECT pg_advisory_xact_lock(:lock_id)"), {"lock_id": user_id}) + + limit = settings.MAX_ACTIVE_DEVICES + active_count = session_repo.count_active_sessions(user_id) + if active_count >= limit: + active_devices = device_repo.get_user_devices(user_id) + devices_data = [{ + "id": d.id, + "os": d.os, + "browser": d.browser, + "device_type": d.device_type, + "city": d.city, + "country": d.country, + "state": d.state, + "last_login": d.last_login.isoformat() if d.last_login else None + } for d in active_devices] + + raise HTTPException(status_code=403, detail={ + "error": "DEVICE_LIMIT_REACHED", + "devices": devices_data + }) + + if not device: + country, state, city, lat, lon = get_location_from_ip(ip_addr) + + new_device = Device( + user_id=user_id, + fingerprint=fingerprint, + browser=browser, + os=os_name, + device_type=device_type, + user_agent=user_agent_str, + ip=ip_addr, + country=country, + state=state, + city=city, + latitude=lat, + longitude=lon, + last_ip=ip_addr + ) + device = device_repo.create(new_device) + + new_session = AuthSession( + user_id=user_id, + device_id=device.id, + refresh_token_hash="pending", + status="ACTIVE", + expires_at=datetime.now(timezone.utc) + timedelta(days=7) + ) + new_session = session_repo.create(new_session) + return new_session.id + + def login(self, username: str, password: str, request: Optional[Any] = None) -> Dict[str, str]: + user = self.user_repo.get_by_email(username) + + generic_error = HTTPException( + status_code=401, detail="Invalid email or password" + ) + + if ( + not user + or user.is_deleted + or (hasattr(user, "is_active") and not user.is_active) + ): + raise HTTPException(status_code=401, detail="User not found or account is inactive") + + if user.password_hash == "SAAS_MANAGED_ACCOUNT_DO_NOT_USE_PASSWORD": + raise HTTPException( + status_code=403, + detail="This account is managed by SSO. Please log in through the main application portal." + ) + + if not verify_password(password, user.password_hash): + raise HTTPException(status_code=401, detail="Incorrect email or password") + + if user.tenant_id: + from app.modules.tenant.repositories.tenant_repository import TenantRepository + tenant_repo = TenantRepository(self.db) + tenant = tenant_repo.get_by_id(user.tenant_id) + if not tenant or not tenant.is_active or tenant.is_deleted: + raise HTTPException(status_code=403, detail="Your organization is currently inactive. Please contact support.") + + try: + session_id = None + if request: + session_id = self._handle_device_login(user.id, request) + except HTTPException as e: + if isinstance(e.detail, dict) and e.detail.get("error") == "DEVICE_LIMIT_REACHED": + temp_payload = {"sub": str(user.id), "type": "device_management"} + from datetime import timedelta + temp_token = create_access_token(temp_payload, expires_delta=timedelta(minutes=10)) + e.detail["temp_token"] = temp_token + raise e + + payload_access = { + "sub": str(user.id), + "tenant_id": str(user.tenant_id) if user.tenant_id else None, + "is_superadmin": bool(getattr(user, "is_superadmin", False)), + } + if session_id: + payload_access["session_id"] = session_id + + payload_refresh = {"sub": str(user.id)} + if session_id: + payload_refresh["session_id"] = session_id + + token = create_access_token(payload_access) + refresh = create_refresh_token(payload_refresh) + + if session_id: + from app.core.security import get_password_hash + from app.modules.auth.repositories.session_repository import SessionRepository + session_repo = SessionRepository(self.db) + session = session_repo.get_by_id(session_id) + if session: + session.refresh_token_hash = get_password_hash(refresh) + + if user.tenant_id: + try: + from app.modules.activity_logs.service import log_event + from app.modules.activity_logs.constants import ActivityLogModule, ActivityLogAction, ActivityLogStatus, ActivityLogTargetType + log_event( + tenant_id=user.tenant_id, + user_id=user.id, + user_email=user.email, + module=ActivityLogModule.TENANT, + action=ActivityLogAction.DEVICE_LOGIN, + target_id=str(session_id) if session_id else str(user.id), + target_type=ActivityLogTargetType.SESSION, + metadata={}, + status=ActivityLogStatus.SUCCESS + ) + except Exception as e: + import logging + logging.getLogger(__name__).error(f"Failed to log login event: {e}") + + return {"access_token": token, "refresh_token": refresh, "token_type": "bearer"} + + + def google_login(self, payload: GoogleLoginIn, request: Optional[Any] = None) -> Dict[str, str]: + client_id = settings.GOOGLE_CLIENT_ID + token_to_verify = payload.token or payload.credential + + if not token_to_verify: + raise HTTPException(status_code=400, detail="Google token missing") + + try: + idinfo = id_token.verify_oauth2_token( + token_to_verify, requests.Request(), client_id, clock_skew_in_seconds=60 + ) + except HTTPException: + raise + except Exception as e: + logger.error(f"Google Auth Error: {e}", exc_info=True) + raise HTTPException(status_code=401, detail="Google authentication failed") + + email = idinfo["email"] + raw_name = idinfo.get("name", email.split("@")[0]) + name = re.sub(r"[^\w\s-]", "", raw_name).strip() + + user = self.user_repo.get_by_email(email) + + if user and user.tenant_id: + from app.modules.tenant.repositories.tenant_repository import TenantRepository + tenant_repo = TenantRepository(self.db) + tenant = tenant_repo.get_by_id(user.tenant_id) + if not tenant or not tenant.is_active or tenant.is_deleted: + raise HTTPException(status_code=403, detail="Your organization is currently inactive. Please contact support.") + + if not user: + tenant_uuid = None + if payload.tenant_id: + try: + tenant_uuid = uuid.UUID(str(payload.tenant_id)) + except Exception: + tenant_uuid = None + + SystemConfigurationService(self.db).ensure_can_create_user( + tenant_id=tenant_uuid + ) + user = User( + name=name, + email=email, + password_hash=get_password_hash(os.urandom(24).hex()), + tenant_id=payload.tenant_id, + ) + user = self.user_repo.create(user) + + from app.modules.drive.services.drive_service import DriveService + + drive_service = DriveService(self.db) + drive_service.get_or_create_root_folder(user) + + from app.modules.storage.models.storage_model import UserStorageUsage + + storage_usage = UserStorageUsage(user_id=user.id) + self.db.add(storage_usage) + + try: + from datetime import datetime + + send_email( + subject="Welcome to DocQube!", + recipient=user.email, + template_name="welcome.html", + template_context={ + "name": user.name, + "login_url": f"{settings.FRONTEND_URL}/login", + "year": datetime.utcnow().year, + }, + db=self.db, + tenant_id=user.tenant_id, + ) + except Exception as e: + logger.error( + f"Failed to send welcome email for google user {user.email}: {e}" + ) + + try: + session_id = None + if request: + session_id = self._handle_device_login(user.id, request) + except HTTPException as e: + if isinstance(e.detail, dict) and e.detail.get("error") == "DEVICE_LIMIT_REACHED": + temp_payload = {"sub": str(user.id), "type": "device_management"} + from datetime import timedelta + temp_token = create_access_token(temp_payload, expires_delta=timedelta(minutes=10)) + e.detail["temp_token"] = temp_token + raise e + + payload_access = { + "sub": str(user.id), + "tenant_id": str(user.tenant_id) if user.tenant_id else None, + "is_superadmin": bool(getattr(user, "is_superadmin", False)), + } + if session_id: + payload_access["session_id"] = session_id + + payload_refresh = {"sub": str(user.id)} + if session_id: + payload_refresh["session_id"] = session_id + + token = create_access_token(payload_access) + refresh = create_refresh_token(payload_refresh) + + if session_id: + from app.core.security import get_password_hash + from app.modules.auth.repositories.session_repository import SessionRepository + session_repo = SessionRepository(self.db) + session = session_repo.get_by_id(session_id) + if session: + session.refresh_token_hash = get_password_hash(refresh) + + if user.tenant_id: + try: + from app.modules.activity_logs.service import log_event + from app.modules.activity_logs.constants import ActivityLogModule, ActivityLogAction, ActivityLogStatus, ActivityLogTargetType + log_event( + tenant_id=user.tenant_id, + user_id=user.id, + user_email=user.email, + module=ActivityLogModule.TENANT, + action=ActivityLogAction.DEVICE_LOGIN, + target_id=str(session_id) if session_id else str(user.id), + target_type=ActivityLogTargetType.SESSION, + metadata={}, + status=ActivityLogStatus.SUCCESS + ) + except Exception as e: + import logging + logging.getLogger(__name__).error(f"Failed to log google login event: {e}") + + return {"access_token": token, "refresh_token": refresh, "token_type": "bearer"} + + def forgot_password(self, payload: ForgotPasswordIn) -> Dict[str, str]: + email = payload.email.lower().strip() + user = self.user_repo.get_by_email(email) + + res = { + "message": "If this email is registered, you will receive a password reset link." + } + + if not user: + logger.info(f"Forgot password request for non-existent email: {email}") + return res + if user.is_deleted or (hasattr(user, "is_active") and not user.is_active): + logger.info(f"Forgot password requested for inactive/deleted user: {email}") + return res + + logger.info(f"Found user {user.id} for forgot password: {email}") + + reset_token = uuid.uuid4().hex + redis_cache.set(f"reset_token:{reset_token}", user.id, ttl=3600) + + reset_link = f"{settings.FRONTEND_URL}/reset-password?token={reset_token}" + + subject = "DocQube - Reset Your Password" + body = f"Hello {user.name},\n\nYou requested a password reset. Use the link below to set a new password:\n\n{reset_link}\n\nThis link expires in 1 hour." + html_body = f""" +
    +

    Reset Your Password

    +

    Hello {user.name},

    +

    You requested a password reset for your DocQube account. Click the button below to continue:

    + +

    Or copy and paste this link:
    {reset_link}

    +

    This link will expire in 1 hour. If you did not request this, please ignore this email.

    +
    + """ + + logger.info(f"Attempting to send reset email to {email}") + send_email( + subject, + email, + body, + html_body, + db=self.db, + tenant_id=user.tenant_id, + ) + logger.info(f"Reset email process completed for {email}") + + return res + + def reset_password(self, payload: ResetPasswordIn) -> Dict[str, str]: + user_id = redis_cache.get(f"reset_token:{payload.token}") + if not user_id: + raise HTTPException( + status_code=400, detail="Invalid or expired reset token" + ) + + user = self.user_repo.get_by_id(user_id) + if not user: + raise HTTPException(status_code=404, detail="User not found") + + user.password_hash = get_password_hash(payload.new_password) + + redis_cache.delete(f"reset_token:{payload.token}") + + return {"message": "Password has been successfully reset."} + + +class UserService: + def __init__(self, db: Session): + self.db = db + self.user_repo = UserRepository(db) + + def update_profile( + self, + user: User, + name: Optional[str] = None, + designation: Optional[str] = None, + preferred_language: Optional[str] = None, + ) -> User: + if name is None and designation is None and preferred_language is None: + raise HTTPException( + status_code=400, + detail="At least one profile field must be provided", + ) + if name is not None: + user.name = name.strip() + if designation is not None: + user.designation = designation.strip() or None + if preferred_language is not None: + user.preferred_language = preferred_language.strip().lower() or "en" + self.db.flush() + self.db.refresh(user) + return user + + def change_password(self, user: User, current_password: str, new_password: str): + if not verify_password(current_password, user.password_hash): + raise HTTPException(status_code=400, detail="Incorrect current password") + + user.password_hash = get_password_hash(new_password) + return {"message": "Password updated successfully"} + + def get_user_stats(self, user: User) -> Dict[str, Any]: + """ + Fetch real-time statistics for the user profile dashboard. + """ + from app.modules.documents.models.document_model import Project + from app.modules.drive.models.drive_model import DriveFile + + total_projects = ( + self.db.query(Project).filter(Project.user_id == user.id).count() + ) + total_files = ( + self.db.query(DriveFile).filter(DriveFile.owner_id == user.id).count() + ) + + return { + "totalProjects": total_projects, + "totalFiles": total_files, + "subscription": user.subscription, + "joined": user.created_at.isoformat() if user.created_at else None, + } + + def delete_account(self, user: User): + user.is_deleted = True + user.is_active = False + from datetime import datetime + + user.deleted_at = datetime.utcnow() + return {"message": "Account deleted successfully"} diff --git a/app/modules/auth/services/permission_service.py b/app/modules/auth/services/permission_service.py new file mode 100644 index 0000000..3377786 --- /dev/null +++ b/app/modules/auth/services/permission_service.py @@ -0,0 +1,386 @@ +from sqlalchemy import func, or_, select +from typing import Set, Optional, List +from fastapi import HTTPException +from sqlalchemy.orm import Session +from app.core.request_cache import request_cached +from app.modules.auth.models.user_model import User +from app.modules.auth.models.access_model import Access + + +class PermissionService: + """ + Centralized permission engine for checking access codes. + + Handles: + - Retrieving user's access codes from their role + - Checking if user has specific access code(s) + - Enforcing access restrictions (raises HTTP 403 on denial) + + **This is the only correct answer to "what may this user do".** There used to + be a second one — the `User.access_codes` property — which read the single + legacy `users.role_id` and nothing else. Because `/api/me/profile` reported + that answer while `require_access` used this one, a user holding a role + through `user_roles` had authority the product would not show them: no menu + entry, no route, no button. `User.access_codes` now defers to whatever + `get_current_user` resolved through here, and + `tests/probes/test_multi_role.py` asserts the two agree. + """ + + def __init__(self, db: Session): + self.db = db + + def user_access_codes(self, user: Optional[User]) -> Set[str]: + """ + Get all access codes for a user. + Includes both local role-based permissions and SaaS-assigned permissions. + + Memoised for the life of the request: `require_access` asks once per + dependency and `get_current_user` asks once more, and the answer cannot + change beneath a half-finished operation. + """ + if not user: + return set() + + return request_cached( + f"access_codes:{user.id}", lambda: self._resolve_codes(user) + ) + + def _resolve_codes(self, user: User) -> Set[str]: + all_access_codes = set() + + saas_permissions = getattr(user, "saas_permissions", set()) + all_access_codes.update(saas_permissions) + + direct_ids: Set[object] = set() + direct_codes: Set[str] = set() + + if user.role: + for ra in user.role.role_accesses: + if ra.access: + direct_ids.add(ra.access.id) + direct_codes.add(ra.access.access_code) + + for access in self._accesses_from_scoped_roles(user): + direct_ids.add(access.id) + direct_codes.add(access.access_code) + + if direct_ids: + all_access_codes.update( + self._expand_with_descendants(direct_ids, direct_codes) + ) + + if any(code.startswith("document.conversion") for code in all_access_codes): + all_access_codes.add("document.conversion") + + return all_access_codes + + def _group_ids_subquery(self, user): + """ + The groups this user belongs to, as a **subquery** rather than a round trip. + + Filtered by tenant on both sides: a membership row naming another + tenant's group must confer nothing, and `user_access_groups` carries its + own `tenant_id` precisely so a forged or stale row cannot reach across. + + A subquery and not a `list` because this used to be one extra SELECT on + every permission check, and permission checks are the hottest path in + the application. + """ + from app.modules.org.models.group_model import AccessGroup, UserAccessGroup + + return ( + select(UserAccessGroup.group_id) + .join(AccessGroup, AccessGroup.id == UserAccessGroup.group_id) + .where( + UserAccessGroup.user_id == user.id, + UserAccessGroup.tenant_id == user.tenant_id, + AccessGroup.tenant_id == user.tenant_id, + ) + .scalar_subquery() + ) + + def _group_ids_for(self, user) -> list: + """Groups the user belongs to. Retained for callers that want the ids.""" + from app.modules.org.models.group_model import AccessGroup, UserAccessGroup + + rows = ( + self.db.query(UserAccessGroup.group_id) + .join(AccessGroup, AccessGroup.id == UserAccessGroup.group_id) + .filter( + UserAccessGroup.user_id == user.id, + UserAccessGroup.tenant_id == user.tenant_id, + AccessGroup.tenant_id == user.tenant_id, + ) + .all() + ) + return [r[0] for r in rows] + + def _accesses_from_scoped_roles(self, user: User) -> list: + """ + Every `Access` reachable through this user's `user_roles` rows. + + Deliberately ignores `org_unit_id`: holding a role in one team still + means holding its codes. Narrowing to a subtree is `ScopeService`'s job, + and conflating the two here would mean an endpoint that has not been + converted to ask about scope silently refuses people who legitimately + hold the permission somewhere. + + **The three predicates below are not optional.** This query originally + filtered on the principal alone, which meant the coarse gate disagreed + with `ScopeService` in two ways that both fail open: + + - an **expired** grant still passed `require_access`, so temporary + elevation did not expire on any endpoint that had not been converted + to ask about scope — which is almost all of them; + - a grant naming **another tenant's role** conferred that role's codes. + This is the defect the C-series found in `ScopeService._resolve` and + fixed there; it lived on here for a release. + + They are expressed in SQL rather than filtered afterwards for the same + reason `ScopeService` gives: an expired grant must be invisible to every + reader, and a post-filter is how one caller ends up honouring a dead + grant. + """ + from app.modules.auth.models.role_access_model import RoleAccess + from app.modules.auth.models.role_model import Role + from app.modules.auth.models.user_role_model import UserRole + + return ( + self.db.query(Access) + .join(RoleAccess, RoleAccess.access_id == Access.id) + .join(UserRole, UserRole.role_id == RoleAccess.role_id) + .join(Role, Role.id == UserRole.role_id) + .filter( + or_( + UserRole.user_id == user.id, + UserRole.group_id.in_(self._group_ids_subquery(user)), + ), + or_( + Role.tenant_id == user.tenant_id, + Role.tenant_id.is_(None), + ), + or_( + UserRole.expires_at.is_(None), + UserRole.expires_at > func.now(), + ), + ) + .distinct() + .all() + ) + + def _access_tree(self): + """ + `(code_by_id, children_by_parent)` for the whole catalogue. + + One query, memoised per request. The catalogue is the same for every + user, so resolving a page of twenty users used to read this table twenty + times. + """ + + def _load(): + rows = self.db.query( + Access.id, Access.parent_id, Access.access_code + ).all() + code_by_id = {row.id: row.access_code for row in rows} + children_by_parent: dict = {} + for row in rows: + if row.parent_id is None: + continue + children_by_parent.setdefault(row.parent_id, set()).add(row.id) + return code_by_id, children_by_parent + + return request_cached("access_tree", _load) + + def _expand_with_descendants( + self, direct_ids: Set[object], direct_codes: Set[str] + ) -> Set[str]: + """ + Expand assigned access codes with all descendants so parent access grants + full module access. + """ + code_by_id, children_by_parent = self._access_tree() + + visited = set(direct_ids) + stack = list(direct_ids) + + while stack: + current = stack.pop() + for child_id in children_by_parent.get(current, set()): + if child_id in visited: + continue + visited.add(child_id) + stack.append(child_id) + + expanded_codes = {code_by_id[access_id] for access_id in visited if access_id in code_by_id} + return direct_codes.union(expanded_codes) + + def has_access(self, user: Optional[User], code: str) -> bool: + """ + Check if user has a specific access code. + + Args: + user: Current user + code: Access code to check (e.g. "project.create") + + Returns: + True if user has the access code, False otherwise + """ + return code in self.user_access_codes(user) + + def require_access(self, user: Optional[User], code: str) -> None: + """ + Assert user has a specific access code. + Raises HTTPException 403 if not authorized. + + Args: + user: Current user + code: Required access code + + Raises: + HTTPException: 403 if user not authenticated or lacks access + """ + if not user: + raise HTTPException(status_code=403, detail="Authentication required") + + if not self.has_access(user, code): + raise HTTPException( + status_code=403, detail=f"Access denied: insufficient permissions" + ) + + def require_any_access(self, user: Optional[User], codes: List[str]) -> None: + """ + Assert user has at least one of the provided access codes. + Raises HTTPException 403 if none match. + + Args: + user: Current user + codes: List of access codes (user needs at least one) + + Raises: + HTTPException: 403 if user lacks all codes + """ + if not user: + raise HTTPException(status_code=403, detail="Authentication required") + + if not any(self.has_access(user, code) for code in codes): + raise HTTPException( + status_code=403, detail="Access denied: insufficient permissions" + ) + + def require_all_access(self, user: Optional[User], codes: List[str]) -> None: + """ + Assert user has ALL of the provided access codes. + Raises HTTPException 403 if any are missing. + + Args: + user: Current user + codes: List of access codes (user must have all) + + Raises: + HTTPException: 403 if user lacks any code + """ + if not user: + raise HTTPException(status_code=403, detail="Authentication required") + + missing = [code for code in codes if not self.has_access(user, code)] + if missing: + raise HTTPException( + status_code=403, detail="Access denied: insufficient permissions" + ) + + def get_all_system_accesses(self) -> List[dict]: + """ + Get all defined access codes in the system grouped by category. + """ + accesses = self.db.query(Access).all() + return [ + { + "id": a.id, + "access_code": a.access_code, + "category": a.category, + "name": a.name, + "parent_id": a.parent_id, + } + for a in accesses + ] + + def granted_by(self, user: Optional[User]) -> dict: + """ + `{access_code: [attribution, ...]}` — which role conferred each code. + + The answer to "why can this person do that", which + `/api/admin/users/{id}/effective-access` exposes. With one role per user + the question answered itself; with several it does not, and support + cannot read the database. + + Attribution names the role, the scope, and whether it arrived through + the primary role or a grant. Descendant expansion is attributed to the + role that holds the parent code, because that is the row an + administrator would edit to take it away. + """ + if not user: + return {} + + attributions: dict = {} + + def _add(code: str, entry: dict) -> None: + existing = attributions.setdefault(code, []) + if entry not in existing: + existing.append(entry) + + from app.modules.auth.services.user_role_service import UserRoleReader + + for assignment in UserRoleReader(self.db).assignments(user): + direct_ids = set() + direct_codes = set() + for access in self._accesses_of_role(assignment["role_id"]): + direct_ids.add(access.id) + direct_codes.add(access.access_code) + if not direct_ids: + continue + entry = { + "role_id": str(assignment["role_id"]), + "role_name": assignment["role_name"], + "source": assignment["source"], + "org_unit_id": ( + str(assignment["org_unit_id"]) + if assignment["org_unit_id"] + else None + ), + "org_unit_name": assignment["org_unit_name"], + "expires_at": ( + assignment["expires_at"].isoformat() + if assignment["expires_at"] + else None + ), + } + for code in self._expand_with_descendants(direct_ids, direct_codes): + _add(code, entry) + + for code in getattr(user, "saas_permissions", set()) or set(): + _add( + code, + { + "role_id": None, + "role_name": "Subscription platform", + "source": "saas", + "org_unit_id": None, + "org_unit_name": None, + "expires_at": None, + }, + ) + + return attributions + + def _accesses_of_role(self, role_id) -> list: + from app.modules.auth.models.role_access_model import RoleAccess + + def _load(): + return ( + self.db.query(Access) + .join(RoleAccess, RoleAccess.access_id == Access.id) + .filter(RoleAccess.role_id == role_id) + .all() + ) + + return request_cached(f"role_accesses:{role_id}", _load) diff --git a/app/modules/auth/services/privilege.py b/app/modules/auth/services/privilege.py new file mode 100644 index 0000000..56f84c0 --- /dev/null +++ b/app/modules/auth/services/privilege.py @@ -0,0 +1,54 @@ +""" +One answer to "is this caller a superadmin", for the routes that branch on it. + +There were five copies of this predicate — in `role_routes`, `admin_user_routes`, +`tenant_routes`, `chat_controller` and `admin_dashboard_service` — each written +as: + + any(acc.startswith("superadmin.") for acc in user.access_codes) + +Identical, and all five wrong in the same way, because `User.access_codes` read +only the legacy `users.role_id`. A user whose superadmin role arrived through a +`user_roles` grant therefore **passed `require_access` and was then treated as a +tenant administrator by the same request**. Getting through the door and then +being scoped as somebody else is the worst failure shape available: it looks like +a data bug, not an authorisation one. + +The property is fixed at its source (`get_current_user` resolves through +`PermissionService`), so these five call sites are now correct. They are +collapsed into one function anyway, because five copies of a security predicate +is five chances for the sixth to be written differently. + +**`users.is_superadmin` is deliberately not consulted here, and the name says so.** +DocQube has two superadmin predicates because it has two questions: + + app.middleware.tenant.is_superadmin(user) the B1 privilege flag — + may this session bypass + the tenant filter and RLS + holds_superadmin_access(user) does this caller hold a + `superadmin.*` code — may + they administer across + tenants through the API + +Two functions both called `is_superadmin` with different semantics is a trap, so +this one is not called that. Folding the flag in here would silently widen five +existing authorisation decisions — a tenant-less operator holding no +`superadmin.*` code would begin passing `_assert_role_in_callers_tenant`, which +today refuses them. That widening may well be desirable; it is a deliberate +product change with its own probe, not a side effect of de-duplicating a helper. +""" + +from __future__ import annotations + +SUPERADMIN_PREFIX = "superadmin." + + +def holds_superadmin_access(user) -> bool: + """True when the caller holds any `superadmin.*` access code.""" + if user is None: + return False + codes = getattr(user, "access_codes", None) or [] + return any(str(code).startswith(SUPERADMIN_PREFIX) for code in codes) + + +__all__ = ["holds_superadmin_access", "SUPERADMIN_PREFIX"] diff --git a/app/modules/auth/services/role_service.py b/app/modules/auth/services/role_service.py new file mode 100644 index 0000000..1708421 --- /dev/null +++ b/app/modules/auth/services/role_service.py @@ -0,0 +1,186 @@ +from sqlalchemy.orm import Session +from sqlalchemy import or_, cast, String +from fastapi import HTTPException, status +from app.modules.auth.models.role_model import Role +from app.modules.auth.models.role_access_model import RoleAccess +from app.modules.auth.models.access_model import Access +from app.modules.auth.models.user_model import User +from app.modules.auth.schemas.role_schema import ( + RoleCreate, + RoleUpdate, + RoleOut, + RolePaginatedOut, +) +from typing import List, Optional +from uuid import UUID + + +class RoleService: + @staticmethod + def create_role(db: Session, role_data: RoleCreate) -> Role: + existing = ( + db.query(Role) + .filter( + Role.name == role_data.role_name, Role.tenant_id == role_data.tenant_id + ) + .first() + ) + if existing: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Role name already exists for this tenant", + ) + role = Role( + name=role_data.role_name, + description=role_data.description, + tenant_id=role_data.tenant_id, + is_default=role_data.is_default or False, + ) + db.add(role) + db.flush() + db.refresh(role) + + if role_data.access_ids: + RoleService.assign_accesses(db, role.id, role_data.access_ids) + + return role + + @staticmethod + def assign_accesses(db: Session, role_id: UUID, access_ids: List[UUID]): + db.query(RoleAccess).filter(RoleAccess.role_id == role_id).delete() + for aid in access_ids: + access = db.query(Access).filter(Access.id == aid).first() + if not access: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Access {aid} not found", + ) + ra = RoleAccess(role_id=role_id, access_id=aid) + db.add(ra) + + @staticmethod + def get_all_roles(db: Session, tenant_id: Optional[UUID] = None): + query = db.query(Role) + if tenant_id is not None: + from app.modules.billing.models.plan_model import TenantSubscription, PlanRole + sub = db.query(TenantSubscription).filter( + TenantSubscription.tenant_id == tenant_id, + TenantSubscription.status == 'active' + ).order_by(TenantSubscription.created_at.desc()).first() + if sub and sub.plan_id: + plan_roles = db.query(PlanRole).filter(PlanRole.plan_id == sub.plan_id).all() + plan_role_ids = [pr.role_id for pr in plan_roles] + query = query.filter(or_(Role.tenant_id == tenant_id, Role.id.in_(plan_role_ids))) + else: + query = query.filter(Role.tenant_id == tenant_id) + return query.all() + + @staticmethod + def get_role_by_id(db: Session, role_id: UUID) -> Role: + role = db.query(Role).filter(Role.id == role_id).first() + if not role: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Role not found" + ) + return role + + @staticmethod + def update_role(db: Session, role_id: UUID, role_data: RoleUpdate) -> Role: + role = RoleService.get_role_by_id(db, role_id) + if getattr(role, "is_system", False): + raise HTTPException(status_code=400, detail="Cannot edit a system role.") + update_dict = role_data.model_dump(exclude_unset=True) + if "role_name" in update_dict: + name = update_dict.pop("role_name") + if name and name != role.name: + conflict = ( + db.query(Role) + .filter(Role.name == name, Role.tenant_id == role.tenant_id) + .first() + ) + if conflict: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Role name already exists", + ) + role.name = name + if "description" in update_dict: + role.description = update_dict.get("description") + if "is_default" in update_dict: + role.is_default = update_dict.get("is_default") + if "access_ids" in update_dict: + access_ids = update_dict.get("access_ids") + if access_ids is not None: + RoleService.assign_accesses(db, role_id, access_ids) + db.flush() + db.refresh(role) + return role + + @staticmethod + def delete_role(db: Session, role_id: UUID): + role = RoleService.get_role_by_id(db, role_id) + if getattr(role, "is_system", False): + raise HTTPException(status_code=400, detail="Cannot delete a system role.") + db.delete(role) + return {"message": "Role deleted successfully"} + + @staticmethod + def get_roles_paginated( + db: Session, + tenant_id: Optional[UUID] = None, + page: int = 1, + page_size: int = 10, + search: Optional[str] = None, + ): + query = db.query(Role) + if tenant_id is not None: + from app.modules.billing.models.plan_model import TenantSubscription, PlanRole + sub = db.query(TenantSubscription).filter( + TenantSubscription.tenant_id == tenant_id, + TenantSubscription.status == 'active' + ).order_by(TenantSubscription.created_at.desc()).first() + if sub and sub.plan_id: + plan_roles = db.query(PlanRole).filter(PlanRole.plan_id == sub.plan_id).all() + plan_role_ids = [pr.role_id for pr in plan_roles] + query = query.filter(or_(Role.tenant_id == tenant_id, Role.id.in_(plan_role_ids))) + else: + query = query.filter(Role.tenant_id == tenant_id) + if search and search.strip(): + search_term = search.strip() + query = query.filter(Role.name.ilike(f"%{search_term}%")) + total = query.count() + offset = (page - 1) * page_size + roles = query.offset(offset).limit(page_size).all() + total_pages = (total + page_size - 1) // page_size if total > 0 else 0 + return { + "items": [r for r in roles], + "total": total, + "page": page, + "page_size": page_size, + "total_pages": total_pages, + } + + @staticmethod + def get_role_with_accesses(db: Session, role_id: UUID) -> Role: + """Get a role with all its access assignments.""" + return RoleService.get_role_by_id(db, role_id) + + @staticmethod + def assign_role_to_user(db: Session, user_id: int, role_id: UUID): + """Assign a role to a specific user.""" + from fastapi import HTTPException, status + + user = db.query(User).filter(User.id == user_id).first() + if not user: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="User not found" + ) + role = RoleService.get_role_by_id(db, role_id) + user.role_id = role.id + db.flush() + db.refresh(user) + return { + "message": "Role assigned successfully", + "user_id": user_id, + "role_id": str(role_id), + } diff --git a/app/modules/auth/services/user_role_service.py b/app/modules/auth/services/user_role_service.py new file mode 100644 index 0000000..7bb8416 --- /dev/null +++ b/app/modules/auth/services/user_role_service.py @@ -0,0 +1,389 @@ +""" +Every role a user actually holds, resolved and named. + +`users.role_id` gives a user one role. `user_roles` gives them any number, at a +scope, optionally expiring, optionally through a group. Both mechanisms are live +and are **unioned, never intersected** — that is the compatibility rule the +C-series migration set, and it is why turning any of this on cannot remove +somebody's access. + +This module answers "which roles", where `PermissionService` answers "which +codes". They are separate because they are asked by different callers for +different reasons: the gate needs codes, and the *interface* needs roles, so an +administrator can see that Alice is both an Editor and an Approver rather than +inferring it from forty-eight access codes. + +**`source` is the field that keeps the UI honest.** + + "primary" the legacy `users.role_id` — changed through PATCH /admin/users, + not revocable as a grant, and (see `ScopeService._resolve`) it + cannot be narrowed by anything while it is still set + "grant" a `user_roles` row — added and revoked freely + "group" a `user_roles` row naming a group this user belongs to; it + confers codes, and it is revoked by editing the group, not the + user + +Rendering all three as one undifferentiated list would imply that removing any +of them is the same action. It is not, and the difference is exactly where a +permissions UI misleads its operator. +""" + +from __future__ import annotations + +import uuid +from typing import Any, Iterable, Optional + +from sqlalchemy import func, or_, select +from sqlalchemy.orm import Session + +from app.core.request_cache import request_cached + + +class UserRoleReader: + """Read-only resolution of role assignments. No writes, no side effects.""" + + def __init__(self, db: Session): + self.db = db + + + def assignments(self, user, include_expired: bool = False) -> list[dict]: + """ + Primary role first, then grants, then group-derived grants. + + Ordered deliberately: the primary role is the one an operator changes + through the existing dropdown, so it belongs at the top of any list that + also offers "remove". + + Memoised per request unless `include_expired` is set — the expired view + is an administrative read, not a hot path, and caching two shapes under + one key is how a stale answer gets served to the wrong caller. + """ + if user is None: + return [] + if include_expired: + return self._resolve(user, include_expired=True) + return request_cached( + f"role_assignments:{user.id}", lambda: self._resolve(user) + ) + + def _resolve(self, user, include_expired: bool = False) -> list[dict]: + out: list[dict] = [] + + primary = getattr(user, "role", None) + if primary is not None: + out.append( + { + "grant_id": None, + "role_id": primary.id, + "role_name": primary.name, + "source": "primary", + "org_unit_id": None, + "org_unit_name": None, + "group_id": None, + "group_name": None, + "expires_at": None, + "assigned_by_id": None, + "created_at": None, + "is_expired": False, + } + ) + + out.extend(self._grants(user, include_expired=include_expired)) + return out + + def _grants(self, user, include_expired: bool = False) -> list[dict]: + from app.modules.auth.models.role_model import Role + from app.modules.auth.models.user_role_model import UserRole + from app.modules.org.models.group_model import AccessGroup, UserAccessGroup + from app.modules.org.models.org_model import OrgUnit + + my_groups = ( + select(UserAccessGroup.group_id) + .join(AccessGroup, AccessGroup.id == UserAccessGroup.group_id) + .where( + UserAccessGroup.user_id == user.id, + UserAccessGroup.tenant_id == user.tenant_id, + AccessGroup.tenant_id == user.tenant_id, + ) + .scalar_subquery() + ) + + query = ( + self.db.query( + UserRole.id, + UserRole.role_id, + Role.name, + UserRole.org_unit_id, + OrgUnit.name, + UserRole.group_id, + AccessGroup.name, + UserRole.expires_at, + UserRole.assigned_by_id, + UserRole.created_at, + ) + .join(Role, Role.id == UserRole.role_id) + .outerjoin(OrgUnit, OrgUnit.id == UserRole.org_unit_id) + .outerjoin(AccessGroup, AccessGroup.id == UserRole.group_id) + .filter( + or_( + UserRole.user_id == user.id, + UserRole.group_id.in_(my_groups), + ), + or_(Role.tenant_id == user.tenant_id, Role.tenant_id.is_(None)), + ) + ) + if not include_expired: + query = query.filter( + or_(UserRole.expires_at.is_(None), UserRole.expires_at > func.now()) + ) + + rows = query.order_by(UserRole.created_at).all() + return [self._row_to_assignment(row) for row in rows] + + @staticmethod + def _row_to_assignment(row) -> dict: + from datetime import datetime, timezone + + ( + grant_id, + role_id, + role_name, + org_unit_id, + org_unit_name, + group_id, + group_name, + expires_at, + assigned_by_id, + created_at, + ) = row + + expired = False + if expires_at is not None: + reference = expires_at + if reference.tzinfo is None: + reference = reference.replace(tzinfo=timezone.utc) + expired = reference <= datetime.now(timezone.utc) + + return { + "grant_id": grant_id, + "role_id": role_id, + "role_name": role_name, + "source": "group" if group_id is not None else "grant", + "org_unit_id": org_unit_id, + "org_unit_name": org_unit_name, + "group_id": group_id, + "group_name": group_name, + "expires_at": expires_at, + "assigned_by_id": assigned_by_id, + "created_at": created_at, + "is_expired": expired, + } + + + def assignments_for_users( + self, users: Iterable[Any] + ) -> dict[int, list[dict]]: + """ + The same resolution for a page of users, in **two** queries rather than + two per user. + + `/api/admin/users` renders up to a hundred rows. Resolving each one + through `assignments()` would be a textbook N+1 that a seeded test + database with three users cannot see and a real tenant feels immediately, + which is precisely the failure `tests/characterization/test_query_budget.py` + exists to catch — so this is written batched from the start. + """ + from app.modules.auth.models.role_model import Role + from app.modules.auth.models.user_model import User + from app.modules.auth.models.user_role_model import UserRole + from app.modules.org.models.group_model import AccessGroup, UserAccessGroup + from app.modules.org.models.org_model import OrgUnit + + users = list(users) + result: dict[int, list[dict]] = {} + for user in users: + primary = getattr(user, "role", None) + entries = [] + if primary is not None: + entries.append( + { + "grant_id": None, + "role_id": primary.id, + "role_name": primary.name, + "source": "primary", + "org_unit_id": None, + "org_unit_name": None, + "group_id": None, + "group_name": None, + "expires_at": None, + "assigned_by_id": None, + "created_at": None, + "is_expired": False, + } + ) + result[user.id] = entries + + user_ids = [u.id for u in users] + if not user_ids: + return result + + rows = ( + self.db.query( + UserRole.user_id, + UserRole.id, + UserRole.role_id, + Role.name, + UserRole.org_unit_id, + OrgUnit.name, + UserRole.expires_at, + UserRole.assigned_by_id, + UserRole.created_at, + ) + .join(Role, Role.id == UserRole.role_id) + .join(User, User.id == UserRole.user_id) + .outerjoin(OrgUnit, OrgUnit.id == UserRole.org_unit_id) + .filter( + UserRole.user_id.in_(user_ids), + or_(Role.tenant_id == User.tenant_id, Role.tenant_id.is_(None)), + or_(UserRole.expires_at.is_(None), UserRole.expires_at > func.now()), + ) + .order_by(UserRole.created_at) + .all() + ) + for row in rows: + user_id = row[0] + if user_id not in result: + continue + result[user_id].append( + self._row_to_assignment( + (row[1], row[2], row[3], row[4], row[5], None, None, row[6], row[7], row[8]) + ) + ) + + group_rows = ( + self.db.query( + UserAccessGroup.user_id, + UserRole.id, + UserRole.role_id, + Role.name, + UserRole.org_unit_id, + OrgUnit.name, + UserRole.group_id, + AccessGroup.name, + UserRole.expires_at, + UserRole.assigned_by_id, + UserRole.created_at, + ) + .join(UserRole, UserRole.group_id == UserAccessGroup.group_id) + .join(Role, Role.id == UserRole.role_id) + .join(AccessGroup, AccessGroup.id == UserRole.group_id) + .join(User, User.id == UserAccessGroup.user_id) + .outerjoin(OrgUnit, OrgUnit.id == UserRole.org_unit_id) + .filter( + UserAccessGroup.user_id.in_(user_ids), + UserAccessGroup.tenant_id == User.tenant_id, + AccessGroup.tenant_id == User.tenant_id, + or_(Role.tenant_id == User.tenant_id, Role.tenant_id.is_(None)), + or_(UserRole.expires_at.is_(None), UserRole.expires_at > func.now()), + ) + .order_by(UserRole.created_at) + .all() + ) + for row in group_rows: + user_id = row[0] + if user_id not in result: + continue + result[user_id].append(self._row_to_assignment(tuple(row[1:]))) + + return result + + + def grant_row(self, tenant_id, grant_id: uuid.UUID): + """One `user_roles` row, refusing to reach outside the caller's tenant.""" + from app.modules.auth.models.role_model import Role + from app.modules.auth.models.user_role_model import UserRole + + return ( + self.db.query(UserRole) + .join(Role, Role.id == UserRole.role_id) + .filter( + UserRole.id == grant_id, + or_(Role.tenant_id == tenant_id, Role.tenant_id.is_(None)), + ) + .first() + ) + + def users_holding_role(self, role_id) -> int: + """ + How many users a role reaches, counting both mechanisms. + + Used to make role deletion say what it is about to do. Deleting a role + cascades its grants away (`user_roles.role_id ON DELETE CASCADE`) and + nulls the primary (`users.role_id ON DELETE SET NULL`), so a silent + delete can remove authority from people nobody was thinking about. + """ + from app.modules.auth.models.user_model import User + from app.modules.auth.models.user_role_model import UserRole + + primary = ( + self.db.query(func.count(User.id)) + .filter(User.role_id == role_id, User.is_deleted.is_(False)) + .scalar() + or 0 + ) + granted = ( + self.db.query(func.count(func.distinct(UserRole.user_id))) + .filter(UserRole.role_id == role_id, UserRole.user_id.isnot(None)) + .scalar() + or 0 + ) + groups = ( + self.db.query(func.count(func.distinct(UserRole.group_id))) + .filter(UserRole.role_id == role_id, UserRole.group_id.isnot(None)) + .scalar() + or 0 + ) + return int(primary) + int(granted) + int(groups) + + +def attach_role_refs(reader: "UserRoleReader", users: Iterable[Any]) -> None: + """ + Give a list of user rows their resolved roles, in two queries for the batch. + + `UserOut.roles` is declared as "every role this user holds". Any endpoint + that serialises a *different* user than the caller has to fill it in, or the + field means one thing on `/api/me/profile` and something narrower elsewhere + — which is precisely the split between the gate and the response that this + work exists to close, reintroduced one endpoint at a time. + + Sets the same `_resolve_role_refs` closure `get_current_user` attaches, so + `User.role_refs` needs no knowledge of where its answer came from. + """ + users = list(users) + if not users: + return + by_user = reader.assignments_for_users(users) + for row in users: + assignments = by_user.get(row.id, []) + setattr(row, "_resolve_role_refs", lambda captured=assignments: captured) + + +def invalidate_for_user(user_id: Optional[int]) -> None: + """ + Forget this request's memoised answers for one user. + + Called after a grant or a revoke. Without it, a POST that adds a role and + then returns the user's roles in the same response serves the pre-grant + answer — which reads as "the button did nothing". + """ + from app.core.request_cache import invalidate_request_cache + + if user_id is None: + invalidate_request_cache("role_assignments:") + invalidate_request_cache("access_codes:") + return + invalidate_request_cache(f"role_assignments:{user_id}") + invalidate_request_cache(f"access_codes:{user_id}") + + +__all__ = ["UserRoleReader", "attach_role_refs", "invalidate_for_user"] diff --git a/app/modules/auth/sso_router.py b/app/modules/auth/sso_router.py new file mode 100644 index 0000000..d35ab0a --- /dev/null +++ b/app/modules/auth/sso_router.py @@ -0,0 +1,356 @@ +from fastapi import APIRouter, Depends, HTTPException, status, Request, Response +from sqlalchemy.orm import Session +import time +import logging +import hmac +import hashlib + +from app.db.database import get_db +from app.core.settings import settings +from app.services.saas_service import SaaSService +from app.modules.tenant.models.tenant_model import Tenant +from app.modules.auth.models.saas_models import SaaSTenantMapping + +router = APIRouter(prefix="/sso", tags=["SSO"]) +logger = logging.getLogger(__name__) + +@router.post("/login") +async def sso_login( + request: Request, + response: Response, + db: Session = Depends(get_db) +): + """ + Endpoint for SaaS to initiate a login session for a user. + Verified via HMAC signature. + """ + try: + data = await read_json_body(request, what="SSO login body") + + user_id = data.get("user_id") + email = data.get("email") + tenant_id = data.get("tenant_id") + timestamp = data.get("timestamp") + signature = request.headers.get("X-Signature") + + if not all([user_id, email, timestamp, signature]): + raise HTTPException(status_code=400, detail="Missing required SSO parameters") + + now = int(time.time() * 1000) + req_timestamp = int(timestamp) + if abs(now - req_timestamp) > 5 * 60 * 1000: + raise HTTPException(status_code=401, detail="Invalid or stale timestamp") + + import json + canonical_string = json.dumps(data, sort_keys=True, separators=(",", ":")) + + if not SaaSService.verify_signature(signature, canonical_string): + raise HTTPException(status_code=401, detail="Invalid signature") + + saas_data = { + "id": user_id, + "email": email, + "name": data.get("first_name") or email.split('@')[0], + "company_id": tenant_id, + "tenant_name": data.get("tenant_name"), + "role": data.get("role"), + "role_id": data.get("role_id"), + "is_superadmin": data.get("is_superadmin"), + "metadata": { + "permissions": data.get("permissions", []), + "subscription": data.get("subscription"), + }, + } + + user, mapping = SaaSService.ensure_saas_user(db, saas_data) + logger.info(f"SSO: User {email} mapped to local user ID {user.id}") + + permissions = data.get("permissions", []) + if mapping: + mapping.metadata_ = { + "permissions": permissions, + "subscription": data.get("subscription"), + } + + is_user_superadmin = bool( + getattr(user, "is_superadmin", False) + or data.get("is_superadmin") + or data.get("role") == "superadmin" + or (isinstance(data.get("permissions"), list) and "superadmin.main.view" in data.get("permissions")) + ) + if is_user_superadmin and not user.is_superadmin: + user.is_superadmin = True + db.flush() + db.refresh(user) + + access_token = create_access_token( + { + "sub": str(user.id), + "tenant_id": str(user.tenant_id) if user.tenant_id else None, + "is_superadmin": is_user_superadmin, + "saas_permissions": permissions, + "saas_subscription": data.get("subscription"), + } + ) + refresh_token = create_refresh_token( + { + "sub": str(user.id), + } + ) + + is_prod = settings.APP_ENV == "production" + + response.set_cookie( + key="docqube_access_token", + value=access_token, + httponly=True, + secure=is_prod, + samesite="lax", + path="/", + max_age=settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60 + ) + + response.set_cookie( + key="docqube_refresh_token", + value=refresh_token, + httponly=True, + secure=is_prod, + samesite="lax", + path="/", + max_age=7 * 24 * 3600 + ) + + return { + "status": "success", + "message": "SSO Login successful", + "access_token": access_token, + "refresh_token": refresh_token, + "user": { + "id": user.id, + "email": user.email, + "name": user.name, + "permissions": permissions + } + } + + except HTTPException as he: + raise he + except Exception as e: + logger.error("SSO Login Failed", exc_info=True) + raise HTTPException(status_code=500, detail="Internal Server Error") + +@router.post("/sync-permissions") +def sync_permissions(request: Request, db: Session = Depends(get_db)): + """ + Return all permission nodes defined in DocQube to the SaaS platform. + """ + signature = request.headers.get("X-SaaS-Signature") or request.headers.get("X-Signature") + if not signature: + raise HTTPException(status_code=401, detail="Missing signature") + + payload_body = "{}" + expected_signature = hmac.new( + settings.SAAS_TRUST_SECRET.encode("utf-8"), + payload_body.encode("utf-8"), + hashlib.sha256 + ).hexdigest() + + if not hmac.compare_digest(signature, expected_signature): + raise HTTPException(status_code=401, detail="Invalid signature") + + try: + from app.modules.auth.models.access_model import Access + all_nodes = db.query(Access).all() + node_map = {str(node.id): node.access_code for node in all_nodes} + + permissions = [] + for node in all_nodes: + parent_code = node_map.get(str(node.parent_id)) if node.parent_id else None + permissions.append({ + "permission_code": node.access_code, + "name": node.name, + "category": node.category, + "parent_code": parent_code + }) + return {"status": "success", "permissions": permissions} + except HTTPException: + raise + except Exception: + logger.exception("Failed to build the permission list for SaaS") + raise HTTPException(status_code=500, detail="Internal Server Error") + +@router.post("/provision") +async def provision_tenant(request: Request, db: Session = Depends(get_db)): + """ + Receive tenant provisioning/update events from SaaS and sync the mapped + DocQube tenant, including max_users_allowed. + """ + raw_body = await request.body() + signature = request.headers.get("X-SaaS-Signature") or request.headers.get("X-Signature") + if not signature: + raise HTTPException(status_code=401, detail="Missing signature") + + expected_signature = hmac.new( + settings.SAAS_TRUST_SECRET.encode("utf-8"), + raw_body, + hashlib.sha256, + ).hexdigest() + if not hmac.compare_digest(signature, expected_signature): + raise HTTPException(status_code=401, detail="Invalid signature") + + payload = await read_json_body(request, what="webhook payload") + event_type = payload.get("event_type") + data = payload.get("data", {}) or {} + + saas_tenant_id = data.get("tenant_id") + tenant_name = data.get("tenant_name") or "SaaS Tenant" + max_users_allowed = data.get("max_users_allowed") + is_active = data.get("is_active") + plan_code = data.get("plan_code") or data.get("plan") + subscription = data.get("subscription") or {} + + try: + if event_type in {"TENANT_PROVISION_REQUESTED", "TENANT_UPDATED", "TENANT_STATUS_CHANGED"}: + if not saas_tenant_id: + raise HTTPException(status_code=400, detail="Missing tenant_id") + tenant = SaaSService.ensure_saas_tenant( + db, + saas_tenant_id=saas_tenant_id, + name=tenant_name, + max_users_allowed=max_users_allowed, + is_active=is_active, + ) + _sync_subscription(db, tenant, plan_code, subscription) + + return { + "status": "success", + "message": f"{event_type} synced", + "tenant_id": str(tenant.id), + "max_users_allowed": tenant.max_users_allowed, + } + + if event_type == "TENANT_DEPROVISION_REQUESTED": + mapping = ( + db.query(SaaSTenantMapping) + .filter(SaaSTenantMapping.saas_tenant_id == str(saas_tenant_id)) + .first() + ) + if mapping: + tenant = db.query(Tenant).filter(Tenant.id == mapping.docqube_tenant_id).first() + if tenant: + tenant.is_active = False + db.flush() + db.refresh(tenant) + return {"status": "success", "message": "Tenant deprovisioned"} + + if event_type == "USER_PROVISION_REQUESTED": + user, _ = SaaSService.provision_user(db, data) + return {"status": "success", "message": "User provisioned", "user_id": str(user.id) if user else None} + + if event_type == "USER_DEPROVISION_REQUESTED": + SaaSService.deprovision_user(db, data) + return {"status": "success", "message": "User deprovisioned"} + + if event_type == "ROLE_PROVISION_REQUESTED": + role, _ = SaaSService.provision_role(db, data) + return {"status": "success", "message": "Role provisioned", "role_id": str(role.id) if role else None} + + if event_type == "ROLE_DEPROVISION_REQUESTED": + SaaSService.deprovision_role(db, data) + return {"status": "success", "message": "Role deprovisioned"} + + if event_type == "PLAN_PROVISION_REQUESTED": + plan = SaaSService.ensure_saas_plan(db, data) + db.commit() + return {"status": "success", "message": "Plan provisioned", "plan_id": str(plan.id)} + + if event_type == "PLAN_UPDATED": + plan = SaaSService.update_saas_plan(db, data) + db.commit() + return {"status": "success", "message": "Plan updated", "plan_id": str(plan.id)} + + if event_type == "PLAN_DEPROVISION_REQUESTED": + SaaSService.deprovision_plan(db, data) + db.commit() + return {"status": "success", "message": "Plan deprovisioned"} + + return {"status": "ignored", "message": f"Unhandled event type: {event_type}"} + except HTTPException: + raise + except Exception as e: + logger.exception("Provisioning webhook failed for event %s", event_type) + raise HTTPException(status_code=500, detail=f"Provisioning error: {str(e)}") + +from app.core.security import create_access_token, create_refresh_token +from app.core.request_body import read_json_body + + +def _sync_subscription(db, tenant, plan_code, subscription: dict) -> None: + """ + Record what the platform says about this tenant's plan. + + **Unknown plan codes are logged and ignored, not created.** Auto-creating a + plan from a webhook would let the other system invent products here — with + no price, no limits, and no one having decided what it includes. A tenant + keeps its current plan until somebody adds the new one deliberately. + + Nothing here is fatal. A provisioning webhook that fails because the plan + name changed would block the tenant from being created at all, which is a + worse outcome than a tenant whose plan is briefly stale. + """ + import logging + + logger = logging.getLogger(__name__) + + from app.modules.billing.models.plan_model import Plan, TenantSubscription + + if not plan_code and not subscription: + return + + row = ( + db.query(TenantSubscription) + .filter(TenantSubscription.tenant_id == tenant.id) + .first() + ) + + plan = None + if plan_code: + plan = db.query(Plan).filter(Plan.code == str(plan_code)).first() + if plan is None: + logger.warning( + "SaaS webhook named plan %r, which does not exist here; leaving " + "tenant %s on its current plan", + plan_code, + tenant.id, + ) + + if row is None: + if plan is None: + return + row = TenantSubscription(tenant_id=tenant.id, plan_id=plan.id) + db.add(row) + elif plan is not None: + row.plan_id = plan.id + + status = subscription.get("status") + if status: + mapped = { + "ACTIVE": "active", + "TRIAL": "trialing", + "TRIALING": "trialing", + "PAST_DUE": "past_due", + "CANCELLED": "cancelled", + "CANCELED": "cancelled", + "EXPIRED": "expired", + }.get(str(status).upper()) + if mapped: + row.status = mapped + else: + logger.warning("SaaS webhook sent unknown status %r; leaving as-is", status) + + external_ref = subscription.get("id") or subscription.get("external_ref") + if external_ref: + row.external_ref = str(external_ref) + + db.flush() + diff --git a/app/modules/auth/utils/device_utils.py b/app/modules/auth/utils/device_utils.py new file mode 100644 index 0000000..a76b041 --- /dev/null +++ b/app/modules/auth/utils/device_utils.py @@ -0,0 +1,105 @@ +import hashlib +import uuid +from fastapi import Request +import re +import requests +import logging + +logger = logging.getLogger(__name__) + +def get_client_ip(request: Request) -> str: + client_host = request.client.host if request.client else "0.0.0.0" + + is_trusted = client_host.startswith("10.") or client_host.startswith("172.") or client_host.startswith("192.168.") or client_host == "127.0.0.1" + + if is_trusted: + x_forwarded_for = request.headers.get("X-Forwarded-For") + if x_forwarded_for: + return x_forwarded_for.split(",")[0].strip() + + return client_host + +def parse_user_agent(user_agent: str): + try: + from user_agents import parse + user_agent_parsed = parse(user_agent) + + browser = user_agent_parsed.browser.family + os = user_agent_parsed.os.family + + device_type = "Desktop" + if user_agent_parsed.is_mobile: + device_type = "Mobile" + elif user_agent_parsed.is_tablet: + device_type = "Tablet" + + return browser, os, device_type + except ImportError: + browser = "Unknown" + os = "Unknown" + device_type = "Desktop" + + ua_lower = user_agent.lower() + + if "mobile" in ua_lower or "android" in ua_lower or "iphone" in ua_lower: + device_type = "Mobile" + if "tablet" in ua_lower or "ipad" in ua_lower: + device_type = "Tablet" + + if "windows" in ua_lower: + os = "Windows" + elif "mac os" in ua_lower or "macos" in ua_lower: + os = "macOS" + elif "android" in ua_lower: + os = "Android" + elif "iphone" in ua_lower or "ipad" in ua_lower: + os = "iOS" + elif "linux" in ua_lower: + os = "Linux" + + if "edg" in ua_lower: + browser = "Edge" + elif "opr" in ua_lower or "opera" in ua_lower: + browser = "Opera" + elif "chrome" in ua_lower: + browser = "Chrome" + elif "safari" in ua_lower: + browser = "Safari" + elif "firefox" in ua_lower: + browser = "Firefox" + + return browser, os, device_type + +def generate_device_fingerprint(request: Request) -> str: + device_id = request.cookies.get("docqube_device_id") + if device_id: + return device_id + + new_device_id = str(uuid.uuid4()) + request.state.new_device_id = new_device_id + return new_device_id + +def get_location_from_ip(ip: str): + """ + Fetch approximate location from IP using free ip-api.com. + Returns (country, state, city, lat, lon) + """ + if not ip or ip in ("127.0.0.1", "0.0.0.0", "localhost"): + return "Unknown", "Unknown", "Unknown", None, None + + try: + response = requests.get(f"http://ip-api.com/json/{ip}", timeout=3) + if response.status_code == 200: + data = response.json() + if data.get("status") == "success": + return ( + data.get("country", "Unknown"), + data.get("regionName", "Unknown"), + data.get("city", "Unknown"), + data.get("lat"), + data.get("lon") + ) + except Exception as e: + logger.warning(f"Failed to fetch location for IP") + + return "Unknown", "Unknown", "Unknown", None, None diff --git a/app/modules/billing/__init__.py b/app/modules/billing/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/modules/billing/models/__init__.py b/app/modules/billing/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/modules/billing/models/plan_model.py b/app/modules/billing/models/plan_model.py new file mode 100644 index 0000000..685aa72 --- /dev/null +++ b/app/modules/billing/models/plan_model.py @@ -0,0 +1,285 @@ +""" +D1 — plans, and what a tenant is entitled to. + +The gap this closes: `src/features/Landing/Pricing/index.tsx` advertises four +priced plans — Starter, Professional, Elite, Custom — and the backend had never +heard of a plan. Meanwhile `tenants` carried four limit columns set per tenant +by hand, so two customers on the same advertised plan could silently have +different limits and nothing would notice. + +So this is not "add subscriptions". It is: **give the limits that already exist +something to be derived from.** + +Four tables: + +- `Plan` what is sold +- `PlanLimit` what it includes +- `TenantSubscription` who is on what +- `TenantLimitOverride` the negotiated exception + +**DocQube does not compute a price.** `price_display` is a string for the +pricing page, and `external_ref` on the subscription is whatever the system that +takes the money calls this. Two systems that both own the money is a +reconciliation problem you cannot test your way out of; one owns it, and this +one owns *entitlement*. +""" + +import uuid +from datetime import datetime, timezone +from typing import Optional + +from sqlalchemy import ( + BigInteger, + Boolean, + DateTime, + ForeignKey, + Index, + Integer, + Numeric, + String, + Text, + UniqueConstraint, + func, +) +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.db.database import Base + + +def _utcnow() -> datetime: + return datetime.now(timezone.utc) + + +class Plan(Base): + """ + Something a customer can be on. + + Not tenant-scoped: a plan is the catalogue, shared by everyone, the same way + `accesses` is. That also keeps it out of `TENANT_TABLES` — it has no + `tenant_id`, so neither the query listener nor RLS applies, which is correct + for a catalogue and would be wrong for anything else here. + """ + + __tablename__ = "plans" + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + code: Mapped[str] = mapped_column(String(50), nullable=False, index=True) + name: Mapped[str] = mapped_column(String(255), nullable=False) + description: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + price_amount: Mapped[Optional[float]] = mapped_column(Numeric(10, 2), nullable=True) + currency: Mapped[Optional[str]] = mapped_column(String(10), nullable=True) + interval: Mapped[Optional[str]] = mapped_column(String(20), nullable=True, default="monthly") + + grace_period_days: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + notify_days_before_expiry: Mapped[int] = mapped_column(Integer, nullable=False, default=7) + + is_public: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + """Whether it appears on the pricing page. A bespoke plan is not public.""" + + is_popular: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + """Whether this is marked as the 'Popular' plan on the pricing page. Only one can be popular.""" + + sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + + saas_plan_id: Mapped[Optional[str]] = mapped_column( + String(255), nullable=True, index=True, unique=True + ) + """The UUID of the corresponding SubscriptionPlan in the SaaS platform. + NULL for plans that were created directly in DocQube. + Used to upsert plan data when a webhook event arrives from SaaS. + """ + + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), onupdate=_utcnow, nullable=False + ) + + limits: Mapped[list["PlanLimit"]] = relationship( + "PlanLimit", back_populates="plan", cascade="all, delete-orphan" + ) + roles: Mapped[list["PlanRole"]] = relationship( + "PlanRole", back_populates="plan", cascade="all, delete-orphan" + ) + + __table_args__ = (UniqueConstraint("code", name="plans_code_key"),) + + def __repr__(self) -> str: # pragma: no cover - debugging aid + return f"" + + +class PlanRole(Base): + """ + Mapping between a Plan and a System Role. + """ + __tablename__ = "plan_roles" + + plan_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("plans.id", ondelete="CASCADE"), primary_key=True + ) + role_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("roles.id", ondelete="CASCADE"), primary_key=True + ) + + plan = relationship("Plan", back_populates="roles") + + +class PlanLimit(Base): + """ + One limit a plan includes. + + A row, not a column. The four that exist today became four columns on + `tenants`, and a fifth would have been a fifth — which is how a schema turns + into a changelog. Adding a limit is now a seed row. + + `value` is a signed 64-bit integer and **-1 means unlimited**, matching the + convention `tenants.envelope_limit` already uses. Null would have been the + tidier spelling and would collide with "not set", which is a different + thing. + """ + + __tablename__ = "plan_limits" + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + plan_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("plans.id", ondelete="CASCADE"), nullable=False, index=True + ) + key: Mapped[str] = mapped_column(String(50), nullable=False) + value: Mapped[int] = mapped_column(BigInteger, nullable=False) + + plan: Mapped["Plan"] = relationship("Plan", back_populates="limits") + + __table_args__ = ( + UniqueConstraint("plan_id", "key", name="uq_plan_limits_plan_key"), + ) + + +class TenantSubscription(Base): + """ + Which plan a tenant is on, and what state that is in. + + **`status` is not `tenant.is_active`.** One is commercial state, set by + billing; the other is suspension, set by an operator. Merging them tells a + customer the wrong thing about why they cannot get in, which is the worst + kind of support call. + """ + + __tablename__ = "tenant_subscriptions" + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + tenant_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("tenants.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + plan_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("plans.id", ondelete="RESTRICT"), nullable=False, index=True + ) + status: Mapped[str] = mapped_column(String(20), nullable=False, default="active") + """active | trialing | past_due | cancelled | expired.""" + + started_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + current_period_end: Mapped[Optional[datetime]] = mapped_column( + DateTime(timezone=True), nullable=True + ) + cancel_at_period_end: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=False + ) + + external_ref: Mapped[Optional[str]] = mapped_column(String(255), nullable=True, index=True) + """ + Whatever the system that bills calls this — a Stripe id, a platform + subscription id. DocQube never interprets it. It exists so that when the two + systems disagree, support has something to compare. + """ + + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), onupdate=_utcnow, nullable=False + ) + + plan: Mapped["Plan"] = relationship("Plan") + + __table_args__ = ( + UniqueConstraint("tenant_id", name="uq_tenant_subscriptions_tenant"), + Index("ix_tenant_subscriptions_status", "status"), + ) + + +class TenantLimitOverride(Base): + """ + The negotiated exception: "Elite, but 500 GB, because the contract says so". + + Deliberately a separate row rather than an edit to the tenant's copy of a + plan limit. Editing loses the fact that it *was* an exception, and that is + exactly what somebody needs to know at renewal. + """ + + __tablename__ = "tenant_limit_overrides" + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + tenant_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("tenants.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + key: Mapped[str] = mapped_column(String(50), nullable=False) + value: Mapped[int] = mapped_column(BigInteger, nullable=False) + note: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + """Why. An exception with no reason recorded becomes folklore.""" + + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + + __table_args__ = ( + UniqueConstraint("tenant_id", "key", name="uq_tenant_limit_overrides"), + ) + + +class PlanPriceHistory(Base): + """ + Tracks pricing changes over time for a plan. + """ + __tablename__ = "plan_price_history" + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + plan_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("plans.id", ondelete="CASCADE"), nullable=False, index=True + ) + old_price: Mapped[Optional[float]] = mapped_column(Numeric(10, 2), nullable=True) + new_price: Mapped[Optional[float]] = mapped_column(Numeric(10, 2), nullable=True) + currency: Mapped[Optional[str]] = mapped_column(String(10), nullable=True) + + changed_by: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + """User ID who made the change.""" + + changed_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + + plan: Mapped["Plan"] = relationship("Plan") + + def __repr__(self) -> str: + return f"" + + +__all__ = ["Plan", "PlanLimit", "TenantSubscription", "TenantLimitOverride", "PlanPriceHistory"] diff --git a/app/modules/billing/routes/__init__.py b/app/modules/billing/routes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/modules/billing/routes/billing_routes.py b/app/modules/billing/routes/billing_routes.py new file mode 100644 index 0000000..d735cf9 --- /dev/null +++ b/app/modules/billing/routes/billing_routes.py @@ -0,0 +1,535 @@ +""" +D4 — the plans, and what a tenant is entitled to. + +Three audiences, three levels of access: + +- **anyone**, including a signed-out visitor on the pricing page — the public + catalogue. This is the endpoint that lets the landing page stop hardcoding + what it sells; +- **any signed-in user** — their own plan and their own limits. Not privileged + information: it is what they bought; +- **a superadmin** — change a tenant's plan, set an override. + +Changing a plan is `superadmin.tenant.update`, not a tenant-level code. A +customer moving themselves onto Elite would be a pricing decision made by the +person who benefits from it. +""" + +import uuid +from typing import List, Optional + +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy.orm import Session + +from app.db.database import get_db +from app.middleware.auth import get_current_user +from app.middleware.tenant import require_superadmin +from app.modules.auth.dependencies.access_dependency import require_access +from app.modules.auth.models.user_model import User +from app.modules.billing.schemas.billing_schema import ( + EntitlementOut, + PlanOut, + CreatePlanIn, + UpdatePlanIn, + PlanPriceHistoryOut, + SetOverrideIn, + SetPlanIn, + SubscriptionOut, +) +from app.modules.billing.services.entitlement_service import EntitlementService + +router = APIRouter(prefix="/billing", tags=["Billing"]) + + +def _plan_out(db: Session, plan) -> dict: + from app.modules.billing.models.plan_model import PlanLimit, PlanRole + from app.modules.auth.models.role_model import Role + from app.modules.auth.models.role_access_model import RoleAccess + from sqlalchemy.orm import selectinload + + limits = { + row.key: row.value + for row in db.query(PlanLimit).filter(PlanLimit.plan_id == plan.id) + } + + roles_out = [] + plan_roles = db.query(PlanRole).filter(PlanRole.plan_id == plan.id).all() + if plan_roles: + role_ids = [pr.role_id for pr in plan_roles] + roles = db.query(Role).options(selectinload(Role.role_accesses).selectinload(RoleAccess.access)).filter(Role.id.in_(role_ids)).all() + for r in roles: + roles_out.append({ + "id": r.id, + "name": r.name, + "description": r.description, + "access_codes": [ra.access.access_code for ra in r.role_accesses if ra.access] + }) + + return { + "id": plan.id, + "code": plan.code, + "name": plan.name, + "description": plan.description, + "price_amount": float(plan.price_amount) if plan.price_amount is not None else None, + "currency": plan.currency, + "interval": plan.interval, + "grace_period_days": plan.grace_period_days, + "notify_days_before_expiry": plan.notify_days_before_expiry, + "is_public": plan.is_public, + "is_popular": plan.is_popular, + "sort_order": plan.sort_order, + "limits": limits, + "roles": roles_out, + } + + +@router.get("/plans", response_model=List[PlanOut]) +def list_plans(db: Session = Depends(get_db)): + """ + The public catalogue. + + **Deliberately unauthenticated.** It is what the pricing page shows to a + visitor who has not signed up yet, and requiring a token would defeat the + purpose. It exposes only what is already printed on that page — names, + prices and limits — and no tenant is named. + + This is the endpoint that closes the original gap: the page advertised four + priced plans and the backend had never heard of a plan. + """ + from app.modules.billing.models.plan_model import Plan + + plans = ( + db.query(Plan) + .filter(Plan.is_public.is_(True)) + .order_by(Plan.sort_order, Plan.name) + .all() + ) + return [_plan_out(db, plan) for plan in plans] + + +@router.get( + "/plans/admin", + response_model=List[PlanOut], + dependencies=[require_access("superadmin.tenant.read"), Depends(require_superadmin)] +) +def list_plans_admin(db: Session = Depends(get_db)): + """ + Returns all plans, including inactive (is_public=False) ones. + Only accessible by superadmins. + """ + from app.modules.billing.models.plan_model import Plan + + plans = ( + db.query(Plan) + .order_by(Plan.sort_order, Plan.name) + .all() + ) + return [_plan_out(db, plan) for plan in plans] + + +@router.get("/me", response_model=EntitlementOut) +def my_entitlement( + db: Session = Depends(get_db), user: User = Depends(get_current_user) +): + """ + What this caller's workspace is entitled to. + + Not gated on an admin code: a user asking what their own workspace includes + is asking what they bought. + """ + service = EntitlementService(db) + tenant = getattr(user, "tenant", None) + subscription = service.subscription(tenant) + plan = subscription.plan if subscription is not None else None + + return { + "plan": _plan_out(db, plan) if plan is not None else None, + "status": subscription.status if subscription is not None else None, + "current_period_end": ( + subscription.current_period_end if subscription is not None else None + ), + "cancel_at_period_end": ( + subscription.cancel_at_period_end if subscription is not None else False + ), + "subscription_active": service.is_active(tenant), + "access_level": service.access_level(tenant), + "limits": service.all_limits(tenant), + } + + +@router.get( + "/tenants/{tenant_id}/subscription", + response_model=SubscriptionOut, + dependencies=[require_access("superadmin.tenant.read"), Depends(require_superadmin)], +) +def get_tenant_subscription( + tenant_id: uuid.UUID, + db: Session = Depends(get_db), + user: User = Depends(get_current_user), +): + from app.modules.billing.models.plan_model import TenantSubscription + + row = ( + db.query(TenantSubscription) + .filter(TenantSubscription.tenant_id == tenant_id) + .first() + ) + if row is None: + raise HTTPException(status_code=404, detail="No subscription for that tenant") + return row + + +@router.put( + "/tenants/{tenant_id}/subscription", + response_model=SubscriptionOut, + dependencies=[require_access("superadmin.tenant.update"), Depends(require_superadmin)], +) +def set_tenant_plan( + tenant_id: uuid.UUID, + payload: SetPlanIn, + db: Session = Depends(get_db), + user: User = Depends(get_current_user), +): + """ + Move a tenant onto a plan. + + A superadmin decision. A tenant admin moving themselves onto Elite would be + a pricing decision made by the person who benefits from it. + """ + from app.modules.billing.models.plan_model import Plan, TenantSubscription + + plan = db.query(Plan).filter(Plan.code == payload.plan_code).first() + if plan is None: + raise HTTPException(status_code=404, detail="No such plan") + + row = ( + db.query(TenantSubscription) + .filter(TenantSubscription.tenant_id == tenant_id) + .first() + ) + if row is None: + row = TenantSubscription(tenant_id=tenant_id, plan_id=plan.id) + db.add(row) + else: + row.plan_id = plan.id + + if payload.status is not None: + row.status = payload.status + if payload.external_ref is not None: + row.external_ref = payload.external_ref + if payload.current_period_end is not None: + row.current_period_end = payload.current_period_end + + # Auto-assign the first plan role to the tenant's first user + from app.modules.billing.models.plan_model import PlanRole + from app.modules.auth.models.user_model import User + + plan_roles = db.query(PlanRole).filter(PlanRole.plan_id == plan.id).all() + if plan_roles: + first_plan_role_id = plan_roles[0].role_id + # Find the first user in this tenant (usually the owner) + tenant_user = db.query(User).filter(User.tenant_id == tenant_id).order_by(User.id.asc()).first() + if tenant_user: + tenant_user.role_id = first_plan_role_id + + db.flush() + db.refresh(row) + return row + + +@router.put( + "/tenants/{tenant_id}/overrides", + response_model=EntitlementOut, + dependencies=[require_access("superadmin.tenant.update"), Depends(require_superadmin)], +) +def set_override( + tenant_id: uuid.UUID, + payload: SetOverrideIn, + db: Session = Depends(get_db), + user: User = Depends(get_current_user), +): + """ + The negotiated exception — "Elite, but 500 GB, because the contract says so". + + Stored as its own row rather than by editing the plan, so the plan stays + honest and the exception stays visible at renewal. `note` is not optional in + spirit: an exception with no reason recorded becomes folklore. + """ + from app.modules.billing.models.plan_model import TenantLimitOverride + from app.modules.billing.services.entitlement_service import DEFAULTS + from app.modules.tenant.models.tenant_model import Tenant + + if payload.key not in DEFAULTS: + raise HTTPException( + status_code=400, + detail=f"Unknown limit. Known limits: {sorted(DEFAULTS)}.", + ) + + tenant = db.query(Tenant).filter(Tenant.id == tenant_id).first() + if tenant is None: + raise HTTPException(status_code=404, detail="Tenant not found") + + row = ( + db.query(TenantLimitOverride) + .filter( + TenantLimitOverride.tenant_id == tenant_id, + TenantLimitOverride.key == payload.key, + ) + .first() + ) + if row is None: + row = TenantLimitOverride( + tenant_id=tenant_id, key=payload.key, value=payload.value, note=payload.note + ) + db.add(row) + else: + row.value = payload.value + row.note = payload.note + db.flush() + + service = EntitlementService(db) + subscription = service.subscription(tenant) + plan = subscription.plan if subscription is not None else None + return { + "plan": _plan_out(db, plan) if plan is not None else None, + "status": subscription.status if subscription is not None else None, + "current_period_end": ( + subscription.current_period_end if subscription is not None else None + ), + "cancel_at_period_end": ( + subscription.cancel_at_period_end if subscription is not None else False + ), + "subscription_active": service.is_active(tenant), + "access_level": service.access_level(tenant), + "limits": service.all_limits(tenant), + } + + +@router.post( + "/plans", + response_model=PlanOut, + dependencies=[require_access("superadmin.tenant.update"), Depends(require_superadmin)], +) +def create_plan( + payload: CreatePlanIn, + db: Session = Depends(get_db), + user: User = Depends(get_current_user), +): + from app.modules.billing.models.plan_model import Plan, PlanLimit, PlanPriceHistory + + plan = db.query(Plan).filter(Plan.code == payload.code).first() + if plan is not None: + raise HTTPException(status_code=400, detail="Plan code already exists") + + plan = Plan( + code=payload.code, + name=payload.name, + description=payload.description, + price_amount=payload.price_amount, + currency=payload.currency, + interval=payload.interval, + grace_period_days=payload.grace_period_days, + notify_days_before_expiry=payload.notify_days_before_expiry, + is_public=payload.is_public, + is_popular=payload.is_popular, + sort_order=payload.sort_order, + ) + db.add(plan) + + if payload.is_popular: + db.query(Plan).filter(Plan.is_popular == True).update({"is_popular": False}) + + db.flush() + + if payload.roles: + from app.modules.auth.models.role_model import Role + from app.modules.auth.models.role_access_model import RoleAccess + from app.modules.auth.models.access_model import Access + from app.modules.billing.models.plan_model import PlanRole + for role_data in payload.roles: + new_role = Role( + name=role_data.name, + description=role_data.description, + is_system=True, + ) + db.add(new_role) + db.flush() + db.add(PlanRole(plan_id=plan.id, role_id=new_role.id)) + for access_code in role_data.access_codes: + acc = db.query(Access).filter(Access.access_code == access_code).first() + if acc: + db.add(RoleAccess(role_id=new_role.id, access_id=acc.id)) + + for key, value in payload.limits.items(): + db.add(PlanLimit(plan_id=plan.id, key=key, value=value)) + + # Log to price history + if payload.price_amount is not None: + history = PlanPriceHistory( + plan_id=plan.id, + old_price=None, + new_price=payload.price_amount, + currency=payload.currency, + changed_by=user.id + ) + db.add(history) + + db.commit() + db.refresh(plan) + return _plan_out(db, plan) + + +@router.put( + "/plans/{plan_id}", + response_model=PlanOut, + dependencies=[require_access("superadmin.tenant.update"), Depends(require_superadmin)], +) +def update_plan( + plan_id: uuid.UUID, + payload: UpdatePlanIn, + db: Session = Depends(get_db), + user: User = Depends(get_current_user), +): + from app.modules.billing.models.plan_model import Plan, PlanLimit, PlanPriceHistory + + plan = db.query(Plan).filter(Plan.id == plan_id).first() + if plan is None: + raise HTTPException(status_code=404, detail="Plan not found") + + old_price = plan.price_amount + old_currency = plan.currency + + update_data = payload.model_dump(exclude_unset=True) + for field, value in update_data.items(): + if field in ["limits", "roles"]: + continue + setattr(plan, field, value) + + if payload.is_popular is True: + db.query(Plan).filter(Plan.id != plan.id, Plan.is_popular == True).update({"is_popular": False}) + + + if payload.limits is not None: + db.query(PlanLimit).filter(PlanLimit.plan_id == plan.id).delete() + for key, value in payload.limits.items(): + db.add(PlanLimit(plan_id=plan.id, key=key, value=value)) + + if payload.roles is not None: + from app.modules.auth.models.role_model import Role + from app.modules.auth.models.role_access_model import RoleAccess + from app.modules.auth.models.access_model import Access + from app.modules.billing.models.plan_model import PlanRole + + existing_plan_roles = db.query(PlanRole).filter(PlanRole.plan_id == plan.id).all() + existing_roles = { + r.name: r + for r in db.query(Role).filter(Role.id.in_([pr.role_id for pr in existing_plan_roles])).all() + } if existing_plan_roles else {} + + processed_role_names = set() + + for role_data in payload.roles: + processed_role_names.add(role_data.name) + if role_data.name in existing_roles: + existing_role = existing_roles[role_data.name] + existing_role.description = role_data.description + + db.query(RoleAccess).filter(RoleAccess.role_id == existing_role.id).delete() + for access_code in role_data.access_codes: + acc = db.query(Access).filter(Access.access_code == access_code).first() + if acc: + db.add(RoleAccess(role_id=existing_role.id, access_id=acc.id)) + else: + new_role = Role( + name=role_data.name, + description=role_data.description, + is_system=True, + ) + db.add(new_role) + db.flush() + db.add(PlanRole(plan_id=plan.id, role_id=new_role.id)) + for access_code in role_data.access_codes: + acc = db.query(Access).filter(Access.access_code == access_code).first() + if acc: + db.add(RoleAccess(role_id=new_role.id, access_id=acc.id)) + + for name, role in existing_roles.items(): + if name not in processed_role_names: + db.query(PlanRole).filter(PlanRole.plan_id == plan.id, PlanRole.role_id == role.id).delete() + db.delete(role) + + # Log to price history if price or currency changed + if plan.price_amount != old_price or plan.currency != old_currency: + history = PlanPriceHistory( + plan_id=plan.id, + old_price=old_price, + new_price=plan.price_amount, + currency=plan.currency, + changed_by=user.id + ) + db.add(history) + + db.commit() + db.refresh(plan) + return _plan_out(db, plan) + + +@router.delete( + "/plans/{plan_id}", + dependencies=[require_access("superadmin.tenant.update"), Depends(require_superadmin)], +) +def delete_plan( + plan_id: uuid.UUID, + db: Session = Depends(get_db), + user: User = Depends(get_current_user), +): + from app.modules.billing.models.plan_model import Plan + + plan = db.query(Plan).filter(Plan.id == plan_id).first() + if plan is None: + raise HTTPException(status_code=404, detail="Plan not found") + + try: + db.delete(plan) + db.commit() + except Exception as e: + db.rollback() + raise HTTPException(status_code=400, detail="Cannot delete plan in use by tenants") + + return {"status": "ok"} + + +@router.get( + "/plans/{plan_id}/price-history", + response_model=List[PlanPriceHistoryOut], + dependencies=[require_access("superadmin.tenant.read"), Depends(require_superadmin)], +) +def get_plan_price_history( + plan_id: uuid.UUID, + db: Session = Depends(get_db), + user: User = Depends(get_current_user), +): + from app.modules.billing.models.plan_model import PlanPriceHistory + + history = ( + db.query(PlanPriceHistory) + .filter(PlanPriceHistory.plan_id == plan_id) + .order_by(PlanPriceHistory.changed_at.desc()) + .all() + ) + return history + + +@router.post( + "/plans/reorder", + dependencies=[require_access("superadmin.tenant.update"), Depends(require_superadmin)], +) +def reorder_plans( + plan_ids: List[uuid.UUID], + db: Session = Depends(get_db), +): + from app.modules.billing.models.plan_model import Plan + for index, plan_id in enumerate(plan_ids): + db.query(Plan).filter(Plan.id == plan_id).update({"sort_order": index + 1}) + db.commit() + return {"message": "Plans reordered successfully"} diff --git a/app/modules/billing/schemas/__init__.py b/app/modules/billing/schemas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/modules/billing/schemas/billing_schema.py b/app/modules/billing/schemas/billing_schema.py new file mode 100644 index 0000000..0a33715 --- /dev/null +++ b/app/modules/billing/schemas/billing_schema.py @@ -0,0 +1,146 @@ +"""D4 — the billing contract.""" + +import uuid +from datetime import datetime +from typing import Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field + + +class PlanRoleIn(BaseModel): + name: str + description: Optional[str] = None + access_codes: List[str] = Field(default_factory=list) + + +class PlanRoleOut(BaseModel): + id: uuid.UUID + name: str + description: Optional[str] = None + access_codes: List[str] + + +class PlanOut(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: uuid.UUID + code: str + name: str + description: Optional[str] = None + currency: Optional[str] = None + is_public: bool + is_popular: bool + sort_order: int + interval: Optional[str] = "monthly" + price_amount: Optional[float] = None + grace_period_days: int = 0 + notify_days_before_expiry: int = 7 + limits: Dict[str, int] + """ + Every limit the plan includes, keyed by name. A dict rather than named + fields, so adding a limit is a seed row rather than a schema change and a + frontend deploy. + """ + roles: List[PlanRoleOut] = Field(default_factory=list) + + +class CreatePlanIn(BaseModel): + code: str + name: str + description: Optional[str] = None + price_amount: Optional[float] = None + currency: Optional[str] = None + interval: Optional[str] = "monthly" + is_public: bool = True + is_popular: bool = False + sort_order: int = 0 + grace_period_days: int = 0 + notify_days_before_expiry: int = 7 + limits: Dict[str, int] = {} + roles: List[PlanRoleIn] = Field(default_factory=list) + + +class UpdatePlanIn(BaseModel): + name: Optional[str] = None + description: Optional[str] = None + price_amount: Optional[float] = None + currency: Optional[str] = None + interval: Optional[str] = None + is_public: Optional[bool] = None + is_popular: Optional[bool] = None + sort_order: Optional[int] = None + grace_period_days: Optional[int] = None + notify_days_before_expiry: Optional[int] = None + limits: Optional[Dict[str, int]] = None + roles: Optional[List[PlanRoleIn]] = None + + +class PlanPriceHistoryOut(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: uuid.UUID + plan_id: uuid.UUID + old_price: Optional[float] = None + new_price: Optional[float] = None + currency: Optional[str] = None + changed_by: Optional[int] = None + changed_at: datetime + + +class SubscriptionOut(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: uuid.UUID + tenant_id: uuid.UUID + plan_id: uuid.UUID + status: str + started_at: datetime + current_period_end: Optional[datetime] = None + cancel_at_period_end: bool + external_ref: Optional[str] = None + + +class EntitlementOut(BaseModel): + """What a workspace is entitled to, resolved.""" + + plan: Optional[PlanOut] = None + status: Optional[str] = None + current_period_end: Optional[datetime] = None + cancel_at_period_end: bool = False + + subscription_active: bool + """ + Commercial state only. Deliberately says nothing about whether the tenant is + suspended — that is an operator's act, and merging the two tells a customer + the wrong thing about why they cannot get in. + """ + + access_level: str + """ + `full` or `read_only` — the same value the middleware enforces on. + + Reported rather than left to be inferred. Without it a client has to + reconstruct the rule from `status` and `subscription_active`, which means + the decision about what a lapsed customer may do exists twice, in two + languages, and drifts the first time the rule gains a case. The middleware + already computes this to decide whether to answer 402; this is that answer. + """ + + limits: Dict[str, int] + """Resolved: override, else plan, else default. `-1` means unlimited.""" + + +class SetPlanIn(BaseModel): + plan_code: str + status: Optional[str] = None + external_ref: Optional[str] = None + current_period_end: Optional[datetime] = None + + +class SetOverrideIn(BaseModel): + key: str + value: int = Field(description="-1 means unlimited") + note: Optional[str] = Field( + default=None, + description="Why. An exception with no reason recorded becomes folklore.", + ) diff --git a/app/modules/billing/services/__init__.py b/app/modules/billing/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/modules/billing/services/billing_tasks.py b/app/modules/billing/services/billing_tasks.py new file mode 100644 index 0000000..c8f03d0 --- /dev/null +++ b/app/modules/billing/services/billing_tasks.py @@ -0,0 +1,62 @@ +import logging +from datetime import datetime, timezone, timedelta +from sqlalchemy.orm import Session +from app.db.database import SessionLocal +from app.modules.billing.models.plan_model import TenantSubscription, Plan +# Assuming you have a notification service, e.g. from app.modules.notifications... + +logger = logging.getLogger(__name__) + +def evaluate_subscriptions(): + """ + Evaluates all active subscriptions and updates their statuses or sends notifications + based on expiry and grace periods. + This should be run daily via a cron job or Celery beat. + """ + db: Session = SessionLocal() + try: + now = datetime.now(timezone.utc) + + subscriptions = db.query(TenantSubscription).filter( + TenantSubscription.status.in_(["active", "trialing", "grace", "past_due"]) + ).all() + + for sub in subscriptions: + plan = sub.plan + if not sub.current_period_end: + continue + + days_until_expiry = (sub.current_period_end - now).days + + if sub.status in ["active", "trialing"]: + if 0 < days_until_expiry <= plan.notify_days_before_expiry: + # TODO: Trigger SUBSCRIPTION_EXPIRING_SOON notification + logger.info(f"Subscription {sub.id} expiring in {days_until_expiry} days. Sending notification.") + + elif days_until_expiry <= 0: + # Expired. Move to grace period or past_due + if plan.grace_period_days > 0: + sub.status = "grace" + logger.info(f"Subscription {sub.id} expired. Moving to grace period.") + # TODO: Trigger SUBSCRIPTION_GRACE_PERIOD notification + else: + sub.status = "past_due" + logger.info(f"Subscription {sub.id} expired. Moving to past_due.") + # TODO: Trigger SUBSCRIPTION_EXPIRED notification + + elif sub.status == "grace": + days_since_expiry = -days_until_expiry + if days_since_expiry > plan.grace_period_days: + sub.status = "past_due" + logger.info(f"Subscription {sub.id} grace period ended. Moving to past_due.") + # TODO: Trigger SUBSCRIPTION_EXPIRED notification + else: + # Still in grace period, maybe send another reminder? + pass + + db.commit() + except Exception as e: + logger.error(f"Error evaluating subscriptions: {e}") + db.rollback() + finally: + db.close() diff --git a/app/modules/billing/services/entitlement_service.py b/app/modules/billing/services/entitlement_service.py new file mode 100644 index 0000000..152e8b5 --- /dev/null +++ b/app/modules/billing/services/entitlement_service.py @@ -0,0 +1,235 @@ +""" +D2 — what is this tenant entitled to? + +One function that every enforcement point asks, instead of four columns that +four places read. + + limit(tenant, key) = + TenantLimitOverride the negotiated exception + else PlanLimit what was actually sold + else the tenant column the transition fallback, one release only + else DEFAULTS the built-in floor + +**It fails open, and that is deliberate.** Everywhere else in this codebase the +answer is fail closed — the tenant filter denies without context, the scope +resolver returns an empty set, `require_access` refuses. Those are about +*authority*, where the risk is showing somebody another tenant's data. + +This is about *capacity*, where the risk is refusing a customer their own. A +tenant with no subscription row is an operational mistake — a failed backfill, a +half-finished signup — and locking a paying customer out of their own documents +is a worse answer to that mistake than serving them. Different question, +different default. + +The one thing it will not do is guess at a key it does not know. An unknown key +raises rather than returning zero, because "limit of zero" and "I have never +heard of that limit" are different, and only one of them should silently switch a +feature off. +""" + +from __future__ import annotations + +from contextvars import ContextVar +from typing import Optional + +from sqlalchemy.orm import Session + +CAPACITY_DEFAULTS: dict[str, int] = { + "storage_bytes": 1024 ** 3, + "seats": -1, + "chat_tokens_daily": 1_000_000, + "envelopes": -1, +} +""" +How much of something. **Fails open**: a resolution failure serves the default +rather than zero, because the risk here is refusing a customer their own data. +""" + +FEATURE_DEFAULTS: dict[str, int] = { +} +""" +Whether something is switched on at all. **Fails closed**: 0 unless a plan says +otherwise. + +Separated from capacity because the two want opposite defaults and sharing a +dict would silently give features the capacity behaviour. On a resolution +failure, "serve them a gigabyte" is generous; "grant them the Elite-only feature" +is giving the product away. Splitting now costs nothing; splitting after the +first feature flag ships means auditing every call site. +""" + +DEFAULTS: dict[str, int] = {**CAPACITY_DEFAULTS, **FEATURE_DEFAULTS} + +LEGACY_COLUMNS: dict[str, str] = { + "storage_bytes": "storage_quota_bytes", + "seats": "max_users_allowed", + "chat_tokens_daily": "chat_token_daily_limit", + "envelopes": "envelope_limit", +} + +UNLIMITED = -1 + +_cache: ContextVar[Optional[dict]] = ContextVar("entitlement_cache", default=None) + + +def reset_entitlement_cache() -> None: + _cache.set({}) + + +def _cached(key: str, compute): + store = _cache.get() + if store is None: + return compute() + if key not in store: + store[key] = compute() + return store[key] + + +class EntitlementService: + def __init__(self, db: Session): + self.db = db + + + def limit(self, tenant, key: str) -> int: + """The effective limit for *key*. `-1` means unlimited.""" + if key not in DEFAULTS: + raise KeyError( + f"Unknown entitlement key {key!r}. Known keys: " + f"{sorted(DEFAULTS)}. A typo must not read as a limit of zero." + ) + if tenant is None: + return DEFAULTS[key] + + return _cached(f"limit:{tenant.id}:{key}", lambda: self._resolve(tenant, key)) + + def is_unlimited(self, tenant, key: str) -> bool: + return self.limit(tenant, key) == UNLIMITED + + def within(self, tenant, key: str, used: int) -> bool: + """Whether *used* is inside the limit. Unlimited is always inside.""" + allowed = self.limit(tenant, key) + return allowed == UNLIMITED or used <= allowed + + def all_limits(self, tenant) -> dict[str, int]: + """Every known limit, for a settings screen or an API response.""" + return {key: self.limit(tenant, key) for key in DEFAULTS} + + + def subscription(self, tenant): + from app.modules.billing.models.plan_model import TenantSubscription + + if tenant is None: + return None + return ( + self.db.query(TenantSubscription) + .filter(TenantSubscription.tenant_id == tenant.id) + .first() + ) + + def is_active(self, tenant) -> bool: + """ + Whether the *subscription* is in good standing. + + Deliberately says nothing about `tenant.is_active`, which is suspension — + an operator's act, not a commercial state. Merging them tells a customer + the wrong thing about why they cannot get in, and "we paid, why are we + locked out?" is the support call that follows. + + A tenant with no subscription row is treated as active, for the same + reason the limits fall back rather than failing closed. + """ + row = self.subscription(tenant) + if row is None: + return True + return row.status in {"active", "trialing"} + + def plan(self, tenant): + row = self.subscription(tenant) + return row.plan if row is not None else None + + def access_level(self, tenant) -> str: + """ + What a lapsed subscription still permits: `full`, `read_only` or `none`. + + Ported from the base, which is also how mature systems behave: an + expired customer is not locked out of their own documents, they are + stopped from creating more. Locking them out is hostile, makes export + impossible, and is bad for renewals — the customer most likely to come + back is the one who can still see what they would be coming back to. + + A tenant with no subscription row gets `full`, for the same reason the + limits fall back rather than failing closed. + """ + row = self.subscription(tenant) + if row is None: + return "full" + if row.status in {"active", "trialing", "grace"}: + return "full" + if row.status in {"past_due"}: + return "read_only" + return "read_only" + + + def _resolve(self, tenant, key: str) -> int: + from app.modules.billing.models.plan_model import ( + PlanLimit, + TenantLimitOverride, + TenantSubscription, + ) + + override = ( + self.db.query(TenantLimitOverride.value) + .filter( + TenantLimitOverride.tenant_id == tenant.id, + TenantLimitOverride.key == key, + ) + .first() + ) + if override is not None: + return int(override[0]) + + row = ( + self.db.query(PlanLimit.value) + .join(TenantSubscription, TenantSubscription.plan_id == PlanLimit.plan_id) + .filter( + TenantSubscription.tenant_id == tenant.id, + PlanLimit.key == key, + ) + .first() + ) + if row is not None: + return int(row[0]) + + column = LEGACY_COLUMNS.get(key) + if column is not None: + value = getattr(tenant, column, None) + if value is not None: + return int(value) + + return DEFAULTS[key] + + def has_feature(self, tenant, key: str) -> bool: + """ + Whether a feature is switched on for this tenant. + + Separate from `limit()` so the caller cannot accidentally read a + feature with capacity semantics — `limit(tenant, 'feature_sso') > 0` + would work today and would quietly grant the feature the moment + resolution failed. + """ + if key not in FEATURE_DEFAULTS: + raise KeyError( + f"Unknown feature {key!r}. Known features: {sorted(FEATURE_DEFAULTS)}." + ) + return self.limit(tenant, key) != 0 + + +__all__ = [ + "EntitlementService", + "DEFAULTS", + "CAPACITY_DEFAULTS", + "FEATURE_DEFAULTS", + "LEGACY_COLUMNS", + "UNLIMITED", + "reset_entitlement_cache", +] diff --git a/app/modules/chat/README.md b/app/modules/chat/README.md new file mode 100644 index 0000000..9d16a83 --- /dev/null +++ b/app/modules/chat/README.md @@ -0,0 +1,41 @@ +# Chatbot Module + +This module implements a robust, conversational RAG (Retrieval-Augmented Generation) chatbot for documents, integrated into the DocQube application. + +## 📂 Structure + +- **`engine/`**: The core RAG orchestration logic (using LlamaIndex and custom adapters). + - **`chatbot_engine.py`**: Main orchestrator handling intent, memory, and path selection. + - **`intent.py`**: Heuristic and LLM-based intent classification. + - **`retrieval.py`**: Qdrant-based retrieval pipeline. + - **`rewrite.py`**: Context-aware query rewriting. + - **`prompts.py`**: Expertly engineered RAG and intent prompts. + - **`formatter.py`**: Dynamic response formatting logic. +- **`services/`**: Integration services. + - **`chat_service.py`**: FastAPI service layer managing session engines and streaming. +- **`models/`**: SQLAlchemy and Pydantic models for chat data. +- **`routes/`**: FastAPI endpoint definitions. + +## 🏗️ Architecture + +``` +User Query → Intent Detection → Query Rewrite → Retrieval → LLM (DeepSeek) → Streaming Response +``` + +- **LLM**: DeepSeek Chat (via OpenAI-compatible adapter). +- **Vector DB**: Qdrant (integrated via LlamaIndex/Custom pipeline). +- **Embeddings**: `BAAI/bge-base-en-v1.5` for semantic search. + +## ⚙️ Configuration + +- **DeepSeek API**: Requires `DEEPSEEK_API_KEY` in `.env`. +- **Vector Store**: Qdrant must be running (configured via `docker-compose`). +- **Persistence**: Engine indices are cached per-document for performance. + +## 🚀 Endpoints + +- `POST /api/chat/upload`: Upload a document for analysis. +- `POST /api/chat/chat`: Synchronous chat response. +- `GET /api/chat/stream`: Streaming chat response (Server-Sent Events). +- `GET /api/chat/documents`: List uploaded documents. +- `DELETE /api/chat/documents/{id}`: Delete a document and its indices. diff --git a/app/modules/chat/controllers/chat_controller.py b/app/modules/chat/controllers/chat_controller.py new file mode 100644 index 0000000..c1bef4a --- /dev/null +++ b/app/modules/chat/controllers/chat_controller.py @@ -0,0 +1,173 @@ +from fastapi import UploadFile, HTTPException +from fastapi.responses import StreamingResponse +from sqlalchemy.orm import Session +from typing import Any + +from app.modules.chat.services.chat_service import ChatbotService +from app.modules.chat.schemas.chat_schema import ChatRequest +from app.modules.auth.models.user_model import User +from app.modules.auth.services import privilege + +import logging + +logger = logging.getLogger(__name__) + + +class ChatbotController: + + @staticmethod + def _get_user(current_user: Any, db: Session) -> User: + user_id = getattr(current_user, "id", None) or current_user.get("id") + + if not user_id: + raise HTTPException(400, "User ID not found") + + if isinstance(current_user, dict): + user = db.query(User).filter(User.id == user_id).first() + else: + user = current_user + + if not user: + raise HTTPException(404, "User not found") + + return user + + @staticmethod + async def upload_document(file: UploadFile, current_user: Any, db: Session): + try: + user = ChatbotController._get_user(current_user, db) + service = ChatbotService(db) + + return await service.upload_document(file, user) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Upload error: {e}", exc_info=True) + raise HTTPException(500, "Upload failed") + + @staticmethod + async def list_documents(current_user: Any, db: Session): + try: + user = ChatbotController._get_user(current_user, db) + service = ChatbotService(db) + + return service.list_documents(user) + + except HTTPException: + raise + except Exception as e: + logger.error(f"List documents error: {e}") + raise HTTPException(500, "Failed to fetch documents") + + @staticmethod + async def ingest_drive_file(file_id: int, current_user: Any, db: Session): + try: + service = ChatbotService(db) + return await service.ingest_drive_file(file_id, current_user) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Drive ingest error: {e}", exc_info=True) + raise HTTPException(500, "Failed to ingest drive file") + + @staticmethod + async def chat(request: ChatRequest, current_user: Any, db: Session): + try: + service = ChatbotService(db) + return await service.chat(current_user, request) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Chat error: {e}", exc_info=True) + raise HTTPException(500, "Chat failed") + + @staticmethod + async def chat_stream(request: ChatRequest, current_user: Any, db: Session): + try: + service = ChatbotService(db) + user = ChatbotController._get_user(current_user, db) + user_id = user.id + tenant_id = user.tenant_id + + return StreamingResponse( + service.chat_stream(user_id, tenant_id, request), + media_type="text/plain", + headers={ + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", + }, + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Streaming error: {e}", exc_info=True) + raise HTTPException(500, "Streaming failed") + + @staticmethod + async def delete_document(doc_id: str, current_user: Any, db: Session): + try: + user = ChatbotController._get_user(current_user, db) + service = ChatbotService(db) + + return service.delete_document(user, doc_id) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Delete error: {e}", exc_info=True) + raise HTTPException(500, "Delete failed") + + @staticmethod + async def get_history(document_id: str, current_user: Any, db: Session): + try: + user = ChatbotController._get_user(current_user, db) + service = ChatbotService(db) + + return service.get_session_history(user.id, user.tenant_id, document_id) + + except HTTPException: + raise + except Exception as e: + logger.error(f"History fetch error: {e}") + raise HTTPException(500, "Failed to fetch history") + + @staticmethod + async def get_usage(current_user: Any, db: Session): + try: + user = ChatbotController._get_user(current_user, db) + service = ChatbotService(db) + + return service.get_user_usage(user.id) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Usage fetch error: {e}") + raise HTTPException(500, "Failed to fetch token usage") + + @staticmethod + async def update_user_quota(user_id: int, daily_limit: int, current_user: Any, db: Session): + try: + admin = ChatbotController._get_user(current_user, db) + target_user = db.query(User).filter(User.id == user_id).first() + is_superadmin = privilege.holds_superadmin_access(admin) + + if not target_user: + raise HTTPException(404, "User not found") + + if not is_superadmin and target_user.tenant_id != admin.tenant_id: + raise HTTPException(403, "Not authorized to update this user's quota") + + service = ChatbotService(db) + return service.update_user_quota(user_id, daily_limit) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Quota update error: {e}") + raise HTTPException(500, "Failed to update user quota") \ No newline at end of file diff --git a/app/modules/chat/engine/__init__.py b/app/modules/chat/engine/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/modules/chat/engine/chatbot_engine.py b/app/modules/chat/engine/chatbot_engine.py new file mode 100644 index 0000000..fba13f3 --- /dev/null +++ b/app/modules/chat/engine/chatbot_engine.py @@ -0,0 +1,692 @@ +"""Conversation orchestration engine — integrated into docQube chat module.""" + +from __future__ import annotations + +import os +import re +from collections import defaultdict, deque +from dataclasses import dataclass +from datetime import datetime, timezone +from threading import Lock +from typing import Any, Protocol +import uuid +import logging + +from app.modules.chat.engine.formatter import FormatDecision, FormatStyle, FormattingEngine +from app.modules.chat.engine.intent import Intent, IntentClassifier +from app.modules.chat.engine.prompts import ANSWER_PROMPT, GROUNDED_SYSTEM_PROMPT, SUMMARY_PROMPT +from app.modules.chat.engine.retrieval import RetrievalConfig, RetrievalPipeline +from app.modules.chat.engine.rewrite import QueryRewriter +from app.modules.chat.services.redis_memory_service import RedisMemoryService + + +logger = logging.getLogger(__name__) + + +class SupportsComplete(Protocol): + def complete(self, prompt: str, system_prompt: str | None = None) -> tuple[str, dict[str, int]]: + ... + + def raw_llm(self) -> Any | None: + ... + + +class OpenRouterAdapter: + """Adapter for OpenRouter API (supports many models including DeepSeek, Mistral, etc.).""" + + def __init__( + self, + model: str, + api_key: str, + api_base: str = "https://openrouter.ai/api/v1", + temperature: float = 0.0, + ): + try: + from openai import OpenAI + except Exception as exc: + raise RuntimeError("Install openai package to use OpenRouter.") from exc + + self._client = OpenAI(api_key=api_key, base_url=api_base) + self._model = model + self._temperature = temperature + + def complete(self, prompt: str, system_prompt: str | None = None) -> tuple[str, dict[str, int]]: + messages: list[dict[str, str]] = [] + if system_prompt: + messages.append({"role": "system", "content": system_prompt}) + messages.append({"role": "user", "content": prompt}) + + response = self._client.chat.completions.create( + model=self._model, + messages=messages, + temperature=self._temperature, + ) + content = response.choices[0].message.content + usage = { + "prompt_tokens": response.usage.prompt_tokens, + "completion_tokens": response.usage.completion_tokens, + "total_tokens": response.usage.total_tokens, + } + return str(content or ""), usage + + def raw_llm(self) -> None: + return None + + +class OpenAILlamaIndexAdapter: + """Adapter around LlamaIndex OpenAI LLM for prompt completion.""" + + def __init__( + self, + model: str = "gpt-4o-mini", + temperature: float = 0.0, + api_base: str | None = None, + api_key: str | None = None, + ): + try: + from llama_index.llms.openai import OpenAI + except Exception as exc: + raise RuntimeError( + "Install llama-index-llms-openai and set OPENAI_API_KEY for production mode." + ) from exc + + kwargs: dict[str, Any] = { + "model": model, + "temperature": temperature, + } + if api_base: + kwargs["api_base"] = api_base + if api_key: + kwargs["api_key"] = api_key + + self._llm = OpenAI(**kwargs) + + def complete(self, prompt: str, system_prompt: str | None = None) -> tuple[str, dict[str, int]]: + usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + + if system_prompt: + try: + from llama_index.core.llms import ChatMessage + + response = self._llm.chat( + [ + ChatMessage(role="system", content=system_prompt), + ChatMessage(role="user", content=prompt), + ] + ) + content = getattr(getattr(response, "message", None), "content", "") + + raw = getattr(response, "raw", {}) + if hasattr(raw, "usage"): + usage = { + "prompt_tokens": raw.usage.prompt_tokens, + "completion_tokens": raw.usage.completion_tokens, + "total_tokens": raw.usage.total_tokens, + } + + if content: + return str(content), usage + except Exception: + pass + + response = self._llm.complete(prompt) + text = getattr(response, "text", "") + + raw = getattr(response, "raw", {}) + if hasattr(raw, "usage"): + usage = { + "prompt_tokens": raw.usage.prompt_tokens, + "completion_tokens": raw.usage.completion_tokens, + "total_tokens": raw.usage.total_tokens, + } + + return str(text), usage + + def raw_llm(self) -> Any: + return self._llm + + +class DeepSeekAdapter: + """Adapter for DeepSeek API (OpenAI-compatible).""" + + def __init__( + self, + model: str = "deepseek-chat", + api_key: str | None = None, + api_base: str = "https://api.deepseek.com/v1", + temperature: float = 0.3, + ): + try: + from openai import OpenAI + except Exception as exc: + raise RuntimeError("Install openai package to use DeepSeek API.") from exc + + self._client = OpenAI(api_key=api_key, base_url=api_base) + self._model = model + self._temperature = temperature + + def complete(self, prompt: str, system_prompt: str | None = None) -> tuple[str, dict[str, int]]: + messages: list[dict[str, str]] = [] + if system_prompt: + messages.append({"role": "system", "content": system_prompt}) + messages.append({"role": "user", "content": prompt}) + + logger.info(f"[DeepSeek] Calling {self._model} (temp={self._temperature})") + logger.debug(f"[DeepSeek] Prompt: {prompt[:100]}...") + + response = self._client.chat.completions.create( + model=self._model, + messages=messages, + temperature=self._temperature, + ) + content = response.choices[0].message.content + usage = { + "prompt_tokens": response.usage.prompt_tokens, + "completion_tokens": response.usage.completion_tokens, + "total_tokens": response.usage.total_tokens, + } + return str(content or ""), usage + + def raw_llm(self) -> None: + return None + + +@dataclass +class ConversationTurn: + user_message: str + assistant_message: str + intent: str + rewritten_query: str + timestamp_utc: str + + +class ConversationMemoryStore: + """Stores recent N interactions per session (conversation buffer).""" + + def __init__(self, max_turns: int = 12): + self.max_turns = max_turns + self._store: dict[str, deque[ConversationTurn]] = defaultdict( + lambda: deque(maxlen=self.max_turns) + ) + self._lock = Lock() + + def add_turn( + self, + session_id: str, + user_message: str, + assistant_message: str, + intent: str, + rewritten_query: str, + ) -> None: + turn = ConversationTurn( + user_message=user_message, + assistant_message=assistant_message, + intent=intent, + rewritten_query=rewritten_query, + timestamp_utc=datetime.now(timezone.utc).isoformat(), + ) + with self._lock: + self._store[session_id].append(turn) + + def get_turns(self, session_id: str) -> list[ConversationTurn]: + with self._lock: + return list(self._store.get(session_id, [])) + + def get_chat_history_messages(self, session_id: str) -> list[dict[str, str]]: + turns = self.get_turns(session_id) + messages: list[dict[str, str]] = [] + for turn in turns: + messages.append({"role": "user", "content": turn.user_message}) + messages.append({"role": "assistant", "content": turn.assistant_message}) + return messages + + def get_last_question(self, session_id: str) -> str | None: + turns = self.get_turns(session_id) + if not turns: + return None + return turns[-1].user_message + + def get_last_answer(self, session_id: str) -> str | None: + turns = self.get_turns(session_id) + if not turns: + return None + return turns[-1].assistant_message + + def resolve_memory_query(self, session_id: str, message: str) -> str: + msg = message.lower() + turns = self.get_turns(session_id) + + if not turns: + return "No previous conversation found for this session." + + if "previous questions" in msg or "all questions" in msg or "list questions" in msg: + qs = [f"{i+1}. **{t.user_message}**" for i, t in enumerate(turns[-5:])] + return "### Previous Questions\n" + "\n".join(qs) + + if "previous question" in msg or "what did i ask" in msg or "last question" in msg: + return f"### Last Question\nYou recently asked: **{turns[-1].user_message}**" + + if "previous answers" in msg or "all answers" in msg or "list answers" in msg: + as_list = [f"{i+1}. {t.assistant_message}" for i, t in enumerate(turns[-5:])] + return "### Previous Answers\n" + "\n".join(as_list) + + if "previous answer" in msg or "last answer" in msg or "what did you answer" in msg: + return f"### Last Answer\nMy previous response was: {turns[-1].assistant_message}" + + if "history" in msg or "all" in msg or "everything" in msg: + formatted = ["### Chat History"] + for idx, turn in enumerate(turns[-5:], start=1): + formatted.append( + f"{idx}. Q: **{turn.user_message}**\nA: {turn.assistant_message}" + ) + return "\n".join(formatted) + + return "Memory query received, but I could not map it to a previous question or answer." + + def clear_session(self, session_id: str) -> None: + with self._lock: + self._store.pop(session_id, None) + + def clear_all(self) -> None: + with self._lock: + self._store.clear() + + +@dataclass +class ChatbotConfig: + memory_turns: int = 12 + not_found_response: str = "Not found in document." + + +@dataclass +class ChatResult: + session_id: str + response: str + intent: str + rewritten_query: str + format_style: str + sources: list[dict[str, Any]] + usage: dict[str, int] = None + + +class ChatbotEngine: + """End-to-end conversational RAG pipeline with intent, rewrite, memory and grounding.""" + + def __init__( + self, + retrieval: RetrievalPipeline, + llm: SupportsComplete, + config: ChatbotConfig | None = None, + memory: ConversationMemoryStore | None = None, + intent_classifier: IntentClassifier | None = None, + rewriter: QueryRewriter | None = None, + formatter: FormattingEngine | None = None, + ): + self.retrieval = retrieval + self.llm = llm + self.config = config or ChatbotConfig() + self.memory = memory or ConversationMemoryStore(max_turns=self.config.memory_turns) + self.intent_classifier = intent_classifier or IntentClassifier(llm=self.llm) + self.rewriter = rewriter or QueryRewriter(llm=self.llm) + self.formatter = formatter or FormattingEngine() + self._session_formats: dict[str, FormatStyle] = {} + + def chat(self, session_id: str, message: str, repo: Any = None, db_session_id: int | None = None, user_id: int | None = None) -> ChatResult: + if repo and db_session_id: + try: + repo.save_message(session_id=db_session_id, role="user", content=message) + + RedisMemoryService.add_message(db_session_id, { + "id": str(uuid.uuid4()), + "role": "user", + "content": message, + "timestamp": datetime.now(timezone.utc).isoformat() + }) + except Exception as e: + logger.warning(f"Failed to persist user message: {e}") + + history = self.memory.get_chat_history_messages(session_id) + + total_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + + intent_decision, intent_usage = self.intent_classifier.classify(message=message, chat_history=history) + self._accumulate_usage(total_usage, intent_usage) + + if repo and user_id and intent_usage: + repo.update_quota_usage(user_id, intent_usage.get("total_tokens", 0)) + + + logger.info(f"[ChatbotEngine] Intent: {intent_decision.intent} (conf: {intent_decision.confidence})") + + if intent_decision.intent == Intent.MEMORY_QUERY: + memory_answer = self.memory.resolve_memory_query(session_id=session_id, message=message) + self.memory.add_turn( + session_id=session_id, + user_message=message, + assistant_message=memory_answer, + intent=intent_decision.intent.value, + rewritten_query=message, + ) + return ChatResult( + session_id=session_id, + response=memory_answer, + intent=intent_decision.intent.value, + rewritten_query=message, + format_style=FormatStyle.PARAGRAPH.value, + sources=[], + usage=total_usage + ) + + if intent_decision.intent == Intent.GREETING: + greeting_resp = "### Welcome to docQube\nHello! I am your **DocQube assistant**. How can I help you with your **documents** today?" + self.memory.add_turn( + session_id=session_id, + user_message=message, + assistant_message=greeting_resp, + intent=intent_decision.intent.value, + rewritten_query=message, + ) + return ChatResult( + session_id=session_id, + response=greeting_resp, + intent=intent_decision.intent.value, + rewritten_query=message, + format_style=FormatStyle.PARAGRAPH.value, + sources=[], + usage=total_usage + ) + + if intent_decision.intent == Intent.CHITCHAT: + chitchat_resp = "### Assistant Information\nI'm here to help you **analyze your documents**. Please ask a question related to the **uploaded content**!" + self.memory.add_turn( + session_id=session_id, + user_message=message, + assistant_message=chitchat_resp, + intent=intent_decision.intent.value, + rewritten_query=message, + ) + return ChatResult( + session_id=session_id, + response=chitchat_resp, + intent=intent_decision.intent.value, + rewritten_query=message, + format_style=FormatStyle.PARAGRAPH.value, + sources=[], + usage=total_usage + ) + + last_q = self.memory.get_last_question(session_id) + last_a = self.memory.get_last_answer(session_id) + + rewrite_result, rewrite_usage = self.rewriter.rewrite( + message=message, + intent=intent_decision.intent, + chat_history=history, + last_question=last_q, + last_answer=last_a, + ) + self._accumulate_usage(total_usage, rewrite_usage) + + if repo and user_id and rewrite_usage: + repo.update_quota_usage(user_id, rewrite_usage.get("total_tokens", 0)) + + previous_style = self._session_formats.get(session_id) + format_decision = self.formatter.detect( + message=message, + intent=intent_decision.intent, + previous_style=previous_style, + ) + self._session_formats[session_id] = format_decision.style + + logger.info(f"[ChatbotEngine] Rewritten Query: {rewrite_result.rewritten_query}") + + if intent_decision.intent == Intent.SUMMARY: + response_text, sources, final_usage = self._run_summary_path( + rewritten_query=rewrite_result.rewritten_query, + history=history, + format_decision=format_decision, + ) + else: + response_text, sources, final_usage = self._run_qa_path( + rewritten_query=rewrite_result.rewritten_query, + history=history, + intent=intent_decision.intent, + format_decision=format_decision, + ) + + self._accumulate_usage(total_usage, final_usage) + + if repo and user_id and final_usage: + repo.update_quota_usage(user_id, final_usage.get("total_tokens", 0)) + + response_text = self._normalize_grounded_response(response_text) + + self.memory.add_turn( + session_id=session_id, + user_message=message, + assistant_message=response_text, + intent=intent_decision.intent.value, + rewritten_query=rewrite_result.rewritten_query, + ) + + if repo and db_session_id: + try: + repo.save_message(session_id=db_session_id, role="assistant", content=response_text) + + RedisMemoryService.add_message(db_session_id, { + "id": str(uuid.uuid4()), + "role": "assistant", + "content": response_text, + "timestamp": datetime.now(timezone.utc).isoformat() + }) + except Exception as e: + logger.warning(f"Failed to persist assistant response: {e}") + + return ChatResult( + session_id=session_id, + response=response_text, + intent=intent_decision.intent.value, + rewritten_query=rewrite_result.rewritten_query, + format_style=format_decision.style.value, + sources=sources, + usage=total_usage + ) + + def _accumulate_usage(self, target: dict[str, int], source: dict[str, int | None] | None) -> None: + if not source: + return + for key in ["prompt_tokens", "completion_tokens", "total_tokens"]: + target[key] += source.get(key, 0) or 0 + + def rebuild_indices(self) -> None: + """Rebuild vector and summary indexes after document updates.""" + self.retrieval.build() + + def clear_session(self, session_id: str) -> None: + """Clear memory and session formatting state for a single session.""" + self.memory.clear_session(session_id) + self._session_formats.pop(session_id, None) + + def clear_all_sessions(self) -> None: + """Clear memory and formatting state for all sessions.""" + self.memory.clear_all() + self._session_formats.clear() + + def _run_qa_path( + self, + rewritten_query: str, + history: list[dict[str, str]], + intent: Intent, + format_decision: FormatDecision, + ) -> tuple[str, list[dict[str, Any]], dict[str, int]]: + logger.info(f"[ChatbotEngine] Running QA path for query: {rewritten_query}") + + nodes = self.retrieval.retrieve(query=rewritten_query, top_k=self.retrieval.config.top_k) + context, sources = self.retrieval.nodes_to_context(nodes) + + if not context.strip(): + return self.config.not_found_response, [], {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + + prompt = ANSWER_PROMPT.format( + chat_history=self._compact_history(history), + intent=intent.value, + format_instruction=format_decision.instruction, + rewritten_query=rewritten_query, + context=context, + ) + response, usage = self.llm.complete(prompt=prompt, system_prompt=GROUNDED_SYSTEM_PROMPT) + if self._looks_like_not_found(response): + fallback = self._term_mention_fallback(rewritten_query, context) + if fallback: + response = fallback + return response, sources, usage + + def _run_summary_path( + self, + rewritten_query: str, + history: list[dict[str, str]], + format_decision: FormatDecision, + ) -> tuple[str, list[dict[str, Any]], dict[str, int]]: + logger.info(f"[ChatbotEngine] Running Summary path for query: {rewritten_query}") + + raw_llm = self.llm.raw_llm() + summary_query = SUMMARY_PROMPT.format( + rewritten_query=rewritten_query, + format_instruction=format_decision.instruction, + ) + + if raw_llm is not None: + try: + response = self.retrieval.summarize(rewritten_query=summary_query, llm=raw_llm) + normalized = response.strip() + if normalized: + return normalized, [], {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + except Exception: + pass + + nodes = self.retrieval.retrieve(query=rewritten_query, top_k=self.retrieval.config.top_k) + context, sources = self.retrieval.nodes_to_context(nodes) + if not context.strip(): + return self.config.not_found_response, [], {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + + prompt = ANSWER_PROMPT.format( + chat_history=self._compact_history(history), + intent=Intent.SUMMARY.value, + format_instruction=format_decision.instruction, + rewritten_query=summary_query, + context=context, + ) + response, usage = self.llm.complete(prompt=prompt, system_prompt=GROUNDED_SYSTEM_PROMPT) + return response, sources, usage + + def _normalize_grounded_response(self, response: str) -> str: + text = (response or "").strip() + if not text: + return self.config.not_found_response + + lowered = text.lower() + if "not found in document" in lowered: + return self.config.not_found_response + + uncertainty_markers = ["i am not sure", "i don't know", "cannot determine"] + if any(marker in lowered for marker in uncertainty_markers): + return self.config.not_found_response + + return text + + @staticmethod + def _looks_like_not_found(response: str) -> bool: + lowered = (response or "").strip().lower() + return not lowered or "not found in document" in lowered + + @staticmethod + def _term_mention_fallback(query: str, context: str) -> str | None: + lowered_query = query.strip().lower() + patterns = [ + r"^\s*what\s+is\s+['\"']?([a-z0-9][a-z0-9\-/\.]{1,30})['\"']?\s*\??\s*$", + r"^\s*define\s+['\"']?([a-z0-9][a-z0-9\-/\.]{1,30})['\"']?\s*\??\s*$", + r"^\s*what\s+does\s+['\"']?([a-z0-9][a-z0-9\-/\.]{1,30})['\"']?\s+stand\s+for\s*\??\s*$", + ] + term = None + for pattern in patterns: + match = re.match(pattern, lowered_query) + if match: + term = match.group(1) + break + + if not term: + return None + + term_regex = re.compile(rf"\b{re.escape(term)}\b", flags=re.IGNORECASE) + if not term_regex.search(context): + return None + + clean_context = re.sub(r"\[Source:\s+.*?\]\n?", "", context) + snippet = "" + first = term_regex.search(clean_context) + if first: + start = max(0, first.start() - 80) + end = min(len(clean_context), first.end() + 80) + snippet = clean_context[start:end].strip().replace("\n", " ") + + if snippet: + return ( + f'### Term Mention Found\n' + f'The document mentions **"{term.upper()}"** (for example: *"{snippet}"*), ' + f"but it does not explicitly define what **{term.upper()}** stands for." + ) + return ( + f'### Term Mention Found\n' + f'The document mentions **"{term.upper()}"**, but it does not explicitly define ' + f"what **{term.upper()}** stands for." + ) + + @staticmethod + def _compact_history(history: list[dict[str, str]]) -> str: + if not history: + return "(empty)" + return "\n".join(f"{m.get('role', 'unknown')}: {m.get('content', '')}" for m in history[-8:]) + + +def build_engine_from_settings( + docs_path: str, + persist_dir: str, + collection_name: str = "chatbot_docs", +) -> ChatbotEngine: + """ + Build a ChatbotEngine using the NVIDIA/DeepSeek credentials already present + in docQube's central settings. Falls back to DeepSeek if NVIDIA key is absent. + """ + from app.core.settings import settings + + embedding_provider = os.getenv("CHATBOT_EMBEDDING_PROVIDER", "bge") + vector_db = os.getenv("CHATBOT_VECTOR_DB", "qdrant") + + retrieval_cfg = RetrievalConfig( + docs_path=docs_path, + vector_db=vector_db, + persist_dir=persist_dir, + collection_name=collection_name, + embedding_provider=embedding_provider, + bge_model=getattr(settings, "EMBEDDING_MODEL", "BAAI/bge-base-en-v1.5"), + ) + + retrieval = RetrievalPipeline(config=retrieval_cfg) + + deepseek_key = getattr(settings, "DEEPSEEK_API_KEY", None) + deepseek_model = getattr(settings, "DEEPSEEK_MODEL", "deepseek-chat") + deepseek_base = getattr(settings, "DEEPSEEK_BASE_URL", "https://api.deepseek.com/v1") + + if deepseek_key: + llm_adapter: SupportsComplete = DeepSeekAdapter( + model=deepseek_model, + api_key=deepseek_key, + api_base=deepseek_base, + temperature=0.3, + ) + else: + raise RuntimeError( + "DeepSeek API key not found. Set DEEPSEEK_API_KEY in your .env file." + ) + + return ChatbotEngine(retrieval=retrieval, llm=llm_adapter) diff --git a/app/modules/chat/engine/formatter.py b/app/modules/chat/engine/formatter.py new file mode 100644 index 0000000..34d0d03 --- /dev/null +++ b/app/modules/chat/engine/formatter.py @@ -0,0 +1,80 @@ +"""Formatting request detection and instruction synthesis.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum + +from app.modules.chat.engine.intent import Intent + + +class FormatStyle(str, Enum): + PARAGRAPH = "paragraph" + BULLETS = "bullets" + SHORT = "short" + DETAILED = "detailed" + TABLE = "table" + COMPARISON = "comparison" + + +@dataclass +class FormatDecision: + style: FormatStyle + instruction: str + + +class FormattingEngine: + """Detects requested answer style without changing factual grounding.""" + + def detect(self, message: str, intent: Intent, previous_style: FormatStyle | None = None) -> FormatDecision: + text = message.lower().strip() + + if intent == Intent.COMPARISON: + return FormatDecision( + style=FormatStyle.COMPARISON, + instruction="Every response MUST start with a relevant Markdown header (###). Use a clear side-by-side comparison with similarities and differences, using **bolding** for key parameters.", + ) + + if "table" in text: + return FormatDecision( + style=FormatStyle.TABLE, + instruction="Every response MUST start with a relevant Markdown header (###). Use a markdown table with concise rows and grounded facts only. Use **bolding** for the first column or key headers.", + ) + + if "bullet" in text or "points" in text: + return FormatDecision( + style=FormatStyle.BULLETS, + instruction="Every response MUST start with a relevant Markdown header (###). Respond in bullet points only, using **bolding** for the start of each point or key facts.", + ) + + if "short" in text or "brief" in text or "concise" in text: + return FormatDecision( + style=FormatStyle.SHORT, + instruction="Respond in 2-4 concise sentences. YOU MUST still include a relevant Markdown header (###) and use **bolding** for key terms.", + ) + + if "detailed" in text or "in depth" in text or "deep" in text: + return FormatDecision( + style=FormatStyle.DETAILED, + instruction="Provide a detailed explanation with structured sections.", + ) + + direct_keywords = ["who", "when", "where", "what is", "how many", "how much", "list", "show"] + is_direct = any(text.startswith(kw) for kw in direct_keywords) or len(text.split()) <= 6 + + if is_direct and intent == Intent.QUESTION: + return FormatDecision( + style=FormatStyle.SHORT, + instruction="Respond in 1-2 concise sentences. Include a relevant Markdown header (###) and use **bolding** for key terms.", + ) + + if previous_style is not None: + return FormatDecision( + style=previous_style, + instruction=f"Continue using {previous_style.value} format unless user changes it.", + ) + + return FormatDecision( + style=FormatStyle.PARAGRAPH, + instruction="Use rich Markdown formatting. You MUST include a relevant Markdown header (###), use **bolding** for key terms, and use bullet points for lists to ensure a professional, structured layout like ChatGPT, even for short responses.", + ) diff --git a/app/modules/chat/engine/intent.py b/app/modules/chat/engine/intent.py new file mode 100644 index 0000000..1f6752a --- /dev/null +++ b/app/modules/chat/engine/intent.py @@ -0,0 +1,222 @@ +"""Intent detection layer for conversational RAG — integrated into docQube chat module.""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from enum import Enum +from typing import Protocol + +from app.modules.chat.engine.prompts import INTENT_CLASSIFIER_PROMPT + + +class SupportsComplete(Protocol): + def complete(self, prompt: str, system_prompt: str | None = None) -> tuple[str, dict[str, int]]: + ... + + +class Intent(str, Enum): + QUESTION = "QUESTION" + FOLLOW_UP = "FOLLOW_UP" + SUMMARY = "SUMMARY" + COMPARISON = "COMPARISON" + FORMAT_REQUEST = "FORMAT_REQUEST" + MEMORY_QUERY = "MEMORY_QUERY" + GREETING = "GREETING" + CHITCHAT = "CHITCHAT" + + +@dataclass +class IntentDecision: + intent: Intent + confidence: float + reason: str + + +def _compact_history(chat_history: list[dict[str, str]]) -> str: + if not chat_history: + return "(empty)" + lines: list[str] = [] + for msg in chat_history[-8:]: + role = msg.get("role", "unknown") + content = msg.get("content", "") + lines.append(f"{role}: {content}") + return "\n".join(lines) + + +class IntentClassifier: + """LLM-first intent classifier with deterministic fallback.""" + + def __init__(self, llm: SupportsComplete): + self.llm = llm + + def classify(self, message: str, chat_history: list[dict[str, str]]) -> tuple[IntentDecision, dict[str, int]]: + if self._is_date_ops_question(message): + return IntentDecision( + intent=Intent.QUESTION, + confidence=0.95, + reason="Date/operations standalone question override", + ), {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + + if self._is_direct_term_question(message): + return IntentDecision( + intent=Intent.QUESTION, + confidence=0.96, + reason="Direct term-definition style question override", + ), {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + + prompt = INTENT_CLASSIFIER_PROMPT.format( + chat_history=_compact_history(chat_history), + message=message, + ) + raw, usage = self.llm.complete(prompt) + + parsed = self._parse_json(raw) + if parsed: + intent = self._normalize_intent(parsed.get("intent", "")) + if intent is not None: + return IntentDecision( + intent=intent, + confidence=float(parsed.get("confidence", 0.7)), + reason=str(parsed.get("reason", "LLM classification")), + ), usage + + fallback_intent, reason = self._heuristic_intent(message) + return IntentDecision(intent=fallback_intent, confidence=0.51, reason=reason), usage + + @staticmethod + def _is_direct_term_question(message: str) -> bool: + text = message.strip() + lowered = text.lower() + + if any( + token in lowered + for token in [ + "previous question", + "previous answer", + "what did i ask", + "what did you answer", + "history", + "i don't understand", + "explain better", + "clarify", + ] + ): + return False + + patterns = [ + r"^\s*what\s+is\s+['\"']?[a-z0-9][a-z0-9\-/\.]{1,30}['\"']?\s*\??\s*$", + r"^\s*define\s+['\"']?[a-z0-9][a-z0-9\-/\.]{1,30}['\"']?\s*\??\s*$", + r"^\s*what\s+does\s+['\"']?[a-z0-9][a-z0-9\-/\.]{1,30}['\"']?\s+stand\s+for\s*\??\s*$", + ] + return any(re.match(pattern, lowered) for pattern in patterns) + + @staticmethod + def _is_date_ops_question(message: str) -> bool: + lowered = message.strip().lower() + + if any( + token in lowered + for token in [ + "i don't understand", + "i dont understand", + "explain better", + "clarify", + "previous question", + "previous answer", + "what do you mean", + ] + ): + return False + + has_ops_keyword = any( + token in lowered + for token in [ + "operational hours", + "operation hours", + "where is the vessel", + "where was the vessel", + "vessel on", + ] + ) + if not has_ops_keyword: + return False + + has_date = bool( + re.search( + r"(\b\d{1,2}[-/\s]?(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\b)|" + r"(\b(january|february|march|april|may|june|july|august|september|october|november|december)\b)|" + r"(\b20\d{2}\b)", + lowered, + ) + ) + return has_date + + @staticmethod + def _parse_json(raw: str) -> dict | None: + try: + return json.loads(raw) + except json.JSONDecodeError: + pass + + match = re.search(r"\{[\s\S]*\}", raw) + if not match: + return None + + try: + return json.loads(match.group(0)) + except json.JSONDecodeError: + return None + + @staticmethod + def _normalize_intent(value: str) -> Intent | None: + normalized = value.strip().upper() + try: + return Intent(normalized) + except ValueError: + return None + + @staticmethod + def _heuristic_intent(message: str) -> tuple[Intent, str]: + text = message.lower().strip() + + import logging + logger = logging.getLogger(__name__) + + if "previous question" in text or "previous answer" in text or "what did i ask" in text: + return Intent.MEMORY_QUERY, "Detected memory lookup phrase" + + summary_keywords = ["summarize the whole", "full summary", "total overview", "summarize everything"] + if any(token in text for token in summary_keywords): + return Intent.SUMMARY, "Detected explicit full summary keyword" + + if text in ["summary", "summarize", "tl;dr", "overview"]: + return Intent.SUMMARY, "Detected standalone summary keyword" + + if any(token in text for token in ["difference", "differentiate", "vs", "compare", "comparison"]): + return Intent.COMPARISON, "Detected comparison keyword" + + if any(token in text for token in ["bullet", "table", "detailed", "format"]): + return Intent.FORMAT_REQUEST, "Detected formatting keyword" + + if any( + token in text + for token in [ + "i don't understand", + "i dont understand", + "explain better", + "explain simply", + "what do you mean", + "clarify", + ] + ): + return Intent.FOLLOW_UP, "Detected follow-up clarifier phrase" + + if text in ["hi", "hello", "hey", "greetings", "good morning", "good afternoon"]: + return Intent.GREETING, "Detected simple greeting" + + if len(text) > 4 and not any(v in text for v in "aeiouy") and not text.isdigit(): + return Intent.CHITCHAT, "Detected nonsensical input (no vowels)" + + return Intent.QUESTION, "Default question classification" diff --git a/app/modules/chat/engine/prompts.py b/app/modules/chat/engine/prompts.py new file mode 100644 index 0000000..ead7109 --- /dev/null +++ b/app/modules/chat/engine/prompts.py @@ -0,0 +1,120 @@ +"""Prompt templates for the production conversational RAG chatbot.""" + +INTENT_CLASSIFIER_PROMPT = """ +You are an intent classifier for a document-grounded conversational RAG system. + +Classify the user message into one of these labels: +- QUESTION +- FOLLOW_UP +- SUMMARY +- COMPARISON +- FORMAT_REQUEST +- MEMORY_QUERY +- GREETING +- CHITCHAT + +Classification guidance: +- QUESTION: first-time factual question about document content. +- FOLLOW_UP: clarification on prior answer (e.g., "I don't understand", "explain better"). +- SUMMARY: asks to summarize full document or section. +- COMPARISON: asks differences/similarities between two concepts. +- FORMAT_REQUEST: asks to reformat output style (bullets/table/short/detailed). +- MEMORY_QUERY: asks what was previously asked/answered in this chat. +- GREETING: simple greetings like "hi", "hello", "good morning". +- CHITCHAT: random words, jokes, or non-document related talk. + +Return JSON ONLY with keys: +{{ + "intent": "