From b14c6b508a861ea1ad7c63c0dd40aba0ebf9c599 Mon Sep 17 00:00:00 2001 From: saquib Date: Tue, 8 Sep 2026 11:00:05 +0530 Subject: [PATCH] latest code --- .dockerignore | 54 + .env.development | 68 + .env.localdev | 132 + .env.production | 97 + .env.test | 68 + .env.uat | 68 + .gitignore | 79 + .pre-commit-config.yaml | 58 + .python-version | 1 + Dockerfile | 80 + README.md | 426 + alembic.ini | 116 + alembic/README | 1 + alembic/env.py | 43 + alembic/script.py.mako | 26 + .../06001599321d_add_collaboration_chat.py | 26 + .../0bffea34ee5f_disable_rls_temporarily.py | 35 + ...a9_add_max_bytes_quota_to_user_storage_.py | 38 + .../14b4aa5f0f5d_merge_multiple_heads.py | 26 + ...5f935213b22_add_tenant_storage_settings.py | 66 + .../1cb413ffaaa0_add_saas_mapping_tables.py | 59 + ...1f4d2a8c9b77_add_missing_vector_columns.py | 67 + .../229359b0d1b2_add_terms_accepted.py | 64 + ...1_add_shared_by_user_id_to_projectshare.py | 32 + .../29c02cb50737_initial_migration.py | 258 + ...4c62c1cf35_add_subscription_fields_and_.py | 51 + ..._add_signature_id_to_signature_imprints.py | 128 + ...nforce_enum_constraint_on_driveacl_role.py | 51 + ...ed9_refactor_documents_and_add_missing_.py | 100 + ...imit_notification_message_to_2000_chars.py | 36 + alembic/versions/49c19df2c1d4_sync_models.py | 371 + .../4a9dc50ce4af_drop_price_display.py | 33 + ...22ee6d91431_add_drive_share_link_fields.py | 30 + ...40c970ff890_add_tenant_chat_token_quota.py | 42 + ...90047e85_increase_file_size_constraints.py | 74 + ...1_add_multiple_files_to_signing_request.py | 34 + .../6c9cc3567866_merge_multiple_heads.py | 26 + .../73dff81eed72_add_tenant_contacts.py | 43 + .../765b053e3dcd_add_vector_index_checksum.py | 30 + ...cc1bb2_refactor_global_signature_config.py | 32 + ...5_add_chat_token_usage_and_quota_tables.py | 70 + .../80be2c61e7fc_add_is_popular_to_plans.py | 30 + ...a486fd287_add_created_at_to_drive_stars.py | 36 + .../8736ab6bd5af_add_notifications_table.py | 41 + .../88717d544229_add_is_default_to_roles.py | 34 + ..._limit_drivecomment_body_to_10000_chars.py | 36 + .../8d4dced54966_add_is_root_column.py | 59 + ...8_add_plan_roles_and_is_system_to_roles.py | 34 + ...7575_tenant_storage_config_and_provider.py | 415 + .../9a6db38b86a6_make_drivecomment_generic.py | 57 + alembic/versions/9af99fc7ca0f_merge_heads.py | 26 + ...9fe120ffb_add_device_and_session_tables.py | 79 + .../9c4a95ac6726_add_saasrolemapping.py | 45 + ...f3_limit_chatbotdocument_file_path_and_.py | 30 + .../versions/a1b2c3d4e5f6_add_rbac_tables.py | 90 + ...e6f7_update_access_model_with_hierarchy.py | 75 + ...e_remove_redundant_id_column_from_user_.py | 34 + ...f57c3d9f1c2_add_signature_config_tables.py | 64 + .../versions/b029d2f5a0c7_add_signature_id.py | 36 + ...3348_add_persistent_chat_history_tables.py | 131 + alembic/versions/b1_0_explicit_superadmin.py | 62 + alembic/versions/b1_3_row_level_security.py | 87 + ...e9f042_add_collaboration_messages_table.py | 55 + alembic/versions/b6_4_index_review.py | 65 + ...b_remove_tenant_storage_config_and_use_.py | 48 + ..._add_encrypted_ai_api_key_to_user_model.py | 38 + .../c1_0_org_units_and_scoped_roles.py | 247 + alembic/versions/c1_1_user_roles_tenant_id.py | 80 + alembic/versions/c3_0_org_unit_path.py | 77 + ...3abf44266de_enforce_b2_storage_provider.py | 40 + alembic/versions/c4_0_grant_expiry.py | 46 + alembic/versions/c4_1_access_log_retention.py | 39 + alembic/versions/c5_0_access_groups.py | 161 + .../c7d4e6f1a9b0_add_tenant_storage_quota.py | 66 + .../ca14c4ab10d9_merge_multiple_heads.py | 26 + ...70ab35793_create_chat_token_usage_table.py | 83 + ...7562afa2e_add_chat_token_usage_tracking.py | 116 + .../cfbee4f91fc2_add_updated_at_defaults.py | 52 + .../versions/d1_0_plans_and_entitlements.py | 219 + .../versions/d2_0_credit_limit_precision.py | 61 + ...add_10mb_check_constraints_on_document_.py | 60 + .../d551c03861f4_rename_quota_to_credits.py | 38 + ...5366d608_add_envelope_limits_to_tenants.py | 34 + ...baa8ef522c0_add_color_to_drive_comments.py | 38 + ..._device_constraints_and_session_updates.py | 28 + alembic/versions/e1_0_add_saas_plan_id.py | 32 + ...1a9d22ca5f_migrate_roles_access_to_uuid.py | 116 + ...57cb_add_cascade_deletes_to_drive_file_.py | 36 + ...6ffb_add_has_completed_tutorial_to_user.py | 34 + ...7834_add_ip_address_to_drive_activities.py | 30 + .../ff9052a97835_add_multi_tenancy.py | 104 + .../g001_add_signing_requests_table.py | 38 + .../g002_replace_docuseal_with_placements.py | 41 + .../versions/h003_add_tenant_smtp_settings.py | 76 + .../h004_add_signing_forensic_columns.py | 48 + .../versions/h004_limits_and_soft_delete.py | 108 + alembic/versions/i001_dual_bucket_storage.py | 56 + alembic/versions/j001_comments_fix.py | 31 + .../j002_allow_unlimited_tenant_quota.py | 38 + .../k001_limit_signature_id_to_15_chars.py | 42 + .../versions/k002_add_docs_tutorial_flag.py | 28 + alembic/versions/l001_merge_heads.py | 25 + .../versions/m001_add_activity_logs_table.py | 87 + alembic/versions/n001_add_user_designation.py | 29 + .../versions/n002_add_signature_imprints.py | 62 + alembic/versions/v3_extraction_data.py | 57 + .../w001_versioning_minor_and_version_fk.py | 137 + .../x001_add_user_smtp_credentials.py | 48 + alembic/versions/y001_add_user_mail_from.py | 39 + alembic/versions/z002_pref_lang.py | 45 + ..._add_disable_pdf_compression_to_tenants.py | 41 + alembic/versions/z004_add_global_workflows.py | 20 + ...dd_virus_scanning_and_quarantine_tables.py | 119 + app/ai_runner.py | 85 + app/ai_service/__init__.py | 0 app/ai_service/model_server.py | 31 + app/api/ws_router.py | 68 + app/assets/stylesheet/Stylesheet.deps.json | 23 + app/assets/stylesheet/Stylesheet.dll | Bin 0 -> 9216 bytes .../stylesheet/Stylesheet.runtimeconfig.json | 18 + app/assets/stylesheet/jats-html.xsl | 4447 ++++ app/core/cancellation.py | 13 + app/core/clamav.py | 118 + app/core/context.py | 4 + app/core/crypto.py | 35 + app/core/mail.py | 250 + app/core/model_manager.py | 70 + app/core/path_validation.py | 88 + app/core/progress.py | 17 + app/core/pubsub_listener.py | 41 + app/core/redis.py | 136 + app/core/request_body.py | 51 + app/core/request_cache.py | 69 + app/core/rls.py | 80 + app/core/sanitization.py | 78 + app/core/schemas.py | 55 + app/core/scope.py | 332 + app/core/security.py | 39 + app/core/settings.py | 268 + app/core/ssrf_protection.py | 79 + app/core/tenant_context.py | 114 + app/core/tenant_filter.py | 180 + app/core/tenant_metrics.py | 73 + app/core/token_blacklist.py | 74 + app/core/transactions.py | 88 + app/core/ws_auth.py | 93 + app/db/all_models.py | 95 + app/db/database.py | 100 + app/db/redis.py | 94 + .../realtime/chat_subscriber.py | 119 + .../realtime/connection_manager.py | 144 + app/infrastructure/vector/__init__.py | 1 + app/infrastructure/vector/vector_client.py | 361 + app/main.py | 449 + app/middleware/__init__.py | 3 + app/middleware/auth.py | 329 + app/middleware/csrf.py | 58 + app/middleware/ip_middleware.py | 16 + app/middleware/rbac.py | 54 + app/middleware/security_headers.py | 15 + app/middleware/tenant.py | 111 + app/middleware/tenant_context_middleware.py | 133 + app/modules/activity_logs/__init__.py | 3 + app/modules/activity_logs/constants.py | 67 + .../activity_logs/models/activity_log.py | 50 + app/modules/activity_logs/routes.py | 112 + app/modules/activity_logs/schemas.py | 227 + app/modules/activity_logs/service.py | 339 + .../auth/controllers/auth_controller.py | 66 + .../auth/controllers/role_controller.py | 78 + app/modules/auth/dependencies/__init__.py | 3 + .../auth/dependencies/access_dependency.py | 91 + app/modules/auth/middleware/hmac.py | 33 + app/modules/auth/models/access_model.py | 42 + app/modules/auth/models/device_model.py | 43 + app/modules/auth/models/role_access_model.py | 21 + app/modules/auth/models/role_model.py | 45 + app/modules/auth/models/saas_models.py | 44 + app/modules/auth/models/session_model.py | 35 + app/modules/auth/models/user_model.py | 226 + app/modules/auth/models/user_role_model.py | 160 + .../auth/repositories/device_repository.py | 45 + .../auth/repositories/session_repository.py | 72 + .../auth/repositories/user_repository.py | 43 + app/modules/auth/routes/access_routes.py | 76 + .../auth/routes/admin_dashboard_routes.py | 43 + app/modules/auth/routes/admin_user_routes.py | 655 + app/modules/auth/routes/auth_routes.py | 499 + app/modules/auth/routes/me_routes.py | 213 + app/modules/auth/routes/role_routes.py | 344 + app/modules/auth/routes/user_routes.py | 33 + app/modules/auth/schemas/access_schema.py | 84 + .../auth/schemas/admin_dashboard_schema.py | 57 + app/modules/auth/schemas/auth_schema.py | 182 + app/modules/auth/schemas/role_schema.py | 141 + .../auth/services/admin_dashboard_service.py | 557 + app/modules/auth/services/auth_service.py | 528 + .../auth/services/permission_service.py | 386 + app/modules/auth/services/privilege.py | 54 + app/modules/auth/services/role_service.py | 186 + .../auth/services/user_role_service.py | 389 + app/modules/auth/sso_router.py | 356 + app/modules/auth/utils/device_utils.py | 105 + app/modules/billing/__init__.py | 0 app/modules/billing/models/__init__.py | 0 app/modules/billing/models/plan_model.py | 285 + app/modules/billing/routes/__init__.py | 0 app/modules/billing/routes/billing_routes.py | 535 + app/modules/billing/schemas/__init__.py | 0 app/modules/billing/schemas/billing_schema.py | 146 + app/modules/billing/services/__init__.py | 0 app/modules/billing/services/billing_tasks.py | 62 + .../billing/services/entitlement_service.py | 235 + app/modules/chat/README.md | 41 + .../chat/controllers/chat_controller.py | 173 + app/modules/chat/engine/__init__.py | 0 app/modules/chat/engine/chatbot_engine.py | 692 + app/modules/chat/engine/formatter.py | 80 + app/modules/chat/engine/intent.py | 222 + app/modules/chat/engine/prompts.py | 120 + app/modules/chat/engine/reranker.py | 85 + app/modules/chat/engine/retrieval.py | 483 + app/modules/chat/engine/rewrite.py | 157 + app/modules/chat/models/chat_message_model.py | 60 + app/modules/chat/models/chat_model.py | 55 + app/modules/chat/models/chat_session_model.py | 75 + app/modules/chat/models/chat_usage_model.py | 47 + .../chat/repositories/chat_repository.py | 387 + app/modules/chat/routes/chat_routes.py | 108 + app/modules/chat/schemas/__init__.py | 0 app/modules/chat/schemas/chat_schema.py | 65 + app/modules/chat/services/chat_service.py | 594 + .../chat/services/redis_memory_service.py | 103 + app/modules/collab/models/collab_model.py | 37 + .../collab/repositories/chat_repository.py | 81 + app/modules/collab/routes/chat_routes.py | 142 + app/modules/collab/schemas/collab_schema.py | 45 + app/modules/collab/services/chat_service.py | 100 + .../models/system_configuration_model.py | 34 + .../routes/system_configuration_routes.py | 93 + .../schemas/system_configuration_schema.py | 41 + .../services/system_configuration_service.py | 225 + .../controllers/document_controller.py | 96 + .../controllers/export_controller.py | 208 + .../controllers/processing_controller.py | 618 + .../documents/models/document_model.py | 118 + .../processors/pdf/image_extractor.py | 219 + .../documents/processors/pdf/pdf_converter.py | 2344 ++ .../processors/pdf/structure_extractor.py | 592 + .../documents/processors/xml/jats_rules.py | 553 + .../documents/processors/xml/xml_compiler.py | 279 + .../repositories/document_repository.py | 81 + .../repositories/version_repository.py | 55 + .../documents/routes/comment_routes.py | 43 + .../documents/routes/details_routes.py | 29 + .../documents/routes/document_routes.py | 69 + app/modules/documents/routes/export_routes.py | 104 + .../documents/routes/sharing_routes.py | 56 + .../documents/routes/version_routes.py | 71 + .../documents/schemas/comment_schema.py | 26 + .../documents/schemas/document_schema.py | 118 + .../documents/schemas/export_schema.py | 28 + .../documents/schemas/processing_schema.py | 32 + .../documents/schemas/sharing_schema.py | 43 + app/modules/documents/schemas/tts_schema.py | 13 + .../documents/schemas/version_schema.py | 28 + .../documents/services/base_service.py | 48 + .../documents/services/document_service.py | 261 + .../documents/services/sharing_service.py | 266 + .../documents/services/version_service.py | 221 + app/modules/documents/tasks/celery_tasks.py | 373 + app/modules/drive/constants.py | 57 + .../drive/controllers/activity_controller.py | 16 + .../drive/controllers/comment_controller.py | 46 + .../drive/controllers/drive_controller.py | 449 + app/modules/drive/models/drive_model.py | 195 + .../drive/permissions/permission_service.py | 164 + .../drive/repositories/drive_repository.py | 822 + app/modules/drive/routes/drive_routes.py | 515 + app/modules/drive/routes/scan_routes.py | 85 + app/modules/drive/schemas/drive_schema.py | 234 + .../drive/services/activity_service.py | 234 + app/modules/drive/services/comment_service.py | 139 + app/modules/drive/services/drive_service.py | 721 + app/modules/drive/services/file_service.py | 1047 + app/modules/drive/services/folder_service.py | 229 + app/modules/drive/services/share_service.py | 695 + app/modules/drive/services/trash_service.py | 437 + app/modules/drive/services/version_service.py | 118 + app/modules/drive/tasks/upload_tasks.py | 209 + app/modules/drive/utils/breadcrumbs.py | 25 + app/modules/drive/utils/compress.py | 134 + app/modules/drive/utils/filename.py | 38 + app/modules/drive/utils/lossy_compress.py | 201 + app/modules/drive/utils/mime.py | 77 + app/modules/drive/utils/quota.py | 44 + app/modules/drive/utils/signature.py | 154 + app/modules/drive/workers/activity_worker.py | 34 + app/modules/drive/workers/delete_worker.py | 28 + .../drive/workers/notification_worker.py | 23 + app/modules/editor/converters.py | 120 + app/modules/editor/pdf_controller.py | 1916 ++ app/modules/editor/pdf_geometry.py | 211 + app/modules/editor/pdf_router.py | 385 + app/modules/editor/utils.py | 8 + app/modules/extraction/__init__.py | 0 app/modules/extraction/models/__init__.py | 0 .../extraction/models/extraction_model.py | 38 + app/modules/extraction/routes/__init__.py | 0 .../extraction/routes/extraction_routes.py | 67 + app/modules/extraction/schemas/__init__.py | 0 .../extraction/schemas/extraction_schema.py | 34 + app/modules/extraction/services/__init__.py | 0 .../extraction/services/docllm_service.py | 176 + .../extraction/services/extraction_service.py | 155 + app/modules/extraction/tasks/__init__.py | 0 .../extraction/tasks/extraction_tasks.py | 45 + .../models/notification_model.py | 33 + .../repositories/notification_repository.py | 112 + .../routes/notification_routes.py | 66 + app/modules/notifications/schemas/__init__.py | 0 .../schemas/notification_schema.py | 33 + .../services/notification_service.py | 191 + app/modules/org/__init__.py | 0 app/modules/org/models/__init__.py | 0 app/modules/org/models/group_model.py | 127 + app/modules/org/models/org_model.py | 240 + app/modules/org/routes/__init__.py | 0 app/modules/org/routes/org_routes.py | 514 + app/modules/org/schemas/__init__.py | 0 app/modules/org/schemas/org_schema.py | 193 + app/modules/org/services/__init__.py | 0 app/modules/org/services/access_audit.py | 354 + .../org/services/access_maintenance.py | 139 + app/modules/org/services/conflicts.py | 57 + app/modules/org/services/grant_service.py | 211 + .../org/services/membership_service.py | 232 + app/modules/org/services/org_service.py | 314 + app/modules/org/tasks.py | 68 + .../controllers/virus_scan_controller.py | 77 + .../security/models/virus_scan_models.py | 175 + .../repositories/virus_scan_repository.py | 210 + .../security/routes/security_routes.py | 118 + .../security/schemas/virus_scan_schemas.py | 75 + .../security/services/virus_scan_service.py | 297 + .../signing/models/signature_config_model.py | 78 + .../signing/models/signature_imprint.py | 47 + app/modules/signing/models/signing_request.py | 53 + app/modules/signing/routes/config_routes.py | 171 + app/modules/signing/routes/signing_routes.py | 467 + app/modules/signing/schemas/config_schema.py | 28 + .../signing/schemas/signing_schemas.py | 106 + .../signing/services/base_signing_provider.py | 38 + .../signing/services/docuseal_service.py | 137 + .../signing/services/provider_registry.py | 29 + .../services/signature_config_service.py | 191 + .../signing/services/signing_service.py | 2023 ++ .../signing/services/verification_service.py | 476 + .../signing/services/zoho_sign_service.py | 359 + app/modules/speech/routes/speech_routes.py | 82 + .../controllers/tenant_contact_controller.py | 45 + .../tenant/controllers/tenant_controller.py | 89 + .../controllers/tenant_smtp_controller.py | 77 + .../controllers/tenant_storage_controller.py | 94 + .../tenant/models/tenant_contact_model.py | 20 + app/modules/tenant/models/tenant_model.py | 72 + .../tenant/models/tenant_smtp_config_model.py | 49 + .../models/tenant_storage_config_model.py | 50 + .../repositories/tenant_contact_repository.py | 44 + .../tenant/repositories/tenant_repository.py | 60 + .../repositories/tenant_smtp_repository.py | 32 + .../repositories/tenant_storage_repository.py | 32 + .../tenant/routes/tenant_contact_routes.py | 130 + app/modules/tenant/routes/tenant_routes.py | 376 + .../tenant/schemas/tenant_contact_schema.py | 29 + app/modules/tenant/schemas/tenant_schema.py | 103 + .../tenant/schemas/tenant_smtp_schema.py | 35 + .../tenant/schemas/tenant_storage_schema.py | 34 + .../tenant/services/tenant_contact_service.py | 59 + app/modules/tenant/services/tenant_service.py | 124 + .../tenant/services/tenant_smtp_service.py | 126 + .../tenant/services/tenant_storage_service.py | 297 + app/services/saas_service.py | 469 + app/tasks/celery_app.py | 83 + app/tasks/routes.py | 250 + app/templates/emails/document_shared.html | 54 + app/templates/emails/email_resource.html | 34 + app/templates/emails/storage_alert_100.html | 24 + app/templates/emails/storage_alert_50.html | 24 + app/templates/emails/storage_alert_75.html | 24 + app/templates/emails/storage_alert_90.html | 24 + app/templates/emails/welcome.html | 558 + docker-compose.development.yml | 148 + docker-compose.production.yml | 148 + docker-compose.test.yml | 148 + manage.py | 116 + openapi.json | 21311 ++++++++++++++++ pyproject.toml | 85 + pytest.ini | 14 + scripts/b1_preflight.py | 251 + scripts/backfill_signature_imprints.py | 261 + scripts/bench_subtree.py | 166 + scripts/check_memory_leaks.py | 72 + scripts/clean_redundant.py | 36 + scripts/create_app_role.py | 112 + scripts/delete_project_accesses.py | 45 + scripts/endpoint_inventory.py | 182 + scripts/export_openapi.py | 70 + scripts/flush_redis.py | 64 + scripts/loadtest.py | 160 + scripts/multi_role_preflight.py | 157 + scripts/plan_review.py | 135 + scripts/seed.py | 322 + scripts/seed_smoke.py | 122 + scripts/smoke_billing.py | 138 + scripts/smoke_org.py | 250 + scripts/strip_comments.py | 112 + uv.lock | 2968 +++ 418 files changed, 86016 insertions(+) create mode 100644 .dockerignore create mode 100644 .env.development create mode 100644 .env.localdev create mode 100644 .env.production create mode 100644 .env.test create mode 100644 .env.uat create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 .python-version create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 alembic.ini create mode 100644 alembic/README create mode 100644 alembic/env.py create mode 100644 alembic/script.py.mako create mode 100644 alembic/versions/06001599321d_add_collaboration_chat.py create mode 100644 alembic/versions/0bffea34ee5f_disable_rls_temporarily.py create mode 100644 alembic/versions/0e1a5f2cdea9_add_max_bytes_quota_to_user_storage_.py create mode 100644 alembic/versions/14b4aa5f0f5d_merge_multiple_heads.py create mode 100644 alembic/versions/15f935213b22_add_tenant_storage_settings.py create mode 100644 alembic/versions/1cb413ffaaa0_add_saas_mapping_tables.py create mode 100644 alembic/versions/1f4d2a8c9b77_add_missing_vector_columns.py create mode 100644 alembic/versions/229359b0d1b2_add_terms_accepted.py create mode 100644 alembic/versions/22dee9a1d631_add_shared_by_user_id_to_projectshare.py create mode 100644 alembic/versions/29c02cb50737_initial_migration.py create mode 100644 alembic/versions/2b4c62c1cf35_add_subscription_fields_and_.py create mode 100644 alembic/versions/39d5fdfb5386_add_signature_id_to_signature_imprints.py create mode 100644 alembic/versions/3aa7d970b8c2_enforce_enum_constraint_on_driveacl_role.py create mode 100644 alembic/versions/41bed7eb6ed9_refactor_documents_and_add_missing_.py create mode 100644 alembic/versions/45c94ee15294_limit_notification_message_to_2000_chars.py create mode 100644 alembic/versions/49c19df2c1d4_sync_models.py create mode 100644 alembic/versions/4a9dc50ce4af_drop_price_display.py create mode 100644 alembic/versions/522ee6d91431_add_drive_share_link_fields.py create mode 100644 alembic/versions/540c970ff890_add_tenant_chat_token_quota.py create mode 100644 alembic/versions/56d090047e85_increase_file_size_constraints.py create mode 100644 alembic/versions/611d6b99ad11_add_multiple_files_to_signing_request.py create mode 100644 alembic/versions/6c9cc3567866_merge_multiple_heads.py create mode 100644 alembic/versions/73dff81eed72_add_tenant_contacts.py create mode 100644 alembic/versions/765b053e3dcd_add_vector_index_checksum.py create mode 100644 alembic/versions/7d7641cc1bb2_refactor_global_signature_config.py create mode 100644 alembic/versions/7fc8dd61bb95_add_chat_token_usage_and_quota_tables.py create mode 100644 alembic/versions/80be2c61e7fc_add_is_popular_to_plans.py create mode 100644 alembic/versions/823a486fd287_add_created_at_to_drive_stars.py create mode 100644 alembic/versions/8736ab6bd5af_add_notifications_table.py create mode 100644 alembic/versions/88717d544229_add_is_default_to_roles.py create mode 100644 alembic/versions/89508c176437_limit_drivecomment_body_to_10000_chars.py create mode 100644 alembic/versions/8d4dced54966_add_is_root_column.py create mode 100644 alembic/versions/905726af73c8_add_plan_roles_and_is_system_to_roles.py create mode 100644 alembic/versions/9557c7307575_tenant_storage_config_and_provider.py create mode 100644 alembic/versions/9a6db38b86a6_make_drivecomment_generic.py create mode 100644 alembic/versions/9af99fc7ca0f_merge_heads.py create mode 100644 alembic/versions/9c49fe120ffb_add_device_and_session_tables.py create mode 100644 alembic/versions/9c4a95ac6726_add_saasrolemapping.py create mode 100644 alembic/versions/9f6521a38bf3_limit_chatbotdocument_file_path_and_.py create mode 100644 alembic/versions/a1b2c3d4e5f6_add_rbac_tables.py create mode 100644 alembic/versions/a2b3c4d5e6f7_update_access_model_with_hierarchy.py create mode 100644 alembic/versions/a902935ea69e_remove_redundant_id_column_from_user_.py create mode 100644 alembic/versions/af57c3d9f1c2_add_signature_config_tables.py create mode 100644 alembic/versions/b029d2f5a0c7_add_signature_id.py create mode 100644 alembic/versions/b0a09cd13348_add_persistent_chat_history_tables.py create mode 100644 alembic/versions/b1_0_explicit_superadmin.py create mode 100644 alembic/versions/b1_3_row_level_security.py create mode 100644 alembic/versions/b3a7c1e9f042_add_collaboration_messages_table.py create mode 100644 alembic/versions/b6_4_index_review.py create mode 100644 alembic/versions/bb224e0b642b_remove_tenant_storage_config_and_use_.py create mode 100644 alembic/versions/be67927ab00d_add_encrypted_ai_api_key_to_user_model.py create mode 100644 alembic/versions/c1_0_org_units_and_scoped_roles.py create mode 100644 alembic/versions/c1_1_user_roles_tenant_id.py create mode 100644 alembic/versions/c3_0_org_unit_path.py create mode 100644 alembic/versions/c3abf44266de_enforce_b2_storage_provider.py create mode 100644 alembic/versions/c4_0_grant_expiry.py create mode 100644 alembic/versions/c4_1_access_log_retention.py create mode 100644 alembic/versions/c5_0_access_groups.py create mode 100644 alembic/versions/c7d4e6f1a9b0_add_tenant_storage_quota.py create mode 100644 alembic/versions/ca14c4ab10d9_merge_multiple_heads.py create mode 100644 alembic/versions/cba70ab35793_create_chat_token_usage_table.py create mode 100644 alembic/versions/cd27562afa2e_add_chat_token_usage_tracking.py create mode 100644 alembic/versions/cfbee4f91fc2_add_updated_at_defaults.py create mode 100644 alembic/versions/d1_0_plans_and_entitlements.py create mode 100644 alembic/versions/d2_0_credit_limit_precision.py create mode 100644 alembic/versions/d2e2c55290ee_add_10mb_check_constraints_on_document_.py create mode 100644 alembic/versions/d551c03861f4_rename_quota_to_credits.py create mode 100644 alembic/versions/daf45366d608_add_envelope_limits_to_tenants.py create mode 100644 alembic/versions/dbaa8ef522c0_add_color_to_drive_comments.py create mode 100644 alembic/versions/dfae38e4ede3_device_constraints_and_session_updates.py create mode 100644 alembic/versions/e1_0_add_saas_plan_id.py create mode 100644 alembic/versions/e81a9d22ca5f_migrate_roles_access_to_uuid.py create mode 100644 alembic/versions/f055f9b157cb_add_cascade_deletes_to_drive_file_.py create mode 100644 alembic/versions/fb6810c36ffb_add_has_completed_tutorial_to_user.py create mode 100644 alembic/versions/ff8052a97834_add_ip_address_to_drive_activities.py create mode 100644 alembic/versions/ff9052a97835_add_multi_tenancy.py create mode 100644 alembic/versions/g001_add_signing_requests_table.py create mode 100644 alembic/versions/g002_replace_docuseal_with_placements.py create mode 100644 alembic/versions/h003_add_tenant_smtp_settings.py create mode 100644 alembic/versions/h004_add_signing_forensic_columns.py create mode 100644 alembic/versions/h004_limits_and_soft_delete.py create mode 100644 alembic/versions/i001_dual_bucket_storage.py create mode 100644 alembic/versions/j001_comments_fix.py create mode 100644 alembic/versions/j002_allow_unlimited_tenant_quota.py create mode 100644 alembic/versions/k001_limit_signature_id_to_15_chars.py create mode 100644 alembic/versions/k002_add_docs_tutorial_flag.py create mode 100644 alembic/versions/l001_merge_heads.py create mode 100644 alembic/versions/m001_add_activity_logs_table.py create mode 100644 alembic/versions/n001_add_user_designation.py create mode 100644 alembic/versions/n002_add_signature_imprints.py create mode 100644 alembic/versions/v3_extraction_data.py create mode 100644 alembic/versions/w001_versioning_minor_and_version_fk.py create mode 100644 alembic/versions/x001_add_user_smtp_credentials.py create mode 100644 alembic/versions/y001_add_user_mail_from.py create mode 100644 alembic/versions/z002_pref_lang.py create mode 100644 alembic/versions/z003_add_disable_pdf_compression_to_tenants.py create mode 100644 alembic/versions/z004_add_global_workflows.py create mode 100644 alembic/versions/z005_add_virus_scanning_and_quarantine_tables.py create mode 100644 app/ai_runner.py create mode 100644 app/ai_service/__init__.py create mode 100644 app/ai_service/model_server.py create mode 100644 app/api/ws_router.py create mode 100644 app/assets/stylesheet/Stylesheet.deps.json create mode 100644 app/assets/stylesheet/Stylesheet.dll create mode 100644 app/assets/stylesheet/Stylesheet.runtimeconfig.json create mode 100644 app/assets/stylesheet/jats-html.xsl create mode 100644 app/core/cancellation.py create mode 100644 app/core/clamav.py create mode 100644 app/core/context.py create mode 100644 app/core/crypto.py create mode 100644 app/core/mail.py create mode 100644 app/core/model_manager.py create mode 100644 app/core/path_validation.py create mode 100644 app/core/progress.py create mode 100644 app/core/pubsub_listener.py create mode 100644 app/core/redis.py create mode 100644 app/core/request_body.py create mode 100644 app/core/request_cache.py create mode 100644 app/core/rls.py create mode 100644 app/core/sanitization.py create mode 100644 app/core/schemas.py create mode 100644 app/core/scope.py create mode 100644 app/core/security.py create mode 100644 app/core/settings.py create mode 100644 app/core/ssrf_protection.py create mode 100644 app/core/tenant_context.py create mode 100644 app/core/tenant_filter.py create mode 100644 app/core/tenant_metrics.py create mode 100644 app/core/token_blacklist.py create mode 100644 app/core/transactions.py create mode 100644 app/core/ws_auth.py create mode 100644 app/db/all_models.py create mode 100644 app/db/database.py create mode 100644 app/db/redis.py create mode 100644 app/infrastructure/realtime/chat_subscriber.py create mode 100644 app/infrastructure/realtime/connection_manager.py create mode 100644 app/infrastructure/vector/__init__.py create mode 100644 app/infrastructure/vector/vector_client.py create mode 100644 app/main.py create mode 100644 app/middleware/__init__.py create mode 100644 app/middleware/auth.py create mode 100644 app/middleware/csrf.py create mode 100644 app/middleware/ip_middleware.py create mode 100644 app/middleware/rbac.py create mode 100644 app/middleware/security_headers.py create mode 100644 app/middleware/tenant.py create mode 100644 app/middleware/tenant_context_middleware.py create mode 100644 app/modules/activity_logs/__init__.py create mode 100644 app/modules/activity_logs/constants.py create mode 100644 app/modules/activity_logs/models/activity_log.py create mode 100644 app/modules/activity_logs/routes.py create mode 100644 app/modules/activity_logs/schemas.py create mode 100644 app/modules/activity_logs/service.py create mode 100644 app/modules/auth/controllers/auth_controller.py create mode 100644 app/modules/auth/controllers/role_controller.py create mode 100644 app/modules/auth/dependencies/__init__.py create mode 100644 app/modules/auth/dependencies/access_dependency.py create mode 100644 app/modules/auth/middleware/hmac.py create mode 100644 app/modules/auth/models/access_model.py create mode 100644 app/modules/auth/models/device_model.py create mode 100644 app/modules/auth/models/role_access_model.py create mode 100644 app/modules/auth/models/role_model.py create mode 100644 app/modules/auth/models/saas_models.py create mode 100644 app/modules/auth/models/session_model.py create mode 100644 app/modules/auth/models/user_model.py create mode 100644 app/modules/auth/models/user_role_model.py create mode 100644 app/modules/auth/repositories/device_repository.py create mode 100644 app/modules/auth/repositories/session_repository.py create mode 100644 app/modules/auth/repositories/user_repository.py create mode 100644 app/modules/auth/routes/access_routes.py create mode 100644 app/modules/auth/routes/admin_dashboard_routes.py create mode 100644 app/modules/auth/routes/admin_user_routes.py create mode 100644 app/modules/auth/routes/auth_routes.py create mode 100644 app/modules/auth/routes/me_routes.py create mode 100644 app/modules/auth/routes/role_routes.py create mode 100644 app/modules/auth/routes/user_routes.py create mode 100644 app/modules/auth/schemas/access_schema.py create mode 100644 app/modules/auth/schemas/admin_dashboard_schema.py create mode 100644 app/modules/auth/schemas/auth_schema.py create mode 100644 app/modules/auth/schemas/role_schema.py create mode 100644 app/modules/auth/services/admin_dashboard_service.py create mode 100644 app/modules/auth/services/auth_service.py create mode 100644 app/modules/auth/services/permission_service.py create mode 100644 app/modules/auth/services/privilege.py create mode 100644 app/modules/auth/services/role_service.py create mode 100644 app/modules/auth/services/user_role_service.py create mode 100644 app/modules/auth/sso_router.py create mode 100644 app/modules/auth/utils/device_utils.py create mode 100644 app/modules/billing/__init__.py create mode 100644 app/modules/billing/models/__init__.py create mode 100644 app/modules/billing/models/plan_model.py create mode 100644 app/modules/billing/routes/__init__.py create mode 100644 app/modules/billing/routes/billing_routes.py create mode 100644 app/modules/billing/schemas/__init__.py create mode 100644 app/modules/billing/schemas/billing_schema.py create mode 100644 app/modules/billing/services/__init__.py create mode 100644 app/modules/billing/services/billing_tasks.py create mode 100644 app/modules/billing/services/entitlement_service.py create mode 100644 app/modules/chat/README.md create mode 100644 app/modules/chat/controllers/chat_controller.py create mode 100644 app/modules/chat/engine/__init__.py create mode 100644 app/modules/chat/engine/chatbot_engine.py create mode 100644 app/modules/chat/engine/formatter.py create mode 100644 app/modules/chat/engine/intent.py create mode 100644 app/modules/chat/engine/prompts.py create mode 100644 app/modules/chat/engine/reranker.py create mode 100644 app/modules/chat/engine/retrieval.py create mode 100644 app/modules/chat/engine/rewrite.py create mode 100644 app/modules/chat/models/chat_message_model.py create mode 100644 app/modules/chat/models/chat_model.py create mode 100644 app/modules/chat/models/chat_session_model.py create mode 100644 app/modules/chat/models/chat_usage_model.py create mode 100644 app/modules/chat/repositories/chat_repository.py create mode 100644 app/modules/chat/routes/chat_routes.py create mode 100644 app/modules/chat/schemas/__init__.py create mode 100644 app/modules/chat/schemas/chat_schema.py create mode 100644 app/modules/chat/services/chat_service.py create mode 100644 app/modules/chat/services/redis_memory_service.py create mode 100644 app/modules/collab/models/collab_model.py create mode 100644 app/modules/collab/repositories/chat_repository.py create mode 100644 app/modules/collab/routes/chat_routes.py create mode 100644 app/modules/collab/schemas/collab_schema.py create mode 100644 app/modules/collab/services/chat_service.py create mode 100644 app/modules/configuration/models/system_configuration_model.py create mode 100644 app/modules/configuration/routes/system_configuration_routes.py create mode 100644 app/modules/configuration/schemas/system_configuration_schema.py create mode 100644 app/modules/configuration/services/system_configuration_service.py create mode 100644 app/modules/documents/controllers/document_controller.py create mode 100644 app/modules/documents/controllers/export_controller.py create mode 100644 app/modules/documents/controllers/processing_controller.py create mode 100644 app/modules/documents/models/document_model.py create mode 100644 app/modules/documents/processors/pdf/image_extractor.py create mode 100644 app/modules/documents/processors/pdf/pdf_converter.py create mode 100644 app/modules/documents/processors/pdf/structure_extractor.py create mode 100644 app/modules/documents/processors/xml/jats_rules.py create mode 100644 app/modules/documents/processors/xml/xml_compiler.py create mode 100644 app/modules/documents/repositories/document_repository.py create mode 100644 app/modules/documents/repositories/version_repository.py create mode 100644 app/modules/documents/routes/comment_routes.py create mode 100644 app/modules/documents/routes/details_routes.py create mode 100644 app/modules/documents/routes/document_routes.py create mode 100644 app/modules/documents/routes/export_routes.py create mode 100644 app/modules/documents/routes/sharing_routes.py create mode 100644 app/modules/documents/routes/version_routes.py create mode 100644 app/modules/documents/schemas/comment_schema.py create mode 100644 app/modules/documents/schemas/document_schema.py create mode 100644 app/modules/documents/schemas/export_schema.py create mode 100644 app/modules/documents/schemas/processing_schema.py create mode 100644 app/modules/documents/schemas/sharing_schema.py create mode 100644 app/modules/documents/schemas/tts_schema.py create mode 100644 app/modules/documents/schemas/version_schema.py create mode 100644 app/modules/documents/services/base_service.py create mode 100644 app/modules/documents/services/document_service.py create mode 100644 app/modules/documents/services/sharing_service.py create mode 100644 app/modules/documents/services/version_service.py create mode 100644 app/modules/documents/tasks/celery_tasks.py create mode 100644 app/modules/drive/constants.py create mode 100644 app/modules/drive/controllers/activity_controller.py create mode 100644 app/modules/drive/controllers/comment_controller.py create mode 100644 app/modules/drive/controllers/drive_controller.py create mode 100644 app/modules/drive/models/drive_model.py create mode 100644 app/modules/drive/permissions/permission_service.py create mode 100644 app/modules/drive/repositories/drive_repository.py create mode 100644 app/modules/drive/routes/drive_routes.py create mode 100644 app/modules/drive/routes/scan_routes.py create mode 100644 app/modules/drive/schemas/drive_schema.py create mode 100644 app/modules/drive/services/activity_service.py create mode 100644 app/modules/drive/services/comment_service.py create mode 100644 app/modules/drive/services/drive_service.py create mode 100644 app/modules/drive/services/file_service.py create mode 100644 app/modules/drive/services/folder_service.py create mode 100644 app/modules/drive/services/share_service.py create mode 100644 app/modules/drive/services/trash_service.py create mode 100644 app/modules/drive/services/version_service.py create mode 100644 app/modules/drive/tasks/upload_tasks.py create mode 100644 app/modules/drive/utils/breadcrumbs.py create mode 100644 app/modules/drive/utils/compress.py create mode 100644 app/modules/drive/utils/filename.py create mode 100644 app/modules/drive/utils/lossy_compress.py create mode 100644 app/modules/drive/utils/mime.py create mode 100644 app/modules/drive/utils/quota.py create mode 100644 app/modules/drive/utils/signature.py create mode 100644 app/modules/drive/workers/activity_worker.py create mode 100644 app/modules/drive/workers/delete_worker.py create mode 100644 app/modules/drive/workers/notification_worker.py create mode 100644 app/modules/editor/converters.py create mode 100644 app/modules/editor/pdf_controller.py create mode 100644 app/modules/editor/pdf_geometry.py create mode 100644 app/modules/editor/pdf_router.py create mode 100644 app/modules/editor/utils.py create mode 100644 app/modules/extraction/__init__.py create mode 100644 app/modules/extraction/models/__init__.py create mode 100644 app/modules/extraction/models/extraction_model.py create mode 100644 app/modules/extraction/routes/__init__.py create mode 100644 app/modules/extraction/routes/extraction_routes.py create mode 100644 app/modules/extraction/schemas/__init__.py create mode 100644 app/modules/extraction/schemas/extraction_schema.py create mode 100644 app/modules/extraction/services/__init__.py create mode 100644 app/modules/extraction/services/docllm_service.py create mode 100644 app/modules/extraction/services/extraction_service.py create mode 100644 app/modules/extraction/tasks/__init__.py create mode 100644 app/modules/extraction/tasks/extraction_tasks.py create mode 100644 app/modules/notifications/models/notification_model.py create mode 100644 app/modules/notifications/repositories/notification_repository.py create mode 100644 app/modules/notifications/routes/notification_routes.py create mode 100644 app/modules/notifications/schemas/__init__.py create mode 100644 app/modules/notifications/schemas/notification_schema.py create mode 100644 app/modules/notifications/services/notification_service.py create mode 100644 app/modules/org/__init__.py create mode 100644 app/modules/org/models/__init__.py create mode 100644 app/modules/org/models/group_model.py create mode 100644 app/modules/org/models/org_model.py create mode 100644 app/modules/org/routes/__init__.py create mode 100644 app/modules/org/routes/org_routes.py create mode 100644 app/modules/org/schemas/__init__.py create mode 100644 app/modules/org/schemas/org_schema.py create mode 100644 app/modules/org/services/__init__.py create mode 100644 app/modules/org/services/access_audit.py create mode 100644 app/modules/org/services/access_maintenance.py create mode 100644 app/modules/org/services/conflicts.py create mode 100644 app/modules/org/services/grant_service.py create mode 100644 app/modules/org/services/membership_service.py create mode 100644 app/modules/org/services/org_service.py create mode 100644 app/modules/org/tasks.py create mode 100644 app/modules/security/controllers/virus_scan_controller.py create mode 100644 app/modules/security/models/virus_scan_models.py create mode 100644 app/modules/security/repositories/virus_scan_repository.py create mode 100644 app/modules/security/routes/security_routes.py create mode 100644 app/modules/security/schemas/virus_scan_schemas.py create mode 100644 app/modules/security/services/virus_scan_service.py create mode 100644 app/modules/signing/models/signature_config_model.py create mode 100644 app/modules/signing/models/signature_imprint.py create mode 100644 app/modules/signing/models/signing_request.py create mode 100644 app/modules/signing/routes/config_routes.py create mode 100644 app/modules/signing/routes/signing_routes.py create mode 100644 app/modules/signing/schemas/config_schema.py create mode 100644 app/modules/signing/schemas/signing_schemas.py create mode 100644 app/modules/signing/services/base_signing_provider.py create mode 100644 app/modules/signing/services/docuseal_service.py create mode 100644 app/modules/signing/services/provider_registry.py create mode 100644 app/modules/signing/services/signature_config_service.py create mode 100644 app/modules/signing/services/signing_service.py create mode 100644 app/modules/signing/services/verification_service.py create mode 100644 app/modules/signing/services/zoho_sign_service.py create mode 100644 app/modules/speech/routes/speech_routes.py create mode 100644 app/modules/tenant/controllers/tenant_contact_controller.py create mode 100644 app/modules/tenant/controllers/tenant_controller.py create mode 100644 app/modules/tenant/controllers/tenant_smtp_controller.py create mode 100644 app/modules/tenant/controllers/tenant_storage_controller.py create mode 100644 app/modules/tenant/models/tenant_contact_model.py create mode 100644 app/modules/tenant/models/tenant_model.py create mode 100644 app/modules/tenant/models/tenant_smtp_config_model.py create mode 100644 app/modules/tenant/models/tenant_storage_config_model.py create mode 100644 app/modules/tenant/repositories/tenant_contact_repository.py create mode 100644 app/modules/tenant/repositories/tenant_repository.py create mode 100644 app/modules/tenant/repositories/tenant_smtp_repository.py create mode 100644 app/modules/tenant/repositories/tenant_storage_repository.py create mode 100644 app/modules/tenant/routes/tenant_contact_routes.py create mode 100644 app/modules/tenant/routes/tenant_routes.py create mode 100644 app/modules/tenant/schemas/tenant_contact_schema.py create mode 100644 app/modules/tenant/schemas/tenant_schema.py create mode 100644 app/modules/tenant/schemas/tenant_smtp_schema.py create mode 100644 app/modules/tenant/schemas/tenant_storage_schema.py create mode 100644 app/modules/tenant/services/tenant_contact_service.py create mode 100644 app/modules/tenant/services/tenant_service.py create mode 100644 app/modules/tenant/services/tenant_smtp_service.py create mode 100644 app/modules/tenant/services/tenant_storage_service.py create mode 100644 app/services/saas_service.py create mode 100644 app/tasks/celery_app.py create mode 100644 app/tasks/routes.py create mode 100644 app/templates/emails/document_shared.html create mode 100644 app/templates/emails/email_resource.html create mode 100644 app/templates/emails/storage_alert_100.html create mode 100644 app/templates/emails/storage_alert_50.html create mode 100644 app/templates/emails/storage_alert_75.html create mode 100644 app/templates/emails/storage_alert_90.html create mode 100644 app/templates/emails/welcome.html create mode 100644 docker-compose.development.yml create mode 100644 docker-compose.production.yml create mode 100644 docker-compose.test.yml create mode 100644 manage.py create mode 100644 openapi.json create mode 100644 pyproject.toml create mode 100644 pytest.ini create mode 100644 scripts/b1_preflight.py create mode 100644 scripts/backfill_signature_imprints.py create mode 100644 scripts/bench_subtree.py create mode 100644 scripts/check_memory_leaks.py create mode 100644 scripts/clean_redundant.py create mode 100644 scripts/create_app_role.py create mode 100644 scripts/delete_project_accesses.py create mode 100644 scripts/endpoint_inventory.py create mode 100644 scripts/export_openapi.py create mode 100644 scripts/flush_redis.py create mode 100644 scripts/loadtest.py create mode 100644 scripts/multi_role_preflight.py create mode 100644 scripts/plan_review.py create mode 100644 scripts/seed.py create mode 100644 scripts/seed_smoke.py create mode 100644 scripts/smoke_billing.py create mode 100644 scripts/smoke_org.py create mode 100644 scripts/strip_comments.py create mode 100644 uv.lock diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..8a11324 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,54 @@ +.git +.gitignore +.gitattributes +.env +.env.* +!.env.example +__pycache__/ +*.py[cod] +*$py.class +*.so +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.coverage +.coverage.* +htmlcov/ +.pytype/ +.pyre/ +*.egg-info/ +dist/ +build/ +.eggs/ +.venv/ +venv/ +env/ +ENV/ +.vscode/ +.idea/ +*.swp +*.swo +*~ +.DS_Store +Thumbs.db +storage_drive/ +storage/ +uploads/ +downloads/ +temp/ +tmp/ +Dockerfile.* +docker-compose*.yml +.docker/ +*.log +logs/ +.pytest_cache/ +htmlcov/ +coverage.xml +redis_data/ +qdrant_data/ +clamav_data/ +*.md +requirements*.txt +openapi.json +.pre-commit-config.yaml diff --git a/.env.development b/.env.development new file mode 100644 index 0000000..da7bce5 --- /dev/null +++ b/.env.development @@ -0,0 +1,68 @@ +APP_ENV=development +HOST=0.0.0.0 +PORT=20001 +DEBUG=False +DB_HOST=106.51.105.22 +DB_PORT=5432 +DB_NAME=docqube_development +DB_USER=docqube_user +DB_PASSWORD=d20kMfypQE8Z +DB_SSLMODE=disable +PGBOUNCER_PORT=6432 +SAAS_TRUST_SECRET=docqube_trust_secret_2026 +SAAS_BASE_URL=http://localhost:8000 +SAAS_PUBLIC_KEY="" +APP_SECRET=Pk7rcnYeSE-9vgTuevzfvKJ5YAs6Bj0QjgFxyJW00oKMnmPlRxUfU8lAm33GAgxL +ALGORITHM=HS256 +ACCESS_TOKEN_EXPIRE_MINUTES=60 +REDIS_HOST=redis +REDIS_PORT=6379 +REDIS_PASSWORD=JVCzB612F573hYwqZZ +CLAMAV_HOST=clamav +CLAMAV_PORT=3310 +CLAMAV_TIMEOUT_SECONDS=20 +CLAMAV_CHUNK_SIZE=1048576 +STORAGE_DRIVE_DIR=storage_drive +API_BASE_URL=https://docqube.maskantech.in +GOOGLE_CLIENT_ID=989963514121-nh9ibk3e4u1dh6jp4ql17tue7f9suk38.apps.googleusercontent.com +GOOGLE_CLIENT_SECRET=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 +DEEPSEEK_API_KEY=sk-95235bd6d3b440d6b6e25bd722813f3c +DEEPSEEK_MODEL=deepseek-chat +DEEPSEEK_BASE_URL=https://api.deepseek.com/v1 +LLM_TEMPERATURE=0.35 +MAX_RESPONSE_TOKENS=1500 +HF_HUB_OFFLINE=0 +TRANSFORMERS_OFFLINE=0 +VECTOR_DB=qdrant +QDRANT_HOST=qdrant +QDRANT_PORT=6333 +QDRANT_COLLECTION=docqube_collection +RERANKER_MODEL=cross-encoder/ms-marco-MiniLM-L-12-v2 +EMBEDDING_MODEL=BAAI/bge-base-en-v1.5 +CHUNK_SIZE=850 +CHUNK_OVERLAP=180 +TOP_K_QA=3 +TOP_K_SUMMARY=4 +MAX_CONTEXT_CHARS=4000 +MIN_SIMILARITY_SCORE=0.18 +LOG_LEVEL=WARNING +CORS_ORIGINS=https://docqube.maskantech.in +SMTP_HOST=smtp.gmail.com +SMTP_PORT=465 +SMTP_USER=info.maskantech@gmail.com +SMTP_PASSWORD=tuthpljtkrwchgxd +MAIL_FROM=info.maskantech@gmail.com +SMTP_SECURE=True +FRONTEND_URL=https://docqube.maskantech.in +CHAT_REDIS_MAX_MESSAGES=10 +CHAT_REDIS_TTL_SECONDS=86400 +CHAT_DAILY_CREDITS_LIMIT=1000 +DOCUSEAL_API_URL=https://api.docuseal.com +DOCUSEAL_API_KEY=prUReaHxSnChGoeeL7xnf5kxsXVLAzfCbn1RvbhnoMz +DOCUSEAL_ACCOUNT_EMAIL=info.maskantech+test@gmail.com +ZOHO_CLIENT_ID="1000.EX179O66BC7FKAWMR17Q4V4ZV4OW9K" +ZOHO_CLIENT_SECRET="9c8e6414aebdbed66b5bc5f9c2a866fb4b4b159a93" +ZOHO_REFRESH_TOKEN="1000.58b1696404de858cd887a29005879ed5.487b427449d87b3fd0f1cb883f4d1b7d" +ZOHO_WEBHOOK_SECRET="0iWuLqg4z41RMWmgI6ql5qbPE3fCE4tc" +ZOHO_DOMAIN="in" +MAX_ACTIVE_DEVICES=5 \ No newline at end of file diff --git a/.env.localdev b/.env.localdev new file mode 100644 index 0000000..38ff103 --- /dev/null +++ b/.env.localdev @@ -0,0 +1,132 @@ +# Environment Configuration +APP_ENV=localdev + +# Server Configuration +HOST=127.0.0.1 +PORT=20001 +DEBUG=True + +# DB_HOST=localhost +# DB_PORT=5432 +# DB_NAME=new-base +# DB_USER=postgres +# DB_PASSWORD=ribai123 +# DB_SSLMODE=disable +# PGBOUNCER_PORT=5432 +DB_HOST=106.51.105.22 +DB_PORT=5432 +DB_NAME=docqube_local +DB_USER=docqube_user +DB_PASSWORD=d20kMfypQE8Z +DB_SSLMODE=disable +PGBOUNCER_PORT=6432 + +# Security Configuration +APP_SECRET=VBN20DIyTbuqRJKKpxB2C4aDVYRUENa1rdzFXPLXwxA +ALGORITHM=HS256 +ACCESS_TOKEN_EXPIRE_MINUTES=60 + +# Redis Configuration +REDIS_HOST=106.51.105.22 +REDIS_PORT=6388 +REDIS_PASSWORD=JVCzB612F573hYwqZZ + +CLAMAV_HOST=106.51.105.22 +CLAMAV_PORT=3310 +CLAMAV_TIMEOUT_SECONDS=20 +CLAMAV_CHUNK_SIZE=1048576 +STORAGE_DRIVE_DIR=storage_drive +API_BASE_URL=http://localhost:20001 + + +#Saas +SAAS_TRUST_SECRET=VBN20DIyTbuqRJKKpxB2C4aDVYRUENa1rdzFXPLXwxA +SAAS_BASE_URL=http://localhost:8000 +SAAS_PUBLIC_KEY="" + +# Google Auth Configuration +GOOGLE_CLIENT_ID=989963514121-nh9ibk3e4u1dh6jp4ql17tue7f9suk38.apps.googleusercontent.com +GOOGLE_CLIENT_SECRET=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 + +# AI & Models Configuration +# ========================= +# AI (DeepSeek) +# ========================= +DEEPSEEK_API_KEY=sk-95235bd6d3b440d6b6e25bd722813f3c +DEEPSEEK_MODEL=deepseek-chat +DEEPSEEK_BASE_URL=https://api.deepseek.com/v1 +LLM_TEMPERATURE=0.35 +MAX_RESPONSE_TOKENS=1500 + +# Extraction / NVIDIA AI Configuration +EXTRACTION_API_KEY="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" +EXTRACTION_BASE_URL=https://integrate.api.nvidia.com/v1 +EXTRACTION_MODEL=mistralai/mistral-medium-3-instruct +EXTRACTION_VISION_MODEL=meta/llama-3.2-90b-vision-instruct + + +# Force Offline Mode for local models (Prevents startup delays/HF checks) +HF_HUB_OFFLINE=0 +TRANSFORMERS_OFFLINE=0 + +# Vector DB Configuration +VECTOR_DB=qdrant +QDRANT_HOST=localhost +QDRANT_PORT=6333 +QDRANT_COLLECTION=docqube_collection +RERANKER_MODEL=cross-encoder/ms-marco-MiniLM-L-12-v2 + +# ========================= +# EMBEDDING +# ========================= +EMBEDDING_MODEL=BAAI/bge-base-en-v1.5 +# RAG SETTINGS +# ========================= +CHUNK_SIZE=850 +CHUNK_OVERLAP=180 +TOP_K_QA=3 +TOP_K_SUMMARY=4 +MAX_CONTEXT_CHARS=4000 +MIN_SIMILARITY_SCORE=0.18 + +# Logging Configuration +LOG_LEVEL=INFO + +# CORS Configuration +CORS_ORIGINS=* + +# Email Configuration +SMTP_HOST=smtp.gmail.com +SMTP_PORT=465 +SMTP_USER=info.maskantech@gmail.com +SMTP_PASSWORD=ftqjghdbthlwpnsh +MAIL_FROM=info.maskantech@gmail.com +SMTP_SECURE=True + + +# Frontend Configuration +FRONTEND_URL=http://localhost:5173 + +# whisper Model Configuration +WHISPER_MODEL=small + + +# Chat Redis Settings +CHAT_REDIS_MAX_MESSAGES=10 +CHAT_REDIS_TTL_SECONDS=86400 +CHAT_DAILY_CREDITS_LIMIT=1000 + +# Docuseal API Settings +# DOCUSEAL_API_URL=https://api.docuseal.com +# DOCUSEAL_API_KEY=prUReaHxSnChGoeeL7xnf5kxsXVLAzfCbn1RvbhnoMz +# DOCUSEAL_ACCOUNT_EMAIL=info.maskantech+test@gmail.com + +#ZOHO API Settings +# ZOHO_CLIENT_ID="1000.EX179O66BC7FKAWMR17Q4V4ZV4OW9K" +# ZOHO_CLIENT_SECRET="9c8e6414aebdbed66b5bc5f9c2a866fb4b4b159a93" +# ZOHO_REFRESH_TOKEN="1000.58b1696404de858cd887a29005879ed5.487b427449d87b3fd0f1cb883f4d1b7d" +ZOHO_WEBHOOK_SECRET="0iWuLqg4z41RMWmgI6ql5qbPE3fCE4tc" +ZOHO_DOMAIN="in" +MAX_ACTIVE_DEVICES=10 + + diff --git a/.env.production b/.env.production new file mode 100644 index 0000000..c78fa98 --- /dev/null +++ b/.env.production @@ -0,0 +1,97 @@ +# Environment Configuration +APP_ENV=production + +# Server Configuration +HOST=127.0.0.1 +PORT=20001 +DEBUG=True + +DB_HOST=127.0.0.1 +DB_PORT=5432 +DB_NAME=prod_docqube +DB_USER=prod_user +DB_PASSWORD='GO5cA8hK3@47Uq9im44' +DB_SSLMODE=disable +PGBOUNCER_PORT=6432 + + +#saas +SAAS_TRUST_SECRET=docqube_trust_secret_2026 +SAAS_BASE_URL=http://localhost:8000 +SAAS_PUBLIC_KEY="" + +# Security Configuration +APP_SECRET=VBN20DIyTbuqRJKKpxB2C4aDVYRUENa1rdzFXPLXwxA +ALGORITHM=HS256 +ACCESS_TOKEN_EXPIRE_MINUTES=60 + +# Redis Configuration +REDIS_HOST=31.57.228.186 +REDIS_PORT=6379 +REDIS_PASSWORD=TP0TmTqyWzBIuvrE + +CLAMAV_HOST=31.57.228.186 +CLAMAV_PORT=3310 +CLAMAV_TIMEOUT_SECONDS=20 +CLAMAV_CHUNK_SIZE=1048576 +STORAGE_DRIVE_DIR=storage_drive +API_BASE_URL=https://api.docqube.com + +# Google Auth Configuration +GOOGLE_CLIENT_ID=989963514121-nh9ibk3e4u1dh6jp4ql17tue7f9suk38.apps.googleusercontent.com +GOOGLE_CLIENT_SECRET=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 + + +# ========================= +# AI (DeepSeek) +# ========================= +DEEPSEEK_API_KEY=sk-f51cbc9eca6b466d949bfdd60a892a97 +DEEPSEEK_MODEL=deepseek-chat +DEEPSEEK_BASE_URL=https://api.deepseek.com/v1 + +# ========================= +# RAG +# ========================= +EMBEDDING_MODEL=BAAI/bge-base-en-v1.5 +CHUNK_SIZE=300 +CHUNK_OVERLAP=80 +TOP_K=3 +# Logging Configuration +LOG_LEVEL=INFO + +# CORS Configuration +CORS_ORIGINS=https://docqube.com,https://api.docqube.com +COOKIE_DOMAIN=.docqube.com +DISABLE_CSRF=True + +# Email Configuration +SMTP_HOST=smtp.hostinger.com +SMTP_PORT=465 +SMTP_USER=support@docqube.com +SMTP_PASSWORD=DQsupport@765$ +MAIL_FROM=support@docqube.com +SMTP_SECURE=True + +# Frontend Configuration +FRONTEND_URL=https://docqube.com + +# whisper Model Configuration +WHISPER_MODEL=small + +# Chat Redis Settings +CHAT_REDIS_MAX_MESSAGES=10 +CHAT_REDIS_TTL_SECONDS=86400 +CHAT_DAILY_CREDITS_LIMIT=1000 + + +# Docuseal API Settings +DOCUSEAL_API_URL=https://api.docuseal.com +DOCUSEAL_API_KEY=prUReaHxSnChGoeeL7xnf5kxsXVLAzfCbn1RvbhnoMz +DOCUSEAL_ACCOUNT_EMAIL=info.maskantech+test@gmail.com + +#ZOHO API Settings +ZOHO_CLIENT_ID="1000.EX179O66BC7FKAWMR17Q4V4ZV4OW9K" +ZOHO_CLIENT_SECRET="9c8e6414aebdbed66b5bc5f9c2a866fb4b4b159a93" +ZOHO_REFRESH_TOKEN="1000.58b1696404de858cd887a29005879ed5.487b427449d87b3fd0f1cb883f4d1b7d" +ZOHO_WEBHOOK_SECRET="0iWuLqg4z41RMWmgI6ql5qbPE3fCE4tc" +ZOHO_DOMAIN="in" \ No newline at end of file diff --git a/.env.test b/.env.test new file mode 100644 index 0000000..0bda686 --- /dev/null +++ b/.env.test @@ -0,0 +1,68 @@ +APP_ENV=test +HOST=0.0.0.0 +PORT=20002 +DEBUG=False +DB_HOST=106.51.105.22 +DB_PORT=5432 +DB_NAME=docqube_test +DB_USER=docqube_user +DB_PASSWORD=d20kMfypQE8Z +DB_SSLMODE=disable +PGBOUNCER_PORT=6432 +SAAS_TRUST_SECRET=docqube_trust_secret_2026 +SAAS_BASE_URL=http://localhost:8000 +SAAS_PUBLIC_KEY="" +APP_SECRET=VBN20DIyTbuqRJKKpxB2C4aDVYRUENa1rdzFXPLXwxA +ALGORITHM=HS256 +ACCESS_TOKEN_EXPIRE_MINUTES=60 +REDIS_HOST=redis +REDIS_PORT=6379 +REDIS_PASSWORD=JVCzB612F573hYwqZZ +CLAMAV_HOST=clamav +CLAMAV_PORT=3310 +CLAMAV_TIMEOUT_SECONDS=20 +CLAMAV_CHUNK_SIZE=1048576 +STORAGE_DRIVE_DIR=storage_drive +API_BASE_URL=https://docqubeapi-test.maskantech.in +GOOGLE_CLIENT_ID=989963514121-nh9ibk3e4u1dh6jp4ql17tue7f9suk38.apps.googleusercontent.com +GOOGLE_CLIENT_SECRET=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 +DEEPSEEK_API_KEY=sk-43e2e5e6d450447687b3b664f3a91f33 +DEEPSEEK_MODEL=deepseek-chat +DEEPSEEK_BASE_URL=https://api.deepseek.com/v1 +LLM_TEMPERATURE=0.35 +MAX_RESPONSE_TOKENS=1500 +HF_HUB_OFFLINE=0 +TRANSFORMERS_OFFLINE=0 +VECTOR_DB=qdrant +QDRANT_HOST=qdrant +QDRANT_PORT=6333 +QDRANT_COLLECTION=docqube_collection +RERANKER_MODEL=cross-encoder/ms-marco-MiniLM-L-12-v2 +EMBEDDING_MODEL=BAAI/bge-base-en-v1.5 +CHUNK_SIZE=850 +CHUNK_OVERLAP=180 +TOP_K_QA=3 +TOP_K_SUMMARY=4 +MAX_CONTEXT_CHARS=4000 +MIN_SIMILARITY_SCORE=0.18 +LOG_LEVEL=INFO +CORS_ORIGINS=https://docqube-test.maskantech.in,https://docqubeapp-test.maskantech.in,https://docqubeapi-test.maskantech.in +SMTP_HOST=smtp.gmail.com +SMTP_PORT=465 +SMTP_USER=info.maskantech@gmail.com +SMTP_PASSWORD=tuthpljtkrwchgxd +MAIL_FROM=info.maskantech@gmail.com +SMTP_SECURE=True +FRONTEND_URL=https://docqube-test.maskantech.in,https://docqubeapp-test.maskantech.in,https://docqubeapi-test.maskantech.in +CHAT_REDIS_MAX_MESSAGES=10 +CHAT_REDIS_TTL_SECONDS=86400 +CHAT_DAILY_CREDITS_LIMIT=1000 +DOCUSEAL_API_URL=https://api.docuseal.com +DOCUSEAL_API_KEY=prUReaHxSnChGoeeL7xnf5kxsXVLAzfCbn1RvbhnoMz +DOCUSEAL_ACCOUNT_EMAIL=info.maskantech+test@gmail.com +ZOHO_CLIENT_ID="1000.EX179O66BC7FKAWMR17Q4V4ZV4OW9K" +ZOHO_CLIENT_SECRET="9c8e6414aebdbed66b5bc5f9c2a866fb4b4b159a93" +ZOHO_REFRESH_TOKEN="1000.58b1696404de858cd887a29005879ed5.487b427449d87b3fd0f1cb883f4d1b7d" +ZOHO_WEBHOOK_SECRET="0iWuLqg4z41RMWmgI6ql5qbPE3fCE4tc" +ZOHO_DOMAIN="in" +MAX_ACTIVE_DEVICES=5 \ No newline at end of file diff --git a/.env.uat b/.env.uat new file mode 100644 index 0000000..ca5430d --- /dev/null +++ b/.env.uat @@ -0,0 +1,68 @@ +# Environment Configuration +APP_ENV=uat + +# Server Configuration +HOST=127.0.0.1 +PORT=20001 +DEBUG=True + +DB_HOST=106.51.105.22 +DB_PORT=5432 +DB_NAME=docqube_uat +DB_USER=docqube_user +DB_PASSWORD=d20kMfypQE8Z +DB_SSLMODE=disable +PGBOUNCER_PORT=6432 + +# Security Configuration +APP_SECRET=VBN20DIyTbuqRJKKpxB2C4aDVYRUENa1rdzFXPLXwxA +ALGORITHM=HS256 +ACCESS_TOKEN_EXPIRE_MINUTES=60 + +# Redis Configuration +REDIS_HOST=106.51.105.22 +REDIS_PORT=6391 +REDIS_PASSWORD=JVCzB612F573hYwqZZ + +CLAMAV_HOST=106.51.105.22 +CLAMAV_PORT=3310 +CLAMAV_TIMEOUT_SECONDS=20 +CLAMAV_CHUNK_SIZE=1048576 +STORAGE_DRIVE_DIR=storage_drive +API_BASE_URL=http://localhost:20001 + +# Google Auth Configuration +GOOGLE_CLIENT_ID=989963514121-nh9ibk3e4u1dh6jp4ql17tue7f9suk38.apps.googleusercontent.com +GOOGLE_CLIENT_SECRET=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 + +# AI & Models Configuration +OPENAI_API_KEY="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" +OPENAI_BASE_URL=https://integrate.api.nvidia.com/v1 +OPENAI_MODEL=mistralai/mistral-medium-3-instruct +OPENAI_VISION_MODEL=meta/llama-3.2-90b-vision-instruct +MODEL_SERVER_URL=http://localhost:9000 + +# Logging Configuration +LOG_LEVEL=INFO + +# CORS Configuration +CORS_ORIGINS=http://localhost:5173,http://127.0.0.1:5173 + +# Email Configuration +SMTP_HOST=smtp.hostinger.com +SMTP_PORT=465 +SMTP_USER=info@maskantech.in +SMTP_PASSWORD=Infomaskan@123 +MAIL_FROM=info@maskantech.in +SMTP_SECURE=True + +# Frontend Configuration +FRONTEND_URL=http://localhost:5173 + +# whisper Model Configuration +WHISPER_MODEL=small + +# Chat Redis Settings +CHAT_REDIS_MAX_MESSAGES=10 +CHAT_REDIS_TTL_SECONDS=86400 +CHAT_DAILY_CREDITS_LIMIT=1000 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..67179d7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,79 @@ +# Python bytecode and caches +__pycache__/ +*.py[cod] +*$py.class +*.so + +# Virtual environments +.venv/ +venv/ +env/ +ENV/ +env1/ + +# Environment files and local secrets +!.env.example + +# Test, coverage, and static-analysis output +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.pytype/ +.pyre/ +.coverage +.coverage.* +coverage.xml +htmlcov/ +tests/ + +# Build and packaging output +build/ +dist/ +*.egg-info/ +.eggs/ +pip-wheel-metadata/ + +# Application runtime data +app/temp_uploads/* +storage_drive/ +storage/ +uploads/ +downloads/ +temp/ +tmp/ +logs/ +*.log + +# Local ML and Hugging Face caches +.cache/ +cache/ +model_cache/ +hub/ +!app/modules/**/models/ +!app/**/models/ + +# Local Docker/service data +.docker/ +redis_data/ +qdrant_data/ +clamav_data/ + +# IDE and editor files +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS files +.DS_Store +Thumbs.db +Desktop.ini +# Local rework environment — never commit +.env.rework +.env.rework_test +app/temp_uploads/* +!app/temp_uploads/.gitkeep +.env.rework_rls +.gitea/workflows/ +.github/workflows/ \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..81ce884 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,58 @@ +# B4.3 — the fast gates, before the commit rather than after the push. +# +# Only checks that run in about a second belong here. The characterization +# suite, the cross-tenant probes and the query budgets need a database and take +# a minute; they stay in CI. A pre-commit hook that takes a minute gets +# `--no-verify`d, and then none of it runs. +# +# pip install pre-commit && pre-commit install +# +# Everything here is also enforced in CI, so an uninstalled hook is a slower +# feedback loop rather than a hole. + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-json + - id: check-toml + - id: check-merge-conflict + - id: check-added-large-files + args: [--maxkb=512] + - id: detect-private-key + + - repo: local + hooks: + # The three rules that are cheap to check and expensive to discover late. + - id: no-exception-text-in-responses + name: No exception text in HTTP responses + entry: python -m pytest tests/probes/test_error_disclosure.py::test_no_exception_text_is_returned_to_callers -q + language: system + pass_filenames: false + files: ^app/.*\.py$ + description: > + `detail=f"...{e}"` sends internal errors to the caller. It reached an + unauthenticated endpoint once already. + + - id: transaction-discipline + name: Route handlers do not commit + entry: python -m pytest tests/probes/test_transaction_discipline.py -q + language: system + pass_filenames: false + files: ^app/.*\.py$ + description: > + The request owns the transaction. A handler that commits splits one + request into several and leaves partial state behind on failure. + + - id: layering + name: Import layering contracts + entry: lint-imports + language: system + pass_filenames: false + files: ^app/.*\.py$ + description: > + routes → controllers → services → repositories → models, one + direction only. diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..e028118 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.11.8 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8b0ce6e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,80 @@ +ARG PYTHON_VERSION=3.11.8 + +FROM python:${PYTHON_VERSION}-slim-bookworm AS builder + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + UV_COMPILE_BYTECODE=1 \ + UV_LINK_MODE=copy \ + UV_PYTHON_DOWNLOADS=0 + +COPY --from=ghcr.io/astral-sh/uv:0.12.8 /uv /uvx /bin/ + +WORKDIR /app + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + pkg-config \ + libcairo2-dev \ + libpango1.0-dev \ + libgdk-pixbuf-2.0-dev \ + libffi-dev \ + libglib2.0-dev \ + libjpeg-dev \ + libopenjp2-7-dev \ + && rm -rf /var/lib/apt/lists/* + +COPY pyproject.toml uv.lock .python-version ./ + +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync \ + --locked \ + --no-install-project \ + --no-editable + +FROM python:${PYTHON_VERSION}-slim-bookworm AS runtime + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app \ + PATH="/app/.venv/bin:$PATH" + +WORKDIR /app + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + libmagic1 \ + poppler-utils \ + libcairo2 \ + libpango-1.0-0 \ + libpangocairo-1.0-0 \ + libgdk-pixbuf-2.0-0 \ + libffi8 \ + libglib2.0-0 \ + libgl1 \ + libjpeg62-turbo \ + libopenjp2-7 \ + fonts-dejavu-core \ + fonts-liberation \ + shared-mime-info \ + && rm -rf /var/lib/apt/lists/* + +RUN addgroup --system app && \ + adduser --system \ + --ingroup app \ + --home /home/app \ + app + +COPY --from=builder --chown=app:app /app/.venv /app/.venv + +COPY --chown=app:app app ./app +COPY --chown=app:app manage.py . +COPY --chown=app:app alembic ./alembic +COPY --chown=app:app alembic.ini . +COPY --chown=app:app scripts ./scripts + +USER app + +CMD ["python", "manage.py", "run"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..b523c1f --- /dev/null +++ b/README.md @@ -0,0 +1,426 @@ +# Docqube Backend + +**FastAPI backend application for Docqube** + +A modern, container-ready backend built with FastAPI, managed by **uv**, and orchestrated with **Docker Compose**. + +--- + +## Table of Contents + +- [Requirements](#requirements) +- [Python Version](#python-version) +- [Installing uv](#installing-uv) +- [Getting Started](#getting-started) +- [Local Development](#local-development) +- [Dependency Management](#dependency-management) +- [Docker Compose](#docker-compose) +- [Running Celery Worker](#running-celery-worker) +- [Useful uv Commands](#useful-uv-commands) +- [Development Workflow](#development-workflow) +- [Source of Truth](#source-of-truth) + +--- + +## Requirements + +| Tool | Version / Notes | +|-------------------|--------------------------| +| **Python** | `3.11.8` (strict) | +| **uv** | Latest | +| **Docker** | Docker Desktop / Engine | +| **Docker Compose**| v2+ | + +--- + +## Python Version + +This project is **locked to Python 3.11**. + +```text +Python 3.11.8 +``` + +| File | Setting | +|-------------------|----------------------------------| +| `pyproject.toml` | `requires-python = ">=3.11,<3.12"` | +| `.python-version` | `3.11.8` | + +**Verify:** + +```bash +uv run python --version +``` + +> **Expected:** `Python 3.11.8` +> Do **not** use Python 3.12+ unless compatibility has been fully verified and the project configuration intentionally updated. + +--- + +## Installing uv + +### Windows + +```powershell +powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex" +``` + +Close and reopen PowerShell, then verify: + +```powershell +uv --version +``` + +### macOS / Linux + +```bash +curl -LsSf https://astral.sh/uv/install.sh | sh +``` + +Restart the terminal and verify: + +```bash +uv --version +``` + +--- + +## Getting Started + +```bash +# 1. Clone the repository +git clone +cd docqube_backend + +# 2. Install all dependencies +uv sync +``` + +`uv sync` will: + +- Create `.venv` automatically +- Use Python 3.11 +- Resolve dependencies from `pyproject.toml` +- Install exact versions from `uv.lock` +- Install Windows-specific packages when on Windows + +> You do **not** need to manually create or activate a virtual environment. + +--- + +## Local Development + +### Install dependencies + +```bash +uv sync +``` + +### Run the application + +```bash +uv run python manage.py run --env local +``` + +The application starts with the local environment configuration +(example: `http://127.0.0.1:20001`). +Host and port are controlled by the application settings. + +--- + +## Database Migrations & Seeding + +All database migrations and seeds are executed through `uv`: + +### 1. Apply Migrations (By Environment) + +**Local Environment:** +```bash +uv run python manage.py migrate --env local +``` + +**Test Environment:** +```bash +uv run python manage.py migrate --env test +``` + +**Production Environment:** +```bash +uv run python manage.py migrate --env production +``` + +*Or directly via Alembic (uses current environment variables):* +```bash +uv run alembic upgrade head +``` + +### 2. Create a New Migration (Autogenerate from Models) + +```bash +uv run alembic revision --autogenerate -m "your_migration_name" +``` + +### 3. Rollback the Last Migration + +```bash +uv run alembic downgrade -1 +``` + +### 4. Seed Database (By Environment) + +**Local:** +```bash +uv run python manage.py seed --env local +``` + +**Production:** +```bash +uv run python manage.py seed --env production +``` + +--- + +## Dependency Management + +All Python dependencies are managed exclusively through: + +- `pyproject.toml` +- `uv.lock` + +> **Do not** maintain a separate `requirements.txt`. + +### Add a production dependency + +```bash +uv add +``` + +Examples: + +```bash +uv add httpx +uv add "httpx==0.28.1" +uv add "pikepdf>=8.0.0,<11.0.0" +``` + +### Add a development dependency + +```bash +uv add --dev pytest +uv add --dev ruff +uv sync +``` + +### Remove a dependency + +```bash +uv remove +uv sync +``` + +### Upgrade dependencies + +**Single package:** + +```bash +uv lock --upgrade-package +uv sync +``` + +Example: + +```bash +uv lock --upgrade-package fastapi +uv sync +``` + +**Specific version:** + +```bash +uv add "fastapi==0.116.1" +``` + +**All packages** (use only for deliberate maintenance): + +```bash +uv lock --upgrade +uv sync +``` + +### Lockfile + +`uv.lock` **must** be committed to Git. +It guarantees identical dependency resolution across developer machines, CI, Docker, staging, and production. + +> **Never** edit `uv.lock` manually. + +### Inspection commands + +```bash +# Full dependency tree +uv tree + +# Find a package (Windows PowerShell) +uv tree | Select-String pikepdf + +# Find a package (Linux / macOS) +uv tree | grep pikepdf + +# Inspect an installed package +uv pip show pikepdf +uv pip show torch +``` + +--- + +## Docker Compose + +### Development environment + +```bash +# Start (build + run) +docker compose -p docqube-development -f docker-compose.development.yml up -d --build + +# Follow logs +docker compose -p docqube-development -f docker-compose.development.yml logs -f + +# Stop +docker compose -p docqube-development -f docker-compose.development.yml down + +# Normal rebuild +docker compose -p docqube-development -f docker-compose.development.yml up -d --build + +# Completely fresh build (no cache) +docker compose -p docqube-development -f docker-compose.development.yml build --no-cache +docker compose -p docqube-development -f docker-compose.development.yml up -d +``` + +--- + +## Running Celery Worker + +### Locally + +```bash +uv run celery -A app.tasks.celery_app:celery_app worker --loglevel=info +``` + +> The worker requires the correct Redis and environment configuration. + +### Recreate the virtual environment + +If `.venv` becomes corrupted: + +**Windows (PowerShell)** + +```powershell +Remove-Item -Recurse -Force .venv +uv sync +``` + +**Linux / macOS** + +```bash +rm -rf .venv +uv sync +``` + +**Force reinstall** + +```bash +uv sync --reinstall +``` + +--- + +## Useful uv Commands + +| Command | Description | +|----------------------------------------------|-------------------------------------------| +| `uv --version` | Check uv version | +| `uv run python --version` | Check Python version | +| `uv python find` | Locate the Python interpreter | +| `uv sync` | Install / synchronize dependencies | +| `uv add ` | Add a production dependency | +| `uv add --dev ` | Add a development dependency | +| `uv remove ` | Remove a dependency | +| `uv lock` | Update the lockfile | +| `uv lock --upgrade-package ` | Upgrade one package | +| `uv lock --upgrade` | Upgrade all packages | +| `uv tree` | View dependency tree | +| `uv pip show ` | Inspect an installed package | +| `uv run python` | Run Python inside the project environment | +| `uv run python manage.py run --env local` | Run the application | + +--- + +## Development Workflow + +### New developer quick start + +1. Clone the repository +2. Install `uv` +3. Run `uv sync` +4. Create the required environment file (e.g. `.env.development`) +5. Verify Python → `uv run python --version` → `Python 3.11.8` +6. Run locally **or** start the full stack with Docker Compose +7. Check services and view logs + +### Day-to-day workflow + +```bash +# Normal local development +uv sync +uv run python manage.py run --env local + +# Add a package +uv add + +# Remove a package +uv remove + +# Upgrade one package +uv lock --upgrade-package +uv sync +``` + +### Before creating a pull request + +```bash +uv sync +uv run python --version +uv run pytest +docker compose -p docqube-development -f docker-compose.development.yml build +docker compose -p docqube-development -f docker-compose.development.yml up -d +``` + +### Important rules + +**Do** + +- Use `uv sync`, `uv add`, `uv remove`, `uv run` +- Commit `pyproject.toml`, `uv.lock`, and `.python-version` +- Use Docker Compose for the full containerized environment + +**Do not** + +- Use `pip install ...` +- Manually create `env/` or `venv/` +- Manually edit `uv.lock` +- Commit `.venv/` or real `.env` files +- Maintain a manually synchronized `requirements.txt` + +--- + +## Source of Truth + +| Concern | Source of Truth | +|--------------------------------|----------------------------| +| Python version | `.python-version` | +| Python dependencies | `pyproject.toml` | +| Locked dependencies | `uv.lock` | +| Python / Docker environment | `Dockerfile` | +| Environment orchestration | `docker-compose.*.yml` | +| Environment configuration | `.env.*` | + + diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..0e0fb5b --- /dev/null +++ b/alembic.ini @@ -0,0 +1,116 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = alembic + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python-dateutil library that can be +# installed by adding `alembic[tz]` to the pip requirements +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = postgresql+psycopg2://dummy + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the exec runner, execute a binary +# hooks = ruff +# ruff.type = exec +# ruff.executable = %(here)s/.venv/bin/ruff +# ruff.options = --fix REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/README b/alembic/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..a918a1b --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,43 @@ +import sys +import os +from logging.config import fileConfig +from sqlalchemy import engine_from_config, pool +from alembic import context + +# Add project root to path +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from app.core.settings import DATABASE_URL +from app.db.database import Base + +# Import all models here so Alembic can detect them. +# +# `app.db.all_models` is the one that matters. Importing `app.main` reaches +# most models by accident — routes import what they use — but it missed +# `user_roles`, `access_groups` and `user_access_groups`, whose every import is +# function-local. Autogenerate reported all three as removed tables and offered +# to drop the SBAC schema. See tests/probes/test_model_registry.py. +import app.db.all_models # noqa: F401 +import app.main +from app.modules.tenant.models.tenant_contact_model import TenantContact + +target_metadata = Base.metadata + +config = context.config +config.set_main_option("sqlalchemy.url", DATABASE_URL.replace('%', '%%')) + +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +def run_migrations_online(): + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + +run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..fbc4b07 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/06001599321d_add_collaboration_chat.py b/alembic/versions/06001599321d_add_collaboration_chat.py new file mode 100644 index 0000000..b0202ae --- /dev/null +++ b/alembic/versions/06001599321d_add_collaboration_chat.py @@ -0,0 +1,26 @@ +"""add collaboration chat + +Revision ID: 06001599321d +Revises: 22dee9a1d631 +Create Date: 2026-02-26 16:13:54.998247 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '06001599321d' +down_revision: Union[str, None] = '22dee9a1d631' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/alembic/versions/0bffea34ee5f_disable_rls_temporarily.py b/alembic/versions/0bffea34ee5f_disable_rls_temporarily.py new file mode 100644 index 0000000..d656678 --- /dev/null +++ b/alembic/versions/0bffea34ee5f_disable_rls_temporarily.py @@ -0,0 +1,35 @@ +"""disable_rls_temporarily + +Revision ID: 0bffea34ee5f +Revises: 14b4aa5f0f5d +Create Date: 2026-09-05 12:01:30.634229 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '0bffea34ee5f' +down_revision: Union[str, None] = '14b4aa5f0f5d' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +RLS_TABLES = [ + "user_org_units", "org_unit_leads", "user_roles", "access_groups", "user_access_groups", + "org_units", "tenant_subscriptions", "tenant_limit_overrides", "chatbot_documents", + "drive_activities", "drive_files", "drive_folders", "notifications", "tenant_smtp_configs", + "user_files", "projects", "roles", "activity_logs", "tenant_storage_configs", + "chat_sessions", "signing_requests", "tenant_signature_configs", "users" +] + +def upgrade() -> None: + for table in RLS_TABLES: + op.execute(f"ALTER TABLE {table} DISABLE ROW LEVEL SECURITY") + +def downgrade() -> None: + for table in RLS_TABLES: + op.execute(f"ALTER TABLE {table} ENABLE ROW LEVEL SECURITY") diff --git a/alembic/versions/0e1a5f2cdea9_add_max_bytes_quota_to_user_storage_.py b/alembic/versions/0e1a5f2cdea9_add_max_bytes_quota_to_user_storage_.py new file mode 100644 index 0000000..f4b7857 --- /dev/null +++ b/alembic/versions/0e1a5f2cdea9_add_max_bytes_quota_to_user_storage_.py @@ -0,0 +1,38 @@ +"""Add max_bytes_quota to user_storage_usage + +Revision ID: 0e1a5f2cdea9 +Revises: 9a6db38b86a6 +Create Date: 2026-03-02 10:48:38.335359 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '0e1a5f2cdea9' +down_revision: Union[str, None] = '9a6db38b86a6' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('project_versions', sa.Column('drive_file_version_id', sa.Integer(), nullable=True)) + op.create_foreign_key(None, 'project_versions', 'drive_file_versions', ['drive_file_version_id'], ['id']) + op.add_column('user_files', sa.Column('is_deleted', sa.Boolean(), nullable=True)) + op.add_column('user_files', sa.Column('deleted_at', sa.TIMESTAMP(), nullable=True)) + op.add_column('user_storage_usage', sa.Column('max_bytes_quota', sa.BigInteger(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('user_storage_usage', 'max_bytes_quota') + op.drop_column('user_files', 'deleted_at') + op.drop_column('user_files', 'is_deleted') + op.drop_constraint(None, 'project_versions', type_='foreignkey') + op.drop_column('project_versions', 'drive_file_version_id') + # ### end Alembic commands ### diff --git a/alembic/versions/14b4aa5f0f5d_merge_multiple_heads.py b/alembic/versions/14b4aa5f0f5d_merge_multiple_heads.py new file mode 100644 index 0000000..fc6d6e1 --- /dev/null +++ b/alembic/versions/14b4aa5f0f5d_merge_multiple_heads.py @@ -0,0 +1,26 @@ +"""Merge multiple heads + +Revision ID: 14b4aa5f0f5d +Revises: e1_0_add_saas_plan_id, z005_add_virus_scan +Create Date: 2026-09-02 19:15:56.213495 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '14b4aa5f0f5d' +down_revision: Union[str, None] = ('e1_0_add_saas_plan_id', 'z005_add_virus_scan') +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/alembic/versions/15f935213b22_add_tenant_storage_settings.py b/alembic/versions/15f935213b22_add_tenant_storage_settings.py new file mode 100644 index 0000000..ea2fbb3 --- /dev/null +++ b/alembic/versions/15f935213b22_add_tenant_storage_settings.py @@ -0,0 +1,66 @@ +"""add tenant storage settings + +Revision ID: 15f935213b22 +Revises: k002_add_docs_tutorial +Create Date: 2026-04-17 15:49:28.721223 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '15f935213b22' +down_revision: Union[str, None] = 'k002_add_docs_tutorial' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _index_names(bind, table_name: str) -> set[str]: + inspector = sa.inspect(bind) + return {idx["name"] for idx in inspector.get_indexes(table_name)} + + +def upgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + + if not inspector.has_table('tenant_storage_configs'): + op.create_table('tenant_storage_configs', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('tenant_id', sa.UUID(), nullable=False), + sa.Column('b2_key_id', sa.String(length=255), nullable=False), + sa.Column('encrypted_b2_application_key', sa.String(length=512), nullable=False), + sa.Column('b2_quarantine_bucket', sa.String(length=255), nullable=False), + sa.Column('b2_clean_bucket', sa.String(length=255), nullable=False), + sa.Column('b2_endpoint', sa.String(length=512), nullable=True), + sa.Column('is_active', sa.Boolean(), nullable=False), + sa.Column('created_by_id', sa.Integer(), nullable=True), + sa.Column('updated_by_id', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['created_by_id'], ['users.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['updated_by_id'], ['users.id'], ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id') + ) + + existing_indexes = _index_names(bind, 'tenant_storage_configs') + if 'ix_tenant_storage_configs_tenant_id' not in existing_indexes: + with op.batch_alter_table('tenant_storage_configs', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_tenant_storage_configs_tenant_id'), ['tenant_id'], unique=True) + + +def downgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + + if inspector.has_table('tenant_storage_configs'): + existing_indexes = _index_names(bind, 'tenant_storage_configs') + if 'ix_tenant_storage_configs_tenant_id' in existing_indexes: + with op.batch_alter_table('tenant_storage_configs', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_tenant_storage_configs_tenant_id')) + + op.drop_table('tenant_storage_configs') diff --git a/alembic/versions/1cb413ffaaa0_add_saas_mapping_tables.py b/alembic/versions/1cb413ffaaa0_add_saas_mapping_tables.py new file mode 100644 index 0000000..a5db058 --- /dev/null +++ b/alembic/versions/1cb413ffaaa0_add_saas_mapping_tables.py @@ -0,0 +1,59 @@ +"""Add SaaS mapping tables + +Revision ID: 1cb413ffaaa0 +Revises: n002_add_signature_imprints +Create Date: 2026-04-23 23:16:32.782986 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '1cb413ffaaa0' +down_revision: Union[str, None] = 'n002_add_signature_imprints' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # --- SaaSTenantMapping --- + op.create_table( + 'saas_tenant_mappings', + sa.Column('id', sa.dialects.postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('saas_tenant_id', sa.String(), nullable=False), + sa.Column('docqube_tenant_id', sa.dialects.postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('metadata', sa.JSON(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), + sa.ForeignKeyConstraint(['docqube_tenant_id'], ['tenants.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_saas_tenant_mappings_id'), 'saas_tenant_mappings', ['id'], unique=False) + op.create_index(op.f('ix_saas_tenant_mappings_saas_tenant_id'), 'saas_tenant_mappings', ['saas_tenant_id'], unique=True) + + # --- SaaSUserMapping --- + op.create_table( + 'saas_user_mappings', + sa.Column('id', sa.dialects.postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('saas_user_id', sa.String(), nullable=False), + sa.Column('docqube_user_id', sa.Integer(), nullable=False), + sa.Column('metadata', sa.JSON(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), + sa.ForeignKeyConstraint(['docqube_user_id'], ['users.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_saas_user_mappings_id'), 'saas_user_mappings', ['id'], unique=False) + op.create_index(op.f('ix_saas_user_mappings_saas_user_id'), 'saas_user_mappings', ['saas_user_id'], unique=True) + + +def downgrade() -> None: + op.drop_index(op.f('ix_saas_user_mappings_saas_user_id'), table_name='saas_user_mappings') + op.drop_index(op.f('ix_saas_user_mappings_id'), table_name='saas_user_mappings') + op.drop_table('saas_user_mappings') + op.drop_index(op.f('ix_saas_tenant_mappings_saas_tenant_id'), table_name='saas_tenant_mappings') + op.drop_index(op.f('ix_saas_tenant_mappings_id'), table_name='saas_tenant_mappings') + op.drop_table('saas_tenant_mappings') diff --git a/alembic/versions/1f4d2a8c9b77_add_missing_vector_columns.py b/alembic/versions/1f4d2a8c9b77_add_missing_vector_columns.py new file mode 100644 index 0000000..ce4314e --- /dev/null +++ b/alembic/versions/1f4d2a8c9b77_add_missing_vector_columns.py @@ -0,0 +1,67 @@ +"""add missing vector columns + +Revision ID: 1f4d2a8c9b77 +Revises: fb6810c36ffb +Create Date: 2026-04-08 22:05:00 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "1f4d2a8c9b77" +down_revision: Union[str, None] = "fb6810c36ffb" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _table_exists(bind, table_name: str) -> bool: + return sa.inspect(bind).has_table(table_name) + + +def _column_exists(bind, table_name: str, column_name: str) -> bool: + inspector = sa.inspect(bind) + return column_name in {col["name"] for col in inspector.get_columns(table_name)} + + +def upgrade() -> None: + bind = op.get_bind() + + if _table_exists(bind, "vector_indices"): + if not _column_exists(bind, "vector_indices", "index_name"): + op.add_column( + "vector_indices", sa.Column("index_name", sa.String(length=255), nullable=True) + ) + if not _column_exists(bind, "vector_indices", "checksum"): + op.add_column( + "vector_indices", sa.Column("checksum", sa.String(length=255), nullable=True) + ) + if not _column_exists(bind, "vector_indices", "metadata_info"): + op.add_column( + "vector_indices", sa.Column("metadata_info", sa.JSON(), nullable=True) + ) + + if _table_exists(bind, "vector_chunks"): + if not _column_exists(bind, "vector_chunks", "embedding_json"): + op.add_column( + "vector_chunks", sa.Column("embedding_json", sa.JSON(), nullable=True) + ) + + +def downgrade() -> None: + bind = op.get_bind() + + if _table_exists(bind, "vector_chunks"): + if _column_exists(bind, "vector_chunks", "embedding_json"): + op.drop_column("vector_chunks", "embedding_json") + + if _table_exists(bind, "vector_indices"): + if _column_exists(bind, "vector_indices", "metadata_info"): + op.drop_column("vector_indices", "metadata_info") + if _column_exists(bind, "vector_indices", "checksum"): + op.drop_column("vector_indices", "checksum") + if _column_exists(bind, "vector_indices", "index_name"): + op.drop_column("vector_indices", "index_name") diff --git a/alembic/versions/229359b0d1b2_add_terms_accepted.py b/alembic/versions/229359b0d1b2_add_terms_accepted.py new file mode 100644 index 0000000..9dfe144 --- /dev/null +++ b/alembic/versions/229359b0d1b2_add_terms_accepted.py @@ -0,0 +1,64 @@ +"""add_terms_accepted + +Revision ID: 229359b0d1b2 +Revises: 823a486fd287 +Create Date: 2026-04-01 11:28:44.953518 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '229359b0d1b2' +down_revision: Union[str, None] = '823a486fd287' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('drive_comments', schema=None) as batch_op: + batch_op.alter_column('rejected_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + type_=sa.DateTime(), + existing_nullable=True) + + with op.batch_alter_table('drive_stars', schema=None) as batch_op: + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + type_=sa.TIMESTAMP(), + nullable=True, + existing_server_default=sa.text('now()')) + + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.add_column(sa.Column('terms_accepted', sa.Boolean(), nullable=False, server_default=sa.text('false'))) + batch_op.drop_index(batch_op.f('ix_users_email')) + batch_op.create_index(batch_op.f('ix_users_email'), ['email'], unique=False) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_users_email')) + batch_op.create_index(batch_op.f('ix_users_email'), ['email'], unique=True) + batch_op.drop_column('terms_accepted') + + with op.batch_alter_table('drive_stars', schema=None) as batch_op: + batch_op.alter_column('created_at', + existing_type=sa.TIMESTAMP(), + type_=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + + with op.batch_alter_table('drive_comments', schema=None) as batch_op: + batch_op.alter_column('rejected_at', + existing_type=sa.DateTime(), + type_=postgresql.TIMESTAMP(timezone=True), + existing_nullable=True) + + # ### end Alembic commands ### diff --git a/alembic/versions/22dee9a1d631_add_shared_by_user_id_to_projectshare.py b/alembic/versions/22dee9a1d631_add_shared_by_user_id_to_projectshare.py new file mode 100644 index 0000000..6ccef18 --- /dev/null +++ b/alembic/versions/22dee9a1d631_add_shared_by_user_id_to_projectshare.py @@ -0,0 +1,32 @@ +"""Add shared_by_user_id to ProjectShare + +Revision ID: 22dee9a1d631 +Revises: dbaa8ef522c0 +Create Date: 2026-02-26 15:33:00.617693 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '22dee9a1d631' +down_revision: Union[str, None] = 'dbaa8ef522c0' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('project_shares', sa.Column('shared_by_user_id', sa.Integer(), nullable=True)) + op.create_foreign_key(None, 'project_shares', 'users', ['shared_by_user_id'], ['id'], ondelete='CASCADE') + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, 'project_shares', type_='foreignkey') + op.drop_column('project_shares', 'shared_by_user_id') + # ### end Alembic commands ### diff --git a/alembic/versions/29c02cb50737_initial_migration.py b/alembic/versions/29c02cb50737_initial_migration.py new file mode 100644 index 0000000..782db29 --- /dev/null +++ b/alembic/versions/29c02cb50737_initial_migration.py @@ -0,0 +1,258 @@ +"""initial migration + +Revision ID: 29c02cb50737 +Revises: +Create Date: 2026-02-16 19:28:02.116622 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '29c02cb50737' +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('drive_acl', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('resource_type', sa.String(length=10), nullable=False), + sa.Column('resource_id', sa.Integer(), nullable=False), + sa.Column('subject_type', sa.String(length=10), nullable=False), + sa.Column('subject_id', sa.Integer(), nullable=True), + sa.Column('role', sa.String(length=20), nullable=False), + sa.Column('inherited_from', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('users', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('email', sa.String(length=255), nullable=False), + sa.Column('password_hash', sa.String(length=255), nullable=False), + sa.Column('subscription', sa.String(length=50), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_users_email'), 'users', ['email'], unique=True) + op.create_table('drive_activities', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('resource_type', sa.String(length=10), nullable=False), + sa.Column('resource_id', sa.Integer(), nullable=False), + sa.Column('actor_id', sa.Integer(), nullable=False), + sa.Column('activity_type', sa.String(length=50), nullable=False), + sa.Column('activity_metadata', sa.Text(), nullable=True), + sa.Column('ip_address', sa.String(length=64), nullable=True), + sa.Column('device_info', sa.Text(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['actor_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('drive_folders', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('owner_id', sa.Integer(), nullable=False), + sa.Column('parent_id', sa.Integer(), nullable=True), + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('color', sa.String(length=20), nullable=True), + sa.Column('is_starred', sa.Boolean(), nullable=False), + sa.Column('is_trashed', sa.Boolean(), nullable=False), + sa.Column('trashed_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['owner_id'], ['users.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['parent_id'], ['drive_folders.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('drive_shares', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('resource_type', sa.String(length=10), nullable=False), + sa.Column('resource_id', sa.Integer(), nullable=False), + sa.Column('invited_email', sa.String(length=255), nullable=True), + sa.Column('link_token', sa.String(length=255), nullable=True), + sa.Column('role', sa.String(length=20), nullable=False), + sa.Column('expires_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('created_by_id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['created_by_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('link_token') + ) + op.create_table('drive_stars', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('resource_type', sa.String(length=10), nullable=False), + sa.Column('resource_id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('resource_type', 'resource_id', 'user_id', name='uq_drive_star') + ) + op.create_table('projects', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('status', sa.String(length=50), nullable=False), + sa.Column('file_type', sa.String(length=50), nullable=False), + sa.Column('output_format', sa.String(length=50), nullable=False), + sa.Column('progress', sa.Integer(), nullable=False), + sa.Column('current_step', sa.String(length=50), nullable=False), + sa.Column('session_id', sa.String(length=255), nullable=True), + sa.Column('html_content', sa.Text(), nullable=True), + sa.Column('markdown_content', sa.Text(), nullable=True), + sa.Column('xml_content', sa.Text(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('user_files', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('file_name', sa.String(length=512), nullable=False), + sa.Column('s3_key', sa.String(length=1024), nullable=False), + sa.Column('file_size', sa.BigInteger(), nullable=False), + sa.Column('mime_type', sa.String(length=255), nullable=True), + sa.Column('is_deleted', sa.Boolean(), nullable=False), + sa.Column('deleted_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('s3_key') + ) + op.create_table('user_storage_usage', + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('total_bytes', sa.BigInteger(), nullable=False), + sa.Column('file_count', sa.Integer(), nullable=False), + sa.Column('last_updated', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('user_id') + ) + op.create_table('drive_files', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('folder_id', sa.Integer(), nullable=False), + sa.Column('owner_id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=512), nullable=False), + sa.Column('mime_type', sa.String(length=255), nullable=True), + sa.Column('size', sa.BigInteger(), nullable=False), + sa.Column('hash', sa.String(length=128), nullable=True), + sa.Column('latest_version_id', sa.Integer(), nullable=True), + sa.Column('is_starred', sa.Boolean(), nullable=False), + sa.Column('is_trashed', sa.Boolean(), nullable=False), + sa.Column('trashed_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['folder_id'], ['drive_folders.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['owner_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('project_activities', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('project_id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('activity_type', sa.String(length=50), nullable=False), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('activity_metadata', sa.Text(), nullable=True), + sa.Column('ip_address', sa.String(length=64), nullable=True), + sa.Column('device_info', sa.Text(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['project_id'], ['projects.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('project_comments', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('project_id', sa.Integer(), nullable=False), + sa.Column('author_id', sa.Integer(), nullable=False), + sa.Column('body', sa.Text(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('resolved_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('anchor', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.ForeignKeyConstraint(['author_id'], ['users.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['project_id'], ['projects.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('project_shares', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('project_id', sa.Integer(), nullable=False), + sa.Column('shared_with_user_id', sa.Integer(), nullable=False), + sa.Column('shared_by_user_id', sa.Integer(), nullable=False), + sa.Column('permission', sa.String(length=50), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['project_id'], ['projects.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['shared_by_user_id'], ['users.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['shared_with_user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('project_id', 'shared_with_user_id', name='unique_project_user_share') + ) + op.create_table('project_versions', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('project_id', sa.Integer(), nullable=False), + sa.Column('created_by_id', sa.Integer(), nullable=False), + sa.Column('version_number', sa.Integer(), nullable=False), + sa.Column('changes_description', sa.String(length=500), nullable=True), + sa.Column('markdown_content', sa.Text(), nullable=True), + sa.Column('xml_content', sa.Text(), nullable=True), + sa.Column('html_content', sa.Text(), nullable=True), + sa.Column('session_id', sa.String(length=255), nullable=True), + sa.Column('is_current', sa.Boolean(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['created_by_id'], ['users.id'], ), + sa.ForeignKeyConstraint(['project_id'], ['projects.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('drive_comments', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('file_id', sa.Integer(), nullable=False), + sa.Column('author_id', sa.Integer(), nullable=False), + sa.Column('body', sa.Text(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('resolved_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('anchor', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.ForeignKeyConstraint(['author_id'], ['users.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['file_id'], ['drive_files.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('drive_file_versions', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('file_id', sa.Integer(), nullable=False), + sa.Column('version_number', sa.Integer(), nullable=False), + sa.Column('storage_key', sa.Text(), nullable=False), + sa.Column('mime_type', sa.String(length=255), nullable=True), + sa.Column('size', sa.BigInteger(), nullable=False), + sa.Column('checksum', sa.String(length=128), nullable=True), + sa.Column('created_by_id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['created_by_id'], ['users.id'], ), + sa.ForeignKeyConstraint(['file_id'], ['drive_files.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('drive_file_versions') + op.drop_table('drive_comments') + op.drop_table('project_versions') + op.drop_table('project_shares') + op.drop_table('project_comments') + op.drop_table('project_activities') + op.drop_table('drive_files') + op.drop_table('user_storage_usage') + op.drop_table('user_files') + op.drop_table('projects') + op.drop_table('drive_stars') + op.drop_table('drive_shares') + op.drop_table('drive_folders') + op.drop_table('drive_activities') + op.drop_index(op.f('ix_users_email'), table_name='users') + op.drop_table('users') + op.drop_table('drive_acl') + # ### end Alembic commands ### diff --git a/alembic/versions/2b4c62c1cf35_add_subscription_fields_and_.py b/alembic/versions/2b4c62c1cf35_add_subscription_fields_and_.py new file mode 100644 index 0000000..7cf296a --- /dev/null +++ b/alembic/versions/2b4c62c1cf35_add_subscription_fields_and_.py @@ -0,0 +1,51 @@ +"""Add subscription fields and PlanPriceHistory + +Revision ID: 2b4c62c1cf35 +Revises: d2_0_credit_limit_precision +Create Date: 2026-08-19 18:04:44.734002 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '2b4c62c1cf35' +down_revision: Union[str, None] = 'd2_0_credit_limit_precision' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('plan_price_history', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('plan_id', sa.UUID(), nullable=False), + sa.Column('old_price', sa.Numeric(precision=10, scale=2), nullable=True), + sa.Column('new_price', sa.Numeric(precision=10, scale=2), nullable=True), + sa.Column('currency', sa.String(length=10), nullable=True), + sa.Column('changed_by', sa.Integer(), nullable=True), + sa.Column('changed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['plan_id'], ['plans.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_plan_price_history_plan_id'), 'plan_price_history', ['plan_id'], unique=False) + + op.add_column('plans', sa.Column('price_amount', sa.Numeric(precision=10, scale=2), nullable=True)) + op.add_column('plans', sa.Column('interval', sa.String(length=20), nullable=True)) + op.add_column('plans', sa.Column('grace_period_days', sa.Integer(), server_default='0', nullable=False)) + op.add_column('plans', sa.Column('notify_days_before_expiry', sa.Integer(), server_default='7', nullable=False)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('plans', 'notify_days_before_expiry') + op.drop_column('plans', 'grace_period_days') + op.drop_column('plans', 'interval') + op.drop_column('plans', 'price_amount') + op.drop_index(op.f('ix_plan_price_history_plan_id'), table_name='plan_price_history') + op.drop_table('plan_price_history') + # ### end Alembic commands ### diff --git a/alembic/versions/39d5fdfb5386_add_signature_id_to_signature_imprints.py b/alembic/versions/39d5fdfb5386_add_signature_id_to_signature_imprints.py new file mode 100644 index 0000000..d293a28 --- /dev/null +++ b/alembic/versions/39d5fdfb5386_add_signature_id_to_signature_imprints.py @@ -0,0 +1,128 @@ +"""add signature_id to signature_imprints + +Revision ID: 39d5fdfb5386 +Revises: 1cb413ffaaa0 +Create Date: 2026-04-25 15:33:32.819817 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '39d5fdfb5386' +down_revision: Union[str, None] = '1cb413ffaaa0' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Legacy tables/indexes — drop only if present (some envs already lack them). + op.execute("DROP TABLE IF EXISTS vector_chunks CASCADE") + op.execute("DROP TABLE IF EXISTS vector_indices CASCADE") + op.execute("DROP TABLE IF EXISTS chat_token_usage CASCADE") + + op.execute("DROP INDEX IF EXISTS ix_chatbot_documents_drive_file_id") + op.execute("DROP INDEX IF EXISTS ix_chatbot_documents_id") + + with op.batch_alter_table('extraction_data', schema=None) as batch_op: + batch_op.alter_column('confidence_score', + existing_type=sa.DOUBLE_PRECISION(precision=53), + nullable=False) + batch_op.alter_column('extraction_status', + existing_type=sa.VARCHAR(length=50), + nullable=False) + op.execute("DROP INDEX IF EXISTS idx_extraction_data_document_id") + op.create_index(op.f('ix_extraction_data_document_id'), 'extraction_data', ['document_id'], unique=False) + + op.execute("DROP INDEX IF EXISTS ix_signature_imprints_file_page") + op.execute("DROP INDEX IF EXISTS ix_signature_imprints_signing_request") + with op.batch_alter_table('signature_imprints', schema=None) as batch_op: + batch_op.add_column(sa.Column('signature_id', sa.String(length=15), nullable=True)) + batch_op.create_index(batch_op.f('ix_signature_imprints_signing_request_id'), ['signing_request_id'], unique=False) + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('signature_imprints', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_signature_imprints_signing_request_id')) + batch_op.create_index(batch_op.f('ix_signature_imprints_signing_request'), ['signing_request_id'], unique=False) + batch_op.create_index(batch_op.f('ix_signature_imprints_file_page'), ['drive_file_id', 'page_number'], unique=False) + batch_op.drop_column('signature_id') + + with op.batch_alter_table('extraction_data', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_extraction_data_document_id')) + batch_op.create_index(batch_op.f('idx_extraction_data_document_id'), ['document_id'], unique=False) + batch_op.alter_column('extraction_status', + existing_type=sa.VARCHAR(length=50), + nullable=True) + batch_op.alter_column('confidence_score', + existing_type=sa.DOUBLE_PRECISION(precision=53), + nullable=True) + + with op.batch_alter_table('chatbot_documents', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_chatbot_documents_id'), ['id'], unique=False) + batch_op.create_index(batch_op.f('ix_chatbot_documents_drive_file_id'), ['drive_file_id'], unique=False) + + op.create_table('chat_token_usage', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('user_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('chat_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('document_id', sa.VARCHAR(length=255), autoincrement=False, nullable=True), + sa.Column('prompt_tokens', sa.BIGINT(), autoincrement=False, nullable=False), + sa.Column('completion_tokens', sa.BIGINT(), autoincrement=False, nullable=False), + sa.Column('total_tokens', sa.BIGINT(), autoincrement=False, nullable=False), + sa.Column('model_name', sa.VARCHAR(length=100), autoincrement=False, nullable=False), + sa.Column('cost_usd', sa.NUMERIC(precision=12, scale=8), autoincrement=False, nullable=False), + sa.Column('timestamp', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), autoincrement=False, nullable=False), + sa.Column('phase', sa.VARCHAR(length=50), autoincrement=False, nullable=True), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('chat_token_usage_user_id_fkey'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('chat_token_usage_pkey')) + ) + with op.batch_alter_table('chat_token_usage', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_chat_token_usage_user_time'), ['user_id', 'timestamp'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_token_usage_user_id'), ['user_id'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_token_usage_timestamp'), ['timestamp'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_token_usage_phase'), ['phase'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_token_usage_model_time'), ['model_name', 'timestamp'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_token_usage_model_name'), ['model_name'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_token_usage_id'), ['id'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_token_usage_document_id'), ['document_id'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_token_usage_chat_id'), ['chat_id'], unique=False) + + op.create_table('vector_indices', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('user_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('index_data', postgresql.BYTEA(), autoincrement=False, nullable=True), + sa.Column('dimension', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('total_vectors', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('updated_at', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False), + sa.Column('index_name', sa.VARCHAR(length=255), autoincrement=False, nullable=True), + sa.Column('checksum', sa.VARCHAR(length=255), autoincrement=False, nullable=True), + sa.Column('metadata_info', postgresql.JSON(astext_type=sa.Text()), autoincrement=False, nullable=True), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('vector_indices_user_id_fkey'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('vector_indices_pkey')) + ) + with op.batch_alter_table('vector_indices', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_vector_indices_user_id'), ['user_id'], unique=True) + batch_op.create_index(batch_op.f('ix_vector_indices_id'), ['id'], unique=False) + + op.create_table('vector_chunks', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('user_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('document_id', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('chunk_text', sa.TEXT(), autoincrement=False, nullable=False), + sa.Column('chunk_index', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False), + sa.Column('embedding_json', postgresql.JSON(astext_type=sa.Text()), autoincrement=False, nullable=True), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('vector_chunks_user_id_fkey'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('vector_chunks_pkey')) + ) + with op.batch_alter_table('vector_chunks', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_vector_chunks_user_id'), ['user_id'], unique=False) + batch_op.create_index(batch_op.f('ix_vector_chunks_id'), ['id'], unique=False) + batch_op.create_index(batch_op.f('ix_vector_chunks_document_id'), ['document_id'], unique=False) + + # ### end Alembic commands ### diff --git a/alembic/versions/3aa7d970b8c2_enforce_enum_constraint_on_driveacl_role.py b/alembic/versions/3aa7d970b8c2_enforce_enum_constraint_on_driveacl_role.py new file mode 100644 index 0000000..fd70e8a --- /dev/null +++ b/alembic/versions/3aa7d970b8c2_enforce_enum_constraint_on_driveacl_role.py @@ -0,0 +1,51 @@ +"""Enforce enum constraint on DriveACL role + +Revision ID: 3aa7d970b8c2 +Revises: 89508c176437 +Create Date: 2026-03-03 22:06:24.190823 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '3aa7d970b8c2' +down_revision: Union[str, None] = '89508c176437' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +# The four valid drive roles +VALID_ROLES = ('viewer', 'commenter', 'editor', 'owner') + + +def upgrade() -> None: + # 1. Create the PostgreSQL enum type (idempotent via IF NOT EXISTS) + op.execute("CREATE TYPE drive_role AS ENUM ('viewer', 'commenter', 'editor', 'owner')") + + # 2. Sanitise any NULL or invalid rows before adding NOT NULL constraint. + # Default unknown/null values to 'viewer' (least-privileged), then log them. + op.execute( + "UPDATE drive_acls SET role = 'viewer' " + "WHERE role IS NULL OR role NOT IN ('viewer', 'commenter', 'editor', 'owner')" + ) + + # 3. Alter the column: cast existing VARCHAR values to the new enum type + op.execute( + "ALTER TABLE drive_acls " + "ALTER COLUMN role TYPE drive_role USING role::drive_role, " + "ALTER COLUMN role SET NOT NULL" + ) + + +def downgrade() -> None: + # Revert column to plain VARCHAR(50), nullable + op.execute( + "ALTER TABLE drive_acls " + "ALTER COLUMN role TYPE VARCHAR(50) USING role::VARCHAR, " + "ALTER COLUMN role DROP NOT NULL" + ) + # Drop the enum type (only if no other table references it) + op.execute("DROP TYPE IF EXISTS drive_role") diff --git a/alembic/versions/41bed7eb6ed9_refactor_documents_and_add_missing_.py b/alembic/versions/41bed7eb6ed9_refactor_documents_and_add_missing_.py new file mode 100644 index 0000000..cf85d17 --- /dev/null +++ b/alembic/versions/41bed7eb6ed9_refactor_documents_and_add_missing_.py @@ -0,0 +1,100 @@ +"""Refactor documents and add missing fields + +Revision ID: 41bed7eb6ed9 +Revises: 06001599321d +Create Date: 2026-02-27 14:21:01.746676 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '41bed7eb6ed9' +down_revision: Union[str, None] = '06001599321d' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('project_activities') + op.execute("DROP INDEX IF EXISTS ix_collaboration_chat_resource_id") + op.execute("DROP INDEX IF EXISTS ix_collaboration_chat_resource_type") + op.execute("DROP TABLE IF EXISTS collaboration_chat CASCADE") + op.drop_table('project_comments') + op.drop_table('project_shares') + op.add_column('project_versions', sa.Column('changes_description', sa.Text(), nullable=True)) + op.add_column('project_versions', sa.Column('session_id', sa.String(length=255), nullable=True)) + op.drop_column('project_versions', 'description') + op.add_column('projects', sa.Column('drive_file_id', sa.Integer(), nullable=True)) + op.create_index('idx_project_session', 'projects', ['session_id'], unique=False) + op.create_index('idx_project_user_status', 'projects', ['user_id', 'status'], unique=False) + op.create_index(op.f('ix_projects_drive_file_id'), 'projects', ['drive_file_id'], unique=False) + op.create_foreign_key(None, 'projects', 'drive_files', ['drive_file_id'], ['id'], ondelete='CASCADE') + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, 'projects', type_='foreignkey') + op.drop_index(op.f('ix_projects_drive_file_id'), table_name='projects') + op.drop_index('idx_project_user_status', table_name='projects') + op.drop_index('idx_project_session', table_name='projects') + op.drop_column('projects', 'drive_file_id') + op.add_column('project_versions', sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True)) + op.drop_column('project_versions', 'session_id') + op.drop_column('project_versions', 'changes_description') + op.create_table('project_shares', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('project_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('shared_with_user_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('permission', sa.VARCHAR(length=20), autoincrement=False, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), autoincrement=False, nullable=False), + sa.Column('shared_by_user_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.ForeignKeyConstraint(['project_id'], ['projects.id'], name='project_shares_project_id_fkey', ondelete='CASCADE'), + sa.ForeignKeyConstraint(['shared_by_user_id'], ['users.id'], name='project_shares_shared_by_user_id_fkey', ondelete='CASCADE'), + sa.ForeignKeyConstraint(['shared_with_user_id'], ['users.id'], name='project_shares_shared_with_user_id_fkey', ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name='project_shares_pkey') + ) + op.create_table('project_comments', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('project_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('author_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('body', sa.TEXT(), autoincrement=False, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), autoincrement=False, nullable=False), + sa.Column('resolved_at', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=True), + sa.Column('anchor', sa.TEXT(), autoincrement=False, nullable=True), + sa.ForeignKeyConstraint(['author_id'], ['users.id'], name='project_comments_author_id_fkey', ondelete='CASCADE'), + sa.ForeignKeyConstraint(['project_id'], ['projects.id'], name='project_comments_project_id_fkey', ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name='project_comments_pkey') + ) + op.create_table('collaboration_chat', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('resource_type', sa.VARCHAR(length=20), autoincrement=False, nullable=False), + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('user_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('message', sa.TEXT(), autoincrement=False, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), autoincrement=False, nullable=True), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], name='collaboration_chat_user_id_fkey', ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name='collaboration_chat_pkey') + ) + op.create_index('ix_collaboration_chat_resource_type', 'collaboration_chat', ['resource_type'], unique=False) + op.create_index('ix_collaboration_chat_resource_id', 'collaboration_chat', ['resource_id'], unique=False) + op.create_table('project_activities', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('project_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('user_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('activity_type', sa.VARCHAR(length=50), autoincrement=False, nullable=False), + sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('activity_metadata', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('ip_address', sa.VARCHAR(length=45), autoincrement=False, nullable=True), + sa.Column('device_info', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint(['project_id'], ['projects.id'], name='project_activities_project_id_fkey', ondelete='CASCADE'), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], name='project_activities_user_id_fkey', ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name='project_activities_pkey') + ) + # ### end Alembic commands ### diff --git a/alembic/versions/45c94ee15294_limit_notification_message_to_2000_chars.py b/alembic/versions/45c94ee15294_limit_notification_message_to_2000_chars.py new file mode 100644 index 0000000..c78a1cb --- /dev/null +++ b/alembic/versions/45c94ee15294_limit_notification_message_to_2000_chars.py @@ -0,0 +1,36 @@ +"""Limit Notification message to 2000 chars + +Revision ID: 45c94ee15294 +Revises: d2e2c55290ee +Create Date: 2026-03-03 22:24:55.795394 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '45c94ee15294' +down_revision: Union[str, None] = 'd2e2c55290ee' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('notifications', 'message', + existing_type=sa.TEXT(), + type_=sa.String(length=2000), + existing_nullable=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('notifications', 'message', + existing_type=sa.String(length=2000), + type_=sa.TEXT(), + existing_nullable=False) + # ### end Alembic commands ### diff --git a/alembic/versions/49c19df2c1d4_sync_models.py b/alembic/versions/49c19df2c1d4_sync_models.py new file mode 100644 index 0000000..6889daa --- /dev/null +++ b/alembic/versions/49c19df2c1d4_sync_models.py @@ -0,0 +1,371 @@ +"""sync_models + +Revision ID: 49c19df2c1d4 +Revises: 29c02cb50737 +Create Date: 2026-02-19 11:21:56.538569 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '49c19df2c1d4' +down_revision: Union[str, None] = '29c02cb50737' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('drive_acls', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('resource_type', sa.String(length=20), nullable=False), + sa.Column('resource_id', sa.Integer(), nullable=False), + sa.Column('subject_id', sa.Integer(), nullable=False), + sa.Column('role', sa.String(length=50), nullable=True), + sa.Column('actor_id', sa.Integer(), nullable=True), + sa.Column('created_at', sa.TIMESTAMP(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('chatbot_documents', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('document_id', sa.UUID(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('drive_file_id', sa.Integer(), nullable=True), + sa.Column('filename', sa.String(), nullable=False), + sa.Column('file_path', sa.String(), nullable=True), + sa.Column('cloud_url', sa.String(), nullable=True), + sa.Column('chunks_count', sa.Integer(), nullable=True), + sa.Column('created_at', sa.TIMESTAMP(), nullable=True), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_chatbot_documents_document_id'), 'chatbot_documents', ['document_id'], unique=True) + op.create_index(op.f('ix_chatbot_documents_drive_file_id'), 'chatbot_documents', ['drive_file_id'], unique=False) + op.create_index(op.f('ix_chatbot_documents_id'), 'chatbot_documents', ['id'], unique=False) + op.create_index(op.f('ix_chatbot_documents_user_id'), 'chatbot_documents', ['user_id'], unique=False) + op.create_table('vector_chunks', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('document_id', sa.String(), nullable=True), + sa.Column('chunk_text', sa.String(), nullable=False), + sa.Column('chunk_index', sa.Integer(), nullable=True), + sa.Column('created_at', sa.TIMESTAMP(), nullable=True), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_vector_chunks_document_id'), 'vector_chunks', ['document_id'], unique=False) + op.create_index(op.f('ix_vector_chunks_id'), 'vector_chunks', ['id'], unique=False) + op.create_index(op.f('ix_vector_chunks_user_id'), 'vector_chunks', ['user_id'], unique=False) + op.create_table('vector_indices', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('index_data', sa.LargeBinary(), nullable=True), + sa.Column('dimension', sa.Integer(), nullable=True), + sa.Column('total_vectors', sa.Integer(), nullable=True), + sa.Column('updated_at', sa.TIMESTAMP(), nullable=True), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_vector_indices_id'), 'vector_indices', ['id'], unique=False) + op.create_index(op.f('ix_vector_indices_user_id'), 'vector_indices', ['user_id'], unique=True) + op.drop_table('drive_acl') + op.alter_column('drive_activities', 'resource_type', + existing_type=sa.VARCHAR(length=10), + type_=sa.String(length=20), + nullable=True) + op.alter_column('drive_activities', 'resource_id', + existing_type=sa.INTEGER(), + nullable=True) + op.alter_column('drive_activities', 'actor_id', + existing_type=sa.INTEGER(), + nullable=True) + op.alter_column('drive_activities', 'activity_type', + existing_type=sa.VARCHAR(length=50), + nullable=True) + op.alter_column('drive_activities', 'created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + type_=sa.TIMESTAMP(), + nullable=True, + existing_server_default=sa.text('now()')) + op.drop_constraint(op.f('drive_activities_actor_id_fkey'), 'drive_activities', type_='foreignkey') + op.create_foreign_key(None, 'drive_activities', 'users', ['actor_id'], ['id']) + op.drop_column('drive_activities', 'device_info') + op.drop_column('drive_activities', 'ip_address') + op.add_column('drive_comments', sa.Column('user_id', sa.Integer(), nullable=False)) + op.add_column('drive_comments', sa.Column('content', sa.Text(), nullable=False)) + op.alter_column('drive_comments', 'created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + type_=sa.TIMESTAMP(), + nullable=True, + existing_server_default=sa.text('now()')) + op.drop_constraint(op.f('drive_comments_file_id_fkey'), 'drive_comments', type_='foreignkey') + op.drop_constraint(op.f('drive_comments_author_id_fkey'), 'drive_comments', type_='foreignkey') + op.create_foreign_key(None, 'drive_comments', 'users', ['user_id'], ['id']) + op.create_foreign_key(None, 'drive_comments', 'drive_files', ['file_id'], ['id']) + op.drop_column('drive_comments', 'anchor') + op.drop_column('drive_comments', 'body') + op.drop_column('drive_comments', 'resolved_at') + op.drop_column('drive_comments', 'author_id') + op.add_column('drive_file_versions', sa.Column('s3_key', sa.String(), nullable=True)) + op.add_column('drive_file_versions', sa.Column('content_type', sa.String(), nullable=True)) + op.add_column('drive_file_versions', sa.Column('uploader_id', sa.Integer(), nullable=True)) + op.alter_column('drive_file_versions', 'file_id', + existing_type=sa.INTEGER(), + nullable=True) + op.alter_column('drive_file_versions', 'version_number', + existing_type=sa.INTEGER(), + nullable=True) + op.alter_column('drive_file_versions', 'size', + existing_type=sa.BIGINT(), + nullable=True) + op.alter_column('drive_file_versions', 'created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + type_=sa.TIMESTAMP(), + nullable=True, + existing_server_default=sa.text('now()')) + op.drop_constraint(op.f('drive_file_versions_file_id_fkey'), 'drive_file_versions', type_='foreignkey') + op.drop_constraint(op.f('drive_file_versions_created_by_id_fkey'), 'drive_file_versions', type_='foreignkey') + op.create_foreign_key(None, 'drive_file_versions', 'drive_files', ['file_id'], ['id']) + op.create_foreign_key(None, 'drive_file_versions', 'users', ['uploader_id'], ['id']) + op.drop_column('drive_file_versions', 'checksum') + op.drop_column('drive_file_versions', 'created_by_id') + op.drop_column('drive_file_versions', 'storage_key') + op.drop_column('drive_file_versions', 'mime_type') + op.alter_column('drive_stars', 'user_id', + existing_type=sa.INTEGER(), + nullable=True) + op.alter_column('drive_stars', 'resource_type', + existing_type=sa.VARCHAR(length=10), + nullable=True) + op.alter_column('drive_stars', 'resource_id', + existing_type=sa.INTEGER(), + nullable=True) + op.drop_constraint(op.f('uq_drive_star'), 'drive_stars', type_='unique') + op.drop_constraint(op.f('drive_stars_user_id_fkey'), 'drive_stars', type_='foreignkey') + op.create_foreign_key(None, 'drive_stars', 'users', ['user_id'], ['id']) + op.drop_column('drive_stars', 'created_at') + op.alter_column('project_activities', 'ip_address', + existing_type=sa.VARCHAR(length=64), + type_=sa.String(length=45), + existing_nullable=True) + op.alter_column('project_comments', 'anchor', + existing_type=postgresql.JSONB(astext_type=sa.Text()), + type_=sa.Text(), + existing_nullable=True) + op.alter_column('project_shares', 'permission', + existing_type=sa.VARCHAR(length=50), + type_=sa.String(length=20), + existing_nullable=False) + op.drop_constraint(op.f('unique_project_user_share'), 'project_shares', type_='unique') + op.drop_constraint(op.f('project_shares_shared_by_user_id_fkey'), 'project_shares', type_='foreignkey') + op.drop_column('project_shares', 'updated_at') + op.drop_column('project_shares', 'shared_by_user_id') + op.add_column('project_versions', sa.Column('version_label', sa.String(length=50), nullable=False)) + op.add_column('project_versions', sa.Column('description', sa.Text(), nullable=True)) + op.alter_column('project_versions', 'created_by_id', + existing_type=sa.INTEGER(), + nullable=True) + op.drop_constraint(op.f('project_versions_created_by_id_fkey'), 'project_versions', type_='foreignkey') + op.create_foreign_key(None, 'project_versions', 'users', ['created_by_id'], ['id'], ondelete='SET NULL') + op.drop_column('project_versions', 'changes_description') + op.drop_column('project_versions', 'session_id') + op.add_column('user_files', sa.Column('filename', sa.String(), nullable=False)) + op.add_column('user_files', sa.Column('size', sa.BigInteger(), nullable=True)) + op.add_column('user_files', sa.Column('content_type', sa.String(), nullable=True)) + op.add_column('user_files', sa.Column('status', sa.String(), nullable=True)) + op.alter_column('user_files', 'created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + type_=sa.TIMESTAMP(), + nullable=True, + existing_server_default=sa.text('now()')) + op.drop_constraint(op.f('user_files_s3_key_key'), 'user_files', type_='unique') + op.create_index(op.f('ix_user_files_id'), 'user_files', ['id'], unique=False) + op.create_index(op.f('ix_user_files_user_id'), 'user_files', ['user_id'], unique=False) + op.drop_constraint(op.f('user_files_user_id_fkey'), 'user_files', type_='foreignkey') + op.create_foreign_key(None, 'user_files', 'users', ['user_id'], ['id']) + op.drop_column('user_files', 'file_name') + op.drop_column('user_files', 'deleted_at') + op.drop_column('user_files', 'file_size') + op.drop_column('user_files', 'is_deleted') + op.drop_column('user_files', 'mime_type') + op.add_column('user_storage_usage', sa.Column('id', sa.Integer(), nullable=False)) + op.add_column('user_storage_usage', sa.Column('total_bytes_used', sa.BigInteger(), nullable=True)) + op.add_column('user_storage_usage', sa.Column('total_files_count', sa.Integer(), nullable=True)) + op.alter_column('user_storage_usage', 'last_updated', + existing_type=postgresql.TIMESTAMP(timezone=True), + type_=sa.TIMESTAMP(), + nullable=True, + existing_server_default=sa.text('now()')) + op.create_index(op.f('ix_user_storage_usage_id'), 'user_storage_usage', ['id'], unique=False) + op.create_unique_constraint(None, 'user_storage_usage', ['user_id']) + op.drop_constraint(op.f('user_storage_usage_user_id_fkey'), 'user_storage_usage', type_='foreignkey') + op.create_foreign_key(None, 'user_storage_usage', 'users', ['user_id'], ['id']) + op.drop_column('user_storage_usage', 'total_bytes') + op.drop_column('user_storage_usage', 'file_count') + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('user_storage_usage', sa.Column('file_count', sa.INTEGER(), autoincrement=False, nullable=False)) + op.add_column('user_storage_usage', sa.Column('total_bytes', sa.BIGINT(), autoincrement=False, nullable=False)) + op.drop_constraint(None, 'user_storage_usage', type_='foreignkey') + op.create_foreign_key(op.f('user_storage_usage_user_id_fkey'), 'user_storage_usage', 'users', ['user_id'], ['id'], ondelete='CASCADE') + op.drop_constraint(None, 'user_storage_usage', type_='unique') + op.drop_index(op.f('ix_user_storage_usage_id'), table_name='user_storage_usage') + op.alter_column('user_storage_usage', 'last_updated', + existing_type=sa.TIMESTAMP(), + type_=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + op.drop_column('user_storage_usage', 'total_files_count') + op.drop_column('user_storage_usage', 'total_bytes_used') + op.drop_column('user_storage_usage', 'id') + op.add_column('user_files', sa.Column('mime_type', sa.VARCHAR(length=255), autoincrement=False, nullable=True)) + op.add_column('user_files', sa.Column('is_deleted', sa.BOOLEAN(), autoincrement=False, nullable=False)) + op.add_column('user_files', sa.Column('file_size', sa.BIGINT(), autoincrement=False, nullable=False)) + op.add_column('user_files', sa.Column('deleted_at', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=True)) + op.add_column('user_files', sa.Column('file_name', sa.VARCHAR(length=512), autoincrement=False, nullable=False)) + op.drop_constraint(None, 'user_files', type_='foreignkey') + op.create_foreign_key(op.f('user_files_user_id_fkey'), 'user_files', 'users', ['user_id'], ['id'], ondelete='CASCADE') + op.drop_index(op.f('ix_user_files_user_id'), table_name='user_files') + op.drop_index(op.f('ix_user_files_id'), table_name='user_files') + op.create_unique_constraint(op.f('user_files_s3_key_key'), 'user_files', ['s3_key']) + op.alter_column('user_files', 'created_at', + existing_type=sa.TIMESTAMP(), + type_=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + op.drop_column('user_files', 'status') + op.drop_column('user_files', 'content_type') + op.drop_column('user_files', 'size') + op.drop_column('user_files', 'filename') + op.add_column('project_versions', sa.Column('session_id', sa.VARCHAR(length=255), autoincrement=False, nullable=True)) + op.add_column('project_versions', sa.Column('changes_description', sa.VARCHAR(length=500), autoincrement=False, nullable=True)) + op.drop_constraint(None, 'project_versions', type_='foreignkey') + op.create_foreign_key(op.f('project_versions_created_by_id_fkey'), 'project_versions', 'users', ['created_by_id'], ['id']) + op.alter_column('project_versions', 'created_by_id', + existing_type=sa.INTEGER(), + nullable=False) + op.drop_column('project_versions', 'description') + op.drop_column('project_versions', 'version_label') + op.add_column('project_shares', sa.Column('shared_by_user_id', sa.INTEGER(), autoincrement=False, nullable=False)) + op.add_column('project_shares', sa.Column('updated_at', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False)) + op.create_foreign_key(op.f('project_shares_shared_by_user_id_fkey'), 'project_shares', 'users', ['shared_by_user_id'], ['id'], ondelete='CASCADE') + op.create_unique_constraint(op.f('unique_project_user_share'), 'project_shares', ['project_id', 'shared_with_user_id']) + op.alter_column('project_shares', 'permission', + existing_type=sa.String(length=20), + type_=sa.VARCHAR(length=50), + existing_nullable=False) + op.alter_column('project_comments', 'anchor', + existing_type=sa.Text(), + type_=postgresql.JSONB(astext_type=sa.Text()), + existing_nullable=True) + op.alter_column('project_activities', 'ip_address', + existing_type=sa.String(length=45), + type_=sa.VARCHAR(length=64), + existing_nullable=True) + op.add_column('drive_stars', sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), autoincrement=False, nullable=False)) + op.drop_constraint(None, 'drive_stars', type_='foreignkey') + op.create_foreign_key(op.f('drive_stars_user_id_fkey'), 'drive_stars', 'users', ['user_id'], ['id'], ondelete='CASCADE') + op.create_unique_constraint(op.f('uq_drive_star'), 'drive_stars', ['resource_type', 'resource_id', 'user_id']) + op.alter_column('drive_stars', 'resource_id', + existing_type=sa.INTEGER(), + nullable=False) + op.alter_column('drive_stars', 'resource_type', + existing_type=sa.VARCHAR(length=10), + nullable=False) + op.alter_column('drive_stars', 'user_id', + existing_type=sa.INTEGER(), + nullable=False) + op.add_column('drive_file_versions', sa.Column('mime_type', sa.VARCHAR(length=255), autoincrement=False, nullable=True)) + op.add_column('drive_file_versions', sa.Column('storage_key', sa.TEXT(), autoincrement=False, nullable=False)) + op.add_column('drive_file_versions', sa.Column('created_by_id', sa.INTEGER(), autoincrement=False, nullable=False)) + op.add_column('drive_file_versions', sa.Column('checksum', sa.VARCHAR(length=128), autoincrement=False, nullable=True)) + op.drop_constraint(None, 'drive_file_versions', type_='foreignkey') + op.drop_constraint(None, 'drive_file_versions', type_='foreignkey') + op.create_foreign_key(op.f('drive_file_versions_created_by_id_fkey'), 'drive_file_versions', 'users', ['created_by_id'], ['id']) + op.create_foreign_key(op.f('drive_file_versions_file_id_fkey'), 'drive_file_versions', 'drive_files', ['file_id'], ['id'], ondelete='CASCADE') + op.alter_column('drive_file_versions', 'created_at', + existing_type=sa.TIMESTAMP(), + type_=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + op.alter_column('drive_file_versions', 'size', + existing_type=sa.BIGINT(), + nullable=False) + op.alter_column('drive_file_versions', 'version_number', + existing_type=sa.INTEGER(), + nullable=False) + op.alter_column('drive_file_versions', 'file_id', + existing_type=sa.INTEGER(), + nullable=False) + op.drop_column('drive_file_versions', 'uploader_id') + op.drop_column('drive_file_versions', 'content_type') + op.drop_column('drive_file_versions', 's3_key') + op.add_column('drive_comments', sa.Column('author_id', sa.INTEGER(), autoincrement=False, nullable=False)) + op.add_column('drive_comments', sa.Column('resolved_at', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=True)) + op.add_column('drive_comments', sa.Column('body', sa.TEXT(), autoincrement=False, nullable=False)) + op.add_column('drive_comments', sa.Column('anchor', postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=True)) + op.drop_constraint(None, 'drive_comments', type_='foreignkey') + op.drop_constraint(None, 'drive_comments', type_='foreignkey') + op.create_foreign_key(op.f('drive_comments_author_id_fkey'), 'drive_comments', 'users', ['author_id'], ['id'], ondelete='CASCADE') + op.create_foreign_key(op.f('drive_comments_file_id_fkey'), 'drive_comments', 'drive_files', ['file_id'], ['id'], ondelete='CASCADE') + op.alter_column('drive_comments', 'created_at', + existing_type=sa.TIMESTAMP(), + type_=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + op.drop_column('drive_comments', 'content') + op.drop_column('drive_comments', 'user_id') + op.add_column('drive_activities', sa.Column('ip_address', sa.VARCHAR(length=64), autoincrement=False, nullable=True)) + op.add_column('drive_activities', sa.Column('device_info', sa.TEXT(), autoincrement=False, nullable=True)) + op.drop_constraint(None, 'drive_activities', type_='foreignkey') + op.create_foreign_key(op.f('drive_activities_actor_id_fkey'), 'drive_activities', 'users', ['actor_id'], ['id'], ondelete='CASCADE') + op.alter_column('drive_activities', 'created_at', + existing_type=sa.TIMESTAMP(), + type_=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + op.alter_column('drive_activities', 'activity_type', + existing_type=sa.VARCHAR(length=50), + nullable=False) + op.alter_column('drive_activities', 'actor_id', + existing_type=sa.INTEGER(), + nullable=False) + op.alter_column('drive_activities', 'resource_id', + existing_type=sa.INTEGER(), + nullable=False) + op.alter_column('drive_activities', 'resource_type', + existing_type=sa.String(length=20), + type_=sa.VARCHAR(length=10), + nullable=False) + op.create_table('drive_acl', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('resource_type', sa.VARCHAR(length=10), autoincrement=False, nullable=False), + sa.Column('resource_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('subject_type', sa.VARCHAR(length=10), autoincrement=False, nullable=False), + sa.Column('subject_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('role', sa.VARCHAR(length=20), autoincrement=False, nullable=False), + sa.Column('inherited_from', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('drive_acl_pkey')) + ) + op.drop_index(op.f('ix_vector_indices_user_id'), table_name='vector_indices') + op.drop_index(op.f('ix_vector_indices_id'), table_name='vector_indices') + op.drop_table('vector_indices') + op.drop_index(op.f('ix_vector_chunks_user_id'), table_name='vector_chunks') + op.drop_index(op.f('ix_vector_chunks_id'), table_name='vector_chunks') + op.drop_index(op.f('ix_vector_chunks_document_id'), table_name='vector_chunks') + op.drop_table('vector_chunks') + op.drop_index(op.f('ix_chatbot_documents_user_id'), table_name='chatbot_documents') + op.drop_index(op.f('ix_chatbot_documents_id'), table_name='chatbot_documents') + op.drop_index(op.f('ix_chatbot_documents_drive_file_id'), table_name='chatbot_documents') + op.drop_index(op.f('ix_chatbot_documents_document_id'), table_name='chatbot_documents') + op.drop_table('chatbot_documents') + op.drop_table('drive_acls') + # ### end Alembic commands ### diff --git a/alembic/versions/4a9dc50ce4af_drop_price_display.py b/alembic/versions/4a9dc50ce4af_drop_price_display.py new file mode 100644 index 0000000..e0d7b97 --- /dev/null +++ b/alembic/versions/4a9dc50ce4af_drop_price_display.py @@ -0,0 +1,33 @@ +"""drop_price_display + +Revision ID: 4a9dc50ce4af +Revises: 2b4c62c1cf35 +Create Date: 2026-08-20 11:40:33.072200 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '4a9dc50ce4af' +down_revision: Union[str, None] = '2b4c62c1cf35' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.execute("UPDATE plans SET price_amount = CAST(SUBSTRING(price_display FROM '[\d.]+') AS NUMERIC) WHERE price_display IS NOT NULL AND price_display != 'Contact Us' AND price_amount IS NULL") + op.drop_column('plans', 'price_display') + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('plans', sa.Column('price_display', sa.VARCHAR(length=50), autoincrement=False, nullable=True)) + op.execute("UPDATE plans SET price_display = '$' || price_amount WHERE price_amount IS NOT NULL") + op.execute("UPDATE plans SET price_display = 'Contact Us' WHERE price_amount IS NULL") + # ### end Alembic commands ### diff --git a/alembic/versions/522ee6d91431_add_drive_share_link_fields.py b/alembic/versions/522ee6d91431_add_drive_share_link_fields.py new file mode 100644 index 0000000..558950b --- /dev/null +++ b/alembic/versions/522ee6d91431_add_drive_share_link_fields.py @@ -0,0 +1,30 @@ +"""add_drive_share_link_fields + +Revision ID: 522ee6d91431 +Revises: 8d4dced54966 +Create Date: 2026-02-24 14:35:17.542820 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '522ee6d91431' +down_revision: Union[str, None] = '8d4dced54966' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('drive_shares', sa.Column('is_active', sa.Boolean(), nullable=False)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('drive_shares', 'is_active') + # ### end Alembic commands ### diff --git a/alembic/versions/540c970ff890_add_tenant_chat_token_quota.py b/alembic/versions/540c970ff890_add_tenant_chat_token_quota.py new file mode 100644 index 0000000..ad07c63 --- /dev/null +++ b/alembic/versions/540c970ff890_add_tenant_chat_token_quota.py @@ -0,0 +1,42 @@ +"""add_tenant_chat_token_quota + +Revision ID: 540c970ff890 +Revises: 7fc8dd61bb95 +Create Date: 2026-05-13 11:21:14.192033 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '540c970ff890' +down_revision: Union[str, None] = '7fc8dd61bb95' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Get current connection and inspector + conn = op.get_bind() + inspector = sa.inspect(conn) + + # Check if columns already exist (defensive migration) + columns = [c['name'] for c in inspector.get_columns('tenants')] + + if 'chat_token_daily_limit' not in columns: + op.add_column('tenants', sa.Column('chat_token_daily_limit', sa.BigInteger(), nullable=False, server_default='1000000')) + + if 'chat_tokens_used_today' not in columns: + op.add_column('tenants', sa.Column('chat_tokens_used_today', sa.BigInteger(), nullable=False, server_default='0')) + + if 'chat_tokens_last_reset_at' not in columns: + op.add_column('tenants', sa.Column('chat_tokens_last_reset_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False)) + + +def downgrade() -> None: + op.drop_column('tenants', 'chat_tokens_last_reset_at') + op.drop_column('tenants', 'chat_tokens_used_today') + op.drop_column('tenants', 'chat_token_daily_limit') diff --git a/alembic/versions/56d090047e85_increase_file_size_constraints.py b/alembic/versions/56d090047e85_increase_file_size_constraints.py new file mode 100644 index 0000000..6247c9b --- /dev/null +++ b/alembic/versions/56d090047e85_increase_file_size_constraints.py @@ -0,0 +1,74 @@ +"""increase_file_size_constraints + +Revision ID: 56d090047e85 +Revises: j002_unlimited_tenant_quota +Create Date: 2026-03-26 15:14:47.276886 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '56d090047e85' +down_revision: Union[str, None] = 'j002_unlimited_tenant_quota' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('projects', schema=None) as batch_op: + # Drop old 10MB constraints + batch_op.drop_constraint('ck_project_markdown_max', type_='check') + batch_op.drop_constraint('ck_project_xml_max', type_='check') + batch_op.drop_constraint('ck_project_html_max', type_='check') + + # Add new 30MB constraints + batch_op.create_check_constraint('ck_project_markdown_max', 'octet_length(markdown_content) <= 31457280') + batch_op.create_check_constraint('ck_project_xml_max', 'octet_length(xml_content) <= 31457280') + batch_op.create_check_constraint('ck_project_html_max', 'octet_length(html_content) <= 31457280') + + + + with op.batch_alter_table('project_versions', schema=None) as batch_op: + # Drop old 10MB constraints + batch_op.drop_constraint('ck_version_markdown_max', type_='check') + batch_op.drop_constraint('ck_version_xml_max', type_='check') + batch_op.drop_constraint('ck_version_html_max', type_='check') + + # Add new 30MB constraints + batch_op.create_check_constraint('ck_version_markdown_max', 'octet_length(markdown_content) <= 31457280') + batch_op.create_check_constraint('ck_version_xml_max', 'octet_length(xml_content) <= 31457280') + batch_op.create_check_constraint('ck_version_html_max', 'octet_length(html_content) <= 31457280') + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('project_versions', schema=None) as batch_op: + # Drop new 30MB constraints + batch_op.drop_constraint('ck_version_markdown_max', type_='check') + batch_op.drop_constraint('ck_version_xml_max', type_='check') + batch_op.drop_constraint('ck_version_html_max', type_='check') + + # Re-add old 10MB constraints + batch_op.create_check_constraint('ck_version_markdown_max', 'octet_length(markdown_content) <= 10485760') + batch_op.create_check_constraint('ck_version_xml_max', 'octet_length(xml_content) <= 10485760') + batch_op.create_check_constraint('ck_version_html_max', 'octet_length(html_content) <= 10485760') + + with op.batch_alter_table('projects', schema=None) as batch_op: + # Drop new 30MB constraints + batch_op.drop_constraint('ck_project_markdown_max', type_='check') + batch_op.drop_constraint('ck_project_xml_max', type_='check') + batch_op.drop_constraint('ck_project_html_max', type_='check') + + # Re-add old 10MB constraints + batch_op.create_check_constraint('ck_project_markdown_max', 'octet_length(markdown_content) <= 10485760') + batch_op.create_check_constraint('ck_project_xml_max', 'octet_length(xml_content) <= 10485760') + batch_op.create_check_constraint('ck_project_html_max', 'octet_length(html_content) <= 10485760') + + + # ### end Alembic commands ### diff --git a/alembic/versions/611d6b99ad11_add_multiple_files_to_signing_request.py b/alembic/versions/611d6b99ad11_add_multiple_files_to_signing_request.py new file mode 100644 index 0000000..3f81ded --- /dev/null +++ b/alembic/versions/611d6b99ad11_add_multiple_files_to_signing_request.py @@ -0,0 +1,34 @@ +"""add_multiple_files_to_signing_request + +Revision ID: 611d6b99ad11 +Revises: d551c03861f4 +Create Date: 2026-07-14 18:00:05.273738 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '611d6b99ad11' +down_revision: Union[str, None] = 'd551c03861f4' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column('signing_requests', sa.Column('drive_file_ids', sa.JSON(), nullable=True)) + op.add_column('signing_requests', sa.Column('signed_file_ids', sa.JSON(), nullable=True)) + op.alter_column('signing_requests', 'drive_file_id', + existing_type=sa.INTEGER(), + nullable=True) + + +def downgrade() -> None: + op.alter_column('signing_requests', 'drive_file_id', + existing_type=sa.INTEGER(), + nullable=False) + op.drop_column('signing_requests', 'signed_file_ids') + op.drop_column('signing_requests', 'drive_file_ids') diff --git a/alembic/versions/6c9cc3567866_merge_multiple_heads.py b/alembic/versions/6c9cc3567866_merge_multiple_heads.py new file mode 100644 index 0000000..309e12b --- /dev/null +++ b/alembic/versions/6c9cc3567866_merge_multiple_heads.py @@ -0,0 +1,26 @@ +"""merge_multiple_heads + +Revision ID: 6c9cc3567866 +Revises: 45c94ee15294, ff9052a97835 +Create Date: 2026-03-05 07:53:09.436096 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '6c9cc3567866' +down_revision: Union[str, None] = ('45c94ee15294', 'ff9052a97835') +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/alembic/versions/73dff81eed72_add_tenant_contacts.py b/alembic/versions/73dff81eed72_add_tenant_contacts.py new file mode 100644 index 0000000..8da7cf2 --- /dev/null +++ b/alembic/versions/73dff81eed72_add_tenant_contacts.py @@ -0,0 +1,43 @@ +"""Add tenant contacts + +Revision ID: 73dff81eed72 +Revises: dfae38e4ede3 +Create Date: 2026-08-12 10:56:21.668158 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '73dff81eed72' +down_revision: Union[str, None] = 'dfae38e4ede3' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('tenant_contacts', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('tenant_id', sa.UUID(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('email', sa.String(length=255), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('tenant_id', 'email', name='uq_tenant_contacts_tenant_id_email'), + ) + op.create_index(op.f('ix_tenant_contacts_email'), 'tenant_contacts', ['email'], unique=False) + op.create_index(op.f('ix_tenant_contacts_tenant_id'), 'tenant_contacts', ['tenant_id'], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_tenant_contacts_tenant_id'), table_name='tenant_contacts') + op.drop_index(op.f('ix_tenant_contacts_email'), table_name='tenant_contacts') + op.drop_table('tenant_contacts') + # ### end Alembic commands ### diff --git a/alembic/versions/765b053e3dcd_add_vector_index_checksum.py b/alembic/versions/765b053e3dcd_add_vector_index_checksum.py new file mode 100644 index 0000000..64fdc02 --- /dev/null +++ b/alembic/versions/765b053e3dcd_add_vector_index_checksum.py @@ -0,0 +1,30 @@ +"""add_vector_index_checksum + +Revision ID: 765b053e3dcd +Revises: be67927ab00d +Create Date: 2026-03-03 20:25:42.849543 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '765b053e3dcd' +down_revision: Union[str, None] = 'be67927ab00d' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/alembic/versions/7d7641cc1bb2_refactor_global_signature_config.py b/alembic/versions/7d7641cc1bb2_refactor_global_signature_config.py new file mode 100644 index 0000000..2e618d7 --- /dev/null +++ b/alembic/versions/7d7641cc1bb2_refactor_global_signature_config.py @@ -0,0 +1,32 @@ +"""refactor_global_signature_config + +Revision ID: 7d7641cc1bb2 +Revises: b3a7c1e9f042 +Create Date: 2026-07-21 07:19:00.000000 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '7d7641cc1bb2' +down_revision: Union[str, None] = 'af57c3d9f1c2' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column('global_signature_configs', sa.Column('global_credentials_json', postgresql.JSON(astext_type=sa.Text()), nullable=True)) + op.add_column('global_signature_configs', sa.Column('enabled_providers', postgresql.JSON(astext_type=sa.Text()), server_default='["docuseal", "zoho_sign"]', nullable=False)) + op.drop_column('global_signature_configs', 'docuseal_credentials') + op.drop_column('global_signature_configs', 'zoho_credentials') + + +def downgrade() -> None: + op.add_column('global_signature_configs', sa.Column('zoho_credentials', postgresql.JSON(astext_type=sa.Text()), autoincrement=False, nullable=True)) + op.add_column('global_signature_configs', sa.Column('docuseal_credentials', postgresql.JSON(astext_type=sa.Text()), autoincrement=False, nullable=True)) + op.drop_column('global_signature_configs', 'enabled_providers') + op.drop_column('global_signature_configs', 'global_credentials_json') diff --git a/alembic/versions/7fc8dd61bb95_add_chat_token_usage_and_quota_tables.py b/alembic/versions/7fc8dd61bb95_add_chat_token_usage_and_quota_tables.py new file mode 100644 index 0000000..974cd55 --- /dev/null +++ b/alembic/versions/7fc8dd61bb95_add_chat_token_usage_and_quota_tables.py @@ -0,0 +1,70 @@ +"""Add chat token usage and quota tables + +Revision ID: 7fc8dd61bb95 +Revises: z002_pref_lang +Create Date: 2026-05-12 18:38:23.935092 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '7fc8dd61bb95' +down_revision: Union[str, None] = 'z002_pref_lang' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Get current connection and inspector + conn = op.get_bind() + inspector = sa.inspect(conn) + + # 1. Create chat_token_usage if missing + if not inspector.has_table('chat_token_usage'): + op.create_table('chat_token_usage', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('session_id', sa.Integer(), nullable=True), + sa.Column('prompt_tokens', sa.BigInteger(), nullable=False, server_default='0'), + sa.Column('completion_tokens', sa.BigInteger(), nullable=False, server_default='0'), + sa.Column('total_tokens', sa.BigInteger(), nullable=False, server_default='0'), + sa.Column('model_name', sa.String(length=100), nullable=False), + sa.Column('cost_usd', sa.DECIMAL(precision=12, scale=8), nullable=False, server_default='0.0'), + sa.Column('timestamp', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('phase', sa.String(length=50), nullable=True), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['session_id'], ['chat_sessions.id'], ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('chat_token_usage', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_chat_token_usage_id'), ['id'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_token_usage_timestamp'), ['timestamp'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_token_usage_user_id'), ['user_id'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_token_usage_session_id'), ['session_id'], unique=False) + + # 2. Create chat_token_quota if missing + if not inspector.has_table('chat_token_quota'): + op.create_table('chat_token_quota', + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('daily_limit', sa.BigInteger(), nullable=False), + sa.Column('tokens_used_today', sa.BigInteger(), nullable=False, server_default='0'), + sa.Column('last_reset_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('total_tokens_ever', sa.BigInteger(), nullable=False, server_default='0'), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('user_id') + ) + + +def downgrade() -> None: + conn = op.get_bind() + inspector = sa.inspect(conn) + + if inspector.has_table('chat_token_quota'): + op.drop_table('chat_token_quota') + + if inspector.has_table('chat_token_usage'): + op.drop_table('chat_token_usage') diff --git a/alembic/versions/80be2c61e7fc_add_is_popular_to_plans.py b/alembic/versions/80be2c61e7fc_add_is_popular_to_plans.py new file mode 100644 index 0000000..31a39ef --- /dev/null +++ b/alembic/versions/80be2c61e7fc_add_is_popular_to_plans.py @@ -0,0 +1,30 @@ +"""add_is_popular_to_plans + +Revision ID: 80be2c61e7fc +Revises: 4a9dc50ce4af +Create Date: 2026-08-20 12:47:53.790237 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '80be2c61e7fc' +down_revision: Union[str, None] = '4a9dc50ce4af' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('plans', sa.Column('is_popular', sa.Boolean(), server_default=sa.text('false'), nullable=False)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('plans', 'is_popular') + # ### end Alembic commands ### diff --git a/alembic/versions/823a486fd287_add_created_at_to_drive_stars.py b/alembic/versions/823a486fd287_add_created_at_to_drive_stars.py new file mode 100644 index 0000000..94fd0a3 --- /dev/null +++ b/alembic/versions/823a486fd287_add_created_at_to_drive_stars.py @@ -0,0 +1,36 @@ +"""add created_at to drive_stars + +Revision ID: 823a486fd287 +Revises: 88717d544229 +Create Date: 2026-03-27 23:24:47.417678 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '823a486fd287' +down_revision: Union[str, None] = '88717d544229' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Add column as nullable first + op.add_column('drive_stars', sa.Column('created_at', sa.DateTime(timezone=True), nullable=True)) + + # Update existing stars to current time so they aren't null + op.execute("UPDATE drive_stars SET created_at = NOW() WHERE created_at IS NULL") + + # Make column non-nullable with server default + op.alter_column('drive_stars', 'created_at', + existing_type=sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text('now()')) + + +def downgrade() -> None: + op.drop_column('drive_stars', 'created_at') diff --git a/alembic/versions/8736ab6bd5af_add_notifications_table.py b/alembic/versions/8736ab6bd5af_add_notifications_table.py new file mode 100644 index 0000000..9b782fb --- /dev/null +++ b/alembic/versions/8736ab6bd5af_add_notifications_table.py @@ -0,0 +1,41 @@ +"""add_notifications_table + +Revision ID: 8736ab6bd5af +Revises: 522ee6d91431 +Create Date: 2026-02-24 15:25:34.045912 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '8736ab6bd5af' +down_revision: Union[str, None] = '522ee6d91431' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + 'notifications', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('notif_type', sa.String(length=50), nullable=False), + sa.Column('title', sa.String(length=255), nullable=False), + sa.Column('message', sa.Text(), nullable=False), + sa.Column('resource_type', sa.String(length=20), nullable=True), + sa.Column('resource_id', sa.Integer(), nullable=True), + sa.Column('is_read', sa.Boolean(), nullable=False, server_default=sa.text('false')), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()')), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + ) + op.create_index('ix_notifications_user_id', 'notifications', ['user_id'], unique=False) + + +def downgrade() -> None: + op.drop_index('ix_notifications_user_id', table_name='notifications') + op.drop_table('notifications') diff --git a/alembic/versions/88717d544229_add_is_default_to_roles.py b/alembic/versions/88717d544229_add_is_default_to_roles.py new file mode 100644 index 0000000..32306f8 --- /dev/null +++ b/alembic/versions/88717d544229_add_is_default_to_roles.py @@ -0,0 +1,34 @@ +"""add is_default to roles + +Revision ID: 88717d544229 +Revises: 56d090047e85 +Create Date: 2026-03-27 14:08:16.581384 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '88717d544229' +down_revision: Union[str, None] = '56d090047e85' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('roles', schema=None) as batch_op: + batch_op.add_column(sa.Column('is_default', sa.Boolean(), nullable=False, server_default=sa.text('false'))) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('roles', schema=None) as batch_op: + batch_op.drop_column('is_default') + + # ### end Alembic commands ### diff --git a/alembic/versions/89508c176437_limit_drivecomment_body_to_10000_chars.py b/alembic/versions/89508c176437_limit_drivecomment_body_to_10000_chars.py new file mode 100644 index 0000000..7c47a5f --- /dev/null +++ b/alembic/versions/89508c176437_limit_drivecomment_body_to_10000_chars.py @@ -0,0 +1,36 @@ +"""Limit DriveComment body to 10000 chars + +Revision ID: 89508c176437 +Revises: ff8052a97834 +Create Date: 2026-03-03 22:02:03.251490 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '89508c176437' +down_revision: Union[str, None] = 'ff8052a97834' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('drive_comments', 'body', + existing_type=sa.TEXT(), + type_=sa.String(length=10000), + existing_nullable=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('drive_comments', 'body', + existing_type=sa.String(length=10000), + type_=sa.TEXT(), + existing_nullable=False) + # ### end Alembic commands ### diff --git a/alembic/versions/8d4dced54966_add_is_root_column.py b/alembic/versions/8d4dced54966_add_is_root_column.py new file mode 100644 index 0000000..d9bcdd7 --- /dev/null +++ b/alembic/versions/8d4dced54966_add_is_root_column.py @@ -0,0 +1,59 @@ +"""add is_root column + +Revision ID: 8d4dced54966 +Revises: cfbee4f91fc2 +Create Date: 2026-02-24 11:15:51.402384 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '8d4dced54966' +down_revision: Union[str, None] = 'cfbee4f91fc2' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('drive_comments', sa.Column('author_id', sa.Integer(), nullable=True)) + op.add_column('drive_comments', sa.Column('body', sa.Text(), nullable=True)) + + # Seeding existing comments if any (though usually clean) + op.execute("UPDATE drive_comments SET author_id = user_id WHERE author_id IS NULL") + op.execute("UPDATE drive_comments SET body = content WHERE body IS NULL") + + op.alter_column('drive_comments', 'author_id', nullable=False) + op.alter_column('drive_comments', 'body', nullable=False) + + op.add_column('drive_comments', sa.Column('anchor', sa.Text(), nullable=True)) + op.add_column('drive_comments', sa.Column('resolved_at', sa.DateTime(), nullable=True)) + op.drop_constraint(op.f('drive_comments_user_id_fkey'), 'drive_comments', type_='foreignkey') + op.create_foreign_key(None, 'drive_comments', 'users', ['author_id'], ['id']) + op.drop_column('drive_comments', 'user_id') + op.drop_column('drive_comments', 'content') + + # 🆕 Add is_root safely + op.add_column('drive_folders', sa.Column('is_root', sa.Boolean(), nullable=True)) + op.execute("UPDATE drive_folders SET is_root = false WHERE is_root IS NULL") + op.execute("UPDATE drive_folders SET is_root = true WHERE parent_id IS NULL") + op.alter_column('drive_folders', 'is_root', nullable=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('drive_folders', 'is_root') + op.add_column('drive_comments', sa.Column('content', sa.TEXT(), autoincrement=False, nullable=False)) + op.add_column('drive_comments', sa.Column('user_id', sa.INTEGER(), autoincrement=False, nullable=False)) + op.drop_constraint(None, 'drive_comments', type_='foreignkey') + op.create_foreign_key(op.f('drive_comments_user_id_fkey'), 'drive_comments', 'users', ['user_id'], ['id']) + op.drop_column('drive_comments', 'resolved_at') + op.drop_column('drive_comments', 'anchor') + op.drop_column('drive_comments', 'body') + op.drop_column('drive_comments', 'author_id') + # ### end Alembic commands ### diff --git a/alembic/versions/905726af73c8_add_plan_roles_and_is_system_to_roles.py b/alembic/versions/905726af73c8_add_plan_roles_and_is_system_to_roles.py new file mode 100644 index 0000000..c51bd43 --- /dev/null +++ b/alembic/versions/905726af73c8_add_plan_roles_and_is_system_to_roles.py @@ -0,0 +1,34 @@ +"""Add plan_roles and is_system to roles + +Revision ID: 905726af73c8 +Revises: 80be2c61e7fc +Create Date: 2026-08-20 19:17:20.939383 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '905726af73c8' +down_revision: Union[str, None] = '80be2c61e7fc' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table('plan_roles', + sa.Column('plan_id', sa.UUID(), nullable=False), + sa.Column('role_id', sa.UUID(), nullable=False), + sa.ForeignKeyConstraint(['plan_id'], ['plans.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['role_id'], ['roles.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('plan_id', 'role_id') + ) + op.add_column('roles', sa.Column('is_system', sa.Boolean(), nullable=False, server_default='false')) + + +def downgrade() -> None: + op.drop_column('roles', 'is_system') + op.drop_table('plan_roles') diff --git a/alembic/versions/9557c7307575_tenant_storage_config_and_provider.py b/alembic/versions/9557c7307575_tenant_storage_config_and_provider.py new file mode 100644 index 0000000..6469cd6 --- /dev/null +++ b/alembic/versions/9557c7307575_tenant_storage_config_and_provider.py @@ -0,0 +1,415 @@ +"""tenant_storage_config_and_provider + +Revision ID: 9557c7307575 +Revises: h004_signing_forensic_v2 +Create Date: 2026-03-16 13:00:58.602085 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '9557c7307575' +down_revision: Union[str, None] = 'h004_signing_forensic_v2' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.execute("UPDATE drive_files SET size = 0 WHERE size IS NULL") + op.execute("UPDATE drive_files SET is_starred = false WHERE is_starred IS NULL") + op.execute("UPDATE drive_files SET is_trashed = false WHERE is_trashed IS NULL") + op.execute("UPDATE drive_files SET created_at = CURRENT_TIMESTAMP WHERE created_at IS NULL") + op.execute("UPDATE drive_files SET updated_at = CURRENT_TIMESTAMP WHERE updated_at IS NULL") + + op.execute("UPDATE drive_folders SET is_starred = false WHERE is_starred IS NULL") + op.execute("UPDATE drive_folders SET is_trashed = false WHERE is_trashed IS NULL") + op.execute("UPDATE drive_folders SET created_at = CURRENT_TIMESTAMP WHERE created_at IS NULL") + op.execute("UPDATE drive_folders SET updated_at = CURRENT_TIMESTAMP WHERE updated_at IS NULL") + + op.execute("UPDATE drive_shares SET created_at = CURRENT_TIMESTAMP WHERE created_at IS NULL") + + op.execute("UPDATE project_versions SET is_current = false WHERE is_current IS NULL") + op.execute("UPDATE project_versions SET created_at = CURRENT_TIMESTAMP WHERE created_at IS NULL") + + op.execute("UPDATE projects SET status = 'processing' WHERE status IS NULL") + op.execute("UPDATE projects SET file_type = 'PDF' WHERE file_type IS NULL") + op.execute("UPDATE projects SET output_format = 'EPUB' WHERE output_format IS NULL") + op.execute("UPDATE projects SET progress = 0 WHERE progress IS NULL") + op.execute("UPDATE projects SET current_step = 'upload' WHERE current_step IS NULL") + op.execute("UPDATE projects SET created_at = CURRENT_TIMESTAMP WHERE created_at IS NULL") + op.execute("UPDATE projects SET updated_at = CURRENT_TIMESTAMP WHERE updated_at IS NULL") + + op.execute("UPDATE signing_requests SET created_at = CURRENT_TIMESTAMP WHERE created_at IS NULL") + + op.execute("UPDATE tenants SET created_at = CURRENT_TIMESTAMP WHERE created_at IS NULL") + op.execute("UPDATE tenants SET updated_at = CURRENT_TIMESTAMP WHERE updated_at IS NULL") + + op.execute("UPDATE users SET subscription = 'free' WHERE subscription IS NULL") + op.execute("UPDATE users SET created_at = CURRENT_TIMESTAMP WHERE created_at IS NULL") + + op.create_table('tenant_storage_configs', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('tenant_id', sa.UUID(), nullable=False), + sa.Column('storage_provider', sa.String(length=50), nullable=False), + sa.Column('b2_key_id', sa.String(length=255), nullable=True), + sa.Column('b2_application_key', sa.String(length=512), nullable=True), + sa.Column('b2_bucket_name', sa.String(length=255), nullable=True), + sa.Column('is_active', sa.Boolean(), nullable=False), + sa.Column('created_by_id', sa.Integer(), nullable=True), + sa.Column('updated_by_id', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['created_by_id'], ['users.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['updated_by_id'], ['users.id'], ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('tenant_storage_configs', schema=None) as batch_op: + op.execute("CREATE UNIQUE INDEX IF NOT EXISTS ix_tenant_storage_configs_tenant_id ON tenant_storage_configs (tenant_id)") + + with op.batch_alter_table('accesses', schema=None) as batch_op: + batch_op.alter_column('name', + existing_type=sa.VARCHAR(length=100), + type_=sa.String(length=255), + existing_nullable=False) + op.execute("DROP INDEX IF EXISTS ix_accesses_name") + batch_op.drop_constraint(batch_op.f('uq_accesses_access_code'), type_='unique') + op.execute("CREATE UNIQUE INDEX IF NOT EXISTS ix_accesses_access_code ON accesses (access_code)") + op.execute("CREATE INDEX IF NOT EXISTS ix_accesses_category ON accesses (category)") + op.execute("ALTER TABLE accesses DROP COLUMN IF EXISTS description") + + op.execute("DROP INDEX IF EXISTS idx_drive_activities_actor") + op.execute("DROP INDEX IF EXISTS idx_drive_activities_res") + op.execute("ALTER TABLE drive_activities DROP COLUMN IF EXISTS metadata") + + with op.batch_alter_table('drive_file_versions', schema=None) as batch_op: + batch_op.add_column(sa.Column('storage_provider', sa.String(length=50), nullable=False, server_default='minio')) + op.execute("DROP INDEX IF EXISTS idx_drive_versions_file") + + with op.batch_alter_table('drive_files', schema=None) as batch_op: + batch_op.alter_column('size', + existing_type=sa.BIGINT(), + nullable=False, + existing_server_default=sa.text('0')) + batch_op.alter_column('is_starred', + existing_type=sa.BOOLEAN(), + nullable=False, + existing_server_default=sa.text('false')) + batch_op.alter_column('is_trashed', + existing_type=sa.BOOLEAN(), + nullable=False, + existing_server_default=sa.text('false')) + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('CURRENT_TIMESTAMP')) + batch_op.alter_column('updated_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + op.execute("DROP INDEX IF EXISTS idx_drive_files_folder") + op.execute("DROP INDEX IF EXISTS idx_drive_files_owner") + op.execute("DROP INDEX IF EXISTS idx_drive_files_trashed") + + with op.batch_alter_table('drive_folders', schema=None) as batch_op: + batch_op.alter_column('is_starred', + existing_type=sa.BOOLEAN(), + nullable=False, + existing_server_default=sa.text('false')) + batch_op.alter_column('is_trashed', + existing_type=sa.BOOLEAN(), + nullable=False, + existing_server_default=sa.text('false')) + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('CURRENT_TIMESTAMP')) + batch_op.alter_column('updated_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + op.execute("DROP INDEX IF EXISTS idx_drive_folders_owner") + op.execute("DROP INDEX IF EXISTS idx_drive_folders_parent") + op.execute("DROP INDEX IF EXISTS idx_drive_folders_trashed") + + with op.batch_alter_table('drive_shares', schema=None) as batch_op: + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('CURRENT_TIMESTAMP')) + op.execute("DROP INDEX IF EXISTS idx_drive_shares_resource") + op.execute("DROP INDEX IF EXISTS uq_drive_shares_link") + batch_op.create_unique_constraint(None, ['link_token']) + batch_op.drop_constraint(batch_op.f('drive_shares_created_by_id_fkey'), type_='foreignkey') + batch_op.create_foreign_key(None, 'users', ['created_by_id'], ['id'], ondelete='CASCADE') + + with op.batch_alter_table('project_versions', schema=None) as batch_op: + batch_op.alter_column('is_current', + existing_type=sa.BOOLEAN(), + nullable=False, + existing_server_default=sa.text('false')) + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('CURRENT_TIMESTAMP')) + op.execute("DROP INDEX IF EXISTS idx_project_versions_created_by") + op.execute("DROP INDEX IF EXISTS idx_project_versions_project_id") + + with op.batch_alter_table('projects', schema=None) as batch_op: + batch_op.alter_column('status', + existing_type=sa.VARCHAR(length=50), + nullable=False, + existing_server_default=sa.text("'processing'::character varying")) + batch_op.alter_column('file_type', + existing_type=sa.VARCHAR(length=50), + nullable=False, + existing_server_default=sa.text("'PDF'::character varying")) + batch_op.alter_column('output_format', + existing_type=sa.VARCHAR(length=50), + nullable=False, + existing_server_default=sa.text("'EPUB'::character varying")) + batch_op.alter_column('progress', + existing_type=sa.INTEGER(), + nullable=False, + existing_server_default=sa.text('0')) + batch_op.alter_column('current_step', + existing_type=sa.VARCHAR(length=50), + nullable=False, + existing_server_default=sa.text("'upload'::character varying")) + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('CURRENT_TIMESTAMP')) + batch_op.alter_column('updated_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + op.execute("DROP INDEX IF EXISTS idx_projects_session_id") + op.execute("DROP INDEX IF EXISTS idx_projects_user_id") + batch_op.create_foreign_key(None, 'users', ['user_id'], ['id'], ondelete='CASCADE') + + with op.batch_alter_table('signing_requests', schema=None) as batch_op: + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + + with op.batch_alter_table('system_configurations', schema=None) as batch_op: + batch_op.drop_constraint(batch_op.f('uq_system_configurations_config_key'), type_='unique') + + with op.batch_alter_table('tenant_smtp_configs', schema=None) as batch_op: + batch_op.drop_constraint(batch_op.f('uq_tenant_smtp_configs_tenant_id'), type_='unique') + op.execute("DROP INDEX IF EXISTS ix_tenant_smtp_configs_tenant_id") + op.execute("CREATE UNIQUE INDEX IF NOT EXISTS ix_tenant_smtp_configs_tenant_id ON tenant_smtp_configs (tenant_id)") + + with op.batch_alter_table('tenants', schema=None) as batch_op: + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + batch_op.alter_column('updated_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + batch_op.drop_constraint(batch_op.f('tenants_slug_key'), type_='unique') + op.execute("CREATE INDEX IF NOT EXISTS ix_tenants_id ON tenants (id)") + op.execute("CREATE UNIQUE INDEX IF NOT EXISTS ix_tenants_slug ON tenants (slug)") + + with op.batch_alter_table('user_files', schema=None) as batch_op: + batch_op.add_column(sa.Column('storage_provider', sa.String(length=50), nullable=False, server_default='minio')) + op.execute("DROP INDEX IF EXISTS idx_user_files_s3_key") + op.execute("DROP INDEX IF EXISTS idx_user_files_user") + + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.alter_column('subscription', + existing_type=sa.VARCHAR(length=50), + nullable=False, + existing_server_default=sa.text("'free'::character varying")) + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('CURRENT_TIMESTAMP')) + op.execute("DROP INDEX IF EXISTS idx_users_email") + op.execute("CREATE INDEX IF NOT EXISTS ix_users_email ON users (email)") + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('users', schema=None) as batch_op: + op.execute("DROP INDEX IF EXISTS ix_users_email") + op.execute("CREATE INDEX IF NOT EXISTS idx_users_email ON users (email)") + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('CURRENT_TIMESTAMP')) + batch_op.alter_column('subscription', + existing_type=sa.VARCHAR(length=50), + nullable=True, + existing_server_default=sa.text("'free'::character varying")) + + with op.batch_alter_table('user_files', schema=None) as batch_op: + op.execute("CREATE INDEX IF NOT EXISTS idx_user_files_user ON user_files (user_id)") + op.execute("CREATE INDEX IF NOT EXISTS idx_user_files_s3_key ON user_files (s3_key)") + op.execute("ALTER TABLE user_files DROP COLUMN IF EXISTS storage_provider") + + with op.batch_alter_table('tenants', schema=None) as batch_op: + op.execute("DROP INDEX IF EXISTS ix_tenants_slug") + op.execute("DROP INDEX IF EXISTS ix_tenants_id") + batch_op.create_unique_constraint(batch_op.f('tenants_slug_key'), ['slug'], postgresql_nulls_not_distinct=False) + batch_op.alter_column('updated_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('now()')) + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('now()')) + + with op.batch_alter_table('tenant_smtp_configs', schema=None) as batch_op: + op.execute("DROP INDEX IF EXISTS ix_tenant_smtp_configs_tenant_id") + op.execute("CREATE INDEX IF NOT EXISTS ix_tenant_smtp_configs_tenant_id ON tenant_smtp_configs (tenant_id)") + batch_op.create_unique_constraint(batch_op.f('uq_tenant_smtp_configs_tenant_id'), ['tenant_id'], postgresql_nulls_not_distinct=False) + + with op.batch_alter_table('system_configurations', schema=None) as batch_op: + batch_op.create_unique_constraint(batch_op.f('uq_system_configurations_config_key'), ['config_key'], postgresql_nulls_not_distinct=False) + + with op.batch_alter_table('signing_requests', schema=None) as batch_op: + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('now()')) + + with op.batch_alter_table('projects', schema=None) as batch_op: + batch_op.drop_constraint(None, type_='foreignkey') + op.execute("CREATE INDEX IF NOT EXISTS idx_projects_user_id ON projects (user_id)") + op.execute("CREATE INDEX IF NOT EXISTS idx_projects_session_id ON projects (session_id)") + batch_op.alter_column('updated_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('now()')) + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('CURRENT_TIMESTAMP')) + batch_op.alter_column('current_step', + existing_type=sa.VARCHAR(length=50), + nullable=True, + existing_server_default=sa.text("'upload'::character varying")) + batch_op.alter_column('progress', + existing_type=sa.INTEGER(), + nullable=True, + existing_server_default=sa.text('0')) + batch_op.alter_column('output_format', + existing_type=sa.VARCHAR(length=50), + nullable=True, + existing_server_default=sa.text("'EPUB'::character varying")) + batch_op.alter_column('file_type', + existing_type=sa.VARCHAR(length=50), + nullable=True, + existing_server_default=sa.text("'PDF'::character varying")) + batch_op.alter_column('status', + existing_type=sa.VARCHAR(length=50), + nullable=True, + existing_server_default=sa.text("'processing'::character varying")) + + with op.batch_alter_table('project_versions', schema=None) as batch_op: + op.execute("CREATE INDEX IF NOT EXISTS idx_project_versions_project_id ON project_versions (project_id)") + op.execute("CREATE INDEX IF NOT EXISTS idx_project_versions_created_by ON project_versions (created_by_id)") + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('CURRENT_TIMESTAMP')) + batch_op.alter_column('is_current', + existing_type=sa.BOOLEAN(), + nullable=True, + existing_server_default=sa.text('false')) + + with op.batch_alter_table('drive_shares', schema=None) as batch_op: + batch_op.drop_constraint(None, type_='foreignkey') + batch_op.create_foreign_key(batch_op.f('drive_shares_created_by_id_fkey'), 'users', ['created_by_id'], ['id']) + batch_op.drop_constraint(None, type_='unique') + op.execute("CREATE INDEX IF NOT EXISTS uq_drive_shares_link ON drive_shares (link_token)") + op.execute("CREATE INDEX IF NOT EXISTS idx_drive_shares_resource ON drive_shares (resource_type,resource_id)") + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('CURRENT_TIMESTAMP')) + + with op.batch_alter_table('drive_folders', schema=None) as batch_op: + op.execute("CREATE INDEX IF NOT EXISTS idx_drive_folders_trashed ON drive_folders (is_trashed)") + op.execute("CREATE INDEX IF NOT EXISTS idx_drive_folders_parent ON drive_folders (parent_id)") + op.execute("CREATE INDEX IF NOT EXISTS idx_drive_folders_owner ON drive_folders (owner_id)") + batch_op.alter_column('updated_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('now()')) + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('CURRENT_TIMESTAMP')) + batch_op.alter_column('is_trashed', + existing_type=sa.BOOLEAN(), + nullable=True, + existing_server_default=sa.text('false')) + batch_op.alter_column('is_starred', + existing_type=sa.BOOLEAN(), + nullable=True, + existing_server_default=sa.text('false')) + + with op.batch_alter_table('drive_files', schema=None) as batch_op: + op.execute("CREATE INDEX IF NOT EXISTS idx_drive_files_trashed ON drive_files (is_trashed)") + op.execute("CREATE INDEX IF NOT EXISTS idx_drive_files_owner ON drive_files (owner_id)") + op.execute("CREATE INDEX IF NOT EXISTS idx_drive_files_folder ON drive_files (folder_id)") + batch_op.alter_column('updated_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('now()')) + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('CURRENT_TIMESTAMP')) + batch_op.alter_column('is_trashed', + existing_type=sa.BOOLEAN(), + nullable=True, + existing_server_default=sa.text('false')) + batch_op.alter_column('is_starred', + existing_type=sa.BOOLEAN(), + nullable=True, + existing_server_default=sa.text('false')) + batch_op.alter_column('size', + existing_type=sa.BIGINT(), + nullable=True, + existing_server_default=sa.text('0')) + + with op.batch_alter_table('drive_file_versions', schema=None) as batch_op: + op.execute("CREATE INDEX IF NOT EXISTS idx_drive_versions_file ON drive_file_versions (file_id)") + op.execute("ALTER TABLE drive_file_versions DROP COLUMN IF EXISTS storage_provider") + + with op.batch_alter_table('drive_activities', schema=None) as batch_op: + batch_op.add_column(sa.Column('metadata', sa.TEXT(), autoincrement=False, nullable=True)) + op.execute("CREATE INDEX IF NOT EXISTS idx_drive_activities_res ON drive_activities (resource_type,resource_id)") + op.execute("CREATE INDEX IF NOT EXISTS idx_drive_activities_actor ON drive_activities (actor_id)") + + with op.batch_alter_table('accesses', schema=None) as batch_op: + batch_op.add_column(sa.Column('description', sa.VARCHAR(length=255), autoincrement=False, nullable=True)) + op.execute("DROP INDEX IF EXISTS ix_accesses_category") + op.execute("DROP INDEX IF EXISTS ix_accesses_access_code") + batch_op.create_unique_constraint(batch_op.f('uq_accesses_access_code'), ['access_code'], postgresql_nulls_not_distinct=False) + op.execute("CREATE INDEX IF NOT EXISTS ix_accesses_name ON accesses (name)") + batch_op.alter_column('name', + existing_type=sa.String(length=255), + type_=sa.VARCHAR(length=100), + existing_nullable=False) + + with op.batch_alter_table('tenant_storage_configs', schema=None) as batch_op: + op.execute("DROP INDEX IF EXISTS ix_tenant_storage_configs_tenant_id") + + op.execute("DROP TABLE IF EXISTS tenant_storage_configs") + # ### end Alembic commands ### diff --git a/alembic/versions/9a6db38b86a6_make_drivecomment_generic.py b/alembic/versions/9a6db38b86a6_make_drivecomment_generic.py new file mode 100644 index 0000000..6b03ba2 --- /dev/null +++ b/alembic/versions/9a6db38b86a6_make_drivecomment_generic.py @@ -0,0 +1,57 @@ +"""Make DriveComment generic + +Revision ID: 9a6db38b86a6 +Revises: 41bed7eb6ed9 +Create Date: 2026-02-27 14:32:05.071480 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '9a6db38b86a6' +down_revision: Union[str, None] = '41bed7eb6ed9' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # 1. Add columns as nullable first + op.add_column('drive_comments', sa.Column('resource_type', sa.String(length=20), nullable=True)) + op.add_column('drive_comments', sa.Column('resource_id', sa.Integer(), nullable=True)) + + # 2. Populate data from existing file_id safely + # Use WHERE file_id IS NOT NULL to avoid unnecessary or invalid updates + op.execute("UPDATE drive_comments SET resource_type = 'file', resource_id = file_id WHERE file_id IS NOT NULL") + + # 3. Set to NOT NULL now that data is populated + op.alter_column('drive_comments', 'resource_type', nullable=False) + op.alter_column('drive_comments', 'resource_id', nullable=False) + + # 4. Cleanup old column and constraint + with op.batch_alter_table('drive_comments') as batch_op: + batch_op.drop_column('file_id') + + +def downgrade() -> None: + # 1. Add column back as nullable first + op.add_column('drive_comments', sa.Column('file_id', sa.Integer(), nullable=True)) + + # 2. Restore data from generic columns back to file_id + # Only migrate records that were originally or are currently 'file' types + op.execute("UPDATE drive_comments SET file_id = resource_id WHERE resource_type = 'file'") + + # 3. Delete or handle non-file comments that cannot exist in the old schema + # Alternatively, keep file_id as nullable if preferred, but here we restore original state + op.execute("DELETE FROM drive_comments WHERE resource_type != 'file'") + + # 4. Restore constraints + op.alter_column('drive_comments', 'file_id', nullable=False) + op.create_foreign_key('drive_comments_file_id_fkey', 'drive_comments', 'drive_files', ['file_id'], ['id']) + + # 5. Cleanup generic columns + op.drop_column('drive_comments', 'resource_id') + op.drop_column('drive_comments', 'resource_type') diff --git a/alembic/versions/9af99fc7ca0f_merge_heads.py b/alembic/versions/9af99fc7ca0f_merge_heads.py new file mode 100644 index 0000000..d4b7c0f --- /dev/null +++ b/alembic/versions/9af99fc7ca0f_merge_heads.py @@ -0,0 +1,26 @@ +"""merge heads + +Revision ID: 9af99fc7ca0f +Revises: 15f935213b22, cba70ab35793, v3_extraction +Create Date: 2026-04-18 14:22:43.985119 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '9af99fc7ca0f' +down_revision: Union[str, None] = ('15f935213b22', 'cba70ab35793', 'v3_extraction') +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/alembic/versions/9c49fe120ffb_add_device_and_session_tables.py b/alembic/versions/9c49fe120ffb_add_device_and_session_tables.py new file mode 100644 index 0000000..9a4dd60 --- /dev/null +++ b/alembic/versions/9c49fe120ffb_add_device_and_session_tables.py @@ -0,0 +1,79 @@ +"""Add device and session tables + +Revision ID: 9c49fe120ffb +Revises: z004_add_global_workflows +Create Date: 2026-08-06 15:27:12.015009 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '9c49fe120ffb' +down_revision: Union[str, None] = 'z004_add_global_workflows' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('devices', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('fingerprint', sa.String(length=255), nullable=False), + sa.Column('browser', sa.String(length=100), nullable=True), + sa.Column('os', sa.String(length=100), nullable=True), + sa.Column('device_type', sa.String(length=50), nullable=True), + sa.Column('user_agent', sa.String(length=512), nullable=True), + sa.Column('ip', sa.String(length=50), nullable=True), + sa.Column('country', sa.String(length=100), nullable=True), + sa.Column('state', sa.String(length=100), nullable=True), + sa.Column('city', sa.String(length=100), nullable=True), + sa.Column('latitude', sa.Float(), nullable=True), + sa.Column('longitude', sa.Float(), nullable=True), + sa.Column('first_login', sa.DateTime(timezone=True), nullable=False), + sa.Column('last_login', sa.DateTime(timezone=True), nullable=False), + sa.Column('last_ip', sa.String(length=50), nullable=True), + sa.Column('is_blocked', sa.Boolean(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_devices_fingerprint'), 'devices', ['fingerprint'], unique=False) + op.create_index(op.f('ix_devices_user_id'), 'devices', ['user_id'], unique=False) + op.create_table('sessions', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('device_id', sa.Integer(), nullable=True), + sa.Column('refresh_token_hash', sa.String(length=255), nullable=False), + sa.Column('status', sa.String(length=50), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('last_activity', sa.DateTime(timezone=True), nullable=False), + sa.Column('expires_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('revoked_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('revoked_by', sa.String(length=50), nullable=True), + sa.Column('revoked_by_user_id', sa.Integer(), nullable=True), + sa.Column('reason', sa.String(length=255), nullable=True), + sa.ForeignKeyConstraint(['device_id'], ['devices.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['revoked_by_user_id'], ['users.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_sessions_device_id'), 'sessions', ['device_id'], unique=False) + op.create_index(op.f('ix_sessions_user_id'), 'sessions', ['user_id'], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_sessions_user_id'), table_name='sessions') + op.drop_index(op.f('ix_sessions_device_id'), table_name='sessions') + op.drop_table('sessions') + op.drop_index(op.f('ix_devices_user_id'), table_name='devices') + op.drop_index(op.f('ix_devices_fingerprint'), table_name='devices') + op.drop_table('devices') + # ### end Alembic commands ### diff --git a/alembic/versions/9c4a95ac6726_add_saasrolemapping.py b/alembic/versions/9c4a95ac6726_add_saasrolemapping.py new file mode 100644 index 0000000..8365943 --- /dev/null +++ b/alembic/versions/9c4a95ac6726_add_saasrolemapping.py @@ -0,0 +1,45 @@ +"""Add SaaSRoleMapping + +Revision ID: 9c4a95ac6726 +Revises: 905726af73c8 +Create Date: 2026-08-28 15:36:54.084287 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '9c4a95ac6726' +down_revision: Union[str, None] = '905726af73c8' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('saas_role_mappings', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('saas_role_id', sa.String(), nullable=False), + sa.Column('docqube_role_id', sa.UUID(), nullable=False), + sa.Column('metadata', sa.JSON(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), + sa.ForeignKeyConstraint(['docqube_role_id'], ['roles.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_saas_role_mappings_id'), 'saas_role_mappings', ['id'], unique=False) + op.create_index(op.f('ix_saas_role_mappings_saas_role_id'), 'saas_role_mappings', ['saas_role_id'], unique=True) + op.execute("ALTER TABLE tenant_contacts DROP CONSTRAINT IF EXISTS uq_tenant_contacts_tenant_id_email") + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_unique_constraint('uq_tenant_contacts_tenant_id_email', 'tenant_contacts', ['tenant_id', 'email']) + op.drop_index(op.f('ix_saas_role_mappings_saas_role_id'), table_name='saas_role_mappings') + op.drop_index(op.f('ix_saas_role_mappings_id'), table_name='saas_role_mappings') + op.drop_table('saas_role_mappings') + # ### end Alembic commands ### diff --git a/alembic/versions/9f6521a38bf3_limit_chatbotdocument_file_path_and_.py b/alembic/versions/9f6521a38bf3_limit_chatbotdocument_file_path_and_.py new file mode 100644 index 0000000..cc03f5f --- /dev/null +++ b/alembic/versions/9f6521a38bf3_limit_chatbotdocument_file_path_and_.py @@ -0,0 +1,30 @@ +"""Limit ChatbotDocument file_path and filename to 512 chars + +Revision ID: 9f6521a38bf3 +Revises: 3aa7d970b8c2 +Create Date: 2026-03-03 22:12:00.390551 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '9f6521a38bf3' +down_revision: Union[str, None] = '3aa7d970b8c2' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/alembic/versions/a1b2c3d4e5f6_add_rbac_tables.py b/alembic/versions/a1b2c3d4e5f6_add_rbac_tables.py new file mode 100644 index 0000000..f81d5da --- /dev/null +++ b/alembic/versions/a1b2c3d4e5f6_add_rbac_tables.py @@ -0,0 +1,90 @@ +"""Add RBAC tables (accesses, roles, role_accesses) and role_id to users + +Revision ID: a1b2c3d4e5f6 +Revises: ff9052a97835 +Create Date: 2026-03-05 11:15:00.000000 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import UUID + + +# revision identifiers, used by Alembic. +revision: str = "a1b2c3d4e5f6" +down_revision: Union[str, None] = ("6c9cc3567866", "ff9052a97835") +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # 1. Create accesses table + op.execute( + """ + CREATE TABLE accesses ( + id SERIAL PRIMARY KEY, + name VARCHAR(100) NOT NULL UNIQUE, + description VARCHAR(255), + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL + ); + """ + ) + op.execute("CREATE INDEX ix_accesses_name ON accesses (name);") + + # 2. Create roles table (tenant_id nullable = system-wide role) + op.execute( + """ + CREATE TABLE roles ( + id SERIAL PRIMARY KEY, + tenant_id UUID REFERENCES tenants(id) ON DELETE CASCADE, + name VARCHAR(100) NOT NULL, + description VARCHAR(255), + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL, + CONSTRAINT uq_role_tenant_name UNIQUE (tenant_id, name) + ); + """ + ) + op.execute("CREATE INDEX ix_roles_tenant_id ON roles (tenant_id);") + + # 3. Create role_accesses join table (composite PK) + op.execute( + """ + CREATE TABLE role_accesses ( + role_id INTEGER NOT NULL REFERENCES roles(id) ON DELETE CASCADE, + access_id INTEGER NOT NULL REFERENCES accesses(id) ON DELETE CASCADE, + CONSTRAINT pk_role_accesses PRIMARY KEY (role_id, access_id) + ); + """ + ) + + # 4. Add role_id to users (nullable FK, SET NULL on delete) + op.execute( + """ + ALTER TABLE users + ADD COLUMN role_id INTEGER REFERENCES roles(id) ON DELETE SET NULL; + """ + ) + op.execute("CREATE INDEX ix_users_role_id ON users (role_id);") + + + +def downgrade() -> None: + # Reverse in dependency order + + # Remove role_id from users + op.execute("DROP INDEX IF EXISTS ix_users_role_id;") + op.execute("ALTER TABLE users DROP COLUMN IF EXISTS role_id;") + + # Drop join table + op.execute("DROP TABLE IF EXISTS role_accesses;") + + # Drop roles table (seed data goes with it) + op.execute("DROP INDEX IF EXISTS ix_roles_tenant_id;") + op.execute("DROP TABLE IF EXISTS roles;") + + # Drop accesses table + op.execute("DROP INDEX IF EXISTS ix_accesses_name;") + op.execute("DROP TABLE IF EXISTS accesses;") diff --git a/alembic/versions/a2b3c4d5e6f7_update_access_model_with_hierarchy.py b/alembic/versions/a2b3c4d5e6f7_update_access_model_with_hierarchy.py new file mode 100644 index 0000000..5dcf1a5 --- /dev/null +++ b/alembic/versions/a2b3c4d5e6f7_update_access_model_with_hierarchy.py @@ -0,0 +1,75 @@ +"""Update Access model with hierarchical permission support + +Revision ID: a2b3c4d5e6f7 +Revises: a1b2c3d4e5f6 +Create Date: 2026-03-05 10:00:00.000000 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'a2b3c4d5e6f7' +down_revision = 'a1b2c3d4e5f6' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # FIRST: Drop any existing unique constraint on `name` (constraints may be named differently depending on DB) + op.execute("ALTER TABLE accesses DROP CONSTRAINT IF EXISTS uq_accesses_name CASCADE") + op.execute("ALTER TABLE accesses DROP CONSTRAINT IF EXISTS accesses_name_key CASCADE") + # also drop postgresql-generated index if it survived + op.execute("DROP INDEX IF EXISTS accesses_name_key") + + # Add new columns to accesses table (all nullable first) + op.add_column('accesses', sa.Column('access_code', sa.String(255), nullable=True)) + op.add_column('accesses', sa.Column('category', sa.String(100), nullable=True)) + op.add_column('accesses', sa.Column('parent_id', sa.Integer(), nullable=True)) + + # Create indexes on new columns + op.create_index('ix_access_code', 'accesses', ['access_code']) + op.create_index('ix_category', 'accesses', ['category']) + op.create_index('ix_accesses_parent_id', 'accesses', ['parent_id']) + + # Set default values from existing 'name' field + op.execute(""" + UPDATE accesses + SET access_code = LOWER(REPLACE(name, ' ', '_')), + category = 'General' + WHERE access_code IS NULL + """) + + # Now make columns NOT NULL + op.alter_column('accesses', 'access_code', existing_type=sa.String(255), nullable=False) + op.alter_column('accesses', 'category', existing_type=sa.String(100), nullable=False) + + # Create unique constraint on access_code + op.create_unique_constraint('uq_accesses_access_code', 'accesses', ['access_code']) + + # Create foreign key for parent_id + op.create_foreign_key('fk_accesses_parent_id', 'accesses', 'accesses', ['parent_id'], ['id'], ondelete='CASCADE') + + +def downgrade() -> None: + # Drop foreign key + op.drop_constraint('fk_accesses_parent_id', 'accesses', type_='foreignkey') + + # Drop unique constraint + op.drop_constraint('uq_accesses_access_code', 'accesses', type_='unique') + + # Drop indexes + op.drop_index('ix_accesses_parent_id', 'accesses') + op.drop_index('ix_category', 'accesses') + op.drop_index('ix_access_code', 'accesses') + + # Drop columns + op.drop_column('accesses', 'parent_id') + op.drop_column('accesses', 'category') + op.drop_column('accesses', 'access_code') + + # Re-add unique constraint on 'name' + op.create_unique_constraint('uq_accesses_name', 'accesses', ['name']) + + diff --git a/alembic/versions/a902935ea69e_remove_redundant_id_column_from_user_.py b/alembic/versions/a902935ea69e_remove_redundant_id_column_from_user_.py new file mode 100644 index 0000000..ee2d947 --- /dev/null +++ b/alembic/versions/a902935ea69e_remove_redundant_id_column_from_user_.py @@ -0,0 +1,34 @@ +"""Remove redundant id column from user_storage_usage + +Revision ID: a902935ea69e +Revises: 0e1a5f2cdea9 +Create Date: 2026-03-02 10:53:47.556155 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'a902935ea69e' +down_revision: Union[str, None] = '0e1a5f2cdea9' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index('ix_user_storage_usage_id', table_name='user_storage_usage') + op.drop_constraint('user_storage_usage_user_id_key', 'user_storage_usage', type_='unique') + op.drop_column('user_storage_usage', 'id') + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('user_storage_usage', sa.Column('id', sa.INTEGER(), autoincrement=False, nullable=False)) + op.create_unique_constraint('user_storage_usage_user_id_key', 'user_storage_usage', ['user_id'], postgresql_nulls_not_distinct=False) + op.create_index('ix_user_storage_usage_id', 'user_storage_usage', ['id'], unique=False) + # ### end Alembic commands ### diff --git a/alembic/versions/af57c3d9f1c2_add_signature_config_tables.py b/alembic/versions/af57c3d9f1c2_add_signature_config_tables.py new file mode 100644 index 0000000..92e64c6 --- /dev/null +++ b/alembic/versions/af57c3d9f1c2_add_signature_config_tables.py @@ -0,0 +1,64 @@ +"""add signature config tables + +Revision ID: af57c3d9f1c2 +Revises: daf45366d608 +Create Date: 2026-07-20 18:53:11.916379 + +""" +from typing import Sequence, Union +import uuid + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +# revision identifiers, used by Alembic. +revision: str = 'af57c3d9f1c2' +down_revision: Union[str, None] = 'daf45366d608' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + 'global_signature_configs', + sa.Column('id', postgresql.UUID(as_uuid=True), primary_key=True), + sa.Column('active_provider', sa.String(length=50), nullable=False, server_default='zoho_sign'), + sa.Column('docuseal_credentials', postgresql.JSON(astext_type=sa.Text()), nullable=True), + sa.Column('zoho_credentials', postgresql.JSON(astext_type=sa.Text()), nullable=True), + sa.Column('created_by_id', sa.Integer(), nullable=True), + sa.Column('updated_by_id', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['created_by_id'], ['users.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['updated_by_id'], ['users.id'], ondelete='SET NULL') + ) + + op.execute( + "INSERT INTO global_signature_configs (id, active_provider, created_at, updated_at) " + "VALUES (gen_random_uuid(), 'zoho_sign', now(), now())" + ) + + op.create_table( + 'tenant_signature_configs', + sa.Column('id', postgresql.UUID(as_uuid=True), primary_key=True), + sa.Column('tenant_id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('use_custom_credentials', sa.Boolean(), nullable=False, server_default='false'), + sa.Column('custom_provider', sa.String(length=50), nullable=False, server_default='docuseal'), + sa.Column('custom_credentials_json', postgresql.JSON(astext_type=sa.Text()), nullable=True), + sa.Column('created_by_id', sa.Integer(), nullable=True), + sa.Column('updated_by_id', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['created_by_id'], ['users.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['updated_by_id'], ['users.id'], ondelete='SET NULL') + ) + op.create_index(op.f('ix_tenant_signature_configs_tenant_id'), 'tenant_signature_configs', ['tenant_id'], unique=True) + + +def downgrade() -> None: + op.drop_index(op.f('ix_tenant_signature_configs_tenant_id'), table_name='tenant_signature_configs') + op.drop_table('tenant_signature_configs') + op.drop_table('global_signature_configs') diff --git a/alembic/versions/b029d2f5a0c7_add_signature_id.py b/alembic/versions/b029d2f5a0c7_add_signature_id.py new file mode 100644 index 0000000..3ef076b --- /dev/null +++ b/alembic/versions/b029d2f5a0c7_add_signature_id.py @@ -0,0 +1,36 @@ +"""add_signature_id + +Revision ID: b029d2f5a0c7 +Revises: fb6810c36ffb +Create Date: 2026-04-14 14:59:09.399640 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'b029d2f5a0c7' +down_revision: Union[str, None] = 'fb6810c36ffb' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('signing_requests', schema=None) as batch_op: + batch_op.add_column(sa.Column('signature_id', sa.String(length=128), nullable=True)) + batch_op.create_index(batch_op.f('ix_signing_requests_signature_id'), ['signature_id'], unique=False) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('signing_requests', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_signing_requests_signature_id')) + batch_op.drop_column('signature_id') + + # ### end Alembic commands ### diff --git a/alembic/versions/b0a09cd13348_add_persistent_chat_history_tables.py b/alembic/versions/b0a09cd13348_add_persistent_chat_history_tables.py new file mode 100644 index 0000000..50509b2 --- /dev/null +++ b/alembic/versions/b0a09cd13348_add_persistent_chat_history_tables.py @@ -0,0 +1,131 @@ +"""Add persistent chat history tables + +Revision ID: b0a09cd13348 +Revises: w001_versioning_minor +Create Date: 2026-05-06 16:38:12.054824 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = 'b0a09cd13348' +down_revision: Union[str, None] = 'w001_versioning_minor' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + from sqlalchemy.engine.reflection import Inspector + conn = op.get_bind() + inspector = Inspector.from_engine(conn) + + if not inspector.has_table('activity_logs'): + op.create_table('activity_logs', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('tenant_id', sa.UUID(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=True), + sa.Column('user_name', sa.String(length=255), nullable=True), + sa.Column('user_email', sa.String(length=255), nullable=True), + sa.Column('module', sa.String(length=50), nullable=False), + sa.Column('action', sa.String(length=100), nullable=False), + sa.Column('target_id', sa.String(length=255), nullable=False), + sa.Column('target_type', sa.String(length=50), nullable=False), + sa.Column('target_name', sa.String(length=512), nullable=True), + sa.Column('metadata', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('ip_address', sa.String(length=64), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('request_id', sa.String(length=255), nullable=True), + sa.Column('status', sa.String(length=20), nullable=False), + sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('activity_logs', schema=None) as batch_op: + batch_op.create_index('ix_activity_logs_module_action', ['module', 'action'], unique=False) + batch_op.create_index('ix_activity_logs_target_id', ['target_id'], unique=False) + batch_op.create_index('ix_activity_logs_tenant_created_at', ['tenant_id', 'created_at'], unique=False) + batch_op.create_index('ix_activity_logs_user_created_at', ['user_id', 'created_at'], unique=False) + + if not inspector.has_table('chat_sessions'): + op.create_table('chat_sessions', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('session_id', sa.UUID(), nullable=False), + sa.Column('tenant_id', sa.UUID(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('chatbot_document_id', sa.Integer(), nullable=True), + sa.Column('title', sa.String(length=255), nullable=True), + sa.Column('created_at', sa.TIMESTAMP(), nullable=False), + sa.Column('updated_at', sa.TIMESTAMP(), nullable=False), + sa.ForeignKeyConstraint(['chatbot_document_id'], ['chatbot_documents.id'], ), + sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('chat_sessions', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_chat_sessions_chatbot_document_id'), ['chatbot_document_id'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_sessions_session_id'), ['session_id'], unique=True) + batch_op.create_index(batch_op.f('ix_chat_sessions_tenant_id'), ['tenant_id'], unique=False) + batch_op.create_index(batch_op.f('ix_chat_sessions_user_id'), ['user_id'], unique=False) + + if not inspector.has_table('chat_messages'): + op.create_table('chat_messages', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('message_id', sa.Uuid(), nullable=False), + sa.Column('session_id', sa.Integer(), nullable=False), + sa.Column('role', sa.String(length=20), nullable=False), + sa.Column('content', sa.Text(), nullable=False), + sa.Column('token_count', sa.Integer(), nullable=True), + sa.Column('created_at', sa.TIMESTAMP(), nullable=False), + sa.ForeignKeyConstraint(['session_id'], ['chat_sessions.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('message_id') + ) + with op.batch_alter_table('chat_messages', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_chat_messages_session_id'), ['session_id'], unique=False) + + with op.batch_alter_table('drive_file_versions', schema=None) as batch_op: + # Check if index exists before dropping + indexes = [idx['name'] for idx in inspector.get_indexes('drive_file_versions')] + if 'ix_drive_file_versions_file_major_minor' in indexes: + batch_op.drop_index('ix_drive_file_versions_file_major_minor') + + with op.batch_alter_table('signature_imprints', schema=None) as batch_op: + # Check if index exists before dropping + indexes = [idx['name'] for idx in inspector.get_indexes('signature_imprints')] + if 'ix_signature_imprints_version_id' in indexes: + batch_op.drop_index('ix_signature_imprints_version_id') + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('signature_imprints', schema=None) as batch_op: + batch_op.create_index('ix_signature_imprints_version_id', ['drive_file_version_id'], unique=False) + + with op.batch_alter_table('drive_file_versions', schema=None) as batch_op: + batch_op.create_index('ix_drive_file_versions_file_major_minor', ['file_id', 'version_number', 'minor_version'], unique=False) + + with op.batch_alter_table('chat_messages', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_chat_messages_session_id')) + + op.drop_table('chat_messages') + with op.batch_alter_table('chat_sessions', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_chat_sessions_user_id')) + batch_op.drop_index(batch_op.f('ix_chat_sessions_tenant_id')) + batch_op.drop_index(batch_op.f('ix_chat_sessions_session_id')) + batch_op.drop_index(batch_op.f('ix_chat_sessions_chatbot_document_id')) + + op.drop_table('chat_sessions') + with op.batch_alter_table('activity_logs', schema=None) as batch_op: + batch_op.drop_index('ix_activity_logs_user_created_at') + batch_op.drop_index('ix_activity_logs_tenant_created_at') + batch_op.drop_index('ix_activity_logs_target_id') + batch_op.drop_index('ix_activity_logs_module_action') + + op.drop_table('activity_logs') + # ### end Alembic commands ### diff --git a/alembic/versions/b1_0_explicit_superadmin.py b/alembic/versions/b1_0_explicit_superadmin.py new file mode 100644 index 0000000..f17a2d6 --- /dev/null +++ b/alembic/versions/b1_0_explicit_superadmin.py @@ -0,0 +1,62 @@ +"""B1.0 — make superadmin an explicit flag instead of a null tenant + +Adds users.is_superadmin and backfills it from the signal it replaces, so the +operators who are superadmins today stay superadmins after deploy. + +Until now privilege was inferred from `tenant_id IS NULL`. That made the +*absence* of tenant context a grant of authority: a bug that dropped the tenant +escalated instead of denying. The flag separates "has no tenant" from "may do +anything". + +This migration only adds and backfills. It deliberately does not drop or alter +`tenant_id`, and the application honours both signals for one release, so this +can be reverted without locking anyone out. + +Revision ID: b1_0_explicit_superadmin +Revises: dfae38e4ede3 +""" + +import sqlalchemy as sa +from alembic import op + +revision = "b1_0_explicit_superadmin" +down_revision = "73dff81eed72" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "users", + sa.Column( + "is_superadmin", + sa.Boolean(), + nullable=False, + server_default=sa.false(), + ), + ) + + # Backfill in the same migration, not a follow-up script. If this ran later + # there would be a window in which every superadmin was demoted. + op.execute( + """ + UPDATE users + SET is_superadmin = true + WHERE tenant_id IS NULL + AND is_deleted = false + """ + ) + + op.create_index( + "ix_users_is_superadmin", + "users", + ["is_superadmin"], + postgresql_where=sa.text("is_superadmin"), + ) + + +def downgrade() -> None: + # Safe to reverse: `tenant_id IS NULL` still identifies the same people, + # and the application still honours it. + op.drop_index("ix_users_is_superadmin", table_name="users") + op.drop_column("users", "is_superadmin") diff --git a/alembic/versions/b1_3_row_level_security.py b/alembic/versions/b1_3_row_level_security.py new file mode 100644 index 0000000..7c03b78 --- /dev/null +++ b/alembic/versions/b1_3_row_level_security.py @@ -0,0 +1,87 @@ +"""B1.3 — row-level security on every tenant-owned table + +Enables **and forces** RLS on the 15 tables carrying a `tenant_id`, with a policy +that compares the row's tenant against a session variable the application sets +per request. + +Why this exists when B1.1 already filters queries: the ORM listener attaches to +entity loading, so it cannot filter aggregates — `query(X).count()` compiles to +`SELECT count(*) FROM (SELECT ...)` and comes back unfiltered. RLS applies in the +database regardless of how the query was composed, so it closes a hole the +listener structurally cannot. + +FORCE matters. Without it the policy does not apply to the table's owner, and +migrations run as the owner — so the tables would look protected while the +application, if it ever connected as that role, saw everything. + +This migration is inert until the application sets `docqube.tenant_id`, and +takes effect fully only once the app connects as a NOSUPERUSER / NOBYPASSRLS +role (B1.4). A superuser connection bypasses RLS unconditionally. + +Revision ID: b1_3_row_level_security +Revises: b1_0_explicit_superadmin +""" + +from alembic import op + +revision = "b1_3_row_level_security" +down_revision = "b1_0_explicit_superadmin" +branch_labels = None +depends_on = None + +POLICY = "docqube_tenant_isolation" + +# Derived from the schema when this was written; asserted against +# information_schema by tests/probes/test_rls.py so drift is caught rather than +# assumed. +TENANT_TABLES = [ + "activity_logs", + "chat_sessions", + "chatbot_documents", + "drive_activities", + "drive_files", + "drive_folders", + "notifications", + "projects", + "roles", + "signing_requests", + "tenant_signature_configs", + "tenant_smtp_configs", + "tenant_storage_configs", + "user_files", + "users", +] + +# The predicate, in words: +# * `docqube.bypass = 'on'` — an explicit system operation (migrations, +# background sweeps). Only the application can set this; it is not reachable +# from user input. +# * the row has no tenant — shared system records such as global roles. +# * the row's tenant matches the session variable. +# +# With no variable set, `current_setting(..., true)` returns NULL, `NULLIF` +# keeps it NULL, and `tenant_id = NULL` is NULL — so only shared rows are +# visible. Losing the tenant denies rather than reveals. +USING = """ + coalesce(current_setting('docqube.bypass', true), '') = 'on' + OR tenant_id IS NULL + OR tenant_id = nullif(current_setting('docqube.tenant_id', true), '')::uuid +""" + + +def upgrade() -> None: + for table in TENANT_TABLES: + op.execute(f"ALTER TABLE {table} ENABLE ROW LEVEL SECURITY") + op.execute(f"ALTER TABLE {table} FORCE ROW LEVEL SECURITY") + op.execute(f"DROP POLICY IF EXISTS {POLICY} ON {table}") + op.execute( + f"CREATE POLICY {POLICY} ON {table} " + f"USING ({USING}) WITH CHECK ({USING})" + ) + + +def downgrade() -> None: + for table in TENANT_TABLES: + op.execute(f"DROP POLICY IF EXISTS {POLICY} ON {table}") + op.execute(f"ALTER TABLE {table} NO FORCE ROW LEVEL SECURITY") + op.execute(f"ALTER TABLE {table} DISABLE ROW LEVEL SECURITY") diff --git a/alembic/versions/b3a7c1e9f042_add_collaboration_messages_table.py b/alembic/versions/b3a7c1e9f042_add_collaboration_messages_table.py new file mode 100644 index 0000000..6aa2e01 --- /dev/null +++ b/alembic/versions/b3a7c1e9f042_add_collaboration_messages_table.py @@ -0,0 +1,55 @@ +"""add collaboration_messages table + +Revision ID: b3a7c1e9f042 +Revises: f055f9b157cb +Create Date: 2026-03-02 15:17:00.000000 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'b3a7c1e9f042' +down_revision: Union[str, None] = 'f055f9b157cb' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "collaboration_messages", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "file_id", + sa.Integer(), + sa.ForeignKey("drive_files.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "sender_id", + sa.Integer(), + sa.ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("message", sa.Text(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + ), + ) + + # Performance indexes + op.create_index("ix_collab_msg_file_id", "collaboration_messages", ["file_id"]) + op.create_index("ix_collab_msg_sender_id", "collaboration_messages", ["sender_id"]) + op.create_index("ix_collab_msg_created_at", "collaboration_messages", ["created_at"]) + + +def downgrade() -> None: + op.drop_index("ix_collab_msg_created_at", table_name="collaboration_messages") + op.drop_index("ix_collab_msg_sender_id", table_name="collaboration_messages") + op.drop_index("ix_collab_msg_file_id", table_name="collaboration_messages") + op.drop_table("collaboration_messages") diff --git a/alembic/versions/b6_4_index_review.py b/alembic/versions/b6_4_index_review.py new file mode 100644 index 0000000..205496d --- /dev/null +++ b/alembic/versions/b6_4_index_review.py @@ -0,0 +1,65 @@ +"""B6.4 — indexes on the foreign keys that queries actually filter by + +The database has 34 foreign keys with no supporting index. PostgreSQL does not +create one automatically, and the cost is invisible on a seeded database: a +sequential scan over five rows is instant, over five hundred thousand it is not. + +This adds indexes only where a measured query path filters or joins on the +column — the ones surfaced by the B6.3 query-budget work. Indexing all 34 would +add write cost for paths nothing traverses; the remainder are listed in +`tests/probes/test_index_coverage.py` so they stay visible rather than +forgotten. + +**Deployment note.** `CREATE INDEX` takes a lock that blocks writes for the +duration. On DocQube's current row counts that is brief, but on a large +`drive_files` it will not be — run these with `CONCURRENTLY` outside a +transaction if the table has grown. Alembic cannot do that inside its own +transaction, which is why this migration is written to be easy to translate. + +Revision ID: b6_4_index_review +Revises: b1_3_row_level_security +""" + +from alembic import op + +revision = "b6_4_index_review" +down_revision = "b1_3_row_level_security" +branch_labels = None +depends_on = None + +# (index name, table, columns) — each justified by a query path measured in +# tests/characterization/test_query_budget.py. +INDEXES = [ + # Every folder listing filters files by folder. + ("ix_drive_files_folder_id", "drive_files", "folder_id"), + ("ix_drive_files_owner_id", "drive_files", "owner_id"), + # Hierarchy walks in get_effective_role and breadcrumbs. + ("ix_drive_folders_parent_id", "drive_folders", "parent_id"), + ("ix_drive_folders_owner_id", "drive_folders", "owner_id"), + # The batched star lookup added in B6.3 filters on all three. + ( + "ix_drive_stars_user_resource", + "drive_stars", + "user_id, resource_type, resource_id", + ), + # Version history, loaded per file. + ("ix_drive_file_versions_file_id", "drive_file_versions", "file_id"), + ("ix_project_versions_project_id", "project_versions", "project_id"), + # Permission resolution joins accesses on every authenticated request. + ("ix_role_accesses_access_id", "role_accesses", "access_id"), + # Signing looks requests up by the file they belong to. + ("ix_signing_requests_drive_file_id", "signing_requests", "drive_file_id"), + # Comments and activity are read per resource. + ("ix_drive_comments_author_id", "drive_comments", "author_id"), + ("ix_drive_activities_actor_id", "drive_activities", "actor_id"), +] + + +def upgrade() -> None: + for name, table, columns in INDEXES: + op.execute(f"CREATE INDEX IF NOT EXISTS {name} ON {table} ({columns})") + + +def downgrade() -> None: + for name, _table, _columns in INDEXES: + op.execute(f"DROP INDEX IF EXISTS {name}") diff --git a/alembic/versions/bb224e0b642b_remove_tenant_storage_config_and_use_.py b/alembic/versions/bb224e0b642b_remove_tenant_storage_config_and_use_.py new file mode 100644 index 0000000..d764cc5 --- /dev/null +++ b/alembic/versions/bb224e0b642b_remove_tenant_storage_config_and_use_.py @@ -0,0 +1,48 @@ +"""remove_tenant_storage_config_and_use_system_config + +Revision ID: bb224e0b642b +Revises: i001_dual_bucket_storage +Create Date: 2026-03-17 09:09:02.599878 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'bb224e0b642b' +down_revision: Union[str, None] = 'i001_dual_bucket_storage' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Drop the tenant-specific storage config table + op.drop_table('tenant_storage_configs') + + +def downgrade() -> None: + # Recreate the tenant-specific storage config table + op.create_table( + 'tenant_storage_configs', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('tenant_id', sa.UUID(), nullable=False), + sa.Column('storage_provider', sa.String(length=50), nullable=False), + sa.Column('b2_key_id', sa.String(length=255), nullable=True), + sa.Column('b2_application_key', sa.String(length=512), nullable=True), + sa.Column('b2_quarantine_bucket', sa.String(length=255), nullable=True), + sa.Column('b2_clean_bucket', sa.String(length=255), nullable=True), + sa.Column('is_active', sa.Boolean(), nullable=False), + sa.Column('created_by_id', sa.Integer(), nullable=True), + sa.Column('updated_by_id', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['created_by_id'], ['users.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['updated_by_id'], ['users.id'], ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('tenant_id') + ) + op.create_index(op.f('ix_tenant_storage_configs_tenant_id'), 'tenant_storage_configs', ['tenant_id'], unique=True) diff --git a/alembic/versions/be67927ab00d_add_encrypted_ai_api_key_to_user_model.py b/alembic/versions/be67927ab00d_add_encrypted_ai_api_key_to_user_model.py new file mode 100644 index 0000000..a9e3536 --- /dev/null +++ b/alembic/versions/be67927ab00d_add_encrypted_ai_api_key_to_user_model.py @@ -0,0 +1,38 @@ +"""Add encrypted_ai_api_key to user_model + +Revision ID: be67927ab00d +Revises: b3a7c1e9f042 +Create Date: 2026-03-03 12:43:12.848347 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = 'be67927ab00d' +down_revision: Union[str, None] = 'b3a7c1e9f042' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('collaboration_messages', 'created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + op.add_column('users', sa.Column('encrypted_ai_api_key', sa.String(length=512), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('users', 'encrypted_ai_api_key') + op.alter_column('collaboration_messages', 'created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('now()')) + # ### end Alembic commands ### diff --git a/alembic/versions/c1_0_org_units_and_scoped_roles.py b/alembic/versions/c1_0_org_units_and_scoped_roles.py new file mode 100644 index 0000000..d813cb4 --- /dev/null +++ b/alembic/versions/c1_0_org_units_and_scoped_roles.py @@ -0,0 +1,247 @@ +"""C1/C3 — organisation units, membership, leadership, and scoped role grants + +Four tables, and a backfill that is deliberately a no-op in behaviour. + +`user_roles` exists because `users.role_id` is single-valued: one role, held +across the whole tenant. Scoped access has nowhere to live until a user can hold +the same role in two units. The backfill turns each existing `users.role_id` +into one `user_roles` row with `org_unit_id = NULL`, which means tenant-wide — +exactly the authority that user already had. Nobody gains or loses anything on +this migration. + +`users.role_id` is **not** dropped. Both mechanisms are honoured for one +release, the same shape as the `is_superadmin` transition and for the same +reason: reverting this migration must not be able to remove everyone's +permissions. Removal is a later, deliberate step with its own preflight. + +All four tables carry `tenant_id`, so `app/core/tenant_filter.py` scopes them +automatically, and they are added to the RLS policy here because +`tests/probes/test_isolation_ratchet.py` fails if a tenant-owned table is left +unprotected. + +Revision ID: c1_0_org_units_and_scoped_roles +Revises: b6_4_index_review +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.postgresql import UUID + +revision = "c1_0_org_units_and_scoped_roles" +down_revision = "b6_4_index_review" +branch_labels = None +depends_on = None + +# Kept in step with b1_3_row_level_security.py. +# Named TENANT_TABLES so `test_isolation_ratchet.py` finds it: the ratchet +# scans every migration for this constant, which is what lets a later +# migration add tenant-owned tables without the check going stale. +TENANT_TABLES = ["org_units", "user_org_units", "org_unit_leads"] +POLICY = "tenant_isolation" +USING = """ + coalesce(current_setting('docqube.bypass', true), '') = 'on' + OR tenant_id IS NULL + OR tenant_id = nullif(current_setting('docqube.tenant_id', true), '')::uuid +""" + + +def upgrade() -> None: + op.create_table( + "org_units", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "tenant_id", + UUID(as_uuid=True), + sa.ForeignKey("tenants.id", ondelete="CASCADE"), + nullable=False, + ), + # SET NULL, not CASCADE: deleting a department must not silently delete + # every team beneath it. What happens to orphans is an application + # decision, and it should be visible as one. + sa.Column( + "parent_id", + UUID(as_uuid=True), + sa.ForeignKey("org_units.id", ondelete="SET NULL"), + nullable=True, + ), + sa.Column("name", sa.String(255), nullable=False), + sa.Column("code", sa.String(50), nullable=True), + sa.Column("unit_type", sa.String(50), nullable=True), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("sort_order", sa.Integer(), nullable=False, server_default="0"), + sa.Column("depth", sa.Integer(), nullable=False, server_default="0"), + sa.Column("is_deleted", sa.Boolean(), nullable=False, server_default=sa.false()), + sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + ) + op.create_index("ix_org_units_tenant_id", "org_units", ["tenant_id"]) + op.create_index("ix_org_units_parent_id", "org_units", ["parent_id"]) + op.create_index("ix_org_units_tenant_parent", "org_units", ["tenant_id", "parent_id"]) + # Codes are unique per tenant among *live* rows only — a soft-deleted unit + # must not reserve its code forever. + op.create_index( + "uq_org_units_tenant_code_live", + "org_units", + ["tenant_id", "code"], + unique=True, + postgresql_where=sa.text("NOT is_deleted AND code IS NOT NULL"), + ) + + op.create_table( + "user_org_units", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "tenant_id", + UUID(as_uuid=True), + sa.ForeignKey("tenants.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "user_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=False + ), + sa.Column( + "org_unit_id", + UUID(as_uuid=True), + sa.ForeignKey("org_units.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("is_primary", sa.Boolean(), nullable=False, server_default=sa.false()), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.UniqueConstraint("user_id", "org_unit_id", name="uq_user_org_units_user_unit"), + ) + op.create_index("ix_user_org_units_tenant_id", "user_org_units", ["tenant_id"]) + op.create_index("ix_user_org_units_user_id", "user_org_units", ["user_id"]) + op.create_index("ix_user_org_units_org_unit_id", "user_org_units", ["org_unit_id"]) + # "Exactly one primary" as an index, not as application logic: written in + # Python it is a race between two concurrent writes, which is the same + # defect class as the user_storage_usage get-or-create found under load. + op.create_index( + "uq_user_org_units_one_primary", + "user_org_units", + ["user_id"], + unique=True, + postgresql_where=sa.text("is_primary"), + ) + + op.create_table( + "org_unit_leads", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "tenant_id", + UUID(as_uuid=True), + sa.ForeignKey("tenants.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "org_unit_id", + UUID(as_uuid=True), + sa.ForeignKey("org_units.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "user_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=False + ), + sa.Column("start_date", sa.Date(), nullable=False), + sa.Column("end_date", sa.Date(), nullable=True), + sa.Column("note", sa.Text(), nullable=True), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + ) + op.create_index("ix_org_unit_leads_tenant_id", "org_unit_leads", ["tenant_id"]) + op.create_index("ix_org_unit_leads_org_unit_id", "org_unit_leads", ["org_unit_id"]) + op.create_index("ix_org_unit_leads_user_id", "org_unit_leads", ["user_id"]) + # At most one *current* lead per unit; history is unconstrained, which is + # what makes it usable as history. + op.create_index( + "uq_org_unit_leads_current", + "org_unit_leads", + ["org_unit_id"], + unique=True, + postgresql_where=sa.text("end_date IS NULL"), + ) + + op.create_table( + "user_roles", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "user_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=False + ), + sa.Column( + "role_id", + UUID(as_uuid=True), + sa.ForeignKey("roles.id", ondelete="CASCADE"), + nullable=False, + ), + # NULL means tenant-wide, which is what every grant is today. + sa.Column( + "org_unit_id", + UUID(as_uuid=True), + sa.ForeignKey("org_units.id", ondelete="CASCADE"), + nullable=True, + ), + sa.Column( + "assigned_by_id", + sa.Integer(), + sa.ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.UniqueConstraint("user_id", "role_id", "org_unit_id", name="uq_user_roles_scoped"), + ) + op.create_index("ix_user_roles_user_id", "user_roles", ["user_id"]) + op.create_index("ix_user_roles_role_id", "user_roles", ["role_id"]) + op.create_index("ix_user_roles_org_unit_id", "user_roles", ["org_unit_id"]) + op.create_index("ix_user_roles_lookup", "user_roles", ["user_id", "role_id"]) + # Every foreign key needs a supporting index or a delete on the parent + # sequentially scans this table. `test_index_coverage.py` enforces it. + op.create_index("ix_user_roles_assigned_by_id", "user_roles", ["assigned_by_id"]) + # Postgres treats NULLs as distinct in a UNIQUE constraint, so the triple + # above does not stop the same role being held tenant-wide twice. + op.create_index( + "uq_user_roles_tenant_wide", + "user_roles", + ["user_id", "role_id"], + unique=True, + postgresql_where=sa.text("org_unit_id IS NULL"), + ) + + # Backfill. Behaviour-neutral by construction: a null org unit is + # tenant-wide, which is the authority every user already has. + op.execute( + """ + INSERT INTO user_roles (id, user_id, role_id, org_unit_id, created_at) + SELECT gen_random_uuid(), u.id, u.role_id, NULL, now() + FROM users u + WHERE u.role_id IS NOT NULL + AND u.is_deleted = false + ON CONFLICT DO NOTHING + """ + ) + + for table in TENANT_TABLES: + op.execute(f"ALTER TABLE {table} ENABLE ROW LEVEL SECURITY") + op.execute(f"ALTER TABLE {table} FORCE ROW LEVEL SECURITY") + op.execute(f"DROP POLICY IF EXISTS {POLICY} ON {table}") + op.execute(f"CREATE POLICY {POLICY} ON {table} USING ({USING}) WITH CHECK ({USING})") + + +def downgrade() -> None: + # Safe to reverse: `users.role_id` was never dropped, so removing these + # tables returns the application to the grants it still reads today. + for table in TENANT_TABLES: + op.execute(f"DROP POLICY IF EXISTS {POLICY} ON {table}") + + op.drop_table("user_roles") + op.drop_table("org_unit_leads") + op.drop_table("user_org_units") + op.drop_table("org_units") diff --git a/alembic/versions/c1_1_user_roles_tenant_id.py b/alembic/versions/c1_1_user_roles_tenant_id.py new file mode 100644 index 0000000..f1cc814 --- /dev/null +++ b/alembic/versions/c1_1_user_roles_tenant_id.py @@ -0,0 +1,80 @@ +"""C3.1 — give user_roles a tenant_id, so isolation is structural + +`user_roles` was created without one. That is not a cosmetic omission: both +isolation layers *derive* their coverage from the presence of that column — +`app/core/tenant_filter.py` scopes any model that has it, and the RLS policy +compares it against the session variable. A table without it is protected by +neither, no matter how careful the queries are. + +It is the same shape of gap as `tenants`, which is where "any holder of +`superadmin.tenant.delete` can delete any tenant" lived: a table outside the +mechanism, guarded only by whatever each call site remembered to write. Three +places currently remember. Three is a number that only goes up, and the failure +mode when one forgets is a grant resolving across tenants. + +After this the explicit predicates stay — they are cheap and they document the +intent — but they stop being the only thing standing there. + +**Backfill from the user, not the role.** `roles.tenant_id` is nullable: a null +means a shared/global role template, and a grant of a shared role still belongs +to the tenant of the person holding it. Rows for tenant-less users (superadmins) +stay null, which the RLS policy already treats as globally visible — the same +way their `users` row is treated. + +Revision ID: c1_1_user_roles_tenant_id +Revises: c1_0_org_units_and_scoped_roles +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.postgresql import UUID + +revision = "c1_1_user_roles_tenant_id" +down_revision = "c1_0_org_units_and_scoped_roles" +branch_labels = None +depends_on = None + +TENANT_TABLES = ["user_roles"] +POLICY = "tenant_isolation" +USING = """ + coalesce(current_setting('docqube.bypass', true), '') = 'on' + OR tenant_id IS NULL + OR tenant_id = nullif(current_setting('docqube.tenant_id', true), '')::uuid +""" + + +def upgrade() -> None: + op.add_column( + "user_roles", + sa.Column( + "tenant_id", + UUID(as_uuid=True), + sa.ForeignKey("tenants.id", ondelete="CASCADE"), + nullable=True, + ), + ) + op.create_index("ix_user_roles_tenant_id", "user_roles", ["tenant_id"]) + + op.execute( + """ + UPDATE user_roles ur + SET tenant_id = u.tenant_id + FROM users u + WHERE u.id = ur.user_id + """ + ) + + # Left nullable on purpose. A superadmin has no tenant, so their grants have + # none either; forcing NOT NULL would mean inventing one. + op.execute(f"ALTER TABLE user_roles ENABLE ROW LEVEL SECURITY") + op.execute(f"ALTER TABLE user_roles FORCE ROW LEVEL SECURITY") + op.execute(f"DROP POLICY IF EXISTS {POLICY} ON user_roles") + op.execute(f"CREATE POLICY {POLICY} ON user_roles USING ({USING}) WITH CHECK ({USING})") + + +def downgrade() -> None: + op.execute(f"DROP POLICY IF EXISTS {POLICY} ON user_roles") + op.execute("ALTER TABLE user_roles NO FORCE ROW LEVEL SECURITY") + op.execute("ALTER TABLE user_roles DISABLE ROW LEVEL SECURITY") + op.drop_index("ix_user_roles_tenant_id", table_name="user_roles") + op.drop_column("user_roles", "tenant_id") diff --git a/alembic/versions/c3_0_org_unit_path.py b/alembic/versions/c3_0_org_unit_path.py new file mode 100644 index 0000000..8e7b981 --- /dev/null +++ b/alembic/versions/c3_0_org_unit_path.py @@ -0,0 +1,77 @@ +"""C3 — a materialised path, so a subtree is a prefix match + +`ScopeService` resolved authority with a recursive CTE, once per access code per +request. That is correct and it is the thing that gets slower as the tree gets +deeper, in the way that shows up as "the admin page feels slow" rather than as +an error — the query-count budgets cannot catch it, because the *count* does not +change. + +A materialised path turns the walk into a prefix match: + + /a1b2.../c3d4.../e5f6.../ + +The subtree of a unit is every row whose path starts with that unit's path. +Indexed with `text_pattern_ops`, that is an index range scan. + +**Text, not `ltree`.** `ltree` is the tidier type and needs an extension, which +is a deployment prerequisite for a performance improvement — a poor trade. Text +with a prefix index needs nothing and behaves identically for this query. + +**Why the delimiters matter.** Ids are stored between slashes, so +`LIKE '/a/b/%'` cannot match `/a/bc/...`. Without them a prefix match would leak +across sibling branches whose ids share a leading substring — which for UUIDs is +rare, and rare is worse than never. + +The column is nullable and the CTE remains as a fallback for any row that has +not been backfilled, so this migration cannot break resolution even if the +backfill is incomplete. + +Revision ID: c3_0_org_unit_path +Revises: c5_0_access_groups +""" + +import sqlalchemy as sa +from alembic import op + +revision = "c3_0_org_unit_path" +down_revision = "c5_0_access_groups" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("org_units", sa.Column("path", sa.Text(), nullable=True)) + + # Prefix matching needs `text_pattern_ops`; the default opclass supports + # equality and ordering but not `LIKE 'prefix%'` as an index scan. + op.execute( + "CREATE INDEX ix_org_units_path_prefix " + "ON org_units (path text_pattern_ops)" + ) + op.create_index("ix_org_units_tenant_path", "org_units", ["tenant_id", "path"]) + + # Backfill, walking down from the roots. Done once here so the application + # never has to compute a path it did not write. + op.execute( + """ + WITH RECURSIVE tree AS ( + SELECT id, '/' || id::text || '/' AS path + FROM org_units + WHERE parent_id IS NULL + UNION ALL + SELECT c.id, t.path || c.id::text || '/' + FROM org_units c + JOIN tree t ON c.parent_id = t.id + ) + UPDATE org_units u + SET path = tree.path + FROM tree + WHERE u.id = tree.id + """ + ) + + +def downgrade() -> None: + op.drop_index("ix_org_units_tenant_path", table_name="org_units") + op.execute("DROP INDEX IF EXISTS ix_org_units_path_prefix") + op.drop_column("org_units", "path") diff --git a/alembic/versions/c3abf44266de_enforce_b2_storage_provider.py b/alembic/versions/c3abf44266de_enforce_b2_storage_provider.py new file mode 100644 index 0000000..623435f --- /dev/null +++ b/alembic/versions/c3abf44266de_enforce_b2_storage_provider.py @@ -0,0 +1,40 @@ +"""enforce_b2_storage_provider + +Revision ID: c3abf44266de +Revises: bb224e0b642b +Create Date: 2026-03-17 10:54:51.326256 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'c3abf44266de' +down_revision: Union[str, None] = 'bb224e0b642b' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # 1. Update existing records from 'minio' to 'b2' + op.execute("UPDATE user_files SET storage_provider = 'b2' WHERE storage_provider = 'minio'") + op.execute("UPDATE drive_file_versions SET storage_provider = 'b2' WHERE storage_provider = 'minio'") + + # 2. Change server default to 'b2' + with op.batch_alter_table('user_files', schema=None) as batch_op: + batch_op.alter_column('storage_provider', server_default='b2') + + with op.batch_alter_table('drive_file_versions', schema=None) as batch_op: + batch_op.alter_column('storage_provider', server_default='b2') + + +def downgrade() -> None: + # 1. Change server default back to 'minio' + with op.batch_alter_table('user_files', schema=None) as batch_op: + batch_op.alter_column('storage_provider', server_default='minio') + + with op.batch_alter_table('drive_file_versions', schema=None) as batch_op: + batch_op.alter_column('storage_provider', server_default='minio') diff --git a/alembic/versions/c4_0_grant_expiry.py b/alembic/versions/c4_0_grant_expiry.py new file mode 100644 index 0000000..3a7331e --- /dev/null +++ b/alembic/versions/c4_0_grant_expiry.py @@ -0,0 +1,46 @@ +"""C4A — grants can expire + +`user_roles.expires_at`, nullable. Null means permanent, which is every existing +row, so this migration changes nobody's authority. + +Temporary elevation, contractor access and just-in-time grants all need this, and +all three are ordinary enterprise asks. Without it the only way to end a grant is +to remember to revoke it, and "remember to" is not an access control. + +Expired rows are **not** deleted. They stay, and they become history — which is +what C4B reads to answer "who had access in March". A swept-away row answers +nothing. + +Revision ID: c4_0_grant_expiry +Revises: c1_1_user_roles_tenant_id +""" + +import sqlalchemy as sa +from alembic import op + +revision = "c4_0_grant_expiry" +down_revision = "c1_1_user_roles_tenant_id" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "user_roles", + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True), + ) + + # Partial: only live grants are ever queried by expiry, and indexing the + # permanent ones (which is all of them today) would be indexing a column + # that is null for every row. + op.create_index( + "ix_user_roles_expires_at", + "user_roles", + ["expires_at"], + postgresql_where=sa.text("expires_at IS NOT NULL"), + ) + + +def downgrade() -> None: + op.drop_index("ix_user_roles_expires_at", table_name="user_roles") + op.drop_column("user_roles", "expires_at") diff --git a/alembic/versions/c4_1_access_log_retention.py b/alembic/versions/c4_1_access_log_retention.py new file mode 100644 index 0000000..2368c2b --- /dev/null +++ b/alembic/versions/c4_1_access_log_retention.py @@ -0,0 +1,39 @@ +"""C4C — index the access timeline, so retention and history are cheap + +Two reads matter and neither had an index: + +- the history endpoint, filtering one tenant's `access` events by date; +- the retention sweeper, finding events older than the policy. + +Both are `(tenant_id, module, created_at)`. Partial on `module = 'access'`, +because ordinary activity is far more numerous and does not share these +queries — indexing it here would triple the index for no reader. + +Retention itself is a **setting**, not a constant: how long access records are +kept is the sort of thing a customer contract specifies, and it must be +answerable without a deploy. See `ACCESS_LOG_RETENTION_DAYS`. + +Revision ID: c4_1_access_log_retention +Revises: c4_0_grant_expiry +""" + +import sqlalchemy as sa +from alembic import op + +revision = "c4_1_access_log_retention" +down_revision = "c4_0_grant_expiry" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_index( + "ix_activity_logs_access_timeline", + "activity_logs", + ["tenant_id", "created_at"], + postgresql_where=sa.text("module = 'access'"), + ) + + +def downgrade() -> None: + op.drop_index("ix_activity_logs_access_timeline", table_name="activity_logs") diff --git a/alembic/versions/c5_0_access_groups.py b/alembic/versions/c5_0_access_groups.py new file mode 100644 index 0000000..c1b5148 --- /dev/null +++ b/alembic/versions/c5_0_access_groups.py @@ -0,0 +1,161 @@ +"""C5.1 — access groups, and grants that can name one + +Two new tables, and one genuinely risky change: `user_roles.user_id` becomes +nullable, so a grant can name a **group** instead. + +That weakens an invariant every existing query relied on. Four readers exist and +all four were reviewed before this migration was written: + + * `app/core/scope.py` — resolution; gains a group branch + * `app/modules/auth/services/permission_service.py` — the coarse gate; likewise + * `app/modules/org/routes/org_routes.py` — lists a user's own grants; unchanged + behaviour, it simply will not list grants a user holds *through* a group + * `app/modules/org/services/access_maintenance.py` — the expiry sweeper; its + audit entry uses the principal, which may now be a group + +The `CHECK` is what keeps the pair honest. "Exactly one of two nullable columns" +is the kind of invariant that survives in the schema and rots in code — one new +call site that sets both, and the resolver counts the grant twice. + +Existing rows all have `user_id`, so the constraint holds the moment it is +added, and nobody's authority changes. + +Revision ID: c5_0_access_groups +Revises: c4_1_access_log_retention +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.postgresql import UUID + +revision = "c5_0_access_groups" +down_revision = "c4_1_access_log_retention" +branch_labels = None +depends_on = None + +TENANT_TABLES = ["access_groups", "user_access_groups"] +POLICY = "tenant_isolation" +USING = """ + coalesce(current_setting('docqube.bypass', true), '') = 'on' + OR tenant_id IS NULL + OR tenant_id = nullif(current_setting('docqube.tenant_id', true), '')::uuid +""" + + +def upgrade() -> None: + op.create_table( + "access_groups", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "tenant_id", + UUID(as_uuid=True), + sa.ForeignKey("tenants.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("name", sa.String(255), nullable=False), + sa.Column("description", sa.Text(), nullable=True), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.UniqueConstraint("tenant_id", "name", name="uq_access_groups_tenant_name"), + ) + op.create_index("ix_access_groups_tenant_id", "access_groups", ["tenant_id"]) + + op.create_table( + "user_access_groups", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "tenant_id", + UUID(as_uuid=True), + sa.ForeignKey("tenants.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "user_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=False + ), + sa.Column( + "group_id", + UUID(as_uuid=True), + sa.ForeignKey("access_groups.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.UniqueConstraint("user_id", "group_id", name="uq_user_access_groups"), + ) + op.create_index("ix_user_access_groups_tenant_id", "user_access_groups", ["tenant_id"]) + op.create_index("ix_user_access_groups_user_id", "user_access_groups", ["user_id"]) + op.create_index("ix_user_access_groups_group_id", "user_access_groups", ["group_id"]) + op.create_index( + "ix_user_access_groups_lookup", "user_access_groups", ["user_id", "group_id"] + ) + + # The risky half. + op.add_column( + "user_roles", + sa.Column( + "group_id", + UUID(as_uuid=True), + # CASCADE: deleting a group takes its grants with it. A grant whose + # principal no longer exists is a row nobody can see, revoke or + # audit — and it must not keep applying. + sa.ForeignKey("access_groups.id", ondelete="CASCADE"), + nullable=True, + ), + ) + op.alter_column("user_roles", "user_id", existing_type=sa.Integer(), nullable=True) + op.create_index("ix_user_roles_group_id", "user_roles", ["group_id"]) + + op.create_check_constraint( + "ck_user_roles_exactly_one_principal", + "user_roles", + "(user_id IS NOT NULL) <> (group_id IS NOT NULL)", + ) + + # The tenant-wide uniqueness guard was written for user grants. Group grants + # need the same protection, and the original index cannot serve both + # because `user_id` is now null on half the rows. + op.create_index( + "uq_user_roles_group_tenant_wide", + "user_roles", + ["group_id", "role_id"], + unique=True, + postgresql_where=sa.text("org_unit_id IS NULL AND group_id IS NOT NULL"), + ) + op.create_index( + "uq_user_roles_group_scoped", + "user_roles", + ["group_id", "role_id", "org_unit_id"], + unique=True, + postgresql_where=sa.text("group_id IS NOT NULL"), + ) + + for table in TENANT_TABLES: + op.execute(f"ALTER TABLE {table} ENABLE ROW LEVEL SECURITY") + op.execute(f"ALTER TABLE {table} FORCE ROW LEVEL SECURITY") + op.execute(f"DROP POLICY IF EXISTS {POLICY} ON {table}") + op.execute(f"CREATE POLICY {POLICY} ON {table} USING ({USING}) WITH CHECK ({USING})") + + +def downgrade() -> None: + for table in TENANT_TABLES: + op.execute(f"DROP POLICY IF EXISTS {POLICY} ON {table}") + + # Group grants cannot survive the column being dropped, and they cannot be + # converted into user grants either — the whole point is that they name no + # single user. Removing them is the only honest reverse. + op.execute("DELETE FROM user_roles WHERE group_id IS NOT NULL") + + op.drop_index("uq_user_roles_group_scoped", table_name="user_roles") + op.drop_index("uq_user_roles_group_tenant_wide", table_name="user_roles") + op.drop_constraint("ck_user_roles_exactly_one_principal", "user_roles", type_="check") + op.drop_index("ix_user_roles_group_id", table_name="user_roles") + op.alter_column("user_roles", "user_id", existing_type=sa.Integer(), nullable=False) + op.drop_column("user_roles", "group_id") + + op.drop_table("user_access_groups") + op.drop_table("access_groups") diff --git a/alembic/versions/c7d4e6f1a9b0_add_tenant_storage_quota.py b/alembic/versions/c7d4e6f1a9b0_add_tenant_storage_quota.py new file mode 100644 index 0000000..bc740a0 --- /dev/null +++ b/alembic/versions/c7d4e6f1a9b0_add_tenant_storage_quota.py @@ -0,0 +1,66 @@ +"""add tenant storage quota bytes + +Revision ID: c7d4e6f1a9b0 +Revises: e81a9d22ca5f +Create Date: 2026-03-07 09:25:00.000000 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "c7d4e6f1a9b0" +down_revision: Union[str, None] = "e81a9d22ca5f" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "tenants", + sa.Column("storage_quota_bytes", sa.BigInteger(), nullable=True), + ) + + # Preserve existing effective quota behavior by summing each tenant's + # current per-user quotas. For tenants with no users, default to 1 GB. + op.execute( + """ + UPDATE tenants t + SET storage_quota_bytes = COALESCE( + ( + SELECT SUM(COALESCE(usu.max_bytes_quota, 1073741824)) + FROM users u + LEFT JOIN user_storage_usage usu ON usu.user_id = u.id + WHERE u.tenant_id = t.id + ), + 1073741824 + ) + """ + ) + + op.alter_column( + "tenants", + "storage_quota_bytes", + existing_type=sa.BigInteger(), + nullable=False, + server_default=sa.text("1073741824"), + ) + + op.create_check_constraint( + "ck_tenants_storage_quota_bytes_positive", + "tenants", + "storage_quota_bytes > 0", + ) + + +def downgrade() -> None: + op.drop_constraint( + "ck_tenants_storage_quota_bytes_positive", + "tenants", + type_="check", + ) + op.drop_column("tenants", "storage_quota_bytes") diff --git a/alembic/versions/ca14c4ab10d9_merge_multiple_heads.py b/alembic/versions/ca14c4ab10d9_merge_multiple_heads.py new file mode 100644 index 0000000..4059914 --- /dev/null +++ b/alembic/versions/ca14c4ab10d9_merge_multiple_heads.py @@ -0,0 +1,26 @@ +"""Merge multiple heads + +Revision ID: ca14c4ab10d9 +Revises: b0a09cd13348, y001_add_user_mail_from +Create Date: 2026-05-08 13:47:42.707120 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'ca14c4ab10d9' +down_revision: Union[str, None] = ('b0a09cd13348', 'y001_add_user_mail_from') +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/alembic/versions/cba70ab35793_create_chat_token_usage_table.py b/alembic/versions/cba70ab35793_create_chat_token_usage_table.py new file mode 100644 index 0000000..43ff6c4 --- /dev/null +++ b/alembic/versions/cba70ab35793_create_chat_token_usage_table.py @@ -0,0 +1,83 @@ +"""Create chat_token_usage table + +Revision ID: cba70ab35793 +Revises: cd27562afa2e +Create Date: 2026-04-16 13:39:46.842540 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'cba70ab35793' +down_revision: Union[str, None] = 'cd27562afa2e' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +# (index_name, columns, unique, use_naming_convention) +# use_naming_convention=True wraps in batch_op.f(); False uses the raw string +# (matches the original auto-generated/hand-written split). +_INDEXES = [ + ('ix_chat_token_usage_chat_id', ['chat_id'], False, True), + ('ix_chat_token_usage_document_id', ['document_id'], False, True), + ('ix_chat_token_usage_id', ['id'], False, True), + ('ix_chat_token_usage_model_name', ['model_name'], False, True), + ('ix_chat_token_usage_model_time', ['model_name', 'timestamp'], False, False), + ('ix_chat_token_usage_phase', ['phase'], False, True), + ('ix_chat_token_usage_timestamp', ['timestamp'], False, True), + ('ix_chat_token_usage_user_id', ['user_id'], False, True), + ('ix_chat_token_usage_user_time', ['user_id', 'timestamp'], False, False), +] + + +def _index_names(bind, table_name: str) -> set[str]: + inspector = sa.inspect(bind) + return {idx["name"] for idx in inspector.get_indexes(table_name)} + + +def upgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + + if not inspector.has_table('chat_token_usage'): + op.create_table('chat_token_usage', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('chat_id', sa.Integer(), nullable=True), + sa.Column('document_id', sa.String(length=255), nullable=True), + sa.Column('prompt_tokens', sa.BigInteger(), nullable=False), + sa.Column('completion_tokens', sa.BigInteger(), nullable=False), + sa.Column('total_tokens', sa.BigInteger(), nullable=False), + sa.Column('model_name', sa.String(length=100), nullable=False), + sa.Column('cost_usd', sa.DECIMAL(precision=12, scale=8), nullable=False), + sa.Column('timestamp', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('phase', sa.String(length=50), nullable=True), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + + existing = _index_names(bind, 'chat_token_usage') + with op.batch_alter_table('chat_token_usage', schema=None) as batch_op: + for name, cols, unique, use_f in _INDEXES: + if name not in existing: + ref = batch_op.f(name) if use_f else name + batch_op.create_index(ref, cols, unique=unique) + + +def downgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + + if inspector.has_table('chat_token_usage'): + existing = _index_names(bind, 'chat_token_usage') + with op.batch_alter_table('chat_token_usage', schema=None) as batch_op: + for name, _, _, use_f in reversed(_INDEXES): + if name in existing: + ref = batch_op.f(name) if use_f else name + batch_op.drop_index(ref) + + op.drop_table('chat_token_usage') diff --git a/alembic/versions/cd27562afa2e_add_chat_token_usage_tracking.py b/alembic/versions/cd27562afa2e_add_chat_token_usage_tracking.py new file mode 100644 index 0000000..43d28ec --- /dev/null +++ b/alembic/versions/cd27562afa2e_add_chat_token_usage_tracking.py @@ -0,0 +1,116 @@ +"""Add chat token usage tracking + +Revision ID: cd27562afa2e +Revises: 1f4d2a8c9b77 +Create Date: 2026-04-16 13:05:09.572442 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = 'cd27562afa2e' +down_revision: Union[str, None] = '1f4d2a8c9b77' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _table_exists(bind, table_name: str) -> bool: + return sa.inspect(bind).has_table(table_name) + + +def upgrade() -> None: + bind = op.get_bind() + + if _table_exists(bind, 'chatbot_documents'): + with op.batch_alter_table('chatbot_documents', schema=None) as batch_op: + batch_op.alter_column('chunks_count', + existing_type=sa.INTEGER(), + nullable=False) + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(), + nullable=False) + + if _table_exists(bind, 'vector_chunks'): + with op.batch_alter_table('vector_chunks', schema=None) as batch_op: + batch_op.alter_column('document_id', + existing_type=sa.VARCHAR(), + nullable=False) + batch_op.alter_column('chunk_text', + existing_type=sa.VARCHAR(), + type_=sa.Text(), + existing_nullable=False) + batch_op.alter_column('chunk_index', + existing_type=sa.INTEGER(), + nullable=False) + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + nullable=False) + batch_op.drop_constraint('vector_chunks_user_id_fkey', type_='foreignkey') + batch_op.create_foreign_key(None, 'users', ['user_id'], ['id'], ondelete='CASCADE') + + if _table_exists(bind, 'vector_indices'): + with op.batch_alter_table('vector_indices', schema=None) as batch_op: + batch_op.alter_column('total_vectors', + existing_type=sa.INTEGER(), + nullable=False) + batch_op.alter_column('updated_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + nullable=False) + batch_op.drop_constraint('vector_indices_user_id_fkey', type_='foreignkey') + batch_op.create_foreign_key(None, 'users', ['user_id'], ['id'], ondelete='CASCADE') + + +def downgrade() -> None: + bind = op.get_bind() + + if _table_exists(bind, 'vector_indices'): + with op.batch_alter_table('vector_indices', schema=None) as batch_op: + batch_op.drop_constraint(None, type_='foreignkey') + batch_op.create_foreign_key('vector_indices_user_id_fkey', 'users', ['user_id'], ['id']) + batch_op.drop_index(batch_op.f('ix_vector_indices_user_id')) + batch_op.create_index('ix_vector_indices_user_id', ['user_id'], unique=True) + batch_op.create_index('ix_vector_indices_id', ['id'], unique=False) + batch_op.alter_column('updated_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + nullable=True) + batch_op.alter_column('total_vectors', + existing_type=sa.INTEGER(), + nullable=True) + + if _table_exists(bind, 'vector_chunks'): + with op.batch_alter_table('vector_chunks', schema=None) as batch_op: + batch_op.drop_constraint(None, type_='foreignkey') + batch_op.create_foreign_key('vector_chunks_user_id_fkey', 'users', ['user_id'], ['id']) + batch_op.create_index('ix_vector_chunks_id', ['id'], unique=False) + batch_op.alter_column('created_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + nullable=True) + batch_op.alter_column('chunk_index', + existing_type=sa.INTEGER(), + nullable=True) + batch_op.alter_column('chunk_text', + existing_type=sa.Text(), + type_=sa.VARCHAR(), + existing_nullable=False) + batch_op.alter_column('document_id', + existing_type=sa.VARCHAR(), + nullable=True) + + if _table_exists(bind, 'chatbot_documents'): + with op.batch_alter_table('chatbot_documents', schema=None) as batch_op: + batch_op.create_index('ix_chatbot_documents_id', ['id'], unique=False) + batch_op.create_index('ix_chatbot_documents_drive_file_id', ['drive_file_id'], unique=False) + batch_op.alter_column('created_at', + existing_type=postgresql.TIMESTAMP(), + nullable=True) + batch_op.alter_column('chunks_count', + existing_type=sa.INTEGER(), + nullable=True) diff --git a/alembic/versions/cfbee4f91fc2_add_updated_at_defaults.py b/alembic/versions/cfbee4f91fc2_add_updated_at_defaults.py new file mode 100644 index 0000000..915bffb --- /dev/null +++ b/alembic/versions/cfbee4f91fc2_add_updated_at_defaults.py @@ -0,0 +1,52 @@ +"""add_updated_at_defaults + +Revision ID: cfbee4f91fc2 +Revises: 49c19df2c1d4 +Create Date: 2026-02-19 11:25:54.874662 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'cfbee4f91fc2' +down_revision: Union[str, None] = '49c19df2c1d4' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('drive_folders', 'updated_at', + existing_type=sa.DateTime(timezone=True), + server_default=sa.text('now()'), + existing_nullable=False) + op.alter_column('drive_files', 'updated_at', + existing_type=sa.DateTime(timezone=True), + server_default=sa.text('now()'), + existing_nullable=False) + op.alter_column('projects', 'updated_at', + existing_type=sa.DateTime(timezone=True), + server_default=sa.text('now()'), + existing_nullable=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('projects', 'updated_at', + existing_type=sa.DateTime(timezone=True), + server_default=None, + existing_nullable=False) + op.alter_column('drive_files', 'updated_at', + existing_type=sa.DateTime(timezone=True), + server_default=None, + existing_nullable=False) + op.alter_column('drive_folders', 'updated_at', + existing_type=sa.DateTime(timezone=True), + server_default=None, + existing_nullable=False) + # ### end Alembic commands ### diff --git a/alembic/versions/d1_0_plans_and_entitlements.py b/alembic/versions/d1_0_plans_and_entitlements.py new file mode 100644 index 0000000..e36b65e --- /dev/null +++ b/alembic/versions/d1_0_plans_and_entitlements.py @@ -0,0 +1,219 @@ +"""D1 — plans, limits, subscriptions and overrides + +The landing page advertises four priced plans and the backend had never heard of +a plan. Meanwhile `tenants` carried four limit columns set per tenant by hand, so +two customers on the same advertised plan could silently have different limits. + +This gives those limits something to be derived from. + +**Behaviour-neutral by construction.** Every tenant's four current values become +**overrides**, not plan limits — so whatever they have today is exactly what they +keep, whichever plan they are placed on. The plan is then chosen to match, or +`custom` when nothing does. Nobody's limits move. + +The `tenants` columns are **not** dropped. They stay for one release and remain +the last-resort fallback in `EntitlementService`, the same transition shape as +`users.role_id` and for the same reason: a rollback must not be able to remove +every tenant's quota. + +`plans` and `plan_limits` are deliberately **not** tenant-scoped — a catalogue is +shared, like `accesses`. `tenant_subscriptions` and `tenant_limit_overrides` are, +and are added to the RLS policy here. + +Revision ID: d1_0_plans_and_entitlements +Revises: c3_0_org_unit_path +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.postgresql import UUID + +revision = "d1_0_plans_and_entitlements" +down_revision = "c3_0_org_unit_path" +branch_labels = None +depends_on = None + +TENANT_TABLES = ["tenant_subscriptions", "tenant_limit_overrides"] +POLICY = "tenant_isolation" +USING = """ + coalesce(current_setting('docqube.bypass', true), '') = 'on' + OR tenant_id IS NULL + OR tenant_id = nullif(current_setting('docqube.tenant_id', true), '')::uuid +""" + +# The four plans the pricing page already sells, with the limits it claims. +# Seeded here rather than in `scripts/seed.py` so that a database which has run +# migrations is one a customer could be placed on — the page and the schema +# start out agreeing. +# +# -1 means unlimited, matching the convention `tenants.envelope_limit` already +# uses. +GB = 1024 ** 3 +PLANS = [ + # code, name, price, public, order + ("starter", "Starter plan", "$12", True, 1), + ("professional", "Professional plan", "$22", True, 2), + ("elite", "Elite plan", "$29", True, 3), + ("custom", "Custom plan", "Contact Us", True, 4), +] +LIMITS = { + "starter": {"storage_bytes": 10 * GB, "seats": 5, + "chat_tokens_daily": 500_000, "envelopes": 25}, + "professional": {"storage_bytes": 100 * GB, "seats": 25, + "chat_tokens_daily": 2_000_000, "envelopes": 250}, + "elite": {"storage_bytes": 500 * GB, "seats": 100, + "chat_tokens_daily": 10_000_000, "envelopes": -1}, + "custom": {"storage_bytes": -1, "seats": -1, + "chat_tokens_daily": -1, "envelopes": -1}, +} + + +def upgrade() -> None: + op.create_table( + "plans", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column("code", sa.String(50), nullable=False, unique=True), + sa.Column("name", sa.String(255), nullable=False), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("price_display", sa.String(50), nullable=True), + sa.Column("currency", sa.String(10), nullable=True), + sa.Column("is_public", sa.Boolean(), nullable=False, server_default=sa.true()), + sa.Column("sort_order", sa.Integer(), nullable=False, server_default="0"), + sa.Column("created_at", sa.DateTime(timezone=True), + server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), + server_default=sa.func.now(), nullable=False), + ) + op.create_index("ix_plans_code", "plans", ["code"]) + + op.create_table( + "plan_limits", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column("plan_id", UUID(as_uuid=True), + sa.ForeignKey("plans.id", ondelete="CASCADE"), nullable=False), + sa.Column("key", sa.String(50), nullable=False), + sa.Column("value", sa.BigInteger(), nullable=False), + sa.UniqueConstraint("plan_id", "key", name="uq_plan_limits_plan_key"), + ) + op.create_index("ix_plan_limits_plan_id", "plan_limits", ["plan_id"]) + + op.create_table( + "tenant_subscriptions", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column("tenant_id", UUID(as_uuid=True), + sa.ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False), + # RESTRICT: deleting a plan somebody is on would silently drop them to + # the defaults. Move them first, deliberately. + sa.Column("plan_id", UUID(as_uuid=True), + sa.ForeignKey("plans.id", ondelete="RESTRICT"), nullable=False), + sa.Column("status", sa.String(20), nullable=False, server_default="active"), + sa.Column("started_at", sa.DateTime(timezone=True), + server_default=sa.func.now(), nullable=False), + sa.Column("current_period_end", sa.DateTime(timezone=True), nullable=True), + sa.Column("cancel_at_period_end", sa.Boolean(), nullable=False, + server_default=sa.false()), + sa.Column("external_ref", sa.String(255), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), + server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), + server_default=sa.func.now(), nullable=False), + # One live subscription per tenant: a second "current" row is an + # ambiguity nothing can resolve. + sa.UniqueConstraint("tenant_id", name="uq_tenant_subscriptions_tenant"), + ) + op.create_index("ix_tenant_subscriptions_tenant_id", "tenant_subscriptions", ["tenant_id"]) + op.create_index("ix_tenant_subscriptions_plan_id", "tenant_subscriptions", ["plan_id"]) + op.create_index("ix_tenant_subscriptions_status", "tenant_subscriptions", ["status"]) + op.create_index("ix_tenant_subscriptions_external_ref", "tenant_subscriptions", ["external_ref"]) + + op.create_table( + "tenant_limit_overrides", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column("tenant_id", UUID(as_uuid=True), + sa.ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False), + sa.Column("key", sa.String(50), nullable=False), + sa.Column("value", sa.BigInteger(), nullable=False), + sa.Column("note", sa.Text(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), + server_default=sa.func.now(), nullable=False), + sa.UniqueConstraint("tenant_id", "key", name="uq_tenant_limit_overrides"), + ) + op.create_index("ix_tenant_limit_overrides_tenant_id", "tenant_limit_overrides", ["tenant_id"]) + + # --- seed the catalogue ------------------------------------------------- + for code, name, price, public, order in PLANS: + op.execute( + sa.text( + "INSERT INTO plans (id, code, name, price_display, currency, " + "is_public, sort_order, created_at, updated_at) " + "VALUES (gen_random_uuid(), :code, :name, :price, 'USD', " + ":public, :order, now(), now())" + ).bindparams(code=code, name=name, price=price, public=public, order=order) + ) + for key, value in LIMITS[code].items(): + op.execute( + sa.text( + "INSERT INTO plan_limits (id, plan_id, key, value) " + "SELECT gen_random_uuid(), id, :key, :value FROM plans " + " WHERE code = :code" + ).bindparams(key=key, value=value, code=code) + ) + + # --- backfill, behaviour-neutral ---------------------------------------- + # + # Current values become **overrides**, so every tenant keeps exactly what it + # has regardless of which plan it lands on. The plan is a label until + # somebody removes the override. + op.execute( + """ + INSERT INTO tenant_limit_overrides (id, tenant_id, key, value, note, created_at) + SELECT gen_random_uuid(), t.id, v.key, v.value, + 'Carried over from the tenant row at D1; not a negotiated exception', + now() + FROM tenants t + CROSS JOIN LATERAL (VALUES + ('storage_bytes', t.storage_quota_bytes), + ('seats', COALESCE(t.max_users_allowed, -1)), + ('chat_tokens_daily', t.chat_token_daily_limit), + ('envelopes', t.envelope_limit) + ) AS v(key, value) + WHERE t.is_deleted = false + ON CONFLICT DO NOTHING + """ + ) + + # Every live tenant is placed on `custom`, not on a guessed plan. + # + # Matching limits to a plan would be inference, and inferring a commercial + # fact is how somebody ends up on Starter because their quota happened to + # look like it. `custom` is honest: nobody has said what they are on yet, + # and their overrides mean it changes nothing. + op.execute( + """ + INSERT INTO tenant_subscriptions (id, tenant_id, plan_id, status, + started_at, created_at, updated_at) + SELECT gen_random_uuid(), t.id, p.id, 'active', now(), now(), now() + FROM tenants t + CROSS JOIN (SELECT id FROM plans WHERE code = 'custom') p + WHERE t.is_deleted = false + ON CONFLICT DO NOTHING + """ + ) + + for table in TENANT_TABLES: + op.execute(f"ALTER TABLE {table} ENABLE ROW LEVEL SECURITY") + op.execute(f"ALTER TABLE {table} FORCE ROW LEVEL SECURITY") + op.execute(f"DROP POLICY IF EXISTS {POLICY} ON {table}") + op.execute(f"CREATE POLICY {POLICY} ON {table} USING ({USING}) WITH CHECK ({USING})") + + +def downgrade() -> None: + for table in TENANT_TABLES: + op.execute(f"DROP POLICY IF EXISTS {POLICY} ON {table}") + + # Safe to reverse: the `tenants` columns were never touched, so removing + # these tables returns the application to the limits it still reads. + op.drop_table("tenant_limit_overrides") + op.drop_table("tenant_subscriptions") + op.drop_table("plan_limits") + op.drop_table("plans") diff --git a/alembic/versions/d2_0_credit_limit_precision.py b/alembic/versions/d2_0_credit_limit_precision.py new file mode 100644 index 0000000..7267d2c --- /dev/null +++ b/alembic/versions/d2_0_credit_limit_precision.py @@ -0,0 +1,61 @@ +"""D2 — a credit limit stops being rounded to a whole number + +`chat_credit_quota.daily_credit_limit` is `bigint` in the database and +`DECIMAL(12, 2)` on the model. PostgreSQL resolves that disagreement by +rounding, silently, on write: + + INSERT ... VALUES (0.60::numeric(12,2)) -> stored: 1 + INSERT ... VALUES (10.75::numeric(12,2)) -> stored: 11 + INSERT ... VALUES (10.25::numeric(12,2)) -> stored: 10 + +So a tenant admin setting a limit of 0.60 credits gets 1 — a 67% overshoot — and +nothing anywhere reports it. Whether that has bitten depends on whether anyone +has entered a fraction, which is not a property worth relying on. + +Widening, so no value is at risk: every bigint currently stored is representable +in `numeric(12, 2)`. `credits_used_today` and `total_credits_ever` are left as +`bigint` deliberately — they count tokens, which are whole. + +The reverse casts back to `bigint` and therefore *does* round. That is the +honest reverse of a widening: going back to a narrower type cannot preserve what +the wider one held. It is written that way rather than left unimplemented so the +migration can actually be reversed, but rolling this back after fractional +limits have been set will round them. + +Found by `alembic revision --autogenerate`, which had never been run against +this schema, and which after the model-registry fix reports this as the single +remaining difference between the models and the database. + +Revision ID: d2_0_credit_limit_precision +Revises: d1_0_plans_and_entitlements +""" + +import sqlalchemy as sa +from alembic import op + +revision = "d2_0_credit_limit_precision" +down_revision = "d1_0_plans_and_entitlements" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.alter_column( + "chat_credit_quota", + "daily_credit_limit", + existing_type=sa.BigInteger(), + type_=sa.DECIMAL(precision=12, scale=2), + existing_nullable=False, + postgresql_using="daily_credit_limit::numeric(12,2)", + ) + + +def downgrade() -> None: + op.alter_column( + "chat_credit_quota", + "daily_credit_limit", + existing_type=sa.DECIMAL(precision=12, scale=2), + type_=sa.BigInteger(), + existing_nullable=False, + postgresql_using="round(daily_credit_limit)::bigint", + ) diff --git a/alembic/versions/d2e2c55290ee_add_10mb_check_constraints_on_document_.py b/alembic/versions/d2e2c55290ee_add_10mb_check_constraints_on_document_.py new file mode 100644 index 0000000..9927d5b --- /dev/null +++ b/alembic/versions/d2e2c55290ee_add_10mb_check_constraints_on_document_.py @@ -0,0 +1,60 @@ +"""Add 10MB CHECK constraints on document content fields + +Revision ID: d2e2c55290ee +Revises: 9f6521a38bf3 +Create Date: 2026-03-03 22:17:00.564459 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'd2e2c55290ee' +down_revision: Union[str, None] = '9f6521a38bf3' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +# 10 MB limit in bytes +_MAX = 10 * 1024 * 1024 # 10485760 + + +def upgrade() -> None: + # projects table — three content columns + op.create_check_constraint( + 'ck_project_markdown_max', 'projects', + f'octet_length(markdown_content) <= {_MAX}' + ) + op.create_check_constraint( + 'ck_project_xml_max', 'projects', + f'octet_length(xml_content) <= {_MAX}' + ) + op.create_check_constraint( + 'ck_project_html_max', 'projects', + f'octet_length(html_content) <= {_MAX}' + ) + + # project_versions table — three content columns + op.create_check_constraint( + 'ck_version_markdown_max', 'project_versions', + f'octet_length(markdown_content) <= {_MAX}' + ) + op.create_check_constraint( + 'ck_version_xml_max', 'project_versions', + f'octet_length(xml_content) <= {_MAX}' + ) + op.create_check_constraint( + 'ck_version_html_max', 'project_versions', + f'octet_length(html_content) <= {_MAX}' + ) + + +def downgrade() -> None: + op.drop_constraint('ck_version_html_max', 'project_versions', type_='check') + op.drop_constraint('ck_version_xml_max', 'project_versions', type_='check') + op.drop_constraint('ck_version_markdown_max', 'project_versions', type_='check') + op.drop_constraint('ck_project_html_max', 'projects', type_='check') + op.drop_constraint('ck_project_xml_max', 'projects', type_='check') + op.drop_constraint('ck_project_markdown_max', 'projects', type_='check') diff --git a/alembic/versions/d551c03861f4_rename_quota_to_credits.py b/alembic/versions/d551c03861f4_rename_quota_to_credits.py new file mode 100644 index 0000000..fc18046 --- /dev/null +++ b/alembic/versions/d551c03861f4_rename_quota_to_credits.py @@ -0,0 +1,38 @@ +"""rename_quota_to_credits + +Revision ID: d551c03861f4 +Revises: 540c970ff890 +Create Date: 2026-05-13 13:01:51.510123 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'd551c03861f4' +down_revision: Union[str, None] = '540c970ff890' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Rename Table + op.rename_table('chat_token_quota', 'chat_credit_quota') + + # Rename Columns in chat_credit_quota + op.alter_column('chat_credit_quota', 'daily_limit', new_column_name='daily_credit_limit') + op.alter_column('chat_credit_quota', 'tokens_used_today', new_column_name='credits_used_today') + op.alter_column('chat_credit_quota', 'total_tokens_ever', new_column_name='total_credits_ever') + + +def downgrade() -> None: + # Rename Columns back + op.alter_column('chat_credit_quota', 'daily_credit_limit', new_column_name='daily_limit') + op.alter_column('chat_credit_quota', 'credits_used_today', new_column_name='tokens_used_today') + op.alter_column('chat_credit_quota', 'total_credits_ever', new_column_name='total_tokens_ever') + + # Rename Table back + op.rename_table('chat_credit_quota', 'chat_token_quota') diff --git a/alembic/versions/daf45366d608_add_envelope_limits_to_tenants.py b/alembic/versions/daf45366d608_add_envelope_limits_to_tenants.py new file mode 100644 index 0000000..9ee0286 --- /dev/null +++ b/alembic/versions/daf45366d608_add_envelope_limits_to_tenants.py @@ -0,0 +1,34 @@ +"""Add envelope limits to tenants + +Revision ID: daf45366d608 +Revises: 611d6b99ad11 +Create Date: 2026-07-18 10:34:04.536445 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'daf45366d608' +down_revision: Union[str, None] = '611d6b99ad11' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column('tenants', sa.Column('envelope_limit', sa.BigInteger(), nullable=False, server_default='-1')) + op.add_column('tenants', sa.Column('envelopes_used', sa.BigInteger(), nullable=False, server_default='0')) + op.add_column('tenants', sa.Column('envelope_reset_days', sa.Integer(), nullable=True)) + op.add_column('tenants', sa.Column('envelope_last_reset_at', sa.DateTime(timezone=True), nullable=False, server_default=sa.text('now()'))) + pass + + +def downgrade() -> None: + op.drop_column('tenants', 'envelope_last_reset_at') + op.drop_column('tenants', 'envelope_reset_days') + op.drop_column('tenants', 'envelopes_used') + op.drop_column('tenants', 'envelope_limit') + pass diff --git a/alembic/versions/dbaa8ef522c0_add_color_to_drive_comments.py b/alembic/versions/dbaa8ef522c0_add_color_to_drive_comments.py new file mode 100644 index 0000000..b66df2c --- /dev/null +++ b/alembic/versions/dbaa8ef522c0_add_color_to_drive_comments.py @@ -0,0 +1,38 @@ +"""add color to drive comments + +Revision ID: dbaa8ef522c0 +Revises: 8736ab6bd5af +Create Date: 2026-02-26 14:23:14.999888 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = 'dbaa8ef522c0' +down_revision: Union[str, None] = '8736ab6bd5af' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('drive_comments', sa.Column('color', sa.String(length=20), nullable=True)) + op.alter_column('notifications', 'created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + existing_server_default=sa.text('now()')) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('notifications', 'created_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + existing_server_default=sa.text('now()')) + op.drop_column('drive_comments', 'color') + # ### end Alembic commands ### diff --git a/alembic/versions/dfae38e4ede3_device_constraints_and_session_updates.py b/alembic/versions/dfae38e4ede3_device_constraints_and_session_updates.py new file mode 100644 index 0000000..0f745b9 --- /dev/null +++ b/alembic/versions/dfae38e4ede3_device_constraints_and_session_updates.py @@ -0,0 +1,28 @@ +"""device constraints and session updates + +Revision ID: dfae38e4ede3 +Revises: 9c49fe120ffb +Create Date: 2026-08-07 11:03:54.251473 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = 'dfae38e4ede3' +down_revision: Union[str, None] = '9c49fe120ffb' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + + op.create_unique_constraint('uq_device_user_fingerprint', 'devices', ['user_id', 'fingerprint']) + + +def downgrade() -> None: + op.drop_constraint('uq_device_user_fingerprint', 'devices', type_='unique') diff --git a/alembic/versions/e1_0_add_saas_plan_id.py b/alembic/versions/e1_0_add_saas_plan_id.py new file mode 100644 index 0000000..9cf4ff4 --- /dev/null +++ b/alembic/versions/e1_0_add_saas_plan_id.py @@ -0,0 +1,32 @@ +"""add saas_plan_id to plans + +Revision ID: e1_0_add_saas_plan_id +Revises: 9c4a95ac6726 +Create Date: 2026-09-01 + +Adds a nullable `saas_plan_id` column to the `plans` table so that DocQube +can trace each plan back to the SaaS SubscriptionPlan that originated it via +the provision webhook. Plans created directly inside DocQube will have NULL. +""" + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = "e1_0_add_saas_plan_id" +down_revision = "9c4a95ac6726" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "plans", + sa.Column("saas_plan_id", sa.String(255), nullable=True), + ) + op.create_index("ix_plans_saas_plan_id", "plans", ["saas_plan_id"], unique=True) + + +def downgrade() -> None: + op.drop_index("ix_plans_saas_plan_id", table_name="plans") + op.drop_column("plans", "saas_plan_id") diff --git a/alembic/versions/e81a9d22ca5f_migrate_roles_access_to_uuid.py b/alembic/versions/e81a9d22ca5f_migrate_roles_access_to_uuid.py new file mode 100644 index 0000000..dec3a3b --- /dev/null +++ b/alembic/versions/e81a9d22ca5f_migrate_roles_access_to_uuid.py @@ -0,0 +1,116 @@ +"""migrate_roles_access_to_uuid + +Revision ID: e81a9d22ca5f +Revises: a2b3c4d5e6f7 +Create Date: 2026-03-05 14:42:53.769667 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'e81a9d22ca5f' +down_revision: Union[str, None] = 'a2b3c4d5e6f7' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # --- Users Table --- + # Drop FK pointing to roles + op.drop_constraint('users_role_id_fkey', 'users', type_='foreignkey') + + # --- RoleAccess Table --- + # Drop FKs + op.drop_constraint('role_accesses_role_id_fkey', 'role_accesses', type_='foreignkey') + op.drop_constraint('role_accesses_access_id_fkey', 'role_accesses', type_='foreignkey') + op.drop_constraint('pk_role_accesses', 'role_accesses', type_='primary') + + # --- Accesses Table --- + # FK was named 'fk_accesses_parent_id' in migration a2b3c4d5e6f7 + op.drop_constraint('fk_accesses_parent_id', 'accesses', type_='foreignkey') + + # --- Alter Columns to UUID --- + # Drop serial defaults before changing type + op.execute('ALTER TABLE accesses ALTER COLUMN id DROP DEFAULT') + op.execute('ALTER TABLE roles ALTER COLUMN id DROP DEFAULT') + + # Change types to UUID (tables should be empty) + op.execute('ALTER TABLE accesses ALTER COLUMN id TYPE uuid USING (gen_random_uuid())') + op.execute('ALTER TABLE accesses ALTER COLUMN parent_id TYPE uuid USING NULL') + + op.execute('ALTER TABLE roles ALTER COLUMN id TYPE uuid USING (gen_random_uuid())') + + op.execute('ALTER TABLE role_accesses ALTER COLUMN role_id TYPE uuid USING (gen_random_uuid())') + op.execute('ALTER TABLE role_accesses ALTER COLUMN access_id TYPE uuid USING (gen_random_uuid())') + + op.execute('ALTER TABLE users ALTER COLUMN role_id TYPE uuid USING NULL') + + # Set UUID defaults for primary keys + op.execute('ALTER TABLE accesses ALTER COLUMN id SET DEFAULT gen_random_uuid()') + op.execute('ALTER TABLE roles ALTER COLUMN id SET DEFAULT gen_random_uuid()') + + # Drop leftover serial sequences + op.execute('DROP SEQUENCE IF EXISTS accesses_id_seq CASCADE') + op.execute('DROP SEQUENCE IF EXISTS roles_id_seq CASCADE') + + # --- Re-add Constraints --- + # Accesses + op.create_foreign_key('fk_accesses_parent_id', 'accesses', 'accesses', ['parent_id'], ['id'], ondelete='CASCADE') + + # RoleAccess + op.create_primary_key('pk_role_accesses', 'role_accesses', ['role_id', 'access_id']) + op.create_foreign_key('role_accesses_role_id_fkey', 'role_accesses', 'roles', ['role_id'], ['id'], ondelete='CASCADE') + op.create_foreign_key('role_accesses_access_id_fkey', 'role_accesses', 'accesses', ['access_id'], ['id'], ondelete='CASCADE') + + # Users + op.create_foreign_key('users_role_id_fkey', 'users', 'roles', ['role_id'], ['id'], ondelete='SET NULL') + + +def downgrade() -> None: + # --- Users Table --- + op.drop_constraint('users_role_id_fkey', 'users', type_='foreignkey') + + # --- RoleAccess Table --- + op.drop_constraint('role_accesses_role_id_fkey', 'role_accesses', type_='foreignkey') + op.drop_constraint('role_accesses_access_id_fkey', 'role_accesses', type_='foreignkey') + op.drop_constraint('pk_role_accesses', 'role_accesses', type_='primary') + + # --- Accesses Table --- + op.drop_constraint('fk_accesses_parent_id', 'accesses', type_='foreignkey') + + # Truncate all tables (UUID -> INT cast is destructive) + op.execute('TRUNCATE TABLE role_accesses CASCADE') + op.execute('TRUNCATE TABLE users CASCADE') + op.execute('TRUNCATE TABLE accesses CASCADE') + op.execute('TRUNCATE TABLE roles CASCADE') + + # Drop UUID defaults + op.execute('ALTER TABLE accesses ALTER COLUMN id DROP DEFAULT') + op.execute('ALTER TABLE roles ALTER COLUMN id DROP DEFAULT') + + # Change types back to integer + op.execute('ALTER TABLE accesses ALTER COLUMN id TYPE integer USING 0') + op.execute('ALTER TABLE accesses ALTER COLUMN parent_id TYPE integer USING NULL') + op.execute('ALTER TABLE roles ALTER COLUMN id TYPE integer USING 0') + op.execute('ALTER TABLE role_accesses ALTER COLUMN role_id TYPE integer USING 0') + op.execute('ALTER TABLE role_accesses ALTER COLUMN access_id TYPE integer USING 0') + op.execute('ALTER TABLE users ALTER COLUMN role_id TYPE integer USING NULL') + + # Recreate serial sequences + op.execute("CREATE SEQUENCE accesses_id_seq OWNED BY accesses.id") + op.execute("ALTER TABLE accesses ALTER COLUMN id SET DEFAULT nextval('accesses_id_seq')") + op.execute("CREATE SEQUENCE roles_id_seq OWNED BY roles.id") + op.execute("ALTER TABLE roles ALTER COLUMN id SET DEFAULT nextval('roles_id_seq')") + + # --- Re-add Constraints --- + op.create_foreign_key('fk_accesses_parent_id', 'accesses', 'accesses', ['parent_id'], ['id'], ondelete='CASCADE') + + op.create_primary_key('pk_role_accesses', 'role_accesses', ['role_id', 'access_id']) + op.create_foreign_key('role_accesses_role_id_fkey', 'role_accesses', 'roles', ['role_id'], ['id'], ondelete='CASCADE') + op.create_foreign_key('role_accesses_access_id_fkey', 'role_accesses', 'accesses', ['access_id'], ['id'], ondelete='CASCADE') + + op.create_foreign_key('users_role_id_fkey', 'users', 'roles', ['role_id'], ['id'], ondelete='SET NULL') diff --git a/alembic/versions/f055f9b157cb_add_cascade_deletes_to_drive_file_.py b/alembic/versions/f055f9b157cb_add_cascade_deletes_to_drive_file_.py new file mode 100644 index 0000000..128e295 --- /dev/null +++ b/alembic/versions/f055f9b157cb_add_cascade_deletes_to_drive_file_.py @@ -0,0 +1,36 @@ +"""Add cascade deletes to drive_file_versions + +Revision ID: f055f9b157cb +Revises: a902935ea69e +Create Date: 2026-03-02 11:08:38.477933 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'f055f9b157cb' +down_revision: Union[str, None] = 'a902935ea69e' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint('drive_file_versions_file_id_fkey', 'drive_file_versions', type_='foreignkey') + op.create_foreign_key(None, 'drive_file_versions', 'drive_files', ['file_id'], ['id'], ondelete='CASCADE') + op.drop_constraint('drive_stars_user_id_fkey', 'drive_stars', type_='foreignkey') + op.create_foreign_key(None, 'drive_stars', 'users', ['user_id'], ['id'], ondelete='CASCADE') + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, 'drive_stars', type_='foreignkey') + op.create_foreign_key('drive_stars_user_id_fkey', 'drive_stars', 'users', ['user_id'], ['id']) + op.drop_constraint(None, 'drive_file_versions', type_='foreignkey') + op.create_foreign_key('drive_file_versions_file_id_fkey', 'drive_file_versions', 'drive_files', ['file_id'], ['id']) + # ### end Alembic commands ### diff --git a/alembic/versions/fb6810c36ffb_add_has_completed_tutorial_to_user.py b/alembic/versions/fb6810c36ffb_add_has_completed_tutorial_to_user.py new file mode 100644 index 0000000..9073fa3 --- /dev/null +++ b/alembic/versions/fb6810c36ffb_add_has_completed_tutorial_to_user.py @@ -0,0 +1,34 @@ +"""Add has_completed_tutorial to User + +Revision ID: fb6810c36ffb +Revises: 229359b0d1b2 +Create Date: 2026-04-02 11:27:02.719590 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'fb6810c36ffb' +down_revision: Union[str, None] = '229359b0d1b2' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.add_column(sa.Column('has_completed_tutorial', sa.Boolean(), server_default=sa.text('false'), nullable=False)) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.drop_column('has_completed_tutorial') + + # ### end Alembic commands ### diff --git a/alembic/versions/ff8052a97834_add_ip_address_to_drive_activities.py b/alembic/versions/ff8052a97834_add_ip_address_to_drive_activities.py new file mode 100644 index 0000000..b3d8593 --- /dev/null +++ b/alembic/versions/ff8052a97834_add_ip_address_to_drive_activities.py @@ -0,0 +1,30 @@ +"""Add ip_address to drive_activities + +Revision ID: ff8052a97834 +Revises: 765b053e3dcd +Create Date: 2026-03-03 21:48:41.907463 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'ff8052a97834' +down_revision: Union[str, None] = '765b053e3dcd' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('drive_activities', sa.Column('ip_address', sa.String(length=64), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('drive_activities', 'ip_address') + # ### end Alembic commands ### diff --git a/alembic/versions/ff9052a97835_add_multi_tenancy.py b/alembic/versions/ff9052a97835_add_multi_tenancy.py new file mode 100644 index 0000000..01dca5f --- /dev/null +++ b/alembic/versions/ff9052a97835_add_multi_tenancy.py @@ -0,0 +1,104 @@ +"""Add multi_tenancy + +Revision ID: ff9052a97835 +Revises: ff8052a97834 +Create Date: 2026-03-05 10:00:00.000000 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import UUID + + +# revision identifiers, used by Alembic. +revision: str = "ff9052a97835" +down_revision: Union[str, None] = "ff8052a97834" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # 1. Create tenants table + op.execute( + """ + CREATE TABLE tenants ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name VARCHAR(255) NOT NULL, + slug VARCHAR(100) NOT NULL UNIQUE, + is_active BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() + ); + """ + ) + + + # 3. Add tenant_id columns (nullable first) + tables = [ + "users", + "projects", + "drive_folders", + "drive_files", + "drive_activities", + "chatbot_documents", + "notifications", + "user_files", + ] + for table in tables: + op.execute( + f"ALTER TABLE {table} ADD COLUMN tenant_id UUID REFERENCES tenants(id);" + ) + + # 4. Backfill existing data + for table in tables: + op.execute( + f"UPDATE {table} SET tenant_id = '00000000-0000-0000-0000-000000000001';" + ) + + # 5. Set NOT NULL (except users) + not_null_tables = [ + "projects", + "drive_folders", + "drive_files", + "drive_activities", + "chatbot_documents", + "notifications", + "user_files", + ] + for table in not_null_tables: + op.execute(f"ALTER TABLE {table} ALTER COLUMN tenant_id SET NOT NULL;") + + # 6. Create indexes + for table in tables: + op.execute(f"CREATE INDEX ix_{table}_tenant_id ON {table}(tenant_id);") + + # 7. Constraint unique (tenant_id, email) + op.execute("ALTER TABLE users DROP CONSTRAINT IF EXISTS users_email_key;") + op.execute("ALTER TABLE users DROP CONSTRAINT IF EXISTS uq_users_email;") + op.execute( + "ALTER TABLE users ADD CONSTRAINT uq_user_tenant_email UNIQUE (tenant_id, email);" + ) + + +def downgrade() -> None: + op.execute("ALTER TABLE users DROP CONSTRAINT IF EXISTS uq_user_tenant_email;") + op.execute("ALTER TABLE users ADD CONSTRAINT users_email_key UNIQUE (email);") + + tables = [ + "users", + "projects", + "drive_folders", + "drive_files", + "drive_activities", + "chatbot_documents", + "notifications", + "user_files", + ] + for table in tables: + op.execute(f"DROP INDEX IF EXISTS ix_{table}_tenant_id;") + op.execute(f"ALTER TABLE {table} DROP COLUMN IF EXISTS tenant_id;") + + op.execute("DROP TABLE IF EXISTS tenants;") diff --git a/alembic/versions/g001_add_signing_requests_table.py b/alembic/versions/g001_add_signing_requests_table.py new file mode 100644 index 0000000..11afce5 --- /dev/null +++ b/alembic/versions/g001_add_signing_requests_table.py @@ -0,0 +1,38 @@ +"""add signing_requests table + +Revision ID: g001_signing +Revises: c7d4e6f1a9b0 +Create Date: 2026-03-06 +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import UUID, JSON + +# revision identifiers +revision = "g001_signing" +down_revision = "c7d4e6f1a9b0" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "signing_requests", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column("tenant_id", UUID(as_uuid=True), sa.ForeignKey("tenants.id"), nullable=False, index=True), + sa.Column("user_id", sa.Integer, sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True), + sa.Column("drive_file_id", sa.Integer, sa.ForeignKey("drive_files.id", ondelete="CASCADE"), nullable=False), + sa.Column("docuseal_template_id", sa.Integer, nullable=True), + sa.Column("docuseal_submission_id", sa.Integer, nullable=True), + sa.Column("docuseal_slug", sa.String(255), nullable=True), + sa.Column("signing_type", sa.String(20), nullable=False), + sa.Column("status", sa.String(20), nullable=False, server_default="pending"), + sa.Column("submitters_json", JSON, nullable=True), + sa.Column("signed_file_id", sa.Integer, sa.ForeignKey("drive_files.id"), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + ) + + +def downgrade(): + op.drop_table("signing_requests") diff --git a/alembic/versions/g002_replace_docuseal_with_placements.py b/alembic/versions/g002_replace_docuseal_with_placements.py new file mode 100644 index 0000000..065c06f --- /dev/null +++ b/alembic/versions/g002_replace_docuseal_with_placements.py @@ -0,0 +1,41 @@ +"""replace docuseal columns with placements_json + +Revision ID: g002_placements +Revises: g001_signing +Create Date: 2026-03-07 +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import JSON + +# revision identifiers +revision = "g002_placements" +down_revision = "g001_signing" +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_column("signing_requests", "docuseal_template_id") + op.drop_column("signing_requests", "docuseal_submission_id") + op.drop_column("signing_requests", "docuseal_slug") + op.add_column( + "signing_requests", + sa.Column("placements_json", JSON, nullable=True), + ) + + +def downgrade(): + op.drop_column("signing_requests", "placements_json") + op.add_column( + "signing_requests", + sa.Column("docuseal_template_id", sa.Integer, nullable=True), + ) + op.add_column( + "signing_requests", + sa.Column("docuseal_submission_id", sa.Integer, nullable=True), + ) + op.add_column( + "signing_requests", + sa.Column("docuseal_slug", sa.String(255), nullable=True), + ) diff --git a/alembic/versions/h003_add_tenant_smtp_settings.py b/alembic/versions/h003_add_tenant_smtp_settings.py new file mode 100644 index 0000000..650633d --- /dev/null +++ b/alembic/versions/h003_add_tenant_smtp_settings.py @@ -0,0 +1,76 @@ +"""add tenant smtp settings + +Revision ID: h003_tenant_smtp +Revises: g002_placements +Create Date: 2026-03-09 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +# revision identifiers, used by Alembic. +revision: str = "h003_tenant_smtp" +down_revision: Union[str, None] = "g002_placements" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + + if not inspector.has_table("tenant_smtp_configs"): + op.create_table( + "tenant_smtp_configs", + sa.Column( + "id", + postgresql.UUID(as_uuid=True), + nullable=False, + server_default=sa.text("gen_random_uuid()"), + ), + sa.Column("tenant_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("smtp_host", sa.String(length=255), nullable=False), + sa.Column("smtp_port", sa.Integer(), nullable=False, server_default=sa.text("587")), + sa.Column("smtp_user", sa.String(length=255), nullable=True), + sa.Column("encrypted_smtp_password", sa.String(length=512), nullable=True), + sa.Column("smtp_secure", sa.Boolean(), nullable=False, server_default=sa.text("false")), + sa.Column("mail_from", sa.String(length=255), nullable=False), + sa.Column("is_active", sa.Boolean(), nullable=False, server_default=sa.text("true")), + sa.Column("created_by_id", sa.Integer(), nullable=True), + sa.Column("updated_by_id", sa.Integer(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["created_by_id"], ["users.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["updated_by_id"], ["users.id"], ondelete="SET NULL"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("tenant_id", name="uq_tenant_smtp_configs_tenant_id"), + ) + + existing_indexes = {idx["name"] for idx in inspector.get_indexes("tenant_smtp_configs")} + if "ix_tenant_smtp_configs_tenant_id" not in existing_indexes: + op.create_index( + "ix_tenant_smtp_configs_tenant_id", + "tenant_smtp_configs", + ["tenant_id"], + unique=False, + ) + + +def downgrade() -> None: + op.drop_index("ix_tenant_smtp_configs_tenant_id", table_name="tenant_smtp_configs") + op.drop_table("tenant_smtp_configs") diff --git a/alembic/versions/h004_add_signing_forensic_columns.py b/alembic/versions/h004_add_signing_forensic_columns.py new file mode 100644 index 0000000..424a3ac --- /dev/null +++ b/alembic/versions/h004_add_signing_forensic_columns.py @@ -0,0 +1,48 @@ +"""add forensic metadata columns to signing_requests + +Revision ID: h004_signing_forensic_v2 +Revises: h004_limits_soft_delete +Create Date: 2026-03-09 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import JSON + + +# revision identifiers, used by Alembic. +revision: str = "h004_signing_forensic_v2" +down_revision: Union[str, None] = "h004_limits_soft_delete" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Safe: only add if column doesn't already exist (idempotent for existing DBs) + conn = op.get_bind() + inspector = sa.inspect(conn) + existing = {c["name"] for c in inspector.get_columns("signing_requests")} + + if "ip_address" not in existing: + op.add_column( + "signing_requests", + sa.Column("ip_address", sa.String(64), nullable=True), + ) + if "user_agent" not in existing: + op.add_column( + "signing_requests", + sa.Column("user_agent", sa.String(512), nullable=True), + ) + if "signing_metadata" not in existing: + op.add_column( + "signing_requests", + sa.Column("signing_metadata", JSON, nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("signing_requests", "signing_metadata") + op.drop_column("signing_requests", "user_agent") + op.drop_column("signing_requests", "ip_address") diff --git a/alembic/versions/h004_limits_and_soft_delete.py b/alembic/versions/h004_limits_and_soft_delete.py new file mode 100644 index 0000000..57f82f0 --- /dev/null +++ b/alembic/versions/h004_limits_and_soft_delete.py @@ -0,0 +1,108 @@ +"""add limits config and soft delete columns + +Revision ID: h004_limits_soft_delete +Revises: h003_tenant_smtp +Create Date: 2026-03-09 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +# revision identifiers, used by Alembic. +revision: str = "h004_limits_soft_delete" +down_revision: Union[str, None] = "h003_tenant_smtp" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "users", sa.Column("is_active", sa.Boolean(), nullable=False, server_default=sa.text("true")) + ) + op.add_column( + "users", sa.Column("is_deleted", sa.Boolean(), nullable=False, server_default=sa.text("false")) + ) + op.add_column("users", sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True)) + op.add_column("users", sa.Column("deleted_by_id", sa.Integer(), nullable=True)) + op.create_foreign_key( + "fk_users_deleted_by_id_users", + "users", + "users", + ["deleted_by_id"], + ["id"], + ondelete="SET NULL", + ) + + op.add_column( + "tenants", sa.Column("is_deleted", sa.Boolean(), nullable=False, server_default=sa.text("false")) + ) + op.add_column("tenants", sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True)) + op.add_column("tenants", sa.Column("deleted_by_id", sa.Integer(), nullable=True)) + op.add_column("tenants", sa.Column("max_users_allowed", sa.Integer(), nullable=True)) + op.create_foreign_key( + "fk_tenants_deleted_by_id_users", + "tenants", + "users", + ["deleted_by_id"], + ["id"], + ondelete="SET NULL", + ) + + op.create_table( + "system_configurations", + sa.Column( + "id", + postgresql.UUID(as_uuid=True), + nullable=False, + server_default=sa.text("gen_random_uuid()"), + ), + sa.Column("config_key", sa.String(length=100), nullable=False), + sa.Column("int_value", sa.BigInteger(), nullable=True), + sa.Column("bool_value", sa.Boolean(), nullable=True), + sa.Column("text_value", sa.Text(), nullable=True), + sa.Column("created_by_id", sa.Integer(), nullable=True), + sa.Column("updated_by_id", sa.Integer(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.ForeignKeyConstraint(["created_by_id"], ["users.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["updated_by_id"], ["users.id"], ondelete="SET NULL"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("config_key", name="uq_system_configurations_config_key"), + ) + op.create_index( + "ix_system_configurations_config_key", + "system_configurations", + ["config_key"], + unique=True, + ) + + +def downgrade() -> None: + op.drop_index("ix_system_configurations_config_key", table_name="system_configurations") + op.drop_table("system_configurations") + + op.drop_constraint("fk_tenants_deleted_by_id_users", "tenants", type_="foreignkey") + op.drop_column("tenants", "max_users_allowed") + op.drop_column("tenants", "deleted_by_id") + op.drop_column("tenants", "deleted_at") + op.drop_column("tenants", "is_deleted") + + op.drop_constraint("fk_users_deleted_by_id_users", "users", type_="foreignkey") + op.drop_column("users", "deleted_by_id") + op.drop_column("users", "deleted_at") + op.drop_column("users", "is_deleted") + op.drop_column("users", "is_active") diff --git a/alembic/versions/i001_dual_bucket_storage.py b/alembic/versions/i001_dual_bucket_storage.py new file mode 100644 index 0000000..40a92fe --- /dev/null +++ b/alembic/versions/i001_dual_bucket_storage.py @@ -0,0 +1,56 @@ +"""Replace b2_bucket_name with b2_quarantine_bucket and b2_clean_bucket + +Revision ID: i001_dual_bucket_storage +Revises: 9557c7307575 +Create Date: 2026-03-17 08:00:00.000000 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'i001_dual_bucket_storage' +down_revision = '9557c7307575' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # 1. Add the two new bucket columns (nullable at first) + op.add_column( + 'tenant_storage_configs', + sa.Column('b2_quarantine_bucket', sa.String(255), nullable=True) + ) + op.add_column( + 'tenant_storage_configs', + sa.Column('b2_clean_bucket', sa.String(255), nullable=True) + ) + + # 2. Migrate existing b2_bucket_name to quarantine (best-effort) + op.execute(""" + UPDATE tenant_storage_configs + SET b2_quarantine_bucket = b2_bucket_name + WHERE b2_bucket_name IS NOT NULL + """) + + # 3. Drop the old column + op.drop_column('tenant_storage_configs', 'b2_bucket_name') + + +def downgrade() -> None: + # 1. Restore the old column + op.add_column( + 'tenant_storage_configs', + sa.Column('b2_bucket_name', sa.String(255), nullable=True) + ) + + # 2. Restore data from quarantine bucket (best-effort) + op.execute(""" + UPDATE tenant_storage_configs + SET b2_bucket_name = b2_quarantine_bucket + WHERE b2_quarantine_bucket IS NOT NULL + """) + + # 3. Drop the new columns + op.drop_column('tenant_storage_configs', 'b2_clean_bucket') + op.drop_column('tenant_storage_configs', 'b2_quarantine_bucket') diff --git a/alembic/versions/j001_comments_fix.py b/alembic/versions/j001_comments_fix.py new file mode 100644 index 0000000..ef7c144 --- /dev/null +++ b/alembic/versions/j001_comments_fix.py @@ -0,0 +1,31 @@ +"""Add rejected_at column to drive_comments + +Revision ID: j001_comments_fix +Revises: c3abf44266de +Create Date: 2026-03-24 + +""" + +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision: str = "j001_comments_fix" +down_revision: Union[str, tuple, None] = "c3abf44266de" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute(""" + ALTER TABLE drive_comments + ADD COLUMN IF NOT EXISTS rejected_at TIMESTAMP WITH TIME ZONE DEFAULT NULL + """) + + +def downgrade() -> None: + op.execute(""" + ALTER TABLE drive_comments + DROP COLUMN IF EXISTS rejected_at + """) diff --git a/alembic/versions/j002_allow_unlimited_tenant_quota.py b/alembic/versions/j002_allow_unlimited_tenant_quota.py new file mode 100644 index 0000000..745ee9e --- /dev/null +++ b/alembic/versions/j002_allow_unlimited_tenant_quota.py @@ -0,0 +1,38 @@ +"""allow unlimited tenant storage quota sentinel + +Revision ID: j002_unlimited_tenant_quota +Revises: j001_comments_fix +Create Date: 2026-03-25 +""" + +from typing import Sequence, Union + +from alembic import op + + +# revision identifiers, used by Alembic. +revision: str = "j002_unlimited_tenant_quota" +down_revision: Union[str, None] = "j001_comments_fix" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +CONSTRAINT_NAME = "ck_tenants_storage_quota_bytes_positive" + + +def upgrade() -> None: + op.drop_constraint(CONSTRAINT_NAME, "tenants", type_="check") + op.create_check_constraint( + CONSTRAINT_NAME, + "tenants", + "storage_quota_bytes = -1 OR storage_quota_bytes > 0", + ) + + +def downgrade() -> None: + op.drop_constraint(CONSTRAINT_NAME, "tenants", type_="check") + op.create_check_constraint( + CONSTRAINT_NAME, + "tenants", + "storage_quota_bytes > 0", + ) diff --git a/alembic/versions/k001_limit_signature_id_to_15_chars.py b/alembic/versions/k001_limit_signature_id_to_15_chars.py new file mode 100644 index 0000000..695957b --- /dev/null +++ b/alembic/versions/k001_limit_signature_id_to_15_chars.py @@ -0,0 +1,42 @@ +"""limit signature_id to 15 characters + +Revision ID: k001_limit_signature_id_15 +Revises: 56d090047e85 +Create Date: 2026-04-15 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "k001_limit_signature_id_15" +down_revision: Union[str, None] = "b029d2f5a0c7" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Truncate any existing values that exceed 15 chars before altering the column + op.execute( + "UPDATE signing_requests SET signature_id = LEFT(signature_id, 15) WHERE LENGTH(signature_id) > 15" + ) + op.alter_column( + "signing_requests", + "signature_id", + existing_type=sa.String(128), + type_=sa.String(15), + existing_nullable=True, + ) + + +def downgrade() -> None: + op.alter_column( + "signing_requests", + "signature_id", + existing_type=sa.String(15), + type_=sa.String(128), + existing_nullable=True, + ) diff --git a/alembic/versions/k002_add_docs_tutorial_flag.py b/alembic/versions/k002_add_docs_tutorial_flag.py new file mode 100644 index 0000000..a074dbc --- /dev/null +++ b/alembic/versions/k002_add_docs_tutorial_flag.py @@ -0,0 +1,28 @@ +"""Add has_completed_docs_tutorial to User + +Revision ID: k002_add_docs_tutorial +Revises: l001_merge_heads +Create Date: 2026-04-17 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'k002_add_docs_tutorial' +down_revision: Union[str, None] = 'l001_merge_heads' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.add_column(sa.Column('has_completed_docs_tutorial', sa.Boolean(), server_default=sa.text('false'), nullable=False)) + + +def downgrade() -> None: + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.drop_column('has_completed_docs_tutorial') diff --git a/alembic/versions/l001_merge_heads.py b/alembic/versions/l001_merge_heads.py new file mode 100644 index 0000000..7a28fe6 --- /dev/null +++ b/alembic/versions/l001_merge_heads.py @@ -0,0 +1,25 @@ +"""merge all heads into single linear chain + +Revision ID: l001_merge_heads +Revises: b029d2f5a0c7, k001_limit_signature_id_15 +Create Date: 2026-04-15 +""" + +from typing import Sequence, Union + +from alembic import op + + +# revision identifiers, used by Alembic. +revision: str = "l001_merge_heads" +down_revision: Union[str, tuple] = ("56d090047e85", "k001_limit_signature_id_15") +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/alembic/versions/m001_add_activity_logs_table.py b/alembic/versions/m001_add_activity_logs_table.py new file mode 100644 index 0000000..0ca9ede --- /dev/null +++ b/alembic/versions/m001_add_activity_logs_table.py @@ -0,0 +1,87 @@ +"""add activity_logs table + +Revision ID: m001_add_activity_logs +Revises: 9af99fc7ca0f +Create Date: 2026-04-20 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +# revision identifiers, used by Alembic. +revision: str = 'm001_add_activity_logs' +down_revision: Union[str, None] = '9af99fc7ca0f' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +_INDEXES = [ + ("ix_activity_logs_tenant_created_at", ["tenant_id", "created_at"]), + ("ix_activity_logs_user_created_at", ["user_id", "created_at"]), + ("ix_activity_logs_module_action", ["module", "action"]), + ("ix_activity_logs_target_id", ["target_id"]), +] + + +def _index_names(bind, table_name: str) -> set[str]: + inspector = sa.inspect(bind) + return {idx["name"] for idx in inspector.get_indexes(table_name)} + + +def upgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + + if not inspector.has_table("activity_logs"): + op.create_table( + "activity_logs", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("tenant_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("user_id", sa.Integer(), nullable=True), + sa.Column("user_name", sa.String(length=255), nullable=True), + sa.Column("user_email", sa.String(length=255), nullable=True), + sa.Column("module", sa.String(length=50), nullable=False), + sa.Column("action", sa.String(length=100), nullable=False), + sa.Column("target_id", sa.String(length=255), nullable=False), + sa.Column("target_type", sa.String(length=50), nullable=False), + sa.Column("target_name", sa.String(length=512), nullable=True), + sa.Column( + "metadata", + postgresql.JSONB(astext_type=sa.Text()), + nullable=False, + ), + sa.Column("ip_address", sa.String(length=64), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("request_id", sa.String(length=255), nullable=True), + sa.Column("status", sa.String(length=20), nullable=False), + sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"]), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="SET NULL"), + sa.PrimaryKeyConstraint("id"), + ) + + existing = _index_names(bind, "activity_logs") + for name, cols in _INDEXES: + if name not in existing: + op.create_index(name, "activity_logs", cols, unique=False) + + +def downgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + + if inspector.has_table("activity_logs"): + existing = _index_names(bind, "activity_logs") + for name, _ in reversed(_INDEXES): + if name in existing: + op.drop_index(name, table_name="activity_logs") + + op.drop_table("activity_logs") diff --git a/alembic/versions/n001_add_user_designation.py b/alembic/versions/n001_add_user_designation.py new file mode 100644 index 0000000..c5bf857 --- /dev/null +++ b/alembic/versions/n001_add_user_designation.py @@ -0,0 +1,29 @@ +"""add designation column to users + +Revision ID: n001_add_user_designation +Revises: m001_add_activity_logs +Create Date: 2026-04-21 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'n001_add_user_designation' +down_revision: Union[str, None] = 'm001_add_activity_logs' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + 'users', + sa.Column('designation', sa.String(length=100), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column('users', 'designation') diff --git a/alembic/versions/n002_add_signature_imprints.py b/alembic/versions/n002_add_signature_imprints.py new file mode 100644 index 0000000..6c4551c --- /dev/null +++ b/alembic/versions/n002_add_signature_imprints.py @@ -0,0 +1,62 @@ +"""add signature_imprints table + +Revision ID: n002_add_signature_imprints +Revises: n001_add_user_designation +Create Date: 2026-04-21 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +# revision identifiers, used by Alembic. +revision: str = 'n002_add_signature_imprints' +down_revision: Union[str, None] = 'n001_add_user_designation' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + 'signature_imprints', + sa.Column('id', postgresql.UUID(as_uuid=True), primary_key=True), + sa.Column( + 'signing_request_id', + postgresql.UUID(as_uuid=True), + sa.ForeignKey('signing_requests.id', ondelete='CASCADE'), + nullable=False, + ), + sa.Column( + 'drive_file_id', + sa.Integer(), + sa.ForeignKey('drive_files.id', ondelete='CASCADE'), + nullable=False, + ), + sa.Column('page_number', sa.Integer(), nullable=False), + sa.Column('bbox_points', postgresql.JSON(), nullable=False), + sa.Column( + 'created_at', + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + ) + op.create_index( + 'ix_signature_imprints_file_page', + 'signature_imprints', + ['drive_file_id', 'page_number'], + ) + op.create_index( + 'ix_signature_imprints_signing_request', + 'signature_imprints', + ['signing_request_id'], + ) + + +def downgrade() -> None: + op.drop_index('ix_signature_imprints_signing_request', table_name='signature_imprints') + op.drop_index('ix_signature_imprints_file_page', table_name='signature_imprints') + op.drop_table('signature_imprints') diff --git a/alembic/versions/v3_extraction_data.py b/alembic/versions/v3_extraction_data.py new file mode 100644 index 0000000..27cd8a7 --- /dev/null +++ b/alembic/versions/v3_extraction_data.py @@ -0,0 +1,57 @@ +"""Add extraction_data table + +Revision ID: v3_extraction +Revises: l001_merge_heads +Create Date: 2026-04-18 12:00:00.000000 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = 'v3_extraction' +down_revision: Union[str, None] = 'l001_merge_heads' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _index_names(bind, table_name: str) -> set[str]: + inspector = sa.inspect(bind) + return {idx["name"] for idx in inspector.get_indexes(table_name)} + + +def upgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + + if not inspector.has_table('extraction_data'): + op.create_table('extraction_data', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('document_id', sa.Integer(), nullable=False), + sa.Column('document_type', sa.String(length=50), nullable=True), + sa.Column('raw_text', sa.Text(), nullable=True), + sa.Column('cleaned_text', sa.Text(), nullable=True), + sa.Column('extracted_json', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('confidence_score', sa.Float(), nullable=True), + sa.Column('extraction_status', sa.String(length=50), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['document_id'], ['drive_files.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + + if 'idx_extraction_data_document_id' not in _index_names(bind, 'extraction_data'): + op.create_index(op.f('idx_extraction_data_document_id'), 'extraction_data', ['document_id'], unique=False) + + +def downgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + + if inspector.has_table('extraction_data'): + if 'idx_extraction_data_document_id' in _index_names(bind, 'extraction_data'): + op.drop_index(op.f('idx_extraction_data_document_id'), table_name='extraction_data') + op.drop_table('extraction_data') diff --git a/alembic/versions/w001_versioning_minor_and_version_fk.py b/alembic/versions/w001_versioning_minor_and_version_fk.py new file mode 100644 index 0000000..6a9e156 --- /dev/null +++ b/alembic/versions/w001_versioning_minor_and_version_fk.py @@ -0,0 +1,137 @@ +"""add minor_version + version-level FKs for signing artefacts + +Revision ID: w001_versioning_minor +Revises: v3_extraction, n002_add_signature_imprints, 39d5fdfb5386 +Create Date: 2026-04-28 + +Schema changes: + - drive_file_versions.minor_version (String(8), nullable) + - signing_requests.drive_file_version_id (Integer, nullable, FK) + - signature_imprints.drive_file_version_id (Integer, nullable, FK) + - Composite index on drive_file_versions(file_id, version_number, minor_version) + +Also serves as the merge revision for the three existing heads +('v3_extraction', 'n002_add_signature_imprints', '39d5fdfb5386'). +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "w001_versioning_minor" +down_revision: Union[str, tuple] = ( + "v3_extraction", + "n002_add_signature_imprints", + "39d5fdfb5386", +) +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _column_names(bind, table_name: str) -> set: + inspector = sa.inspect(bind) + return {c["name"] for c in inspector.get_columns(table_name)} + + +def _index_names(bind, table_name: str) -> set: + inspector = sa.inspect(bind) + return {idx["name"] for idx in inspector.get_indexes(table_name)} + + +def upgrade() -> None: + bind = op.get_bind() + + # 1) drive_file_versions.minor_version + if "minor_version" not in _column_names(bind, "drive_file_versions"): + op.add_column( + "drive_file_versions", + sa.Column("minor_version", sa.String(length=8), nullable=True), + ) + + if "ix_drive_file_versions_file_major_minor" not in _index_names( + bind, "drive_file_versions" + ): + op.create_index( + "ix_drive_file_versions_file_major_minor", + "drive_file_versions", + ["file_id", "version_number", "minor_version"], + unique=False, + ) + + # 2) signing_requests.drive_file_version_id + if "drive_file_version_id" not in _column_names(bind, "signing_requests"): + op.add_column( + "signing_requests", + sa.Column("drive_file_version_id", sa.Integer(), nullable=True), + ) + op.create_foreign_key( + "fk_signing_requests_drive_file_version_id", + "signing_requests", + "drive_file_versions", + ["drive_file_version_id"], + ["id"], + ondelete="SET NULL", + ) + + # 3) signature_imprints.drive_file_version_id + if "drive_file_version_id" not in _column_names(bind, "signature_imprints"): + op.add_column( + "signature_imprints", + sa.Column("drive_file_version_id", sa.Integer(), nullable=True), + ) + op.create_foreign_key( + "fk_signature_imprints_drive_file_version_id", + "signature_imprints", + "drive_file_versions", + ["drive_file_version_id"], + ["id"], + ondelete="SET NULL", + ) + if "ix_signature_imprints_version_id" not in _index_names( + bind, "signature_imprints" + ): + op.create_index( + "ix_signature_imprints_version_id", + "signature_imprints", + ["drive_file_version_id"], + ) + + +def downgrade() -> None: + bind = op.get_bind() + + if "drive_file_version_id" in _column_names(bind, "signature_imprints"): + if "ix_signature_imprints_version_id" in _index_names( + bind, "signature_imprints" + ): + op.drop_index( + "ix_signature_imprints_version_id", + table_name="signature_imprints", + ) + op.drop_constraint( + "fk_signature_imprints_drive_file_version_id", + "signature_imprints", + type_="foreignkey", + ) + op.drop_column("signature_imprints", "drive_file_version_id") + + if "drive_file_version_id" in _column_names(bind, "signing_requests"): + op.drop_constraint( + "fk_signing_requests_drive_file_version_id", + "signing_requests", + type_="foreignkey", + ) + op.drop_column("signing_requests", "drive_file_version_id") + + if "minor_version" in _column_names(bind, "drive_file_versions"): + if "ix_drive_file_versions_file_major_minor" in _index_names( + bind, "drive_file_versions" + ): + op.drop_index( + "ix_drive_file_versions_file_major_minor", + table_name="drive_file_versions", + ) + op.drop_column("drive_file_versions", "minor_version") diff --git a/alembic/versions/x001_add_user_smtp_credentials.py b/alembic/versions/x001_add_user_smtp_credentials.py new file mode 100644 index 0000000..bca35ca --- /dev/null +++ b/alembic/versions/x001_add_user_smtp_credentials.py @@ -0,0 +1,48 @@ +"""add user smtp credential columns + +Revision ID: x001_user_smtp_credentials +Revises: w001_versioning_minor +Create Date: 2026-04-30 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "x001_user_smtp_credentials" +down_revision: Union[str, None] = "w001_versioning_minor" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _column_names(bind, table_name: str) -> set: + inspector = sa.inspect(bind) + return {column["name"] for column in inspector.get_columns(table_name)} + + +def upgrade() -> None: + bind = op.get_bind() + column_names = _column_names(bind, "users") + + if "smtp_user" not in column_names: + op.add_column("users", sa.Column("smtp_user", sa.String(length=255), nullable=True)) + + if "encrypted_smtp_password" not in column_names: + op.add_column( + "users", + sa.Column("encrypted_smtp_password", sa.String(length=512), nullable=True), + ) + + +def downgrade() -> None: + bind = op.get_bind() + column_names = _column_names(bind, "users") + + if "encrypted_smtp_password" in column_names: + op.drop_column("users", "encrypted_smtp_password") + + if "smtp_user" in column_names: + op.drop_column("users", "smtp_user") diff --git a/alembic/versions/y001_add_user_mail_from.py b/alembic/versions/y001_add_user_mail_from.py new file mode 100644 index 0000000..33f2130 --- /dev/null +++ b/alembic/versions/y001_add_user_mail_from.py @@ -0,0 +1,39 @@ +"""add user mail_from column + +Revision ID: y001_add_user_mail_from +Revises: x001_user_smtp_credentials +Create Date: 2026-04-30 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "y001_add_user_mail_from" +down_revision: Union[str, None] = "x001_user_smtp_credentials" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _column_names(bind, table_name: str) -> set: + inspector = sa.inspect(bind) + return {column["name"] for column in inspector.get_columns(table_name)} + + +def upgrade() -> None: + bind = op.get_bind() + column_names = _column_names(bind, "users") + + if "mail_from" not in column_names: + op.add_column("users", sa.Column("mail_from", sa.String(length=255), nullable=True)) + + +def downgrade() -> None: + bind = op.get_bind() + column_names = _column_names(bind, "users") + + if "mail_from" in column_names: + op.drop_column("users", "mail_from") diff --git a/alembic/versions/z002_pref_lang.py b/alembic/versions/z002_pref_lang.py new file mode 100644 index 0000000..f377765 --- /dev/null +++ b/alembic/versions/z002_pref_lang.py @@ -0,0 +1,45 @@ +"""add preferred_language to users + +Revision ID: z002_pref_lang +Revises: ca14c4ab10d9 +Create Date: 2026-05-08 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "z002_pref_lang" +down_revision: Union[str, None] = "ca14c4ab10d9" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _column_names(bind, table_name: str) -> set[str]: + inspector = sa.inspect(bind) + return {column["name"] for column in inspector.get_columns(table_name)} + + +def upgrade() -> None: + bind = op.get_bind() + column_names = _column_names(bind, "users") + + if "preferred_language" not in column_names: + op.add_column( + "users", + sa.Column("preferred_language", sa.String(length=16), nullable=True), + ) + op.execute( + "UPDATE users SET preferred_language = 'en' WHERE preferred_language IS NULL" + ) + op.alter_column("users", "preferred_language", nullable=False) + + +def downgrade() -> None: + bind = op.get_bind() + column_names = _column_names(bind, "users") + + if "preferred_language" in column_names: + op.drop_column("users", "preferred_language") diff --git a/alembic/versions/z003_add_disable_pdf_compression_to_tenants.py b/alembic/versions/z003_add_disable_pdf_compression_to_tenants.py new file mode 100644 index 0000000..16ede8b --- /dev/null +++ b/alembic/versions/z003_add_disable_pdf_compression_to_tenants.py @@ -0,0 +1,41 @@ +"""add disable_pdf_compression to tenants + +Revision ID: z003_disable_pdf_compression +Revises: 7d7641cc1bb2 +Create Date: 2026-07-23 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "z003_disable_pdf_compression" +down_revision: Union[str, None] = "7d7641cc1bb2" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _column_names(bind, table_name: str) -> set[str]: + inspector = sa.inspect(bind) + return {column["name"] for column in inspector.get_columns(table_name)} + + +def upgrade() -> None: + bind = op.get_bind() + column_names = _column_names(bind, "tenants") + + if "disable_pdf_compression" not in column_names: + op.add_column( + "tenants", + sa.Column("disable_pdf_compression", sa.Boolean(), nullable=False, server_default="false"), + ) + + +def downgrade() -> None: + bind = op.get_bind() + column_names = _column_names(bind, "tenants") + + if "disable_pdf_compression" in column_names: + op.drop_column("tenants", "disable_pdf_compression") diff --git a/alembic/versions/z004_add_global_workflows.py b/alembic/versions/z004_add_global_workflows.py new file mode 100644 index 0000000..bfbe030 --- /dev/null +++ b/alembic/versions/z004_add_global_workflows.py @@ -0,0 +1,20 @@ +"""dummy z004 + +Revision ID: z004_add_global_workflows +Revises: z003_disable_pdf_compression +Create Date: 2026-08-06 + +""" +from alembic import op +import sqlalchemy as sa + +revision = 'z004_add_global_workflows' +down_revision = 'z003_disable_pdf_compression' +branch_labels = None +depends_on = None + +def upgrade(): + pass + +def downgrade(): + pass diff --git a/alembic/versions/z005_add_virus_scanning_and_quarantine_tables.py b/alembic/versions/z005_add_virus_scanning_and_quarantine_tables.py new file mode 100644 index 0000000..83f5841 --- /dev/null +++ b/alembic/versions/z005_add_virus_scanning_and_quarantine_tables.py @@ -0,0 +1,119 @@ +"""add virus scanning and quarantine tables + +Revision ID: z005_add_virus_scan +Revises: 73dff81eed72 +Create Date: 2026-08-31 19:15:00.000000 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = 'z005_add_virus_scan' +down_revision = '73dff81eed72' +branch_labels = None +depends_on = None + + +def upgrade(): + # 1. virus_scan_settings table + op.create_table( + 'virus_scan_settings', + sa.Column('tenant_id', postgresql.UUID(as_uuid=True), sa.ForeignKey('tenants.id', ondelete='CASCADE'), primary_key=True), + sa.Column('enabled', sa.Boolean(), nullable=False, server_default=sa.text('true')), + sa.Column('file_types', postgresql.JSONB(astext_type=sa.Text()), nullable=False, server_default=sa.text("'[]'::jsonb")), + sa.Column('max_file_size_mb', sa.Integer(), nullable=False, server_default=sa.text('100')), + sa.Column('action_on_detect', sa.String(length=50), nullable=False, server_default=sa.text("'quarantine'")), + sa.Column('notify_admin', sa.Boolean(), nullable=False, server_default=sa.text('true')), + sa.Column('notify_uploader', sa.Boolean(), nullable=False, server_default=sa.text('false')), + sa.Column('auto_suspend_uploader', sa.Boolean(), nullable=False, server_default=sa.text('false')), + sa.Column('suspend_threshold', sa.Integer(), nullable=False, server_default=sa.text('3')), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + ) + + # 2. virus_scan_jobs table + op.create_table( + 'virus_scan_jobs', + sa.Column('id', sa.BigInteger(), primary_key=True, autoincrement=True), + sa.Column('task_id', sa.String(length=255), nullable=True), + sa.Column('tenant_id', postgresql.UUID(as_uuid=True), sa.ForeignKey('tenants.id', ondelete='CASCADE'), nullable=True), + sa.Column('file_id', sa.BigInteger(), nullable=True), + sa.Column('version_id', sa.BigInteger(), nullable=True), + sa.Column('uploader_id', sa.Integer(), sa.ForeignKey('users.id', ondelete='SET NULL'), nullable=True), + sa.Column('storage_key', sa.String(length=1000), nullable=False), + sa.Column('file_size', sa.BigInteger(), nullable=False, server_default=sa.text('0')), + sa.Column('status', sa.String(length=50), nullable=False, server_default=sa.text("'pending'")), + sa.Column('retry_count', sa.Integer(), nullable=False, server_default=sa.text('0')), + sa.Column('error_message', sa.Text(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column('completed_at', sa.DateTime(timezone=True), nullable=True), + ) + op.create_index('ix_virus_scan_jobs_tenant_id', 'virus_scan_jobs', ['tenant_id']) + op.create_index('ix_virus_scan_jobs_task_id', 'virus_scan_jobs', ['task_id']) + op.create_index('ix_virus_scan_jobs_status', 'virus_scan_jobs', ['status']) + + # 3. virus_scan_results table + op.create_table( + 'virus_scan_results', + sa.Column('id', sa.BigInteger(), primary_key=True, autoincrement=True), + sa.Column('tenant_id', postgresql.UUID(as_uuid=True), sa.ForeignKey('tenants.id', ondelete='CASCADE'), nullable=True), + sa.Column('file_id', sa.BigInteger(), nullable=True), + sa.Column('version_id', sa.BigInteger(), nullable=True), + sa.Column('uploader_id', sa.Integer(), sa.ForeignKey('users.id', ondelete='SET NULL'), nullable=True), + sa.Column('status', sa.String(length=50), nullable=False), + sa.Column('threat_name', sa.String(length=255), nullable=True), + sa.Column('action_taken', sa.String(length=50), nullable=False, server_default=sa.text("'none'")), + sa.Column('scan_duration_ms', sa.Integer(), nullable=False, server_default=sa.text('0')), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + ) + op.create_index('ix_virus_scan_results_tenant_id', 'virus_scan_results', ['tenant_id']) + op.create_index('ix_virus_scan_results_status', 'virus_scan_results', ['status']) + op.create_index('ix_virus_scan_results_created_at', 'virus_scan_results', ['created_at']) + + # 4. quarantined_files table + op.create_table( + 'quarantined_files', + sa.Column('id', sa.BigInteger(), primary_key=True, autoincrement=True), + sa.Column('tenant_id', postgresql.UUID(as_uuid=True), sa.ForeignKey('tenants.id', ondelete='CASCADE'), nullable=True), + sa.Column('file_id', sa.BigInteger(), nullable=True), + sa.Column('version_id', sa.BigInteger(), nullable=True), + sa.Column('uploader_id', sa.Integer(), sa.ForeignKey('users.id', ondelete='SET NULL'), nullable=True), + sa.Column('filename', sa.String(length=500), nullable=False), + sa.Column('file_size', sa.BigInteger(), nullable=False, server_default=sa.text('0')), + sa.Column('mime_type', sa.String(length=255), nullable=True), + sa.Column('threat_name', sa.String(length=255), nullable=False), + sa.Column('quarantine_storage_key', sa.String(length=1000), nullable=False), + sa.Column('status', sa.String(length=50), nullable=False, server_default=sa.text("'quarantined'")), + sa.Column('quarantined_at', sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column('released_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('released_by_id', sa.Integer(), sa.ForeignKey('users.id', ondelete='SET NULL'), nullable=True), + sa.Column('deleted_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('deleted_by_id', sa.Integer(), sa.ForeignKey('users.id', ondelete='SET NULL'), nullable=True), + ) + op.create_index('ix_quarantined_files_tenant_id', 'quarantined_files', ['tenant_id']) + op.create_index('ix_quarantined_files_status', 'quarantined_files', ['status']) + + # 5. user_malware_counts table + op.create_table( + 'user_malware_counts', + sa.Column('id', sa.BigInteger(), primary_key=True, autoincrement=True), + sa.Column('tenant_id', postgresql.UUID(as_uuid=True), sa.ForeignKey('tenants.id', ondelete='CASCADE'), nullable=True), + sa.Column('user_id', sa.Integer(), sa.ForeignKey('users.id', ondelete='CASCADE'), nullable=False), + sa.Column('malware_count', sa.Integer(), nullable=False, server_default=sa.text('0')), + sa.Column('last_detected_at', sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column('is_suspended', sa.Boolean(), nullable=False, server_default=sa.text('false')), + sa.Column('suspension_reason', sa.Text(), nullable=True), + ) + op.create_index('ix_user_malware_counts_user_id', 'user_malware_counts', ['user_id']) + op.create_index('ix_user_malware_counts_tenant_id', 'user_malware_counts', ['tenant_id']) + + +def downgrade(): + op.drop_table('user_malware_counts') + op.drop_table('quarantined_files') + op.drop_table('virus_scan_results') + op.drop_table('virus_scan_jobs') + op.drop_table('virus_scan_settings') diff --git a/app/ai_runner.py b/app/ai_runner.py new file mode 100644 index 0000000..72d38f0 --- /dev/null +++ b/app/ai_runner.py @@ -0,0 +1,85 @@ +import sys +import json +import logging +import os +import re + +sys.path.append(os.getcwd()) + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("AI_Runner") + +def main(): + if len(sys.argv) < 4: + print(json.dumps({"status": "failed", "error": "Invalid arguments. Usage: python ai_runner.py "})) + sys.exit(1) + + input_path = sys.argv[1] + session_id = sys.argv[2] + user_id = sys.argv[3] + + try: + from app.core.model_manager import ModelManager + from app.modules.documents.processors.pdf.pdf_converter import UniversalContentIntelligence + from app.modules.documents.processors.pdf.image_extractor import ImageExtractor + + logger.info(f"🧠 Step 1: Running Marker AI on {input_path}") + with ModelManager() as ai: + result = ai.process_document(input_path) + + if hasattr(result, "markdown"): + content = result.markdown + elif isinstance(result, dict) and "markdown" in result: + content = result["markdown"] + else: + try: + from marker.output import text_from_rendered + content = text_from_rendered(result) + except: + content = str(result) + + logger.info("🔍 Step 2: Running Universal Content Intelligence") + metadata = UniversalContentIntelligence.extract_universal_metadata(content) + + figure_image_map = {} + pattern1 = r'!\[Image\s+(\d+)\]\((http[s]?://[^)]+)\)\s*\n\s*\*\*Figure\s+(\d+):' + matches1 = re.findall(pattern1, content, re.IGNORECASE | re.MULTILINE) + for _, image_url, figure_num in matches1: + figure_image_map[figure_num] = image_url + + pattern2 = r'!\[Image\s+(\d+)\]\(([^)]+)\)' + matches2 = re.findall(pattern2, content, re.IGNORECASE) + for image_num, image_url in matches2: + if image_num not in figure_image_map: + figure_image_map[image_num] = image_url + + logger.info("🖼️ Step 3: Extracting images") + images = [] + try: + extractor = ImageExtractor() + images = extractor.process_markdown_output_images(result, session_id=session_id, user_id=user_id) + logger.info(f"✅ Processed {len(images)} images") + except Exception as img_err: + logger.error(f"⚠️ Image extraction failed: {img_err}") + + print(json.dumps({ + "status": "success", + "markdown_content": content, + "session_id": session_id, + "images": images, + "metadata": { + "figures": figure_image_map, + "title": metadata.title if hasattr(metadata, 'title') else "" + } + })) + + except Exception as e: + logger.error(f"❌ AI Runner Failed: {e}") + print(json.dumps({ + "status": "failed", + "error": str(e) + })) + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/app/ai_service/__init__.py b/app/ai_service/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/ai_service/model_server.py b/app/ai_service/model_server.py new file mode 100644 index 0000000..2675fe2 --- /dev/null +++ b/app/ai_service/model_server.py @@ -0,0 +1,31 @@ +from fastapi import FastAPI +from pydantic import BaseModel +from app.core.model_manager import ModelManager + +app = FastAPI(title="AI Model Server") + +model = ModelManager() + + +class ProcessRequest(BaseModel): + file_path: str + + +@app.get("/health") +def health(): + return {"status": "ok"} + + +@app.post("/process") +def process_document(req: ProcessRequest): + """ + Heavy AI processing happens here. + This process should be run separately (e.g., on port 9000). + """ + result = model.process_document(req.file_path) + return {"result": result} + + +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="0.0.0.0", port=9000) diff --git a/app/api/ws_router.py b/app/api/ws_router.py new file mode 100644 index 0000000..f6c918a --- /dev/null +++ b/app/api/ws_router.py @@ -0,0 +1,68 @@ +from fastapi import APIRouter, WebSocket, Query, WebSocketDisconnect +from jose import JWTError, jwt +from sqlalchemy.orm import Session +import logging + +from app.infrastructure.realtime.connection_manager import manager +from app.core.settings import settings +from typing import Optional + +from app.core.ws_auth import authenticate_websocket +from app.db.database import SessionLocal +from app.modules.auth.models.user_model import User +from app.modules.documents.models.document_model import Project +from app.core.token_blacklist import TokenBlacklist + +logger = logging.getLogger(__name__) + +router = APIRouter() + +def _authenticate_ws(token: str, db: Session) -> Optional[User]: + """ + Kept as a thin delegation, not deleted. + + Both WebSocket modules carried their own copy of this and drifted apart; + `app/core/ws_auth.py` is now the single implementation. The name stays + because the handshake below calls it and the probes resolve it by name -- + the point was to remove the second set of *rules*, not the local symbol. + """ + return authenticate_websocket(token, db) + +@router.websocket("/ws/{task_id}") +async def websocket_progress(websocket: WebSocket, task_id: str, token: str = Query(None)): + auth_token = token or websocket.cookies.get("docqube_access_token") + + if auth_token and TokenBlacklist.is_token_blacklisted(auth_token): + await websocket.accept() + await websocket.close(code=4001, reason="Token revoked") + return + + db = SessionLocal() + try: + user = _authenticate_ws(auth_token, db) if auth_token else None + if not user: + logger.warning(f"WebSocket progress auth failed for task {task_id}") + await websocket.close(code=4001, reason="Authentication failed") + return + + project = db.query(Project).filter(Project.session_id == task_id, Project.user_id == user.id).first() + if not project: + logger.warning(f"Unauthorized WebSocket progress access: User {user.id} task {task_id}") + await websocket.close(code=4003, reason="Access denied to this task") + return + + user_id = user.id + finally: + db.close() + + await manager.connect_task(task_id, websocket) + + try: + while True: + await websocket.receive_text() + except WebSocketDisconnect: + logger.info(f"WebSocket progress disconnected: user={user_id} task={task_id}") + except Exception as e: + logger.error(f"WebSocket progress error: user={user_id} task={task_id}: {e}") + finally: + manager.disconnect(websocket) \ No newline at end of file diff --git a/app/assets/stylesheet/Stylesheet.deps.json b/app/assets/stylesheet/Stylesheet.deps.json new file mode 100644 index 0000000..2b4c555 --- /dev/null +++ b/app/assets/stylesheet/Stylesheet.deps.json @@ -0,0 +1,23 @@ +{ + "runtimeTarget": { + "name": ".NETCoreApp,Version=v8.0", + "signature": "" + }, + "compilationOptions": {}, + "targets": { + ".NETCoreApp,Version=v8.0": { + "Stylesheet/1.0.0": { + "runtime": { + "Stylesheet.dll": {} + } + } + } + }, + "libraries": { + "Stylesheet/1.0.0": { + "type": "project", + "serviceable": false, + "sha512": "" + } + } +} \ No newline at end of file diff --git a/app/assets/stylesheet/Stylesheet.dll b/app/assets/stylesheet/Stylesheet.dll new file mode 100644 index 0000000000000000000000000000000000000000..8003c5ed6f935aad1c5a9a59bf266d9130e3d0e2 GIT binary patch literal 9216 zcmeHMYj7OLaqhV{jwHxCkN`!BQ5kM&I5%HLp8L-nQm!HK$}$)3jV=MpNv9spzIMIG$E=R#r=`SP|V)Ry{F9 zG%Eb`{@dHXUm5K^O3*UVPP7h`h(~?pF4PL1UOWsGR6fscw50u9Qwa~~TzqumQKD7y zzgCw~kA*VOJI?K1qHM5CtglWGEeCGgNifbXl5Nx9mh3;b(00iaLL zwQM&qSrnoZDci9#z@%-xXoT?GjHj;j!upi085S7XRyu))btUlBm0qI$I#f2FX1}7a zZ9WRpFJocan}`JPFh+a1vlbJ~_fh2m-|B}U8}KFLL{aEW!r%~XjnYHqZC@v&}^+{-iEc-8Yz=(M&-9!0GEZGwWyXyLaSHXkju9?a9A{n$cQ!t$XU;I z(1AmuO`%vQXl+1?Vj*iIYHJgd!?AF)r7$@V3nw=-DFTjIAQp^8EVu|(URn{0v_*in zGWyqy2Y?5vc(AE~uB_V%+mr04M=1UztzcjHXo&8A&TI3`Ub;X**dW%o2kFmcKhL!} z>O13yS1fopn!E*IS-5RE+3;}QxfSR`RL^i5D9a;lA!>=zwq^BP@K)R5CA_c)o`?nP zz0hT!1lY7H`7ufK#R7F&?Hu}&m_jtU4Ztok+Quj>!y?(|S;3c>x*gP8>C)P!AjN`K zin;6QSV|JuRy^NVU!%1=-==mSK`oypG4i)9^PS-+SbfF6O*BVCZH=&*37$jA=TGi{ zRw!y}_tSaw$l`Ob>7x%LbOS!8gPZI$U-EY5^_^k55(4w4NoOZW)~*`eeiMdUyK96^ z&Zai!4(37xSEO4kY~u=I|A0jh{}n=ub$1Q%T?qx?7v~<3tS*nv)@*9objy;_nP~q{ z-RMkER5r6^i2+PsTQ&e;_g*hRzR&4~?&KahKufYm*2<%hRvw2>q=ICxq$c}Nl^3+X zp2B{yJW5M)K=w2z2g^0iSIp0j39PH<9=jFo^1LxiNPDwP4wpp`Nf1fmNWcmmpk##O z!+}Ymej7I?ShuUX!tEgj{pXu+bFa(;Y*H5vBOT3bsPZV z;;8kNm<6%apT4(W@Tq{p=hwonRC{V?`_2w|HU-H5?Pf%1>mx+Jg8G{Jh2gwsjp7v`ydxSw&NAN##JM34Bs7Yxy537-&u5?o0)!D5_yVy8bu-}b!` z2+>{%0}^hQ@Cyd8V>lwjbsPk@JH!i(2mkqBmQuRz9L8YrEm{R zg?a*U`h(C_jIuSbh2nH2aMd5D{|v7OeSd)AAEP%;4+oj&a)70V1;hIU+w6*C!DiYd zo(AXN_@y3yS0qle5tjNIF7B4YypjGn{D6NWt@A&Cd94;wL$D{X5xh)^O8#FbfA~xE zmxz=oT|zrSzYW+-TVW@A-t#+0k3go^0_VcrvR%Rf)d}n%RxV78{azWvssa1EP%`u= z`WnEU5AVeddFTyv1<|THP?w}*!GRoLGdH|9~H#6s1 zy5K(|LbNZz*f*(DbmH<-lGs_g1V1#;<}KX!6rBh(iU!&(vCq;bKVzdF7WraW1D27P zH+lmZp4{5kgUrdZj_?K|KDq_rfJXZX7g* z3Hu}*k?^2|_erQqs7rVXP|!1gYv?C{A^NF=LR<{4qoQnoxeSkoJL${xJ%1P71aA)j z9t@4pYv>)P&EP*ouTzi6AbwcSJ7qY8HGGHO4dv-wY7`7v&)eIdX|d947@;R#b^C5p&b%0&^d8B^84VwfE6gp z^*JZUeM-WEq9GEZXJFgcq5so?HvoSGd!8>-!v42F`9wqj9*nG_7sRtc%n`kR0m|?C z-=){keve)kYs3El_!;^kMp-4L{yp@s^p3cfJ^+6ToJ z*l<>S5QvDgVo#_M@TG7Z@Of&%dBwiEEDlhsxQ$NIW4;v5DNCfNi53CZ)6;;h^c>(e zx&*kBegt?A{S1he?rx++1z+Q4_v+sn2B?&)8pQ77*Uo68XsLl5todHFm z$LS(%6hXut(VO81g%5GVT{{H=5<#~=b3agXLyaQHmep4cf@pD)kKHG!xvmDoly-9mT8pM+xVva_0E2)vur<|%x~aaYc{HiSN0VJr* z!wh?`W@bk0=0ZgzaF{sTH=H_?;HCmtJXt^1qIHkMwQ)`TjZBPJ`8o=}Ysyf9u8Eyg*%z`$-p?p2Be9Nk z&{j_~I+;#S+Zr#>^wIR_)HFMMLUrefx7nyQSH(HX)2?c}g}m(NO$MtnCuzntryY)E zXI=x7M=Wnpo*UxVba|&3-c+z2ylmK8quK(9avw<9RMr6MZEfsF1gz<**aS8=Y^%O6 zb<{CvOv_nz5m!)6%@G!azju`7<#|A0^43$!PT8vI@W3<%l{9PTrgPjFm{-j?ZD`Te zOpdbp?Iq0}wY^<)pIZHD9-{>W@s6XJUJQ6VZbsfl%d6pmY03r_9g@bpF@+_ zwVI95O|C%T^TFX$YD_h8%GfuwCAGwW3g^%QoWD;sv)K1Fv8*;||)ic35``sv*~|U(KAvv@Dxa8CODCm0d8Xt8=F1xO&E^7*G!U zssU-uUeGfd`m14-sw@z`s5n-^=3ehKLez3FEn&vm=QZ2aK^-!A6V_Sh&|b~$FS;z@ zg;qKCQxmqH!(Ljb^(+n78!#Nv&8&6WNev^AocjLivQ^IHnkTCi-a_LAH(zk8B)L3& zHeR5(la~P@QZ;nRXfhhBiD(cxLI!M=c?yRPR%)(bsP@oe9&Y1!c98Stw8e25XO6c4 z=QXU?ylSf%tg}Oz${vOD7InvUXnf{D1Z~}FT(ifev4dWo(R*6fiM&6n0R$;N+oeR4rNs7xQq?RYoQ>oA+`V za~Iud-GpB&E6ip{BW1&ZYh{DuxX&E z=4CFGT$<mQWT4sEZ}QUuG< zC)NyL^c?i_I4+(nkFlW-chWJ8n+B~Y`y9w>l7}|TfNIh-G~A$f%?)LmT#V>KMu%jc z3TRVo7IV*mmV~X>zhGiqObr&qNm`mt0lqL_-l<#A+SGxesR7H@a7k?|-$J?gABHs)kzRv=(6r z1sp2{MQg`DzB334$^#!CqcB1TzD42Bqf3N{MaoBHCwJkk7={GcnEM3nAJa3oF7+|<&BT&^2arlm+^e- zPv8F|9`7kUMm6V`6BbTmujS>>HYMj|EZfj$+LVfK_!Arc+LQsDjRjllHnoClt45nL zQJBH)YQI*TvQBDd_e^J}x;wM`j*h!_?$X-tysP#4!MqoT%9xm*4@OnWMBgC)v&~$- z@M7@-iLnW0`2?PC;DhWp_;;Ri;W^=l0lE!O7ry^>;R}C#!RNdsWimaOzPw+2JMqfs zy`|wRZ~bcE#Rb+eaQAU`CvpvLdedXa(=wIwOUCiO0iYB5!1#gfNB3?AHhn-leY_R| z$E}$MkIM)+ZfY(fWIHaboP6teQu*u*rT6vSzWWX;55KN5JWp}Ur%zluHvX;O{O?#t z{0aFP)3f?_xUG2YYhmT57>|yejBJXF7p{tO`7}-5Qf2SeTS)1>R{rOJ`|!O(+E>D_ zE;K<@TB=mWIbq4Jqch(%L{L2b`RlrPh47mej*=&PiM~@8FW>R)LeF8G&C`HGG>O&- zexW-6d<1aV`?Ze#F!25r*6%&2WE+W(n9r^GIQ86o6Oxn(<;f+PL;o1G=nzL9oWDEInT?5JVp^S zQjwW7sr`mI4Ly6LT*cx+aO3U-D|O_!diy?1z8&YOsM>iL^0?#I&8q|1xV`%Dcq_rr zp5a_=qvqQ}&C(lC!oiI6$SXlt+SRsG|#u=`?Wxn*UMe&otke5anFA2gYWoOmBXqUn3aM! zUFy-A4EvSH9U%HgFXW|v-oN_DjUh-@{7*^OKjo{kx)!z%kIQjvF`g%O3mjJJ3I}pU2yQxYS#K0 z?YTW62oe^^Z+09NU1-&nWS3J0JRy7>0JSnMU#|^Bhn7(6)?Oi*nP5?vKj>-z*lKcx_ X5TN{<+pBd+>KEST|2OdG=7IkO^ZiH@ literal 0 HcmV?d00001 diff --git a/app/assets/stylesheet/Stylesheet.runtimeconfig.json b/app/assets/stylesheet/Stylesheet.runtimeconfig.json new file mode 100644 index 0000000..1dc0145 --- /dev/null +++ b/app/assets/stylesheet/Stylesheet.runtimeconfig.json @@ -0,0 +1,18 @@ +{ + "runtimeOptions": { + "tfm": "net8.0", + "frameworks": [ + { + "name": "Microsoft.NETCore.App", + "version": "8.0.0" + }, + { + "name": "Microsoft.WindowsDesktop.App", + "version": "8.0.0" + } + ], + "configProperties": { + "System.Runtime.Serialization.EnableUnsafeBinaryFormatterSerialization": true + } + } +} \ No newline at end of file diff --git a/app/assets/stylesheet/jats-html.xsl b/app/assets/stylesheet/jats-html.xsl new file mode 100644 index 0000000..02632ed --- /dev/null +++ b/app/assets/stylesheet/jats-html.xsl @@ -0,0 +1,4447 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <xsl:variable name="authors"> + <xsl:call-template name="author-string"/> + </xsl:variable> + <xsl:value-of select="normalize-space(string($authors))"/> + <xsl:if test="normalize-space(string($authors))">: + </xsl:if> + JATS-HTML Viewer + <!--<xsl:value-of select="/article/front/article-meta/title-group/article-title[1]"/>--> + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + +
+ +
+ + + + +
+ +
+ + + + --> + + + +
+ + + Floating objects + + + +
+ + + + + + + + + + + + +
+
+ + + +
+

+ Journal Information +

+
+ + + + + + +
+   +
+ +   +
+
+ + + + + + +
+

+ Article Information +

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ +
+
+ +
+ + + +
+ +
+ + + + +
+
+
+
+
+   +
+
+
+
+ + + + + +
+ + + + + + +
+ + +

+ + + + Translated + Abstract + + +

+ + +
+ +
+
+ + + + + + + +
+ +
+ +
+ + + + + + + + + + +
+
+

+ Article Information (continued) +

+
+   + +   + +   +
+
+ + + + + + + + +
+
+ +
+
+

+ Process warnings +

+

Warnings reported by the processor due to problematic markup follow:

+
+ +
+
+
+
+ + + + + + + + + + + + + + + + Journal ID + + ( + + + + ) + + + + + + + + + + + + + + + ISSN + + + + + + + + + + ISSN-L + + + + + + + + + + ISBN + + + + + + + + Publisher + + + + + + + + + + + + + + + ( + + ) + + + + + + Notes + + + + + + + + + Title + + + + + + + Subtitle + + + + + + + + + + + + Abbreviated Title + + + + + + + + + + Translated Title + + ( + + + + ) + + + + + + + + + + Translated Subtitle + + ( + + + + ) + + + + + + + + + + + + + + External link + + ( + + + + ) + + + + + + + + + + + + Email + + + + + + + + + + URI + + + + + + + + + + + Self URI + + + + + + + + + + + + + + + + + + + + + Product Information + + + + + + + + + + + + + + + + + + + + + + + Copyright + + + + , + + + + + + + + + + + Copyright statement + + + + + + + Copyright + + + + + + + + License + + ( + + + + , + + + + + + + ) + + + + + + + + + + Date + + + accepted + received + revision requested + revision received + + + + + + + + + + + + + + Publication date + + + + + + + + + + + + + + + + + Volume + + + + + + + + + + + + + + + + + ( + + + + + + + ID: + + + ) + + + + + + , + + + + + + + + + Volume + + + + + + + + Volume ID + + ( + + + + ) + + + + + + + + + Series + + + + + + + + + + + + Issue + + + + + + + + + + + + + + + + , + + + + + + + + + Issue + + + + + + + + Issue ID + + ( + + + + ) + + + + + + + + + Issue title + + + + + + + Issue sponsor + + + + + + + Issue part + + + + + + + + + + Page + + + s + + + + + + - + + + + (pp. + + ) + + + + + + + + + + Electronic Location + Identifier + + + + + + + + + Supplement Info + + + + + + + + Related + + object + article + + + ( + + + + ) + + + + + + + + + + + + + + + + + + + + + + + + + + Conference + + + + + ( + + , + + + ) + + + + + + + + + + + + + + Conference date + + + + + + + Conference + + + + + + + Conference + + + + + + + Conference number + + + + + + + Conference location + + + + + + + Conference sponsor + + + + + + + Conference theme + + + + + + + + + + ( + + + + ) + + + + + + + + + + + + + + + + + + + + Accession ID + Coden + DOI + Manuscript ID + Medline ID + Publisher Item ID + PubMed ID + Publisher ID + Serial Item and Contribution ID + DOAJ ID + arXiv.org ID + + Article Id + + ( + + + + ) + + + + + + + + + + + + Contract + + + + + + + Contract Sponsor + + + + + + + Grant Number + + + + + + + Grant Sponsor + + + + + + + + + + + + + Funded by + + + + + + + Award ID + + + + + + + Award Recipient + + + + + + + Principal Investigator + + + + + + + Funding + + + + + + + Open Access + + + + + + + + + +
+ +
+ + + + + “” + + + +

+ + : +

+ + + +

+ +

+ + + + + + + + + + + + + + Alternative title + + ( + + + + ) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+ + + +
+
+ + +
+
+
+ + + + + +
+
+
 
+
+
+ +
+
+
+
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + (Equal contributor) + + + +
+ + + + + Anonymous + + + + + + + + + + + + + + + , + + + + + , + + + + + + + + + + + + + [ + + ] + + + + + +
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Correspondence should be addressed to

  + + + + + + + + Comment + + + + + + + Bio + + + + + + + On behalf of + + + + + + + + + + + + Author notes + + + + + + + + + + + + + Correspondence to + + + + + + + + + + + + + + + + + + + + + Supplementary material + + + + + + + + + + + + Categories + + + + + + + + + + + + + +
+ +
+ + + + + + Subject + + + + + + + Series title + + + + + + + Series description + + + + + + + + + Keywords + + + + + + + + + + + + + + + + Keyword + + + + +
    + +
+ + + +
  • + +
    • +     + + + + + Compound keyword + + + + + + + + + + + Keyword part + + ( + + + + ) + + + + + + + + + + Counts + + + + + + + + + + + + + + + + + + + + + Count + + ( + + + + ) + + + + + Figures + + + Tables + + + Equations + + + References + + + Pages + + + Words + + + + + Custom metadata + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + + + + + +
    +     + + + + + + + + + + + +
    + + + +
    +     + + + +
    + + + + +
    + +
    +     + +
    +     + + + +
    + + +
    +     + + + + + + + + + + + + + + + + + + + + + + + + +

    + +

    +     +     + + + + + + + + +

    +   +

    +     +     + + + + + + + + +

    +   +

    +     +     + + + + + + + + +

    + +

    +     +     + + + + + +

    + +

    +     +     + + + + +
    + +
    +     +     + + + + + + + + + + + + + + +
    + +
    +     +     +     + + + + + + , + + + + + + +

    + +

    +     + + +
    + +
    +     + + +

    + +

    +     + + + + + + + chem-struct-wrap + + + + + +
    + + + display: float; clear: both + + + + + +
    +     + + + +
    + +
    +     + + + +
    + + + +
    +     + + + + +
    + + + + + + Glossary + + + + +
    +     + + + +

    + [Textual form] + +

    +     + + + + + +
    + + +
    +     + + + + + + + + + + + + + + + + +
    + + + +
    +     + + + +
      + +
    +     + + + +
      + +
    +     + + + + + + lower-alpha + upper-alpha + lower-roman + upper-roman + decimal + + +
      + +
    +     + + + +
  • + +
    • +     + + + + + + + + + + + + + + + + + + + + +

    + + first + + + + +

    +     + + + + + + + + +

    + + + + + + + + + +

    +     + + + +

    + + +

    +     + + + +
    + + +

    + Copyright + + + + , + + +

    +     + +
    +     + + + +

    + +

    +     + + + +
    + + +
    + +
    +
    + +
    +
    + +
    +
    +     + +
    + +
    +     + + + +
    + + +
    +     + + + +
    + +

    + +

    +
    +     + + + +
    + + +
    +     + + + +
    + + +
    +     + + + + 
    +      
+
    +     + + + + + + + + + + + + + + + +
    +
    + + +   + + + +
    +
    + +
    +
    +     + + + + +

    + + +

    +     + + + + + +

    + +

    +     +
    + + +
    +     + + + + + + + + + + + + + +
    + + +
    +     + + + + + + +

    + + +

    +     + + + + + + + : + + + + +
    + + + +
    +     + + + + + + + + + + + + + + + + + + + + + + + +
    + + +
    +     + + + +

    + +

    +     + + + + + [ + + ] + + + + + + + + + + + + + + + + +
    + +
    +     + + + + + + + + + + + TableData + + + + + + + + + + + + + +
    +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + [ + + ] + + + + + + + + + + , + + + + + + + + + + + + + + + + + + +
    +     + + + + + + + + + + + [ + + + + + + + + + + + + + + + ] + + + + + + + + + + + +
    +     + + + + + + + + + + + + + +
    + +
    +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + [Private character + + + + + ] + + + + + + (Glyph not rendered) + + + + + [Related article:] + + + + + + [Related object:] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Notes + + + + + + + + + + Acknowledgements + + + + + + + Appendices + + + + + + + Biography + + + + + + + Notes + + + + + + + Glossary + + + + + + + References + + + + + + + + + + Notes + + + + + + + + + +
    + + + + + + + + + + + + + + + + + +
    +     + + + + + + + + + + + + + + + + + + + + + + + + + +
    + + +
    +     + + + +

    + + + + + + + +

    +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + +
    +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Box + + + + + + + + + + + + + + Formula + + + + + + + + + + + + + + Formula (chemical) + + + + + + + + + + + + + + Figure + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + [ + + ] + + + + + + + + + + + + + + + + [ + + ] + + + + + + + Abbreviation + + + + Communicated by + + + + Contributed by + + + + Conflicts of interest + + + + Corresponding author + + + + Current affiliation + + + + Deceased + + + + Edited by + + + + Equal contributor + + + + Financial disclosure + + + + On leave + + + + + + Participating researcher + + + + Current address + + + + Presented at + + + + Presented by + + + + Previously at + + + + Study group member + + + + Supplementary material + + + + Supported by + + + + + + + + + + + + + + + + + + + + + + + + + + Statement + + + + + + + + + + + + + + Supplementary Material + + + + + + + + + + + + + + Table + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +   +   +   +   +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + etal. + + + + + + + + + + + + + ( + + + electronic + print + print and electronic + electronic preprint + print preprint + corrected, electronic + corrected, print + retracted, electronic + retracted, print + + + + + + ) + + + + + + + + + + + + + + + : + + + + + + + + + + + + +

    + +

    +     + + + + + + + +
    + + + + + + + + +
    + +
    +
    +     + + + + + + + + + + + + + + + + + { label + (or @symbol) + needed for + + + [@id=' + + '] + + } + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + named anchor + + + + + + + + + + + + + + + + + + + + + + +
  • + + + +
    • +     +     +     + + +

    Elements are cross-referenced without labels.

    +

    Either the element should be provided a label, or their cross-reference(s) should + have literal text content.

    +
      + +
    +     + + + + + +
  • + In + +
      + +
    • + +
      • +       +
    + +
    • +     +     + + +

    Elements with different 'specific-use' assignments appearing together

    +
      + +
    +     +     + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + January + February + March + April + May + June + July + August + September + October + November + December + + + + + + + + + + + + + + + + + , + + + and + + + + + + + + + + + +
    +
    +

    + This display is generated from + NISO JATS XML with + + + + . The XSLT engine is + + . +

    +
    +     + + + + + + + + + + + + + + + + [@ + + =' + + '] + + + + + + + + + + + + + + + + / + + + [ + + ] + + + + + + /@ + + + + + + /comment() + + [ + + ] + + + + + + /processing-instruction() + + [ + + ] + + + + + + /text() + + [ + + ] + + + + +   + + DOI: + PMID: + Scopus: + + +   + + + + +
    + + +

    Query

    + + + + + + + +
    + + + + + + + + + + + + + + + +
    +     +     + + + + + + + + + + + vol. + + , + + + + + + + + + + + no. + + , + + + + + + + + + + + pp. + + + , + + + + + + + + + + + + + + - + + pp. + + + + , + + + + + + + + + + + + “” + + , + + + + + + + , + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/app/core/cancellation.py b/app/core/cancellation.py new file mode 100644 index 0000000..ea760ba --- /dev/null +++ b/app/core/cancellation.py @@ -0,0 +1,13 @@ +import redis +from app.core.settings import settings + +redis_client = redis.from_url( + settings.REDIS_URL or "redis://localhost:6379/0", + decode_responses=True +) + +def cancel_task(task_id: str): + redis_client.set(f"cancel:{task_id}", "1", ex=86400) + +def is_cancelled(task_id: str): + return redis_client.exists(f"cancel:{task_id}") == 1 diff --git a/app/core/clamav.py b/app/core/clamav.py new file mode 100644 index 0000000..5691f9b --- /dev/null +++ b/app/core/clamav.py @@ -0,0 +1,118 @@ +import io +import socket +import struct +import logging +from typing import BinaryIO, Iterable, Optional + +logger = logging.getLogger(__name__) + +from app.core.settings import settings + +class ClamAVError(Exception): + """Base exception for ClamAV scanning failures.""" + + +class ClamAVConnectionError(ClamAVError): + """Raised when the ClamAV service cannot be reached.""" + + +class MalwareDetectedError(ClamAVError): + """Raised when ClamAV reports malware in a payload.""" + + def __init__(self, virus_name: str, response: str): + self.virus_name = virus_name + self.response = response + super().__init__(f"Malware detected: {virus_name}") + + +def _response_status(response: str) -> tuple[str, Optional[str]]: + text = response.strip().strip("\x00") + + if " FOUND" in text: + payload = text.split(": ", 1)[1] if ": " in text else text + virus_name = payload.rsplit(" FOUND", 1)[0].strip() or "Unknown threat" + return "found", virus_name + + if text.endswith("OK") or " OK" in text: + return "ok", None + + return "error", None + + +def _recv_clamav_response(sock: socket.socket) -> str: + chunks = [] + while True: + data = sock.recv(4096) + if not data: + break + chunks.append(data) + if b"\x00" in data: + break + return b"".join(chunks).decode("utf-8", errors="replace") + + +def _scan_chunks(chunks: Iterable[bytes]) -> None: + host = settings.CLAMAV_HOST + port = settings.CLAMAV_PORT + timeout = float(settings.CLAMAV_TIMEOUT_SECONDS) + + try: + logger.info(f"Connecting to ClamAV for scan at {host}:{port}...") + with socket.create_connection((host, port), timeout=timeout) as sock: + sock.settimeout(timeout) + sock.sendall(b"zINSTREAM\0") + + total_bytes = 0 + for chunk in chunks: + if not chunk: + continue + total_bytes += len(chunk) + sock.sendall(struct.pack("!I", len(chunk))) + sock.sendall(chunk) + + sock.sendall(struct.pack("!I", 0)) + response = _recv_clamav_response(sock) + logger.info(f"ClamAV scan completed. Sent {total_bytes} bytes. Response: {response.strip()}") + except (OSError, socket.timeout) as exc: + raise ClamAVConnectionError( + f"Unable to connect to ClamAV at {host}:{port}" + ) from exc + + status, virus_name = _response_status(response) + if status == "found": + logger.warning(f"SCAN RESULT: MALWARE FOUND - {virus_name}") + raise MalwareDetectedError(virus_name=virus_name or "Unknown threat", response=response) + if status != "ok": + logger.error(f"SCAN RESULT: ERROR - Unexpected response: {response}") + raise ClamAVError(f"Unexpected ClamAV response: {response}") + + logger.info("SCAN RESULT: CLEAN - Document is safe.") + + +def scan_file_for_malware(file_obj: BinaryIO) -> None: + """ + Scan a file-like object using ClamAV INSTREAM and rewind it for later reads. + """ + chunk_size = max(1, int(settings.CLAMAV_CHUNK_SIZE)) + + can_seek = hasattr(file_obj, "seek") + if can_seek: + file_obj.seek(0) + + try: + def iter_chunks(): + while True: + chunk = file_obj.read(chunk_size) + if not chunk: + break + yield chunk + + _scan_chunks(iter_chunks()) + finally: + if can_seek: + file_obj.seek(0) + + +def scan_bytes_for_malware(payload: bytes) -> None: + """Scan an in-memory payload using ClamAV.""" + scan_file_for_malware(io.BytesIO(payload)) diff --git a/app/core/context.py b/app/core/context.py new file mode 100644 index 0000000..8b70f91 --- /dev/null +++ b/app/core/context.py @@ -0,0 +1,4 @@ +from contextvars import ContextVar + +client_ip_ctx_var: ContextVar[str] = ContextVar("client_ip", default=None) +user_agent_ctx_var: ContextVar[str] = ContextVar("user_agent", default=None) diff --git a/app/core/crypto.py b/app/core/crypto.py new file mode 100644 index 0000000..8876caf --- /dev/null +++ b/app/core/crypto.py @@ -0,0 +1,35 @@ +import base64 +import hashlib +import os +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from app.core.settings import settings + +def _get_key() -> bytes: + """Derive a consistent 32-byte key from APP_SECRET for AES-256.""" + return hashlib.sha256(settings.APP_SECRET.encode('utf-8')).digest() + +def encrypt_data(data: str) -> str: + """Encrypts a string using AES-256-GCM and returns a base64 encoded string.""" + if not data: + return data + aesgcm = AESGCM(_get_key()) + nonce = os.urandom(12) + ciphertext = aesgcm.encrypt(nonce, data.encode('utf-8'), None) + combined = nonce + ciphertext + return base64.b64encode(combined).decode('utf-8') + +def decrypt_data(token: str) -> str: + """Decrypts a base64 encoded AES-256-GCM string.""" + if not token: + return token + try: + combined = base64.b64decode(token.encode('utf-8')) + if len(combined) < 12: + return "" + nonce = combined[:12] + ciphertext = combined[12:] + aesgcm = AESGCM(_get_key()) + decrypted = aesgcm.decrypt(nonce, ciphertext, None) + return decrypted.decode('utf-8') + except Exception: + return "" diff --git a/app/core/mail.py b/app/core/mail.py new file mode 100644 index 0000000..803743f --- /dev/null +++ b/app/core/mail.py @@ -0,0 +1,250 @@ +import smtplib +import logging +import os +from email.mime.text import MIMEText +from email.mime.multipart import MIMEMultipart +from email.mime.base import MIMEBase +from email import encoders +from typing import Optional, Dict, Any, List +from uuid import UUID +from jinja2 import Environment, FileSystemLoader +from sqlalchemy.orm import Session + +from app.core.settings import settings +from app.core.crypto import decrypt_data + +logger = logging.getLogger(__name__) + +template_dir = os.path.join( + os.path.dirname(os.path.dirname(__file__)), "templates", "emails" +) +try: + jinja_env = Environment(loader=FileSystemLoader(template_dir)) +except Exception as e: + logger.warning(f"Failed to initialize Jinja template environment: {e}") + jinja_env = None + + +def send_email( + subject: str, + recipient: str, + body: Optional[str] = None, + html_body: Optional[str] = None, + template_name: Optional[str] = None, + template_context: Optional[Dict[str, Any]] = None, + db: Optional[Session] = None, + tenant_id: Optional[UUID] = None, + user_id: Optional[int] = None, + attachments: Optional[List[Dict[str, Any]]] = None, +): + """ + Sends an email using the configured SMTP settings. + In development, if SMTP is not configured, it logs the email. + """ + if template_name and template_context is not None and jinja_env: + try: + template = jinja_env.get_template(template_name) + html_body = template.render(**template_context) + if not body: + body = f"Please view this email in an HTML-compatible client. Subject: {subject}" + except Exception as e: + logger.error(f"Failed to render email template {template_name}: {e}") + return + elif template_name and not jinja_env: + logger.error("Jinja2 environment not initialized. Cannot render template '%s'", template_name) + return + + if not body and not html_body: + logger.error("Email must have either a body or html_body") + return + + smtp_config = _resolve_smtp_config(db=db, tenant_id=tenant_id, user_id=user_id) + logger.info( + "SMTP source selected: %s (tenant_id=%s)", + smtp_config.get("source", "unknown"), + str(tenant_id) if tenant_id else "none", + ) + + if not smtp_config["smtp_host"]: + return + + try: + _send_with_config( + smtp_config=smtp_config, + subject=subject, + recipient=recipient, + body=body, + html_body=html_body, + attachments=attachments, + ) + logger.info( + "Email sent successfully to %s via %s SMTP", + recipient, + smtp_config.get("source", "unknown"), + ) + return + except Exception as e: + logger.error(f"Failed to send email to {recipient} via {smtp_config.get('source', 'unknown')} SMTP: {e}") + + fallback_config = None + if smtp_config.get("source") == "user": + fallback_config = _resolve_smtp_config(db=db, tenant_id=tenant_id, user_id=None) + elif smtp_config.get("source") == "tenant": + fallback_config = _resolve_smtp_config(db=None, tenant_id=None, user_id=None) + + if fallback_config and fallback_config.get("smtp_host"): + logger.warning( + "Retrying email delivery via fallback SMTP after %s SMTP failure (tenant_id=%s)", + smtp_config.get("source", "unknown"), + str(tenant_id) if tenant_id else "none", + ) + try: + _send_with_config( + smtp_config=fallback_config, + subject=subject, + recipient=recipient, + body=body, + html_body=html_body, + attachments=attachments, + ) + logger.info( + "Email sent successfully to %s via fallback SMTP after %s SMTP failure", + recipient, + smtp_config.get("source", "unknown"), + ) + return + except Exception as fallback_error: + logger.error( + "Fallback SMTP also failed for %s: %s", + recipient, + fallback_error, + ) + + +def _send_with_config( + smtp_config: Dict[str, Any], + subject: str, + recipient: str, + body: Optional[str], + html_body: Optional[str], + attachments: Optional[List[Dict[str, Any]]] = None, +) -> None: + msg = MIMEMultipart() + msg["Subject"] = subject + msg["From"] = smtp_config["mail_from"] + msg["To"] = recipient + + if body or html_body: + content_part = MIMEMultipart("alternative") + if body: + content_part.attach(MIMEText(body, "plain")) + if html_body: + content_part.attach(MIMEText(html_body, "html")) + msg.attach(content_part) + + if attachments: + for attachment in attachments: + part = MIMEBase("application", "octet-stream") + part.set_payload(attachment["content"]) + encoders.encode_base64(part) + part.add_header( + "Content-Disposition", + f"attachment; filename={attachment['filename']}", + ) + msg.attach(part) + + if smtp_config["smtp_secure"] or smtp_config["smtp_port"] == 465: + with smtplib.SMTP_SSL(smtp_config["smtp_host"], smtp_config["smtp_port"]) as server: + if smtp_config["smtp_user"] and smtp_config["smtp_password"]: + server.login(smtp_config["smtp_user"], smtp_config["smtp_password"]) + server.send_message(msg) + else: + with smtplib.SMTP(smtp_config["smtp_host"], smtp_config["smtp_port"]) as server: + if smtp_config["smtp_user"] and smtp_config["smtp_password"]: + server.starttls() + server.login(smtp_config["smtp_user"], smtp_config["smtp_password"]) + server.send_message(msg) + + +def _resolve_smtp_config( + db: Optional[Session] = None, + tenant_id: Optional[UUID] = None, + user_id: Optional[int] = None, +) -> Dict[str, Any]: + fallback = { + "source": "fallback", + "smtp_host": settings.SMTP_HOST, + "smtp_port": settings.SMTP_PORT or 587, + "smtp_user": settings.SMTP_USER, + "smtp_password": settings.SMTP_PASSWORD, + "smtp_secure": settings.SMTP_SECURE, + "mail_from": settings.MAIL_FROM, + } + + close_db = False + if db is None and (tenant_id is not None or user_id is not None): + try: + from app.db.database import SessionLocal + db = SessionLocal() + close_db = True + except Exception as err: + logger.error("Failed to create temporary DB session for SMTP resolution: %s", err) + + try: + base_config = fallback + + if db is not None and tenant_id is not None: + try: + from app.modules.tenant.repositories.tenant_smtp_repository import TenantSMTPRepository + + cfg = TenantSMTPRepository(db).get_by_tenant_id(tenant_id) + if cfg and cfg.is_active: + base_config = { + "source": "tenant", + "smtp_host": cfg.smtp_host, + "smtp_port": cfg.smtp_port, + "smtp_user": cfg.smtp_user, + "smtp_password": ( + decrypt_data(cfg.encrypted_smtp_password) + if cfg.encrypted_smtp_password + else None + ), + "smtp_secure": cfg.smtp_secure, + "mail_from": cfg.mail_from, + } + except Exception as e: + logger.error( + "SMTP config lookup failed for tenant_id=%s, using fallback SMTP: %s", + tenant_id, + e, + ) + + if db is None or user_id is None: + return base_config + + try: + from app.modules.auth.models.user_model import User + + user = db.query(User).filter(User.id == user_id).first() + if not user or not user.smtp_user or not user.encrypted_smtp_password: + return base_config + + return { + "source": "user", + "smtp_host": base_config["smtp_host"], + "smtp_port": base_config["smtp_port"], + "smtp_user": user.smtp_user, + "smtp_password": decrypt_data(user.encrypted_smtp_password), + "smtp_secure": base_config["smtp_secure"], + "mail_from": user.mail_from or base_config["mail_from"], + } + except Exception as e: + logger.error( + "User SMTP credential lookup failed for user_id=%s, using base SMTP: %s", + user_id, + e, + ) + return base_config + finally: + if close_db and db is not None: + db.close() diff --git a/app/core/model_manager.py b/app/core/model_manager.py new file mode 100644 index 0000000..ad431f3 --- /dev/null +++ b/app/core/model_manager.py @@ -0,0 +1,70 @@ +import logging +import torch + +logger = logging.getLogger("AI_Model_Manager") + + +class ModelManager: + + _instance = None + _initialized = False + + def __new__(cls): + if cls._instance is None: + cls._instance = super(ModelManager, cls).__new__(cls) + return cls._instance + + def __init__(self): + pass + + def load_models(self): + + if ModelManager._initialized: + return + + logger.info("🧠 Lazy loading AI models...") + + torch.set_num_threads(2) + + from marker.converters.pdf import PdfConverter + from marker.models import create_model_dict + + self.model_cache = create_model_dict() + + config = { + "use_llm": False, + "extract_images": True, + "extract_tables": True, + "extract_equations": True, + "extract_layout": True, + } + + self.converter = PdfConverter( + artifact_dict=self.model_cache, + config=config + ) + + ModelManager._initialized = True + logger.info("✅ Models ready") + + def process_document(self, file_path): + + self.load_models() + + ext = file_path.lower() + + if ext.endswith(".pdf"): + return self.converter(file_path) + + if ext.endswith((".jpg", ".jpeg", ".png")): + from surya.ocr import run_ocr + return run_ocr(file_path) + + if ext.endswith(".docx"): + from docx import Document + doc = Document(file_path) + return {"markdown": "\n".join(p.text for p in doc.paragraphs)} + + raise ValueError("Unsupported file") + +model_manager = ModelManager() diff --git a/app/core/path_validation.py b/app/core/path_validation.py new file mode 100644 index 0000000..2537e6e --- /dev/null +++ b/app/core/path_validation.py @@ -0,0 +1,88 @@ +""" +path_validation.py — server-side file path and filename sanitization utilities. + +These helpers prevent path traversal attacks (C-017 / H-038) by rejecting +dangerous patterns before paths are stored in the DB or used in file I/O. +""" +import re +import logging + +logger = logging.getLogger(__name__) + +_TRAVERSAL_PATTERN = re.compile( + r""" + \.\. # double-dot (parent directory traversal) + | \x00 # null byte + | [<>:"|?*] # Windows-forbidden characters + """, + re.VERBOSE, +) + +_SEPARATOR_PATTERN = re.compile(r"[/\\]") + +MAX_PATH_LENGTH = 512 +MAX_FILENAME_LENGTH = 255 + + +def validate_storage_path(path: str) -> str: + """ + Validate a storage key / relative path before writing it to the DB or + using it in an S3 / B2 operation. + + Accepts: + - B2 / S3 object keys (e.g. "chatbot/42/abc123.pdf") + - Internal sentinel URIs (e.g. "drive://123") + + Raises: + ValueError – with a human-readable reason on rejection. + + Returns: + The original `path` unchanged if it passes all checks. + """ + if not path: + raise ValueError("File path must not be empty.") + + if len(path) > MAX_PATH_LENGTH: + raise ValueError( + f"File path exceeds the maximum allowed length " + f"({len(path)} > {MAX_PATH_LENGTH})." + ) + + if path.startswith("drive://"): + return path + + if _TRAVERSAL_PATTERN.search(path): + logger.warning("Rejected dangerous storage path: %r", path) + raise ValueError("File path contains disallowed characters or sequences.") + + return path + + +def validate_filename(filename: str) -> str: + """ + Validate an upload filename before storing it or using it on disk. + + Raises: + ValueError – with a human-readable reason on rejection. + + Returns: + The original `filename` unchanged if it passes all checks. + """ + if not filename: + raise ValueError("Filename must not be empty.") + + if len(filename) > MAX_FILENAME_LENGTH: + raise ValueError( + f"Filename exceeds the maximum allowed length " + f"({len(filename)} > {MAX_FILENAME_LENGTH})." + ) + + if _SEPARATOR_PATTERN.search(filename): + logger.warning("Rejected filename containing path separator: %r", filename) + raise ValueError("Filename must not contain path separators.") + + if _TRAVERSAL_PATTERN.search(filename): + logger.warning("Rejected dangerous filename: %r", filename) + raise ValueError("Filename contains disallowed characters or sequences.") + + return filename diff --git a/app/core/progress.py b/app/core/progress.py new file mode 100644 index 0000000..0efcd1d --- /dev/null +++ b/app/core/progress.py @@ -0,0 +1,17 @@ +import redis +import json +from app.core.settings import settings + +redis_client = redis.from_url( + settings.REDIS_URL or "redis://localhost:6379/0", + decode_responses=True +) + +def publish_progress(task_id, progress, message): + redis_client.publish( + f"task:{task_id}", + json.dumps({ + "progress": progress, + "message": message + }) + ) diff --git a/app/core/pubsub_listener.py b/app/core/pubsub_listener.py new file mode 100644 index 0000000..9533158 --- /dev/null +++ b/app/core/pubsub_listener.py @@ -0,0 +1,41 @@ +import asyncio +import json +import redis +import logging +from app.infrastructure.realtime.connection_manager import manager +from app.core.settings import settings + +logger = logging.getLogger(__name__) + +async def start_listener(): + redis_client = redis.from_url( + settings.REDIS_URL or "redis://localhost:6379/0", + decode_responses=True + ) + + pubsub = redis_client.pubsub() + pubsub.psubscribe("task:*") + + logger.info("📡 Redis Pub/Sub listener started") + + while True: + try: + message = await asyncio.get_event_loop().run_in_executor(None, pubsub.get_message, False, 1.0) + + if message and message["type"] == "pmessage": + parts = message["channel"].split(":") + if len(parts) != 2: + continue + task_id = parts[1] + try: + data = json.loads(message["data"]) + except (json.JSONDecodeError, TypeError): + logger.error(f"❌ Received malformed JSON on channel {message['channel']}") + continue + + await manager.send_task(task_id, data) + + await asyncio.sleep(0.01) + except Exception as e: + logger.error(f"Error in Pub/Sub listener: {e}") + await asyncio.sleep(1) diff --git a/app/core/redis.py b/app/core/redis.py new file mode 100644 index 0000000..ffeec75 --- /dev/null +++ b/app/core/redis.py @@ -0,0 +1,136 @@ +""" +Async Redis Pub/Sub client for real-time collaboration messaging. + +This module is SEPARATE from app/db/redis.py (sync cache client). +It uses redis.asyncio for non-blocking publish/subscribe operations +intended for WebSocket fan-out across backend workers. + +Usage: + from app.core.redis import redis_pubsub + + # Publish a message to a file's chat channel + await redis_pubsub.publish(file_id=42, message={...}) + + # Subscribe and iterate over incoming messages + async for msg in redis_pubsub.subscribe(file_id=42): + await websocket.send_json(msg) +""" +import json +import logging +from typing import Any, AsyncIterator, Optional + +import redis.asyncio as aioredis + +from app.core.settings import settings + +logger = logging.getLogger(__name__) + +CHANNEL_PREFIX = "chat:file" + + +def _channel(file_id: int) -> str: + """Build the canonical Redis channel name for a file's chat room.""" + return f"{CHANNEL_PREFIX}:{file_id}" + + +class RedisPubSub: + """ + Lightweight async wrapper around redis.asyncio for chat pub/sub. + + • Lazily creates a shared connection pool on first use. + • publish() and subscribe() are the only public API. + • Graceful degradation: if Redis is unreachable, publish is a no-op + and subscribe yields nothing (the app stays up). + """ + + def __init__(self) -> None: + self._pool: Optional[aioredis.ConnectionPool] = None + self._client: Optional[aioredis.Redis] = None + + + async def connect(self) -> None: + """Create the shared async connection pool (idempotent).""" + if self._client is not None: + return + try: + url = settings.REDIS_URL or "redis://localhost:6379/0" + self._pool = aioredis.ConnectionPool.from_url( + url, decode_responses=True + ) + self._client = aioredis.Redis(connection_pool=self._pool) + await self._client.ping() + logger.info("✅ Async Redis Pub/Sub connected") + except Exception as e: + logger.warning(f"Async Redis Pub/Sub unavailable: {e}") + self._client = None + self._pool = None + + async def disconnect(self) -> None: + """Tear down the pool (call on app shutdown).""" + if self._client: + await self._client.aclose() + self._client = None + if self._pool: + await self._pool.disconnect() + self._pool = None + logger.info("Async Redis Pub/Sub disconnected") + + + async def publish(self, file_id: int, message: dict[str, Any]) -> None: + """ + Publish a JSON message to ``chat:file:{file_id}``. + + No-op if Redis is not connected. + """ + if not self._client: + await self.connect() + if not self._client: + return + try: + channel = _channel(file_id) + payload = json.dumps(message, default=str) + await self._client.publish(channel, payload) + except Exception as e: + logger.error(f"Redis publish error: {e}") + + async def subscribe(self, file_id: int) -> AsyncIterator[dict[str, Any]]: + """ + Subscribe to ``chat:file:{file_id}`` and yield parsed messages. + + Intended to be consumed inside a WebSocket handler:: + + async for msg in redis_pubsub.subscribe(file_id): + await ws.send_json(msg) + + Yields nothing (returns immediately) if Redis is not available. + """ + if not self._client: + await self.connect() + if not self._client: + return + + channel = _channel(file_id) + pubsub = self._client.pubsub() + + try: + await pubsub.subscribe(channel) + logger.debug(f"Subscribed to {channel}") + + async for raw in pubsub.listen(): + if raw["type"] != "message": + continue + try: + yield json.loads(raw["data"]) + except (json.JSONDecodeError, TypeError): + logger.warning(f"Bad payload on {channel}: {raw['data']}") + except Exception as e: + logger.error(f"Redis subscribe error on {channel}: {e}") + finally: + try: + await pubsub.unsubscribe(channel) + await pubsub.aclose() + except Exception: + pass + + +redis_pubsub = RedisPubSub() diff --git a/app/core/request_body.py b/app/core/request_body.py new file mode 100644 index 0000000..069d4ae --- /dev/null +++ b/app/core/request_body.py @@ -0,0 +1,51 @@ +""" +Reading a JSON request body without turning the caller's mistake into our fault. + +`await request.json()` raises `JSONDecodeError` on a malformed body. Nothing in +these routes caught it, so it fell through to the generic handler and became a +**500**. That is wrong in three separate ways: + +- it reports a client error as a server fault, so it pages whoever is on call + and buries genuine failures in noise; +- on the webhook endpoints it tells the sender to retry, because 5xx means + "try again later" and 4xx means "your request is wrong" — a malformed + DocuSeal payload would be redelivered on a schedule, forever; +- `POST /api/sso/login` is unauthenticated, so anyone could produce a 500 with + a one-byte body. + +The discovery harness found this by sending a form body to a JSON endpoint — +which is exactly what a misconfigured integration does on its first day. +""" + +import json + +from fastapi import HTTPException, Request, status + + +async def read_json_body(request: Request, *, what: str = "request body") -> dict: + """ + Parse the body as a JSON object, or raise 400. + + Returns a `dict` specifically: every caller immediately does `.get(...)`, so + a bare list or string would fail with `AttributeError` a line later and land + back in the 500 handler this exists to avoid. + """ + raw = await request.body() + if not raw: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Expected a JSON {what}, but it was empty.", + ) + try: + parsed = json.loads(raw) + except (json.JSONDecodeError, UnicodeDecodeError): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"The {what} is not valid JSON.", + ) from None + if not isinstance(parsed, dict): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"The {what} must be a JSON object.", + ) + return parsed diff --git a/app/core/request_cache.py b/app/core/request_cache.py new file mode 100644 index 0000000..3575587 --- /dev/null +++ b/app/core/request_cache.py @@ -0,0 +1,69 @@ +""" +A memo that lives exactly as long as one request. + +Access resolution is asked the same question many times in a single request. +`require_access` runs once per dependency — several times on a route that +declares more than one — `get_current_user` now resolves the caller's codes so +the response can report them, and `/api/admin/users` resolves assignments for a +page of users. Without a memo each of those repeats the same three queries, one +of which reads the whole `accesses` table. + +Modelled on `app/core/scope.py`, deliberately, and for the same reason: a +`ContextVar` reset once per request is the only cache shape that cannot outlive +the authority it caches. A process-wide cache of "what may this user do" is the +bug where revoking a role takes effect after a restart. + +**Default of `None` means "not in a request".** Celery tasks, scripts and any +code path that never passes through `TenantContextMiddleware` compute every +time. That is correct rather than merely safe — a long-running task must not +pin a permission decision made at its start. + +The trap worth naming, because it is the same one `ScopeService` documents: a +grant that expires *mid-request* stays honoured to the end of that request. +Permissions must not change under a half-finished operation, so this is +intended. Do not "fix" it by shortening the cache. +""" + +from __future__ import annotations + +from contextvars import ContextVar +from typing import Any, Callable, Optional + +_cache: ContextVar[Optional[dict]] = ContextVar("request_cache", default=None) + + +def reset_request_cache() -> None: + """Drop everything memoised. Called once per request, before routing.""" + _cache.set({}) + + +def request_cached(key: str, compute: Callable[[], Any]) -> Any: + """ + Return `compute()`, memoised under *key* for the rest of this request. + + Outside a request the memo is absent and `compute` runs every time. + """ + store = _cache.get() + if store is None: + return compute() + if key not in store: + store[key] = compute() + return store[key] + + +def invalidate_request_cache(prefix: str) -> None: + """ + Forget every memoised key starting with *prefix*. + + Needed because a request may **change** the thing it cached. Granting a role + and then returning the user's roles in the same response would otherwise + serve the pre-grant answer, which reads as "the grant did not work". + """ + store = _cache.get() + if not store: + return + for key in [k for k in store if k.startswith(prefix)]: + del store[key] + + +__all__ = ["reset_request_cache", "request_cached", "invalidate_request_cache"] diff --git a/app/core/rls.py b/app/core/rls.py new file mode 100644 index 0000000..35b3f9c --- /dev/null +++ b/app/core/rls.py @@ -0,0 +1,80 @@ +""" +B1.3 — hand the tenant context to PostgreSQL. + +The RLS policy compares each row's `tenant_id` against `docqube.tenant_id`, a +session variable. Something has to set it, and it has to be set on the same +connection the query runs on — which is why this hooks the session rather than +the request. + +`set_config(..., true)` scopes the setting to the current **transaction**, so it +cannot leak to the next request through a pooled connection. That matters more +than it looks: a connection-scoped setting left behind by one tenant would be +inherited by whoever picked the connection up next. + +Off by the same switch as the ORM listener. The policy itself is inert without a +variable, so an unset switch means RLS is enabled in the database and permissive +in practice — which is the safe way round while it is being rolled out. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from sqlalchemy import event, text +from sqlalchemy.orm import Session + +logger = logging.getLogger(__name__) + + +def _enabled() -> bool: + from app.core.settings import settings + + return bool(getattr(settings, "TENANT_FILTER_ENABLED", False)) + + +def apply_tenant_context(session: Session) -> None: + """Push the current tenant context onto the session's connection.""" + from app.core.tenant_context import ( + current_tenant_id, + is_bypassed, + is_super_admin, + ) + + bypass = is_bypassed() or is_super_admin() + tenant_id = current_tenant_id() + + session.execute( + text("SELECT set_config('docqube.bypass', :v, true)"), + {"v": "on" if bypass else "off"}, + ) + session.execute( + text("SELECT set_config('docqube.tenant_id', :v, true)"), + {"v": str(tenant_id) if tenant_id else ""}, + ) + + +@event.listens_for(Session, "after_begin") +def _set_tenant_on_begin(session: Session, transaction: Any, connection: Any) -> None: + if not _enabled(): + return + try: + from app.core.tenant_context import ( + current_tenant_id, + is_bypassed, + is_super_admin, + ) + + bypass = is_bypassed() or is_super_admin() + tenant_id = current_tenant_id() + + connection.execute( + text("SELECT set_config('docqube.bypass', :v, true)"), + {"v": "on" if bypass else "off"}, + ) + connection.execute( + text("SELECT set_config('docqube.tenant_id', :v, true)"), + {"v": str(tenant_id) if tenant_id else ""}, + ) + except Exception: # noqa: BLE001 - never break a request over this + logger.exception("Failed to apply tenant context to the connection") diff --git a/app/core/sanitization.py b/app/core/sanitization.py new file mode 100644 index 0000000..6cb866a --- /dev/null +++ b/app/core/sanitization.py @@ -0,0 +1,78 @@ +from bs4 import BeautifulSoup +import logging + +logger = logging.getLogger(__name__) + +ALLOWED_TAGS = [ + 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', + 'p', 'br', 'hr', 'pre', 'code', 'blockquote', + 'ul', 'ol', 'li', 'dl', 'dt', 'dd', + 'b', 'strong', 'i', 'em', 'u', 's', 'small', 'sub', 'sup', + 'table', 'thead', 'tbody', 'tfoot', 'tr', 'th', 'td', 'caption', 'colgroup', 'col', + 'div', 'span', 'section', 'article', 'aside', 'header', 'footer', + 'img', 'a', 'figure', 'figcaption', + 'math', 'mrow', 'mi', 'mo', 'mn', 'msup', 'msub', 'msubsup', 'mfrac', 'msqrt', 'mroot', 'mover', 'munder', 'munderover', 'mtd', 'mtr', 'mtable', 'mstyle', 'merror', 'mpadded', 'mphantom', 'mfenced', 'menclose' +] + +ALLOWED_ATTRIBUTES = { + '*': ['id', 'class', 'title', 'lang'], + 'a': ['href', 'target', 'rel', 'name'], + 'img': ['src', 'alt', 'width', 'height', 'loading'], + 'table': ['border', 'cellpadding', 'cellspacing', 'frame', 'rules', 'summary', 'width'], + 'th': ['align', 'valign', 'colspan', 'rowspan', 'scope', 'width'], + 'td': ['align', 'valign', 'colspan', 'rowspan', 'width'], + 'col': ['span', 'width'], + 'colgroup': ['span', 'width'], + 'section': ['sec-type'], + 'div': ['id', 'class'], + 'span': ['id', 'class'], + 'math': ['display', 'xmlns'], + 'mtd': ['columnspan', 'rowspan', 'align'], + 'mtr': ['rowalign'], + 'mtable': ['equalrows', 'equalcolumns', 'displaystyle'], +} + +def sanitize_html(html_content: str) -> str: + """ + Sanitize HTML content using BeautifulSoup4. + Removes potentially dangerous tags (script, iframe, object, etc.) and attributes. + """ + if not html_content: + return "" + + try: + soup = BeautifulSoup(html_content, 'lxml') + + for tag in soup.findAll(): + if tag.name.lower() not in ALLOWED_TAGS: + if tag.name.lower() in ['script', 'style', 'iframe', 'object', 'embed', 'applet', 'link']: + tag.decompose() + else: + tag.unwrap() + + for tag in soup.findAll(): + allowed_attrs = ALLOWED_ATTRIBUTES.get(tag.name.lower(), []) + global_attrs = ALLOWED_ATTRIBUTES.get('*', []) + + attrs_to_remove = [attr for attr in tag.attrs if attr.lower() not in allowed_attrs and attr.lower() not in global_attrs] + + for attr in ['href', 'src']: + if attr in tag.attrs: + val = tag[attr].lower().strip() + if val.startswith('javascript:') or val.startswith('data:text/html') or val.startswith('vbscript:'): + attrs_to_remove.append(attr) + + for attr in list(tag.attrs.keys()): + if attr.lower().startswith('on'): + attrs_to_remove.append(attr) + + for attr in attrs_to_remove: + del tag[attr] + + if soup.body: + return "".join([str(c) for c in soup.body.contents]) + return str(soup) + + except Exception as e: + logger.error(f"HTML Sanitization failed: {e}") + return "Sanitization Error - Access Denied" diff --git a/app/core/schemas.py b/app/core/schemas.py new file mode 100644 index 0000000..6bfebb8 --- /dev/null +++ b/app/core/schemas.py @@ -0,0 +1,55 @@ +""" +Shared response envelopes. + +DocQube's mutations return a small number of repeated shapes — `{"message": …}`, +`{"status": …}`, `{"success": …}` and a couple of counters. Declaring them once +means the generated frontend client sees one type per shape instead of a +hundred anonymous inline objects. + +Every model here matches a response observed in +`tests/characterization/discovered_mutation_shapes.json`. None of them is a +guess, and none carries an optional field the endpoint does not return — a +declared field is *added* as null rather than omitted, so a wider model changes +the payload just as surely as a narrower one drops it. +""" + +from typing import Optional + +from pydantic import BaseModel + + +class MessageOut(BaseModel): + """`{"message": "..."}` — the most common acknowledgement in the codebase.""" + + message: str + + +class StatusOut(BaseModel): + """`{"status": "ok"}` / `{"status": "success"}`.""" + + status: str + + +class StatusMessageOut(BaseModel): + """`{"status": "...", "message": "..."}`.""" + + status: str + message: Optional[str] = None + + +class SuccessOut(BaseModel): + """`{"success": true}`.""" + + success: bool + + +class DeletedCountOut(BaseModel): + """`{"deleted": n}` — bulk delete.""" + + deleted: int + + +class MarkedCountOut(BaseModel): + """`{"marked": n}` — bulk mark-as-read.""" + + marked: int diff --git a/app/core/scope.py b/app/core/scope.py new file mode 100644 index 0000000..41515f9 --- /dev/null +++ b/app/core/scope.py @@ -0,0 +1,332 @@ +""" +C4 — where does this user hold this permission? + +Ported from `docqube_platform_backend/app/services/scope_service.py`, adapted to +resolve through DocQube's `RoleAccess`/`Access` rather than the base's +`RolePermission`/`Permission`. + +The whole design rests on one separation, and it is worth stating plainly +because it is what makes the rollout incremental: + +- `require_access("code")` stays exactly as it is — the **coarse** gate, + answering "do you hold this code at all". Every one of the 227 endpoints + keeps working, unchanged, today and after. +- `ScopeService.authority(user, code)` answers the **scope** question + separately, at the data layer, for endpoints that have been converted. + +So this file adds a capability without altering a single existing decision. An +endpoint becomes org-aware when someone converts it, one at a time, with a test. + +`authority()` returns one of two things, and the difference is deliberate: + + TENANT_WIDE a sentinel — the user holds this everywhere in their tenant + set[UUID] the org units they hold it in, *including descendants* + +It never returns None. A caller must not be able to confuse "no authority" with +"authority everywhere" by testing falsiness — an empty set is falsy and +`TENANT_WIDE` is truthy, which is the safe way round, but the sentinel is a +distinct object so `== TENANT_WIDE` is unambiguous. + +**Subtree inheritance is the point.** Grant a role at *Engineering* and it +applies to *Platform* and *Infrastructure* without further assignment. That is a +recursive CTE walking `parent_id` **downward**; a join written the other way +matches ancestors just as happily and silently promotes every team lead to a +department head. `tests/probes/test_org_scope.py` asserts the direction +explicitly for exactly that reason. +""" + +from __future__ import annotations + +import uuid +from contextvars import ContextVar +from typing import Union + +from sqlalchemy import func, or_, select +from sqlalchemy.orm import Session + +TENANT_WIDE = "TENANT_WIDE" + +Authority = Union[str, set] + +_cache: ContextVar[dict] = ContextVar("scope_cache", default=None) + + +def reset_scope_cache() -> None: + """Drop any memoised authority. Called once per request.""" + _cache.set({}) + + +def _cached(key: str, compute): + store = _cache.get() + if store is None: + return compute() + if key not in store: + store[key] = compute() + return store[key] + + +class ScopeService: + def __init__(self, db: Session): + self.db = db + + + def authority(self, user, access_code: str) -> Authority: + """`TENANT_WIDE`, or the set of org units where the user holds *code*.""" + key = f"{user.id}:{access_code}" + return _cached(key, lambda: self._resolve(user, access_code)) + + def can_reach(self, user, access_code: str, org_unit_id) -> bool: + """Does the user hold *code* over this particular unit?""" + auth = self.authority(user, access_code) + if auth == TENANT_WIDE: + return True + if org_unit_id is None: + return False + return org_unit_id in auth + + def can_access_user(self, actor, target_user_id: int, access_code: str) -> bool: + """ + May *actor* act on this user, under this access code? + + Three ways to yes, and the order matters: + + 1. the actor holds the code tenant-wide — the state every user is in + today, so this is the answer that keeps the application unchanged; + 2. the target is the actor themselves. Someone scoped to one team must + still be able to read their own record, or scoping a user removes + their own profile from them; + 3. the target belongs to a unit the actor has authority over. + + A user in **no** unit is reachable only by (1) or (2). That is + deliberate: an unfiled user belongs to the tenant, not to a team, and a + manager scoped to one team should not inherit everyone nobody has + filed yet. + """ + auth = self.authority(actor, access_code) + if auth == TENANT_WIDE: + return True + if int(target_user_id) == int(actor.id): + return True + if not auth: + return False + + from app.modules.org.models.org_model import UserOrgUnit + + return ( + self.db.query(UserOrgUnit.id) + .filter( + UserOrgUnit.user_id == target_user_id, + UserOrgUnit.org_unit_id.in_(auth), + ) + .first() + is not None + ) + + def visible_user_ids(self, actor, access_code: str): + """ + The users *actor* may see, or `None` for "no restriction". + + `None` and `set()` mean opposite things and both are falsy, which is + exactly the confusion that makes scoped access fail open. Callers must + branch on `is None`, never on truthiness — and there is a probe that + fails if a converted endpoint gets it wrong. + """ + auth = self.authority(actor, access_code) + if auth == TENANT_WIDE: + return None + if not auth: + return {int(actor.id)} + + from app.modules.org.models.org_model import UserOrgUnit + + rows = ( + self.db.query(UserOrgUnit.user_id) + .filter(UserOrgUnit.org_unit_id.in_(auth)) + .distinct() + .all() + ) + return {int(r[0]) for r in rows} | {int(actor.id)} + + def can_delegate(self, actor, access_code: str, target_org_unit_id) -> bool: + """ + May *actor* grant a role that applies at `target_org_unit_id`? + + The escalation this closes: without it, a manager scoped to one team can + grant **themselves** a tenant-wide role and become an administrator of + the whole tenant. Every scoped-permission system has this hole unless it + is closed deliberately, because granting is just another write and looks + like one. + + Two rules: + + - **tenant-wide can only be granted by tenant-wide.** `None` means "this + role, everywhere", and nobody may hand out more authority than they + hold; + - a scoped actor may only grant within the subtree they already have + authority over. + + Mirrors `_can_delegate` in the base's `user_service.py`. + """ + auth = self.authority(actor, access_code) + if auth == TENANT_WIDE: + return True + if target_org_unit_id is None: + return False + return target_org_unit_id in auth + + def visible_unit_ids(self, user, access_code: str): + """ + `None` when the user's authority is tenant-wide, else the id set. + + Shaped for query building: `None` means "add no predicate", a set means + `.filter(Model.org_unit_id.in_(ids))`, and an **empty** set means the + user sees nothing — which callers must handle rather than treating as + "no filter". That distinction is the whole failure mode of scoped + access, so it is expressed in the return type. + """ + auth = self.authority(user, access_code) + return None if auth == TENANT_WIDE else auth + + + def _group_ids(self, user) -> list: + """ + The groups this user belongs to, within their own tenant. + + Filtered by tenant on **both** sides. A membership row naming another + tenant's group must confer nothing, and `user_access_groups` carries its + own `tenant_id` precisely so that a forged or stale row cannot reach + across. + """ + from app.modules.org.models.group_model import AccessGroup, UserAccessGroup + + rows = ( + self.db.query(UserAccessGroup.group_id) + .join(AccessGroup, AccessGroup.id == UserAccessGroup.group_id) + .filter( + UserAccessGroup.user_id == user.id, + UserAccessGroup.tenant_id == user.tenant_id, + AccessGroup.tenant_id == user.tenant_id, + ) + .all() + ) + return [r[0] for r in rows] + + def _resolve(self, user, access_code: str) -> Authority: + from app.modules.auth.models.access_model import Access + from app.modules.auth.models.role_access_model import RoleAccess + from app.modules.auth.models.role_model import Role + from app.modules.auth.models.user_role_model import UserRole + + rows = ( + self.db.query(UserRole.org_unit_id) + .join(Role, Role.id == UserRole.role_id) + .join(RoleAccess, RoleAccess.role_id == Role.id) + .join(Access, Access.id == RoleAccess.access_id) + .filter( + or_( + UserRole.user_id == user.id, + UserRole.group_id.in_(self._group_ids(user)), + ), + Access.access_code == access_code, + or_( + Role.tenant_id == user.tenant_id, + Role.tenant_id.is_(None), + ), + or_( + UserRole.expires_at.is_(None), + UserRole.expires_at > func.now(), + ), + ) + .distinct() + .all() + ) + + if user.role_id and self._legacy_role_grants(user, access_code): + return TENANT_WIDE + + if not rows: + return set() + + anchors: set[uuid.UUID] = set() + for (org_unit_id,) in rows: + if org_unit_id is None: + return TENANT_WIDE + anchors.add(org_unit_id) + + return self._subtree_ids(user.tenant_id, anchors) + + def _legacy_role_grants(self, user, access_code: str) -> bool: + from app.modules.auth.models.access_model import Access + from app.modules.auth.models.role_access_model import RoleAccess + + return ( + self.db.query(RoleAccess.role_id) + .join(Access, Access.id == RoleAccess.access_id) + .filter( + RoleAccess.role_id == user.role_id, + Access.access_code == access_code, + ) + .first() + is not None + ) + + def _subtree_ids(self, tenant_id, root_ids: set) -> set: + """ + Every live unit at or beneath *root_ids*, within this tenant. + + Both filters matter. `tenant_id` means a grant naming another tenant's + unit resolves to nothing rather than reaching into it — defence against + a forged or stale `org_unit_id`. `NOT is_deleted` means a removed unit + stops conferring access immediately, including for the branch beneath + it, because the walk cannot pass *through* a deleted node. + """ + from app.modules.org.models.org_model import OrgUnit + + if not root_ids: + return set() + + anchors = ( + self.db.query(OrgUnit.id, OrgUnit.path) + .filter( + OrgUnit.id.in_(root_ids), + OrgUnit.tenant_id == tenant_id, + OrgUnit.is_deleted.is_(False), + ) + .all() + ) + if anchors and all(path for _, path in anchors): + found: set = set() + for _, path in anchors: + rows = ( + self.db.query(OrgUnit.id) + .filter( + OrgUnit.tenant_id == tenant_id, + OrgUnit.is_deleted.is_(False), + OrgUnit.path.like(f"{path}%"), + ) + .all() + ) + found |= {r[0] for r in rows} + return found + + anchor = ( + select(OrgUnit.id.label("id")) + .where( + OrgUnit.id.in_(root_ids), + OrgUnit.tenant_id == tenant_id, + OrgUnit.is_deleted.is_(False), + ) + .cte("org_subtree", recursive=True) + ) + + descendants = select(OrgUnit.id).where( + OrgUnit.parent_id == anchor.c.id, + OrgUnit.tenant_id == tenant_id, + OrgUnit.is_deleted.is_(False), + ) + + tree = anchor.union(descendants) + return set(self.db.execute(select(tree.c.id)).scalars().all()) + + +__all__ = ["ScopeService", "TENANT_WIDE", "reset_scope_cache"] diff --git a/app/core/security.py b/app/core/security.py new file mode 100644 index 0000000..0533a43 --- /dev/null +++ b/app/core/security.py @@ -0,0 +1,39 @@ +import uuid +from datetime import datetime, timedelta +from typing import Optional +from jose import jwt +from passlib.context import CryptContext +from app.core.settings import settings + +pwd_context = CryptContext(schemes=["pbkdf2_sha256"], deprecated="auto") + +def create_access_token(data: dict, expires_delta: Optional[timedelta] = None): + to_encode = data.copy() + expire = datetime.utcnow() + ( + expires_delta or timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES) + ) + to_encode.update({ + "exp": expire, + "jti": str(uuid.uuid4()) + }) + encoded_jwt = jwt.encode(to_encode, settings.APP_SECRET, algorithm=settings.ALGORITHM) + return encoded_jwt + +def create_refresh_token(data: dict, expires_delta: Optional[timedelta] = None): + to_encode = data.copy() + expire = datetime.utcnow() + ( + expires_delta or timedelta(days=7) + ) + to_encode.update({ + "exp": expire, + "jti": str(uuid.uuid4()), + "type": "refresh" + }) + encoded_jwt = jwt.encode(to_encode, settings.APP_SECRET, algorithm=settings.ALGORITHM) + return encoded_jwt + +def get_password_hash(password: str): + return pwd_context.hash(password) + +def verify_password(plain_password: str, hashed_password: str): + return pwd_context.verify(plain_password, hashed_password) diff --git a/app/core/settings.py b/app/core/settings.py new file mode 100644 index 0000000..db3543e --- /dev/null +++ b/app/core/settings.py @@ -0,0 +1,268 @@ +""" +Central Application Settings +Loads environment variables using Pydantic v2 +""" + +import os +from typing import Optional, List +from pydantic_settings import BaseSettings, SettingsConfigDict +from pydantic import ConfigDict, model_validator + + +class Settings(BaseSettings): + + APP_ENV: str + + HOST: str + PORT: int + DEBUG: bool + + DB_HOST: str + ZOHO_CLIENT_ID: str = "" + ZOHO_CLIENT_SECRET: str = "" + ZOHO_REFRESH_TOKEN: str = "" + ZOHO_WEBHOOK_SECRET: str = "" + ZOHO_DOMAIN: str = "in" + DB_PORT: int + DB_NAME: str + DB_USER: str + DB_PASSWORD: str + DB_SSLMODE: str + + PGBOUNCER_PORT: Optional[int] = None + + @property + def DATABASE_URL(self) -> str: + import urllib.parse + user = urllib.parse.quote(self.DB_USER, safe='') + password = urllib.parse.quote(self.DB_PASSWORD, safe='') + return ( + f"postgresql+psycopg2://{user}:" + f"{password}@" + f"{self.DB_HOST}:{self.DB_PORT}/" + f"{self.DB_NAME}?sslmode={self.DB_SSLMODE}" + ) + + @property + def PGBOUNCER_URL(self) -> Optional[str]: + if not self.PGBOUNCER_PORT: + return None + import urllib.parse + user = urllib.parse.quote(self.DB_USER, safe='') + password = urllib.parse.quote(self.DB_PASSWORD, safe='') + return ( + f"postgresql+psycopg2://{user}:" + f"{password}@" + f"{self.DB_HOST}:{self.PGBOUNCER_PORT}/" + f"{self.DB_NAME}?sslmode={self.DB_SSLMODE}" + ) + + APP_SECRET: str + ALGORITHM: str + ACCESS_TOKEN_EXPIRE_MINUTES: int + MAX_ACTIVE_DEVICES: int = 3 + + REDIS_HOST: Optional[str] + REDIS_PORT: Optional[int] + REDIS_PASSWORD: Optional[str] + + @property + def REDIS_URL(self) -> Optional[str]: + if not self.REDIS_HOST: + return None + + import urllib.parse + + encoded_password = ( + urllib.parse.quote( + self.REDIS_PASSWORD) if self.REDIS_PASSWORD else None + ) + + if encoded_password: + return f"redis://:{encoded_password}@{self.REDIS_HOST}:{self.REDIS_PORT}/0" + return f"redis://{self.REDIS_HOST}:{self.REDIS_PORT}/0" + + CHAT_REDIS_MAX_MESSAGES: int + CHAT_REDIS_TTL_SECONDS: int + + STORAGE_DRIVE_DIR: str + + CLAMAV_HOST: str + CLAMAV_PORT: int + CLAMAV_TIMEOUT_SECONDS: float = 20.0 + CLAMAV_CHUNK_SIZE: int = 1024 * 1024 + + API_BASE_URL: str + + GOOGLE_CLIENT_ID: Optional[str] = None + GOOGLE_CLIENT_SECRET: Optional[str] = None + + SAAS_TRUST_SECRET: Optional[str] = None + SAAS_BASE_URL: Optional[str] = None + SAAS_PUBLIC_KEY: Optional[str] = None + + DOCUSEAL_API_URL: str = "http://localhost:3000" + DOCUSEAL_API_KEY: Optional[str] = None + DOCUSEAL_ACCOUNT_EMAIL: Optional[str] = None + + DEEPSEEK_API_KEY: Optional[str] = None + DEEPSEEK_BASE_URL: str = "https://api.deepseek.com/v1" + DEEPSEEK_MODEL: str = "deepseek-chat" + LLM_TEMPERATURE: float = 0.2 + + EMBEDDING_MODEL: str = "BAAI/bge-base-en-v1.5" + + EXTRACTION_API_KEY: Optional[str] = None + EXTRACTION_BASE_URL: str = "https://integrate.api.nvidia.com/v1" + EXTRACTION_MODEL: str = "mistralai/mistral-medium-3-instruct" + EXTRACTION_VISION_MODEL: str = "meta/llama-3.2-90b-vision-instruct" + + HF_HUB_OFFLINE: bool = False + TRANSFORMERS_OFFLINE: bool = False + + CHUNK_SIZE: int = 850 + CHUNK_OVERLAP: int = 180 + TOP_K_QA: int = 3 + TOP_K_RETRIEVAL: int = 12 + TOP_K_RERANK: int = 6 + MAX_CHAT_HISTORY: int = 5 + TOP_K_SUMMARY: int = 4 + MAX_CONTEXT_CHARS: int = 4000 + MAX_RESPONSE_TOKENS: int = 1024 + CHAT_DAILY_CREDITS_LIMIT: int + MIN_SIMILARITY_SCORE: float = 0.18 + RAG_DOMAIN_PROFILE: str = "auto" + RAG_K_SCHEDULE: str = "8,16,24,36,48" + RAG_MAX_CANDIDATES: int = 180 + RAG_STAGNATION_PATIENCE: int = 2 + RAG_RERANK_MAX_DOCS: int = 48 + RAG_SEMANTIC_WEIGHT: float = 1.0 + RAG_KEYWORD_HIT_WEIGHT: float = 0.08 + RAG_KEYWORD_COVERAGE_WEIGHT: float = 0.35 + RAG_COVERAGE_TARGET_RATIO: float = 0.75 + RAG_MAX_SELECTED_DOCS: int = 40 + RAG_CONTEXT_BASE_BUDGET: int = 4200 + RAG_CONTEXT_MAX_BUDGET: int = 9000 + RAG_PER_SOURCE_CHAR_CAP: int = 680 + RAG_ENABLE_EXHAUSTIVE_COVERAGE: bool = True + RAG_COVERAGE_HARD_FAIL: bool = False + RAG_MIN_COVERAGE_TERM_LEN: int = 4 + RAG_COVERAGE_MAX_POINTS: int = 20000 + RAG_COVERAGE_MAX_PASSES: int = 4 + RAG_SCOPE_CACHE_TTL_SECONDS: int = 300 + RAG_SCOPE_CACHE_MAX_DOCS: int = 24 + RAG_MICRO_MAX_SENTENCES: int = 6 + RAG_MICRO_CHUNK_CHAR_CAP: int = 1200 + RAG_STRICT_CONTEXT_MAX_BUDGET: int = 18000 + RAG_STRICT_PER_SOURCE_CHAR_CAP: int = 2000 + RAG_STRICT_EVIDENCE_ONLY_CONTEXT: bool = True + RAG_SECTION_FULL_CHUNK_MAX_CHARS: int = 2600 + RAG_SECTION_FOCUS_ONLY: bool = True + RAG_SECTION_FOCUS_CHUNK_WINDOW: int = 0 + RAG_SECTION_PROBE_NEIGHBOR_WINDOW: int = 2 + RAG_ENABLE_SECTION_FAST_PATH: bool = True + RAG_ENABLE_FAST_PATH_AUTO_DOC: bool = True + RAG_FAST_PATH_AUTO_DOC_TOP_K: int = 10 + RAG_FAST_PATH_AUTO_DOC_MAX_TRY_DOCS: int = 4 + RAG_ENABLE_EARLY_OOD_CHECK: bool = True + RAG_SECTION_FAST_MAX_CHARS: int = 7000 + RAG_SECTION_FAST_MAX_BULLETS: int = 32 + RAG_SECTION_FAST_PER_SUBSECTION_BULLETS: int = 8 + RAG_SECTION_FAST_COMPACT_ROOT: bool = True + RAG_SECTION_FAST_SUBSECTION_CHAR_CAP: int = 900 + RAG_SECTION_FAST_USE_LLM_FOR_ROOT: bool = False + RAG_SECTION_FAST_LLM_MAX_CHARS: int = 7000 + RAG_REQUIRE_DOCUMENT_ID: bool = True + RAG_DOC_OVERVIEW_CONTEXT_CHARS: int = 16000 + RAG_DOC_OVERVIEW_MAX_SECTIONS: int = 14 + RAG_DOC_OVERVIEW_POINTS_PER_SECTION: int = 3 + RAG_RESCUE_CONTEXT_CHAR_CAP: int = 2600 + RAG_RESPONSE_TOKEN_SMALL: int = 180 + RAG_RESPONSE_TOKEN_MEDIUM: int = 280 + RAG_RESPONSE_TOKEN_LARGE: int = 420 + RAG_RESPONSE_TOKEN_HARD_CAP: int = 520 + VECTOR_DB: str = "qdrant" + QDRANT_HOST: str = "localhost" + QDRANT_PORT: int = 6333 + QDRANT_COLLECTION: str = "documents" + RERANKER_MODEL: str = "cross-encoder/ms-marco-MiniLM-L-12-v2" + + SMTP_HOST: Optional[str] = None + SMTP_PORT: Optional[int] = 587 + SMTP_USER: Optional[str] = None + SMTP_PASSWORD: Optional[str] = None + SMTP_SECURE: bool = False + MAIL_FROM: Optional[str] = "noreply@docqube.com" + + FRONTEND_URL: str = "http://localhost:5173" + + LOG_LEVEL: str + + CORS_ORIGINS: str + + @property + def CORS_ORIGINS_LIST(self) -> List[str]: + return [origin.strip() for origin in self.CORS_ORIGINS.split(",") if origin] + + model_config = SettingsConfigDict( + env_file=os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))), + f".env.{os.getenv('APP_ENV', 'development')}", + ), + extra="ignore", + ) + + @model_validator(mode="after") + def validate_production_config(self) -> "Settings": + if self.MAX_ACTIVE_DEVICES < 1: + raise ValueError("MAX_ACTIVE_DEVICES must be at least 1") + + if self.APP_ENV == "production": + if not self.REDIS_HOST: + raise ValueError( + "REDIS_HOST must be explicitly set in production environment " + "to prevent silent fallback to insecure defaults." + ) + # Enforce PgBouncer in production: app/db/database.py silently + # falls back to a direct PostgreSQL connection (DATABASE_URL, + # DB_PORT) whenever PGBOUNCER_PORT is unset — with no error and + # no log warning at the point it matters. That silent fallback + # is what let the API run for a time connected straight to + # Postgres on 5432 instead of through PgBouncer on 6432 during + # a prior incident. Failing fast here, before the DB engine is + # ever created, converts that into a startup error instead. + if not self.PGBOUNCER_PORT: + raise ValueError( + "PGBOUNCER_PORT must be explicitly set in production environment " + "to prevent silently falling back to a direct PostgreSQL connection." + ) + # CORS check + if "*" in self.CORS_ORIGINS_LIST: + raise ValueError( + "CORS_ORIGINS must not contain '*' in production environment " + "to prevent unauthorized cross-origin requests." + ) + if self.ZOHO_CLIENT_ID and not self.ZOHO_WEBHOOK_SECRET: + import logging + logging.getLogger(__name__).warning( + "ZOHO_WEBHOOK_SECRET is empty in production. This silently disables webhook authentication " + "and leaves your Zoho Sign integration vulnerable to spoofed requests." + ) + return self + + COOKIE_DOMAIN: Optional[str] = None + DISABLE_CSRF: bool = False + + TENANT_FILTER_ENABLED: bool = False + + ORG_SCOPE_ENABLED: bool = False + + SUBSCRIPTION_REQUIRED_FOR_MAPPED_TENANTS: bool = False + + SUBSCRIPTION_ENFORCEMENT_ENABLED: bool = False + + ACCESS_LOG_RETENTION_DAYS: int = 365 + +settings = Settings() + +DATABASE_URL = settings.DATABASE_URL diff --git a/app/core/ssrf_protection.py b/app/core/ssrf_protection.py new file mode 100644 index 0000000..e2b8527 --- /dev/null +++ b/app/core/ssrf_protection.py @@ -0,0 +1,79 @@ +import socket +import logging +from urllib.parse import urlparse +from ipaddress import ip_address, ip_network +from fastapi import HTTPException + +logger = logging.getLogger(__name__) + +PRIVATE_IP_RANGES = [ + ip_network('10.0.0.0/8'), + ip_network('172.16.0.0/12'), + ip_network('192.168.0.0/16'), + ip_network('127.0.0.0/8'), + ip_network('169.254.0.0/16'), + ip_network('::1/128'), + ip_network('fc00::/7'), + ip_network('fe80::/10'), +] + +ALLOWED_SCHEMES = ['http', 'https'] +ALLOWED_HOSTS = [] + +def is_safe_url(url: str, allowed_hosts: list = None, allowed_schemes: list = None, require_https: bool = False) -> bool: + """ + Check if a URL is safe from SSRF by validating the scheme, host, and IP. + """ + try: + parsed = urlparse(url) + + schemes = allowed_schemes if allowed_schemes is not None else ALLOWED_SCHEMES + if require_https: + schemes = ['https'] + + if parsed.scheme.lower() not in schemes: + logger.warning(f"SSRF Prevention: Invalid scheme {parsed.scheme} in URL {url}") + return False + + host = parsed.hostname + if not host: + logger.warning(f"SSRF Prevention: No hostname in URL {url}") + return False + + if allowed_hosts and host not in allowed_hosts: + logger.warning(f"SSRF Prevention: Host {host} not in whitelist for URL {url}") + return False + + try: + ips = socket.getaddrinfo(host, parsed.port or (443 if parsed.scheme == 'https' else 80)) + for family, _, _, _, sockaddr in ips: + addr_str = sockaddr[0] + if '%' in addr_str: + addr_str = addr_str.split('%')[0] + + addr = ip_address(addr_str) + for private_range in PRIVATE_IP_RANGES: + if addr in private_range: + logger.warning(f"SSRF Prevention: URL {url} resolved to private IP {addr}") + return False + except (socket.gaierror, ValueError) as e: + logger.warning(f"SSRF Prevention: Could not resolve or parse host {host} for URL {url}: {e}") + return False + + return True + except Exception as e: + logger.error(f"SSRF Prevention: Error validating URL {url}: {e}") + return False + +def get_safe_fetcher(): + """ + Returns a URL fetcher function compatible with WeasyPrint. + """ + from weasyprint import default_url_fetcher + + def safe_fetcher(url, timeout=10, **kwargs): + if not is_safe_url(url): + raise ValueError(f"SSRF Prevention: Blocked request to unsafe URL: {url}") + return default_url_fetcher(url, timeout=timeout, **kwargs) + + return safe_fetcher diff --git a/app/core/tenant_context.py b/app/core/tenant_context.py new file mode 100644 index 0000000..bdcdb9a --- /dev/null +++ b/app/core/tenant_context.py @@ -0,0 +1,114 @@ +""" +B1.1 — tenant context. + +The request's tenant, held in context variables so the query listener can reach +it without every call site passing it down. + +The important state is not the tenant id but **whether the context was set at +all**. DocQube's defect is that a lost tenant currently escalates: `tenant_id IS +NULL` means super admin, so code that forgets to scope a query sees everything. +Distinguishing "no tenant, because nobody set one" from "tenant is None, which +means superuser" is what lets the listener deny rather than grant. +""" + +from __future__ import annotations + +import functools +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from contextvars import ContextVar, Token +from typing import Any, TypeVar +from uuid import UUID + +_tenant_id: ContextVar[UUID | None] = ContextVar("tenant_id", default=None) +_is_super_admin: ContextVar[bool] = ContextVar("is_super_admin", default=False) +_bypass: ContextVar[bool] = ContextVar("tenant_bypass", default=False) +_context_set: ContextVar[bool] = ContextVar("tenant_context_set", default=False) + +F = TypeVar("F", bound=Callable[..., Any]) + + +def current_tenant_id() -> UUID | None: + return _tenant_id.get() + + +def is_super_admin() -> bool: + return _is_super_admin.get() + + +def is_bypassed() -> bool: + return _bypass.get() + + +def context_was_set() -> bool: + """True when something deliberately established a tenant context.""" + return _context_set.get() + + +def set_context( + tenant_id: UUID | None, is_super: bool = False, bypass: bool = False +) -> tuple[Token, Token, Token, Token]: + return ( + _tenant_id.set(tenant_id), + _is_super_admin.set(is_super), + _bypass.set(bypass), + _context_set.set(True), + ) + + +def reset_context(tokens: tuple[Token, Token, Token, Token]) -> None: + t_tid, t_super, t_bypass, t_set = tokens + _tenant_id.reset(t_tid) + _is_super_admin.reset(t_super) + _bypass.reset(t_bypass) + _context_set.reset(t_set) + + +@contextmanager +def scoped_to(tenant_id: UUID | None, is_super: bool = False) -> Iterator[None]: + """Run a block scoped to one tenant.""" + tokens = set_context(tenant_id, is_super=is_super, bypass=False) + try: + yield + finally: + reset_context(tokens) + + +@contextmanager +def unscoped() -> Iterator[None]: + """ + Run a block with tenant filtering switched off. + + For genuine system work — migrations, background sweeps, the seed script. + Every use is a place where the guarantee does not apply, so they should be + few and obvious. + """ + tokens = (_bypass.set(True), _context_set.set(True)) + try: + yield + finally: + _bypass.reset(tokens[0]) + _context_set.reset(tokens[1]) + + +def system_operation(fn: F) -> F: + """Decorator form of `unscoped()`.""" + + @functools.wraps(fn) + def wrapper(*args: Any, **kwargs: Any) -> Any: + with unscoped(): + return fn(*args, **kwargs) + + return wrapper + + +class TenantUnscoped: + """ + Opt a model *out* of tenant filtering. + + Scoping is derived from the presence of a `tenant_id` column rather than + from a marker mixin, so a new tenant-owned model is protected the day it is + written and nobody has to remember anything. This marker exists for the rare + model that carries a `tenant_id` but must be readable across tenants — and + applying it should require an argument in review. + """ diff --git a/app/core/tenant_filter.py b/app/core/tenant_filter.py new file mode 100644 index 0000000..90aaf6b --- /dev/null +++ b/app/core/tenant_filter.py @@ -0,0 +1,180 @@ +""" +B1.1 — the fail-closed query listener. + +Every SELECT against a tenant-owned table gets a tenant predicate added. With no +tenant context it matches **nothing**, which is the whole point: a bug that loses +the tenant used to see everything, because `tenant_id IS NULL` doubled as super +admin. Now the same bug returns an empty page. + +**Off by default, but no longer blocked.** The ordering constraint is satisfied: +the `tenant_id IS NULL` fallback in `is_superadmin()` is gone, privilege is an +explicit flag, and `TenantContextMiddleware` (B1.4) sets both the tenant and the +flag on every request. The whole suite passes with `TENANT_FILTER_ENABLED=true`, +and CI runs it that way. + +It stays off by default because the switch is the rollback — turning it off takes +effect without a deploy — and because enabling it needs the *data* to be ready, +not just the code. `scripts/b1_preflight.py` is the check. + +Which models are scoped is **derived from the schema**, not from a marker list. +A model with a `tenant_id` column is protected the day it is written. Opting out +requires inheriting `TenantUnscoped`, which is visible in review. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from sqlalchemy import event, false, or_ +from sqlalchemy.orm import Session, with_loader_criteria + +logger = logging.getLogger(__name__) + +_scoped_cache: dict[str, Any] | None = None +_scoped_cache_mappers: int = -1 + + +def scoped_models() -> dict[str, Any]: + """ + Mapped classes carrying a `tenant_id` column, by table name. + + Cached, but **invalidated when the registry grows** — and that second half + is load-bearing, not tidiness. + + The cache used to be computed once and kept forever. The first call happens + on the first query, which in a running application is early: before every + model module has necessarily been imported. Anything mapped after that point + was absent from the set and therefore **silently never filtered** — a + tenant-owned table with no tenant predicate, and nothing anywhere to say so. + + Found when `user_roles` gained a `tenant_id` and the isolation ratchet + reported it unscoped with the filter enabled and scoped with it disabled. + The difference was import order: enabling the filter makes the listener run + during startup, which populated the cache before the model was loaded. + + Counting mappers is enough. SQLAlchemy's registry only grows, so a changed + count means new classes and a stale set. + """ + global _scoped_cache, _scoped_cache_mappers + + from app.core.tenant_context import TenantUnscoped + from app.db.database import Base + + mapper_count = len(Base.registry.mappers) + if _scoped_cache is not None and _scoped_cache_mappers == mapper_count: + return _scoped_cache + + found: dict[str, Any] = {} + for mapper in Base.registry.mappers: + cls = mapper.class_ + if issubclass(cls, TenantUnscoped): + continue + if "tenant_id" in mapper.columns: + found[mapper.local_table.name] = cls + _scoped_cache = found + _scoped_cache_mappers = mapper_count + return found + + +def reset_scoped_cache() -> None: + """Forget the derived set — used by tests that define models on the fly.""" + global _scoped_cache, _scoped_cache_mappers + _scoped_cache = None + _scoped_cache_mappers = -1 + + +def _table_names(statement: Any) -> set[str]: + """ + Every table a statement touches, including through subqueries. + + Walking `get_final_froms()` by hand is not enough: `query(X).count()` + compiles to SELECT count(*) FROM (SELECT ...) and the outer FROM is an + anonymous subquery, so a naive walk sees only the alias. SQLAlchemy's own + `find_tables` handles the nesting. + """ + from sqlalchemy.sql.util import find_tables + + try: + return { + table.name + for table in find_tables( + statement, + include_aliases=True, + include_joins=True, + include_selects=True, + include_crud=True, + ) + } + except Exception: # noqa: BLE001 - not every construct can be walked + return set() + + +def _enabled() -> bool: + from app.core.settings import settings + + return bool(getattr(settings, "TENANT_FILTER_ENABLED", False)) + + +@event.listens_for(Session, "do_orm_execute") +def _apply_tenant_filter(execute_state: Any) -> None: + if not _enabled(): + return + if not execute_state.is_select: + return + if execute_state.is_column_load or execute_state.is_relationship_load: + return + + from app.core.tenant_context import ( + context_was_set, + current_tenant_id, + is_bypassed, + is_super_admin, + ) + + if is_bypassed() or is_super_admin(): + from app.core.tenant_metrics import metrics + + metrics.record_bypassed() + return + + by_table = scoped_models() + if not by_table: + return + + targets = { + mapper.class_ + for mapper in execute_state.all_mappers + if mapper.local_table is not None + and mapper.local_table.name in by_table + } + + if not targets: + targets = { + by_table[name] + for name in _table_names(execute_state.statement) + if name in by_table + } + if not targets: + return + + fail_closed = not context_was_set() + tenant_id = current_tenant_id() + + from app.core.tenant_metrics import metrics + + if fail_closed: + names = sorted(cls.__name__ for cls in targets) + metrics.record_fail_closed(names) + logger.warning("Tenant filter closed a query with no tenant context: %s", names) + else: + metrics.record_scoped() + + for cls in targets: + if fail_closed: + criteria = false() + else: + criteria = or_(cls.tenant_id == tenant_id, cls.tenant_id.is_(None)) + execute_state.statement = execute_state.statement.options( + with_loader_criteria(cls, criteria, include_aliases=True) + ) diff --git a/app/core/tenant_metrics.py b/app/core/tenant_metrics.py new file mode 100644 index 0000000..c0e495b --- /dev/null +++ b/app/core/tenant_metrics.py @@ -0,0 +1,73 @@ +""" +B1.6 / B4.5 — see what the tenant filter is doing. + +After B1 lands, the failure mode is not a crash. It is a listener quietly +returning nothing for a legitimate query, and the first report arrives from a +customer saying a page is empty. Counters make that visible in minutes instead. + +Three numbers, deliberately few: + + ``fail_closed`` a query ran with no tenant context and was denied. In + steady state this should be **zero**. Anything above zero + is a code path that lost the tenant — a bug that used to + escalate silently and now denies loudly. + ``scoped`` queries filtered normally. The denominator. + ``bypassed`` explicit system operations. Should be small and stable; a + rise means someone is reaching for ``unscoped()`` to make + a problem go away. + +In-process and reset on restart. That is enough to answer "is the rollout +safe?", which is what B1.6 needs. Wiring these to the metrics backend is B4.5. +""" + +from __future__ import annotations + +import threading +from dataclasses import dataclass, field + + +@dataclass +class TenantFilterMetrics: + fail_closed: int = 0 + scoped: int = 0 + bypassed: int = 0 + fail_closed_by_model: dict[str, int] = field(default_factory=dict) + _lock: threading.Lock = field(default_factory=threading.Lock, repr=False) + + def record_fail_closed(self, models: list[str]) -> None: + with self._lock: + self.fail_closed += 1 + for name in models: + self.fail_closed_by_model[name] = ( + self.fail_closed_by_model.get(name, 0) + 1 + ) + + def record_scoped(self) -> None: + with self._lock: + self.scoped += 1 + + def record_bypassed(self) -> None: + with self._lock: + self.bypassed += 1 + + def snapshot(self) -> dict: + with self._lock: + total = self.fail_closed + self.scoped + self.bypassed + return { + "fail_closed": self.fail_closed, + "scoped": self.scoped, + "bypassed": self.bypassed, + "total": total, + "fail_closed_rate": (self.fail_closed / total) if total else 0.0, + "fail_closed_by_model": dict(self.fail_closed_by_model), + } + + def reset(self) -> None: + with self._lock: + self.fail_closed = 0 + self.scoped = 0 + self.bypassed = 0 + self.fail_closed_by_model.clear() + + +metrics = TenantFilterMetrics() diff --git a/app/core/token_blacklist.py b/app/core/token_blacklist.py new file mode 100644 index 0000000..71292a9 --- /dev/null +++ b/app/core/token_blacklist.py @@ -0,0 +1,74 @@ +from app.db.redis import redis_cache +import logging + +logger = logging.getLogger(__name__) + +class TokenBlacklist: + """ + Service for managing revoked JWT tokens using Redis. + Uses the 'jti' (JWT ID) claim to identify tokens for blacklisting. + """ + + @staticmethod + def add(jti: str, expires_in: int = 3600): + """ + Add a token's JTI to the blacklist with an expiration time. + + Args: + jti: The unique JWT ID (jti claim) + expires_in: Time in seconds until the token naturally expires + """ + if not jti: + return + + try: + key = f"blacklist:{jti}" + redis_cache.set(key, {"revoked": True}, ttl=expires_in) + except Exception as e: + logger.warning(f"Failed to blacklist JTI: {e}") + + @staticmethod + def is_token_blacklisted(token: str) -> bool: + """ + Check whether a raw JWT has been revoked, by reading its `jti` claim. + + Callers that hold a raw token must use this rather than passing the + token to `is_blacklisted`: the blacklist is keyed on `jti`, so a raw + token builds the key `blacklist:`, which is never written + and therefore never matches. Both WebSocket endpoints did exactly that, + which silently disabled revocation on the whole real-time surface. + """ + if not token: + return False + try: + from jose import jwt + + from app.core.settings import settings + + payload = jwt.decode( + token, + settings.APP_SECRET, + algorithms=[settings.ALGORITHM], + options={"verify_exp": False}, + ) + except Exception: + return False + return TokenBlacklist.is_blacklisted(payload.get("jti")) + + @staticmethod + def is_blacklisted(jti: str) -> bool: + """ + Check if a token's JTI is in the blacklist. + + Takes a `jti` claim, not a raw JWT. See `is_token_blacklisted`. + """ + if not jti: + return False + + try: + key = f"blacklist:{jti}" + value = redis_cache.get(key) + return value is not None + except Exception as e: + logger.warning(f"Failed to check JTI blacklist: {e}") + return False diff --git a/app/core/transactions.py b/app/core/transactions.py new file mode 100644 index 0000000..0444f30 --- /dev/null +++ b/app/core/transactions.py @@ -0,0 +1,88 @@ +""" +B2.4 — keeping object storage and the database consistent. + +B2 made the request own the database transaction. Object storage has no such +boundary: an upload to Backblaze is durable the moment it returns, and no +rollback undoes it. So making the database atomic on its own moves the problem +rather than solving it — + + * a **write** that succeeds in storage and then fails in the database leaves + an object with no row pointing at it, and now not even a half-written row to + find it by; + * a **delete** that removes the object first and then fails in the database + leaves a row pointing at nothing, which is worse: the application will try + to serve it. + +Two primitives, used at the point where the storage call happens: + + compensate_on_rollback(session, lambda: delete(key)) + The object is already written. If the transaction rolls back, remove it. + + defer_until_commit(session, lambda: delete(key)) + Do not touch storage yet. If the transaction commits, do it then. + +Both are per-session and fire once. A failure inside a callback is logged and +swallowed: compensation is best-effort by nature, and raising from it would +replace a tidy-up problem with a failed request. + +This is the cheap half of the outbox pattern. The full version — durable intent +that survives a process restart — is B5.1, and is worth having if the roadmap +ever needs guaranteed delivery. What is here covers the common case: a crash +between the two systems within a single request. +""" + +from __future__ import annotations + +import logging +from typing import Callable, List + +from sqlalchemy import event +from sqlalchemy.orm import Session + +logger = logging.getLogger(__name__) + +_ON_ROLLBACK = "_docqube_compensate_on_rollback" +_ON_COMMIT = "_docqube_defer_until_commit" + + +def _bucket(session: Session, key: str) -> List[Callable[[], None]]: + actions = session.info.get(key) + if actions is None: + actions = [] + session.info[key] = actions + return actions + + +def compensate_on_rollback(session: Session, action: Callable[[], None]) -> None: + """Run `action` if this session's transaction is rolled back.""" + _bucket(session, _ON_ROLLBACK).append(action) + + +def defer_until_commit(session: Session, action: Callable[[], None]) -> None: + """Run `action` only once this session's transaction has committed.""" + _bucket(session, _ON_COMMIT).append(action) + + +def _drain(session: Session, key: str, what: str) -> None: + actions = session.info.pop(key, None) + if not actions: + return + for action in actions: + try: + action() + except Exception: # noqa: BLE001 - tidy-up must not fail the request + logger.exception("Storage %s action failed", what) + + +@event.listens_for(Session, "after_commit") +def _run_deferred(session: Session) -> None: + _drain(session, _ON_COMMIT, "post-commit") + session.info.pop(_ON_ROLLBACK, None) + + +@event.listens_for(Session, "after_soft_rollback") +def _run_compensation(session: Session, previous_transaction) -> None: + if getattr(previous_transaction, "nested", False): + return + _drain(session, _ON_ROLLBACK, "compensating") + session.info.pop(_ON_COMMIT, None) diff --git a/app/core/ws_auth.py b/app/core/ws_auth.py new file mode 100644 index 0000000..7522aa7 --- /dev/null +++ b/app/core/ws_auth.py @@ -0,0 +1,93 @@ +""" +B0.4.5 — one authenticator for the WebSocket surface. + +There were two `_authenticate_ws` functions, in `app/api/ws_router.py` and +`app/modules/collab/routes/chat_routes.py`, near-identical and independently +maintained. That is how they drifted, and the drift was not cosmetic: + +- both keyed the token blacklist on the raw JWT where a `jti` was expected, so + revocation silently did nothing on the entire real-time surface; +- neither rejected `refresh` tokens, so a long-lived refresh token could open a + socket it could not have opened an HTTP request with; +- neither checked whether the user was deleted or deactivated; +- and neither checked the **tenant**, so suspending a tenant closed no sockets. + A suspended customer kept live document collaboration and chat for as long as + the connection stayed open, which on a WebSocket is indefinitely. + +The first two were fixed in both copies. Fixing the rest in both copies would +have been the third round of the same edit, so this is the shared version the +plan called for: one function, one set of checks, both callers delegating. + +**Deliberately not `get_current_user`.** That is a FastAPI dependency built on +`Request` and `Depends`; a WebSocket handshake has neither. What it *does* have +is the same token, so the checks are what get shared, not the plumbing. Every +rule here mirrors one in `get_current_user`, and +`tests/probes/test_websocket_auth.py` asserts the two surfaces agree. +""" + +import logging +from typing import Optional + +from jose import JWTError, jwt +from sqlalchemy.orm import Session + +from app.core.settings import settings +from app.core.token_blacklist import TokenBlacklist +from app.modules.auth.models.user_model import User + +logger = logging.getLogger(__name__) + + +def authenticate_websocket(token: str, db: Session) -> Optional[User]: + """ + The user this token authorises for a socket, or None. + + Returns None for every failure rather than raising: the caller's only + remedy is to close the handshake, and distinguishing *why* would tell an + unauthenticated peer whether an account exists. + """ + if not token: + return None + + try: + payload = jwt.decode( + token, settings.APP_SECRET, algorithms=[settings.ALGORITHM] + ) + except JWTError: + return None + + if payload.get("type") in ("refresh", "device_management"): + return None + + jti = payload.get("jti") + if jti and TokenBlacklist.is_blacklisted(jti): + return None + + sub = payload.get("sub") + if not sub: + return None + + try: + user = db.get(User, int(sub)) + except (ValueError, TypeError): + user = db.query(User).filter(User.email == str(sub)).first() + + if user is None: + return None + + if getattr(user, "is_deleted", False) or not getattr(user, "is_active", True): + return None + + tenant = getattr(user, "tenant", None) + if tenant is not None and not getattr(tenant, "is_active", True): + logger.info( + "WebSocket auth refused: tenant %s is not active", getattr(tenant, "id", "?") + ) + return None + if tenant is not None and getattr(tenant, "is_deleted", False): + return None + + return user + + +__all__ = ["authenticate_websocket"] diff --git a/app/db/all_models.py b/app/db/all_models.py new file mode 100644 index 0000000..1fffe37 --- /dev/null +++ b/app/db/all_models.py @@ -0,0 +1,95 @@ +""" +Every mapped class, registered — importing this module is the guarantee. + +SQLAlchemy only knows about a model once its module has been imported. Nothing +was making that happen for the whole set: `alembic/env.py` imported `app.main` +and trusted that starting the application would drag every model in behind it. +Mostly it did, because routes import the models they use at module level. + +Three did not. `UserRole`, `AccessGroup` and `UserAccessGroup` are imported +*inside functions* everywhere they appear — a deliberate style elsewhere in the +codebase, and harmless right up until something needs the full registry: + +- **`alembic revision --autogenerate` proposed `DROP TABLE user_roles`, + `access_groups` and `user_access_groups`.** Autogenerate diffs the database + against `Base.metadata`; a table present in one and absent from the other + reads as "deleted". The three tables missing are the three that carry scoped + role grants and group membership — the entire SBAC layer — so the next person + to run autogenerate and apply the result without reading it drops + authorization and keeps the app booting. +- `Base.metadata.create_all()` would not create them. +- Anything deriving behaviour from the registry sees a partial picture. The + tenant filter is safe here only because `scoped_models()` invalidates on + mapper count, which was itself added after this class of bug bit once. + +So the imports are explicit and in one place. `tests/probes/test_model_registry.py` +fails if a model module exists and is not listed here, which is what keeps the +list honest — a new model is registered on the day it is written, not the day +someone notices it missing from a migration. +""" + +from app.modules.activity_logs.models import activity_log as _activity_log +from app.modules.auth.models import access_model as _access +from app.modules.auth.models import device_model as _device +from app.modules.auth.models import role_access_model as _role_access +from app.modules.auth.models import role_model as _role +from app.modules.auth.models import saas_models as _saas +from app.modules.auth.models import session_model as _session +from app.modules.auth.models import user_model as _user +from app.modules.auth.models import user_role_model as _user_role +from app.modules.billing.models import plan_model as _plan +from app.modules.chat.models import chat_message_model as _chat_message +from app.modules.chat.models import chat_model as _chat +from app.modules.chat.models import chat_session_model as _chat_session +from app.modules.chat.models import chat_usage_model as _chat_usage +from app.modules.collab.models import collab_model as _collab +from app.modules.configuration.models import ( + system_configuration_model as _system_configuration, +) +from app.modules.documents.models import document_model as _document +from app.modules.drive.models import drive_model as _drive +from app.modules.extraction.models import extraction_model as _extraction +from app.modules.notifications.models import notification_model as _notification +from app.modules.org.models import group_model as _group +from app.modules.org.models import org_model as _org +from app.modules.signing.models import signature_config_model as _signature_config +from app.modules.signing.models import signature_imprint as _signature_imprint +from app.modules.signing.models import signing_request as _signing_request +from app.modules.storage.models import storage_model as _storage +from app.modules.tenant.models import tenant_contact_model as _tenant_contact +from app.modules.tenant.models import tenant_model as _tenant +from app.modules.tenant.models import tenant_smtp_config_model as _tenant_smtp +from app.modules.tenant.models import tenant_storage_config_model as _tenant_storage + +__all__ = [ + "_access", + "_activity_log", + "_chat", + "_chat_message", + "_chat_session", + "_chat_usage", + "_collab", + "_device", + "_document", + "_drive", + "_extraction", + "_group", + "_notification", + "_org", + "_plan", + "_role", + "_role_access", + "_saas", + "_session", + "_signature_config", + "_signature_imprint", + "_signing_request", + "_storage", + "_system_configuration", + "_tenant", + "_tenant_contact", + "_tenant_smtp", + "_tenant_storage", + "_user", + "_user_role", +] diff --git a/app/db/database.py b/app/db/database.py new file mode 100644 index 0000000..c4b5dec --- /dev/null +++ b/app/db/database.py @@ -0,0 +1,100 @@ +import logging +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker, DeclarativeBase, Session +from contextlib import contextmanager +from app.core.settings import DATABASE_URL, settings + +from sqlalchemy.pool import NullPool, QueuePool + +logger = logging.getLogger(__name__) + +class Base(DeclarativeBase): + pass + +active_url = settings.PGBOUNCER_URL if (settings.PGBOUNCER_URL and settings.PGBOUNCER_PORT) else DATABASE_URL + +if settings.PGBOUNCER_URL and settings.PGBOUNCER_PORT: + engine = create_engine( + active_url, + poolclass=NullPool, + echo=False, + ) +elif settings.APP_ENV == "development": + engine = create_engine( + active_url, + poolclass=NullPool, + echo=False, + ) +else: + engine = create_engine( + active_url, + poolclass=QueuePool, + pool_pre_ping=True, + pool_recycle=900, + pool_size=3, + max_overflow=7, + pool_timeout=30, + echo=False, + ) + +SessionLocal = sessionmaker( + bind=engine, + autocommit=False, + autoflush=False, +) + +def get_db(): + db = SessionLocal() + try: + yield db + db.commit() + except Exception: + db.rollback() + raise + finally: + db.close() + +@contextmanager +def get_db_connection() -> Session: + db = SessionLocal() + try: + yield db + db.commit() + except Exception: + db.rollback() + raise + finally: + db.close() + +def _to_dict(obj): + if not obj: return None + if hasattr(obj, '__table__'): + return {c.name: getattr(obj, c.name) for c in obj.__table__.columns} + if isinstance(obj, dict): + return obj + return None + +def init_db_pool(): + """Log pool configuration on startup.""" + pool = engine.pool + + if settings.PGBOUNCER_URL and settings.PGBOUNCER_PORT: + logger.info(f"✅ PgBouncer connection successfully routed! (Strict Port {settings.PGBOUNCER_PORT})") + + logger.info( + f"DB pool initialized: size={getattr(pool, 'size', 'N/A')}, " + f"overflow={getattr(pool, '_max_overflow', 'N/A')}, " + f"class={pool.__class__.__name__}" + ) + +def close_db_pool(): + engine.dispose() + logger.info("DB pool disposed — all connections closed") + +def init_db(): + pass + + +from app.core import tenant_filter # noqa: E402,F401 + +from app.core import rls # noqa: E402,F401 diff --git a/app/db/redis.py b/app/db/redis.py new file mode 100644 index 0000000..c39cf3c --- /dev/null +++ b/app/db/redis.py @@ -0,0 +1,94 @@ +import redis +import os +import json +import logging +from typing import Optional, Any, Dict +from app.core.settings import settings + +logger = logging.getLogger(__name__) + +REDIS_HOST = settings.REDIS_HOST or "localhost" +REDIS_PORT = settings.REDIS_PORT or 6379 +REDIS_PASSWORD = settings.REDIS_PASSWORD + +class RedisClient: + def __init__(self): + self.client = None + try: + self.client = redis.Redis( + host=REDIS_HOST, + port=REDIS_PORT, + password=REDIS_PASSWORD, + decode_responses=True, + socket_connect_timeout=5 + ) + self.client.ping() + logger.info(f"✅ Redis Connected: {REDIS_HOST}:{REDIS_PORT}") + except Exception as e: + logger.info(f"Redis not available: {e}. Running without caching.") + self.client = None + + def get(self, key: str) -> Optional[Any]: + if not self.client: + return None + try: + val = self.client.get(key) + return json.loads(val) if val else None + except Exception as e: + logger.debug(f"Redis get error: {e}") + return None + + def set(self, key: str, value: Any, ttl: int = 300): + if not self.client: + return + try: + self.client.setex(key, ttl, json.dumps(value, default=str)) + except Exception as e: + logger.debug(f"Redis set error: {e}") + + def delete(self, key: str): + if not self.client: + return + try: + self.client.delete(key) + except Exception: + pass + + def hgetall(self, key: str) -> Dict[str, str]: + if not self.client: + return {} + try: + return self.client.hgetall(key) or {} + except Exception as e: + logger.debug(f"Redis hgetall error: {e}") + return {} + + def hset(self, key: str, field: str, value: str): + if not self.client: + return + try: + self.client.hset(key, field, value) + except Exception as e: + logger.debug(f"Redis hset error: {e}") + + def hdel(self, key: str, *fields: str): + if not self.client or not fields: + return + try: + self.client.hdel(key, *fields) + except Exception as e: + logger.debug(f"Redis hdel error: {e}") + + def invalidate_pattern(self, pattern: str): + """Invalidate keys matching pattern (e.g. 'drive:user:1:*')""" + if not self.client: + return + try: + keys = list(self.client.scan_iter(pattern)) + if keys: + self.client.delete(*keys) + logger.debug(f"Invalidated {len(keys)} keys for pattern {pattern}") + except Exception as e: + logger.error(f"Redis invalidate error: {e}") + +redis_cache = RedisClient() diff --git a/app/infrastructure/realtime/chat_subscriber.py b/app/infrastructure/realtime/chat_subscriber.py new file mode 100644 index 0000000..8ed705d --- /dev/null +++ b/app/infrastructure/realtime/chat_subscriber.py @@ -0,0 +1,119 @@ +""" +Redis Chat Subscriber Worker — single-per-process background task. + +Subscribes to the Redis pattern ``chat:file:*`` via PSUBSCRIBE and +forwards incoming messages to the local ChatConnectionManager rooms. + +Why one global worker instead of one per WebSocket connection? + • One Redis subscription per process (not per connection) — O(1) vs O(N). + • Works across multiple FastAPI / Uvicorn workers: each worker runs its + own instance of this task, so every local room receives messages + published by any other worker. + • The ``_origin_worker`` tag prevents echo: messages that originated on + THIS worker are skipped (they were already broadcast locally). + +Usage (in main.py lifespan): + from app.infrastructure.realtime.chat_subscriber import start_chat_subscriber, stop_chat_subscriber + + # Startup + await start_chat_subscriber() + + # Shutdown + await stop_chat_subscriber() +""" +import asyncio +import json +import logging +from typing import Optional + +import redis.asyncio as aioredis + +from app.core.settings import settings +from app.infrastructure.realtime.connection_manager import manager + +logger = logging.getLogger(__name__) + +_CHANNEL_PATTERN = "chat:file:*" + +_subscriber_task: Optional[asyncio.Task] = None + + +async def _listen_forever() -> None: + """ + Connect to Redis, pattern-subscribe, and loop forever. + + Automatically reconnects on transient errors with exponential backoff. + """ + backoff = 1 + + while True: + client: Optional[aioredis.Redis] = None + try: + url = settings.REDIS_URL or "redis://localhost:6379/0" + client = aioredis.from_url(url, decode_responses=True) + await client.ping() + logger.info(f"📡 Chat subscriber connected (pattern: {_CHANNEL_PATTERN})") + backoff = 1 + + pubsub = client.pubsub() + await pubsub.psubscribe(_CHANNEL_PATTERN) + + async for raw in pubsub.listen(): + if raw["type"] != "pmessage": + continue + + channel: str = raw["channel"] + try: + file_id = int(channel.rsplit(":", 1)[-1]) + except (ValueError, IndexError): + logger.warning(f"Bad channel name: {channel}") + continue + + try: + data: dict = json.loads(raw["data"]) + except (json.JSONDecodeError, TypeError): + logger.warning(f"Bad payload on {channel}") + continue + + await manager.broadcast_chat(file_id, data) + + except asyncio.CancelledError: + logger.info("Chat subscriber task cancelled — shutting down") + break + + except Exception as e: + logger.error( + f"Chat subscriber error: {e} — reconnecting in {backoff}s" + ) + await asyncio.sleep(backoff) + backoff = min(backoff * 2, 30) + + finally: + if client: + try: + await client.aclose() + except Exception: + pass + + +async def start_chat_subscriber() -> None: + """Launch the subscriber as a background task (idempotent).""" + global _subscriber_task + if _subscriber_task is not None and not _subscriber_task.done(): + return + _subscriber_task = asyncio.create_task(_listen_forever()) + logger.info("✅ Chat subscriber worker started") + + +async def stop_chat_subscriber() -> None: + """Cancel the background task gracefully.""" + global _subscriber_task + if _subscriber_task is None: + return + _subscriber_task.cancel() + try: + await _subscriber_task + except asyncio.CancelledError: + pass + _subscriber_task = None + logger.info("Chat subscriber worker stopped") diff --git a/app/infrastructure/realtime/connection_manager.py b/app/infrastructure/realtime/connection_manager.py new file mode 100644 index 0000000..0666649 --- /dev/null +++ b/app/infrastructure/realtime/connection_manager.py @@ -0,0 +1,144 @@ +""" +ConnectionManager — unified WebSocket manager for all real-time features. + +Currently handles two types of connectivity: +1. Room-based (Chat): file-based collaboration where multiple users share a room. +2. Direct/Task-based: 1:1 connections for tracking specific backend task progress. + +This replaces the old infrastructure/websocket/ws_manager.py for a +unified, production-ready implementation. +""" +import logging +from collections import defaultdict +from dataclasses import dataclass +from typing import Any, Optional, Set, Dict + +from fastapi import WebSocket +from starlette.websockets import WebSocketState + +logger = logging.getLogger(__name__) + + +@dataclass(eq=False) +class Connection: + """Metadata associated with a single WebSocket connection.""" + websocket: WebSocket + room_id: str + user_id: Optional[int] = None + conn_type: str = 'chat' + + +class ConnectionManager: + """ + Manages WebSocket connections partitioned by room_id. + """ + + def __init__(self) -> None: + self._rooms: Dict[str, Set[Connection]] = defaultdict(set) + self._ws_index: Dict[int, Connection] = {} + + + async def connect( + self, + room_id: str, + websocket: WebSocket, + user_id: Optional[int] = None, + conn_type: str = 'chat' + ) -> Connection: + """Accept the WebSocket and register it in the specified room.""" + await websocket.accept() + + conn = Connection( + websocket=websocket, + room_id=room_id, + user_id=user_id, + conn_type=conn_type + ) + self._rooms[room_id].add(conn) + self._ws_index[id(websocket)] = conn + + logger.info( + f"WS connected: type={conn_type} room={room_id} user={user_id} " + f"(room size: {len(self._rooms[room_id])})" + ) + return conn + + async def connect_chat(self, user_id: int, file_id: int, websocket: WebSocket) -> Connection: + """Helper to connect to a chat room.""" + return await self.connect( + room_id=f"chat:{file_id}", + websocket=websocket, + user_id=user_id, + conn_type='chat' + ) + + async def connect_task(self, task_id: str, websocket: WebSocket) -> Connection: + """Helper to connect to a task status channel.""" + return await self.connect( + room_id=f"task:{task_id}", + websocket=websocket, + conn_type='task' + ) + + def disconnect(self, websocket: WebSocket) -> None: + """Remove a WebSocket from its room.""" + conn = self._ws_index.pop(id(websocket), None) + if conn is None: + return + + room = self._rooms.get(conn.room_id) + if room: + room.discard(conn) + if not room: + del self._rooms[conn.room_id] + + logger.info( + f"WS disconnected: room={conn.room_id} user={conn.user_id} " + f"(room size: {len(self._rooms.get(conn.room_id, []))})" + ) + + async def broadcast(self, room_id: str, message: Dict[str, Any]) -> None: + """Send a JSON payload to every connection in the room.""" + room = self._rooms.get(room_id) + if not room: + return + + stale: list[Connection] = [] + + for conn in room: + try: + if conn.websocket.client_state == WebSocketState.CONNECTED: + await conn.websocket.send_json(message) + else: + stale.append(conn) + except Exception: + stale.append(conn) + + for conn in stale: + self.disconnect(conn.websocket) + + async def broadcast_chat(self, file_id: int, data: Dict[str, Any]) -> None: + """Legacy helper for broadcast on chat:file:{file_id}.""" + await self.broadcast(f"chat:{file_id}", data) + + async def send_task(self, task_id: str, data: Dict[str, Any]) -> None: + """Send message to a specific task status channel.""" + await self.broadcast(f"task:{task_id}", data) + + + def get_room_users(self, file_id: int) -> list[int]: + """Return a list of user IDs currently in a file's chat room.""" + room_id = f"chat:{file_id}" + return list({conn.user_id for conn in self._rooms.get(room_id, set()) if conn.user_id}) + + def get_room_size(self, room_id: str) -> int: + """Number of active connections in a specific room.""" + return len(self._rooms.get(room_id, set())) + + @property + def total_connections(self) -> int: + """Total number of active WebSocket connections across all rooms.""" + return len(self._ws_index) + + +manager = ConnectionManager() diff --git a/app/infrastructure/vector/__init__.py b/app/infrastructure/vector/__init__.py new file mode 100644 index 0000000..eaa6e2b --- /dev/null +++ b/app/infrastructure/vector/__init__.py @@ -0,0 +1 @@ +from .vector_client import VectorStoreManager, get_model, get_qdrant_client diff --git a/app/infrastructure/vector/vector_client.py b/app/infrastructure/vector/vector_client.py new file mode 100644 index 0000000..b8a1a77 --- /dev/null +++ b/app/infrastructure/vector/vector_client.py @@ -0,0 +1,361 @@ +import logging +import uuid +import threading +import numpy as np +from typing import List, Optional + +from qdrant_client import QdrantClient +from qdrant_client.models import ( + Distance, + VectorParams, + PointStruct, + Filter, + FieldCondition, + MatchValue, + FilterSelector, + PayloadSchemaType, +) + +from app.core.settings import settings + +logger = logging.getLogger(__name__) + +_model = None +_model_lock = threading.Lock() + + +def get_model(): + """ + Get the embedding library (SentenceTransformer) for generating vector representations. + Uses centralized setting settings.EMBEDDING_MODEL. + Thread-safe singleton — the model is loaded only once. + """ + global _model + if _model is None: + with _model_lock: + if _model is None: + try: + from sentence_transformers import SentenceTransformer + logger.info(f"Loading SentenceTransformer: {settings.EMBEDDING_MODEL}") + _model = SentenceTransformer(settings.EMBEDDING_MODEL, device='cpu') + _model.to('cpu') + logger.info(f"SentenceTransformer loaded successfully (dim={_model.get_sentence_embedding_dimension()})") + except Exception as e: + logger.error(f"Failed to load embedding model: {e}") + raise + return _model + + +_qdrant_client = None +_qdrant_lock = threading.Lock() + + +def get_qdrant_client() -> QdrantClient: + """Get or create a singleton Qdrant client.""" + global _qdrant_client + if _qdrant_client is None: + with _qdrant_lock: + if _qdrant_client is None: + _qdrant_client = QdrantClient( + host=settings.QDRANT_HOST, + port=settings.QDRANT_PORT, + ) + logger.info( + f"Connected to Qdrant at {settings.QDRANT_HOST}:{settings.QDRANT_PORT}" + ) + return _qdrant_client + + +_manager_instance = None +_manager_lock = threading.Lock() + + +class VectorStoreManager: + """ + Handles all vector operations via Qdrant. + Uses singleton pattern — only one instance exists per process. + + Architecture: + Upload -> Chunk -> Embed (BGE) -> Qdrant -> Search -> DeepSeek + + Each Qdrant point stores: + - id: unique UUID + - vector: BGE embedding + - payload: { user_id, document_id, chunk_text, chunk_index } + """ + + _collection_verified = False + + def __new__(cls): + """Singleton: reuse the same instance across all callers.""" + global _manager_instance + if _manager_instance is None: + with _manager_lock: + if _manager_instance is None: + instance = super().__new__(cls) + instance._initialized = False + _manager_instance = instance + return _manager_instance + + def __init__(self): + if self._initialized: + return + self.client = get_qdrant_client() + self.collection_name = settings.QDRANT_COLLECTION + self._ensure_collection() + self._initialized = True + + def _ensure_collection(self): + """Create the Qdrant collection if it doesn't exist. Runs only once.""" + if VectorStoreManager._collection_verified: + return + + try: + collections = self.client.get_collections().collections + exists = any(c.name == self.collection_name for c in collections) + + if not exists: + model = get_model() + dimension = model.get_sentence_embedding_dimension() + + self.client.create_collection( + collection_name=self.collection_name, + vectors_config=VectorParams( + size=dimension, + distance=Distance.COSINE, + ), + ) + + self.client.create_payload_index( + collection_name=self.collection_name, + field_name="user_id", + field_schema=PayloadSchemaType.INTEGER, + ) + self.client.create_payload_index( + collection_name=self.collection_name, + field_name="document_id", + field_schema=PayloadSchemaType.KEYWORD, + ) + + logger.info( + f"Created Qdrant collection '{self.collection_name}' " + f"(dim={dimension}, distance=COSINE)" + ) + else: + logger.info(f"Qdrant collection '{self.collection_name}' ready") + + VectorStoreManager._collection_verified = True + + except Exception as e: + logger.error(f"Failed to ensure Qdrant collection: {e}") + raise + + def create_index(self, user_id: int, document_id: str, chunks: List[str]): + """ + Calculates embeddings for chunks and upserts them into Qdrant. + Returns the number of indexed chunks. + """ + if not chunks: + return 0 + + user_id = int(user_id) + model = get_model() + + embeddings = model.encode(chunks, show_progress_bar=False) + embeddings = np.array(embeddings).astype("float32") + + points = [] + for idx, (chunk_text, embedding) in enumerate(zip(chunks, embeddings)): + point_id = str(uuid.uuid4()) + points.append( + PointStruct( + id=point_id, + vector=embedding.tolist(), + payload={ + "user_id": user_id, + "document_id": document_id, + "chunk_text": chunk_text, + "chunk_index": idx, + }, + ) + ) + + try: + batch_size = 100 + for i in range(0, len(points), batch_size): + batch = points[i : i + batch_size] + self.client.upsert( + collection_name=self.collection_name, + points=batch, + ) + + logger.info( + f"Indexed {len(chunks)} chunks in Qdrant " + f"(user={user_id}, doc={document_id})" + ) + return len(chunks) + + except Exception as e: + error_str = str(e) + if "doesn't exist" in error_str or "Not found: Collection" in error_str: + logger.warning(f"Qdrant collection '{self.collection_name}' missing. Re-creating...") + VectorStoreManager._collection_verified = False + self._ensure_collection() + + batch_size = 100 + for i in range(0, len(points), batch_size): + batch = points[i : i + batch_size] + self.client.upsert( + collection_name=self.collection_name, + points=batch, + ) + + logger.info( + f"Indexed {len(chunks)} chunks in Qdrant after recreation " + f"(user={user_id}, doc={document_id})" + ) + return len(chunks) + + logger.error(f"Failed to upsert vectors to Qdrant: {e}") + raise + + def search( + self, + user_id: int, + question: str, + document_id: Optional[str] = None, + top_k: int = 10, + ) -> List[dict]: + """ + Searches Qdrant for chunks most similar to the question. + Returns the top_k most relevant chunks with their scores. + """ + try: + user_id = int(user_id) + model = get_model() + + query_embedding = model.encode([question], show_progress_bar=False) + query_vector = query_embedding[0].tolist() + + must_conditions = [ + FieldCondition(key="user_id", match=MatchValue(value=user_id)) + ] + + if document_id: + must_conditions.append( + FieldCondition( + key="document_id", match=MatchValue(value=document_id) + ) + ) + + query_filter = Filter(must=must_conditions) + + results = self.client.search( + collection_name=self.collection_name, + query_vector=query_vector, + query_filter=query_filter, + limit=top_k, + score_threshold=None, + ) + + min_score = settings.MIN_SIMILARITY_SCORE or 0.2 + filtered = [] + + for hit in results: + print(f"SCORE: {hit.score:.4f} | TEXT: {hit.payload['chunk_text'][:80]}...") + + if hit.score >= min_score: + filtered.append({ + "text": hit.payload["chunk_text"], + "score": hit.score + }) + + logger.info( + f"Qdrant search: {len(filtered)} filtered results (from {len(results)}) for user {user_id}" + ) + return filtered + + except Exception as e: + error_str = str(e) + if "doesn't exist" in error_str or "Not found: Collection" in error_str: + logger.warning(f"Qdrant collection '{self.collection_name}' missing during search. Re-creating...") + VectorStoreManager._collection_verified = False + self._ensure_collection() + return [] + + logger.error(f"Search failed for user {user_id}: {e}") + return [] + + def delete_document(self, user_id: int, document_id: str): + """ + Deletes all vectors associated with a specific document from Qdrant. + """ + try: + self.client.delete( + collection_name=self.collection_name, + points_selector=FilterSelector( + filter=Filter( + must=[ + FieldCondition( + key="user_id", match=MatchValue(value=user_id) + ), + FieldCondition( + key="document_id", + match=MatchValue(value=document_id), + ), + ] + ) + ), + ) + logger.info(f"Deleted vectors for user {user_id}, doc {document_id}") + + except Exception as e: + error_str = str(e) + if "doesn't exist" in error_str or "Not found: Collection" in error_str: + logger.warning(f"Qdrant collection '{self.collection_name}' missing during delete. Ignoring.") + VectorStoreManager._collection_verified = False + self._ensure_collection() + else: + logger.error(f"Delete failed: {e}") + + def delete_user_data(self, user_id: int): + """ + Deletes ALL vectors for a user (used for account cleanup / reset). + """ + try: + self.client.delete( + collection_name=self.collection_name, + points_selector=FilterSelector( + filter=Filter( + must=[ + FieldCondition( + key="user_id", match=MatchValue(value=user_id) + ), + ] + ) + ), + ) + logger.info(f"Deleted all vectors for user {user_id}") + + except Exception as e: + error_str = str(e) + if "doesn't exist" in error_str or "Not found: Collection" in error_str: + logger.warning(f"Qdrant collection '{self.collection_name}' missing during delete_user_data. Ignoring.") + VectorStoreManager._collection_verified = False + self._ensure_collection() + else: + logger.error(f"Delete all failed for user {user_id}: {e}") + + +def preload(): + """ + Preload the embedding model and verify Qdrant connection at startup. + Call this during app initialization to avoid cold-start latency. + """ + try: + logger.info("Preloading embedding model and Qdrant connection...") + get_model() + VectorStoreManager() + logger.info("Preload complete - ready for queries") + except Exception as e: + logger.warning(f"Preload failed (will retry on first request): {e}") diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..b8973a6 --- /dev/null +++ b/app/main.py @@ -0,0 +1,449 @@ +import os +import logging +import asyncio +import tempfile +import shutil +import uuid +from contextlib import asynccontextmanager +from datetime import datetime, timezone + +from fastapi import FastAPI, UploadFile, File, HTTPException, Depends, Request, Response, status +from fastapi.middleware.cors import CORSMiddleware +from fastapi.staticfiles import StaticFiles +from sqlalchemy import text +from sqlalchemy.orm import Session +from app.middleware.csrf import CSRFMiddleware +from app.middleware.ip_middleware import IPMiddleware + +from app.core.settings import settings +from app.db.database import init_db, init_db_pool, close_db_pool, get_db + +import app.db.all_models # noqa: F401 +from app.middleware.auth import get_current_user +from app.modules.auth.models.user_model import User +from app.modules.auth.models.access_model import Access +from app.modules.auth.models.role_model import Role +from app.modules.auth.models.role_access_model import RoleAccess +from app.modules.documents.models.document_model import Project + +from app.modules.auth.routes.auth_routes import router as auth_router +from app.modules.auth.routes.user_routes import router as user_router +from app.modules.auth.routes.me_routes import router as me_router +from app.modules.auth.routes.access_routes import router as access_router +from app.modules.auth.routes.role_routes import router as role_router +from app.modules.auth.routes.admin_dashboard_routes import router as admin_dashboard_router +from app.modules.auth.routes.admin_user_routes import router as admin_user_router +from app.modules.documents.routes.document_routes import router as projects_router +from app.modules.documents.routes.sharing_routes import router as sharing_router +from app.modules.documents.routes.version_routes import router as version_router +from app.modules.documents.routes.comment_routes import ( + router as project_comments_router, +) +from app.modules.documents.routes.details_routes import router as project_details_router +from app.modules.drive.routes.drive_routes import ( + router as drive_router, + public_router as public_drive_router, +) +from app.modules.storage.routes.storage_routes import router as storage_router +from app.modules.chat.routes.chat_routes import router as chatbot_router +from app.tasks.routes import router as task_results_router +from app.modules.notifications.routes.notification_routes import ( + router as notifications_router, +) +from app.modules.documents.routes.export_routes import router as export_router + +from app.modules.speech.routes.speech_routes import router as speech_router +from app.api.ws_router import router as ws_router +from app.core.pubsub_listener import start_listener +from app.core.redis import redis_pubsub +from app.modules.collab.routes.chat_routes import ( + router as collab_chat_router, + rest_router as collab_rest_router, +) +from app.modules.tenant.routes.tenant_routes import router as tenant_router +from app.modules.tenant.routes.tenant_contact_routes import router as tenant_contact_router +from app.modules.signing.routes.signing_routes import router as signing_router +from app.modules.signing.routes.config_routes import router as signing_config_router +from app.modules.configuration.routes.system_configuration_routes import ( + router as system_configuration_router, +) +from app.modules.activity_logs.routes import router as activity_logs_router +from app.infrastructure.realtime.chat_subscriber import ( + start_chat_subscriber, + stop_chat_subscriber, +) +from app.modules.drive.routes.scan_routes import router as scan_router +from app.modules.extraction.routes.extraction_routes import router as extraction_router +from app.modules.editor.pdf_router import router as pdf_editor_router +from app.modules.auth.sso_router import router as sso_router +from app.modules.security.routes.security_routes import router as security_router + +TEMP_DIR = tempfile.mkdtemp(prefix="ai_uploads_") +processing_semaphore = asyncio.Semaphore(1) + +logging.basicConfig( + level=getattr(logging, settings.LOG_LEVEL), + format="%(asctime)s - %(levelname)s - %(message)s", +) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + logging.info(f"Server is starting up in {settings.APP_ENV} mode...") + try: + init_db_pool() + init_db() + + asyncio.create_task(start_listener()) + + await redis_pubsub.connect() + + await start_chat_subscriber() + + from app.infrastructure.vector.vector_client import preload as preload_vector + loop = asyncio.get_running_loop() + loop.run_in_executor(None, preload_vector) + + except Exception as e: + logging.error(f"Critical Startup failure: {e}") + raise e + + yield + + logging.info("Server is shutting down...") + try: + await stop_chat_subscriber() + await redis_pubsub.disconnect() + logging.info("✅ Chat subscriber & Redis Pub/Sub disconnected") + except Exception as e: + logging.error(f"Shutdown error during subscriber/redis stop: {e}") + try: + close_db_pool() + logging.info("✅ Database connection pool closed") + except Exception as e: + logging.error(f"Error closing database pool: {e}") + + try: + shutil.rmtree(TEMP_DIR, ignore_errors=True) + logging.info("🧹 Temp directory cleaned") + except Exception as e: + logging.error(f"Shutdown error during temp cleanup: {e}") + + +app = FastAPI( + lifespan=lifespan, + title="DocQube Document Processing Pipeline API", + description="An interactive API and UI to process PDFs into JATS XML.", + version="2.0.0", +) + +app.add_middleware(CSRFMiddleware) +app.add_middleware(IPMiddleware) + +from app.middleware.tenant_context_middleware import TenantContextMiddleware # noqa: E402 + +app.add_middleware(TenantContextMiddleware) + +app.add_middleware( + CORSMiddleware, + allow_origins=settings.CORS_ORIGINS_LIST, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + # Browsers only expose a small header safelist to JS on cross-origin + # responses by default (Content-Range/Accept-Ranges/Content-Length are + # NOT in it) — pdf.js reads these itself to decide whether it can use + # HTTP Range requests for progressive PDF loading, so without this it + # falls back to downloading the entire file before rendering anything. + expose_headers=["Content-Range", "Accept-Ranges", "Content-Length"], +) + + +# File Serving Internal Helper +async def _get_s3_file_response(key: str, db: Session = None, tenant_id: str = None): + from app.infrastructure.storage.local_storage_handler import ( + get_storage_client, + ) + from fastapi.responses import StreamingResponse + import mimetypes + + storage, quarantine_bucket, clean_bucket = get_storage_client( + db=db, tenant_id=tenant_id) + try: + try: + response = storage.get_object(Bucket=clean_bucket, Key=key) + except Exception: + response = storage.get_object(Bucket=quarantine_bucket, Key=key) + + body = response["Body"] + + content_type, _ = mimetypes.guess_type(key) + if not content_type: + content_type = "application/octet-stream" + + return StreamingResponse(body, media_type=content_type) + except HTTPException: + raise + except Exception as e: + logging.error(f"Storage File Proxy Error: {e}") + raise HTTPException(status_code=404, detail="File not found") + + +@app.get("/api/storage_drive/{key:path}") +async def serve_storage_file(key: str, current_user: User = Depends(get_current_user), db: Session = Depends(get_db)): + if ".." in key: + raise HTTPException(status_code=400, detail="Invalid path") + + tenant_part = f"tenants/{current_user.tenant_id}/" if current_user.tenant_id else "" + user_prefix1 = f"{tenant_part}user_{current_user.id}/" + user_prefix2 = f"{tenant_part}users/{current_user.id}/" + conv_prefix = f"{tenant_part}doc_conversion/{current_user.id}/" + chatbot_prefix = f"{tenant_part}chatbot_documents/{current_user.id}/" + images_prefix = f"extracted_images/{current_user.id}/" + + allowed_prefixes = [ + user_prefix1, + user_prefix2, + conv_prefix, + chatbot_prefix, + images_prefix, + f"user_{current_user.id}/", + f"users/{current_user.id}/", + f"doc_conversion/{current_user.id}/", + f"chatbot_documents/{current_user.id}/", + f"extracted_images/{current_user.id}/", + ] + + if not any(key.startswith(p) for p in allowed_prefixes): + raise HTTPException( + status_code=403, detail="Access denied to this storage key") + + return await _get_s3_file_response(key, db=db, tenant_id=current_user.tenant_id) + + +@app.get("/api/serve-image/{session_id}/{filename}") +async def serve_image( + session_id: str, + filename: str, + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db), +): + """ + Alias for serving images extracted during document processing. + """ + if ".." in session_id or ".." in filename: + raise HTTPException(status_code=400, detail="Invalid parameters") + + project = ( + db.query(Project) + .filter(Project.session_id == session_id, Project.user_id == current_user.id) + .first() + ) + + if not project: + raise HTTPException( + status_code=403, detail="Access denied to this session's resources" + ) + + key = f"extracted_images/{current_user.id}/{session_id}/{filename}" + return await _get_s3_file_response(key, db=db, tenant_id=current_user.tenant_id) + + +def _check_database_ready(db: Session) -> bool: + """Lightweight DB reachability check shared by /api/health/ready and the + legacy /api/health. On failure it rolls back before returning — without + this, the session handed back to Depends(get_db)'s own cleanup (which + unconditionally calls db.commit() after a successful return from this + function) is left in an aborted-transaction state, and that commit() + itself raises. That turns a DB failure this function already caught + into an unhandled 500 from the dependency's own teardown — which is + exactly what took the original /api/health down during the PgBouncer + incident, even though it appeared to catch the error internally. + """ + try: + db.execute(text("SELECT 1")) + return True + except Exception: + db.rollback() + return False + + +@app.get("/api/health/live") +def health_live(): + """Liveness: is the API process able to serve HTTP at all. No database, + no PgBouncer, no external dependency of any kind — this is what the + Docker HEALTHCHECK now uses, so a temporarily unreachable database can + no longer get the whole API container killed as unhealthy. + """ + return { + "status": "ok", + "service": "api", + "timestamp": datetime.now(timezone.utc).isoformat(), + } + + +@app.get("/api/health/ready") +def health_ready(response: Response, db: Session = Depends(get_db)): + """Readiness: is the API ready to serve real traffic, i.e. can it reach + the database through the application's actual configured pool/engine + (app.db.database.engine via get_db) — the same one every request uses, + not a separate health-only connection. + """ + if _check_database_ready(db): + return { + "status": "ready", + "database": "connected", + "timestamp": datetime.now(timezone.utc).isoformat(), + } + response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE + return { + "status": "not_ready", + "database": "disconnected", + "timestamp": datetime.now(timezone.utc).isoformat(), + } + + +@app.get("/api/health") +def health_check(db: Session = Depends(get_db)): + """Backward-compatible general health endpoint — response shape kept + identical to before this refactor. Reuses the same DB-reachability + check as /api/health/ready instead of duplicating it; the Docker + HEALTHCHECK no longer depends on this endpoint (see /api/health/live). + """ + if not _check_database_ready(db): + return {"status": "degraded", "database": "error", "error": "database unreachable"} + + health = {"status": "ok", "database": "connected"} + try: + version = db.execute(text("SELECT version_num FROM alembic_version")).scalar() + health["migration_version"] = version + except Exception: + db.rollback() + health["status"] = "degraded" + health["error"] = "migration version lookup failed" + + return health + + +app.include_router(tenant_router, prefix="/api") +app.include_router(tenant_contact_router, prefix="/api/tenant") +app.include_router(auth_router, prefix="/api") +app.include_router(sso_router, prefix="/api") +app.include_router(me_router, prefix="/api") +app.include_router(access_router) +app.include_router(role_router) +app.include_router(admin_dashboard_router) +app.include_router(admin_user_router) +app.include_router(user_router, prefix="/api") +app.include_router(projects_router, prefix="/api") +app.include_router(sharing_router, prefix="/api") +app.include_router(version_router, prefix="/api") +app.include_router(project_comments_router, prefix="/api") +app.include_router(project_details_router, prefix="/api") +app.include_router(drive_router, prefix="/api") +app.include_router(public_drive_router, prefix="/api") +app.include_router(storage_router, prefix="/api") +app.include_router(task_results_router, prefix="/api") +app.include_router(chatbot_router) +app.include_router(notifications_router, prefix="/api") +app.include_router(system_configuration_router) +app.include_router(activity_logs_router) +app.include_router(export_router, prefix="/api") + +app.include_router(speech_router) +app.include_router(ws_router, prefix="/api") +app.include_router(collab_chat_router, prefix="/api") +app.include_router(collab_rest_router, prefix="/api") +app.include_router(signing_router, prefix="/api") +app.include_router(signing_config_router, prefix="/api") +app.include_router(pdf_editor_router, prefix="/api") +app.include_router(scan_router, prefix="/api") +app.include_router(extraction_router, prefix="/api") + +from app.modules.org.routes.org_routes import router as org_router # noqa: E402 + +app.include_router(org_router, prefix="/api") + +from app.modules.billing.routes.billing_routes import router as billing_router # noqa: E402 + +app.include_router(billing_router, prefix="/api") +app.include_router(security_router, prefix="/api") + + +@app.post("/api/extract") +async def extract( + request: Request, + file: UploadFile = File(...), + current_user: User = Depends(get_current_user), +): + """ + On-demand document extraction endpoint. + Loads models only when called and unloads after. + """ + file_id = str(uuid.uuid4()) + ext = os.path.splitext(file.filename)[1].lower() + temp_path = os.path.join(TEMP_DIR, f"{file_id}{ext}") + + MAX_FILE_SIZE = 50 * 1024 * 1024 + + try: + total_size = 0 + with open(temp_path, "wb") as buffer: + while True: + chunk = await file.read(1024 * 1024) + if not chunk: + break + total_size += len(chunk) + if total_size > MAX_FILE_SIZE: + raise HTTPException( + status_code=413, detail="File too large (Max 50MB)" + ) + buffer.write(chunk) + + # Convert PDF to HTML locally, then derive markdown + async with processing_semaphore: + from app.modules.editor.converters import process_pdf_to_html + + loop = asyncio.get_running_loop() + + def _convert(): + html = process_pdf_to_html(temp_path) + from bs4 import BeautifulSoup + markdown = BeautifulSoup( + html, "html.parser").get_text("\n") + return {"html": html, "markdown": markdown} + + result = await loop.run_in_executor(None, _convert) + + return { + "status": "success", + "file_id": file_id, + "data": result, + } + + except HTTPException: + raise + except Exception as e: + error_id = str(uuid.uuid4()) + logging.error( + f"Extraction failed [Ref: {error_id}]: {e}", exc_info=True) + raise HTTPException( + status_code=500, + detail=f"An internal error occurred during processing. Please contact support and provide reference: {error_id}", + ) + + finally: + if os.path.exists(temp_path): + try: + os.remove(temp_path) + except Exception as e: + logging.error(f"Cleanup error removing {temp_path}: {e}") + + +if __name__ == "__main__": + import uvicorn + + uvicorn.run( + "main:app", host=settings.HOST, port=settings.PORT, reload=settings.DEBUG + ) + diff --git a/app/middleware/__init__.py b/app/middleware/__init__.py new file mode 100644 index 0000000..d369da1 --- /dev/null +++ b/app/middleware/__init__.py @@ -0,0 +1,3 @@ +""" +Middleware module for application-wide processing. +""" diff --git a/app/middleware/auth.py b/app/middleware/auth.py new file mode 100644 index 0000000..f37a0ba --- /dev/null +++ b/app/middleware/auth.py @@ -0,0 +1,329 @@ +from typing import Optional +from fastapi import Depends, HTTPException, Request +from fastapi.security import OAuth2PasswordBearer +from jose import JWTError, jwt +from sqlalchemy.orm import Session, selectinload +import logging +from datetime import datetime, timezone + +from app.db.database import get_db +from app.modules.auth.models.role_model import Role +from app.modules.auth.models.role_access_model import RoleAccess +from app.modules.auth.models.user_model import User +from app.core.settings import settings +from app.core.tenant_context import set_context +from app.core.token_blacklist import TokenBlacklist + +logger = logging.getLogger(__name__) + +oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/auth/login", auto_error=False) + + +def _validate_saas_subscription(subscription_details: Optional[dict]) -> None: + if not subscription_details: + return + + status_value = str(subscription_details.get("status") or "").upper() + is_active = subscription_details.get("is_active") + today = datetime.now(timezone.utc).date() + + start_date_raw = subscription_details.get("start_date") + end_date_raw = subscription_details.get("end_date") + + try: + start_date = ( + datetime.fromisoformat(start_date_raw).date() if start_date_raw else None + ) + except ValueError: + start_date = None + + try: + end_date = datetime.fromisoformat(end_date_raw).date() if end_date_raw else None + except ValueError: + end_date = None + + if is_active is False or status_value in {"INACTIVE", "EXPIRED"}: + raise HTTPException(status_code=403, detail="Tenant subscription is inactive") + + if start_date and today < start_date: + raise HTTPException(status_code=403, detail="Tenant subscription is not active yet") + + if end_date and today > end_date: + raise HTTPException(status_code=403, detail="Tenant subscription has expired") + + +def _check_missing_subscription_claim(db, user, subscription_details) -> None: + """ + A platform-managed tenant arriving without a subscription claim. + + `_validate_saas_subscription` only checks a claim that is present. Direct + login sets none, so subscription state is enforced for SSO and not for + direct logins — which is either deliberate or a hole, depending on how the + product is sold. + + `saas_tenant_mappings` is the evidence that settles it per tenant: a mapped + tenant is billed on the platform, an unmapped one is not. So this only ever + looks at mapped tenants, and it cannot affect a customer who was never on + the platform. + + **Logs always, blocks only when configured.** Leave the setting off, watch + the warnings, and enable it once the logs show mapped tenants always arrive + with a claim. Enforcing a billing rule at the login door is the change that + locks real customers out if the assumption is wrong — so the observation + comes first and the enforcement is a deliberate second step. + + Only runs when the claim is *absent*, which is the exceptional path, so the + extra query is not on the hot path for SSO users. + """ + if subscription_details: + return + if not user.tenant_id: + return + + from app.modules.auth.models.saas_models import SaaSTenantMapping + + mapped = ( + db.query(SaaSTenantMapping.id) + .filter(SaaSTenantMapping.docqube_tenant_id == user.tenant_id) + .first() + is not None + ) + if not mapped: + return + + logger.warning( + "Tenant %s is managed by the SaaS platform but this token carries no " + "subscription claim; subscription state is not being enforced for this " + "request (user %s)", + user.tenant_id, + user.id, + ) + + if getattr(settings, "SUBSCRIPTION_REQUIRED_FOR_MAPPED_TENANTS", False): + raise HTTPException( + status_code=403, + detail="Your subscription could not be verified. Please sign in again.", + ) + + +_WRITE_METHODS = frozenset({"POST", "PUT", "PATCH", "DELETE"}) + + +def _enforce_subscription_access(db, request, user) -> None: + """ + A lapsed subscription degrades to read-only. It does not lock anybody out. + + `EntitlementService.is_active()` existed, was tested, and enforced nothing — + it was read by two API responses and by nothing else, so a cancelled + subscription restricted precisely nothing. That is the same failure as the + tenant context in B1: machinery built, machinery proven, machinery never + connected. + + **Read-only rather than refused**, ported from the base and matching how + mature systems behave. An expired customer keeps access to their own + documents and loses the ability to create more. Locking them out makes + export impossible and is bad for renewals: the customer most likely to come + back is the one who can still see what they would be coming back to. + + Off by default, like every other enforcement added in this work. Turning it + on is a commercial decision, and it should be made after somebody has looked + at how many tenants are currently in a lapsed state. + """ + if not getattr(settings, "SUBSCRIPTION_ENFORCEMENT_ENABLED", False): + return + if request.method not in _WRITE_METHODS: + return + + tenant = getattr(user, "tenant", None) + if tenant is None: + return + + from app.modules.billing.services.entitlement_service import EntitlementService + + level = EntitlementService(db).access_level(tenant) + if level == "full": + return + + raise HTTPException( + status_code=402, + detail=( + "Your subscription has lapsed. You can still read and export your " + "documents; renewing restores the ability to make changes." + ), + ) + + +def _attach_resolved_access(db, user) -> None: + """ + Give the user object a way to answer "what may I do" — **lazily**. + + Before this existed there were two answers to that question. The gate + (`require_access` -> `PermissionService`) unioned the legacy `users.role_id` + with every live `user_roles` grant. The *response* (`UserOut.accesses` -> + `User.access_codes`) read the legacy role alone. So granting somebody a + second role gave them API authority the interface would not render — no menu + entry, no route, no button — and multi-role looked broken while being + enforced correctly. + + **Closures, not results.** Resolving eagerly here cost three queries on every + authenticated request, including the many that never ask — `/api/storage/usage` + neither gates on an access code nor serialises the user, and it went from 20 + queries to 23 for an answer nobody read. Attaching the resolvers instead + means the cost lands only where the question is asked, and + `app/core/request_cache.py` makes the second asker free. Net effect on a + *gated* endpoint is a reduction: `PermissionService` used to issue three + queries per `require_access` dependency with no memo at all. + + **Closures and not the session**, because `User` is a model and + `.importlinter` forbids models from importing services — including inside a + function body, which import-linter reads. Keeping the import here is what + keeps the ORM registry independent of request handling. + + **Runs after `set_context`.** Both `user_roles` and `roles` are tenant-owned, + so with `TENANT_FILTER_ENABLED` resolution must happen under the correct + tenant context or it silently returns nothing — which would present as "the + user lost all permissions" rather than as an error. + """ + from app.modules.auth.services.permission_service import PermissionService + from app.modules.auth.services.user_role_service import UserRoleReader + + setattr( + user, + "_resolve_access_codes", + lambda: PermissionService(db).user_access_codes(user), + ) + setattr( + user, + "_resolve_role_refs", + lambda: UserRoleReader(db).assignments(user), + ) + + +def get_current_user( + request: Request, + db: Session = Depends(get_db), + token: Optional[str] = Depends(oauth2_scheme), +) -> User: + token_candidates = [] + if token: + token_candidates.append(token) + if request.cookies.get("docqube_access_token"): + token_candidates.append(request.cookies.get("docqube_access_token")) + if request.cookies.get("access_token"): + token_candidates.append(request.cookies.get("access_token")) + + if not token_candidates: + raise HTTPException(status_code=401, detail="Authentication required") + + for tok in token_candidates: + try: + payload = jwt.decode( + tok, settings.APP_SECRET, algorithms=[settings.ALGORITHM] + ) + jti = payload.get("jti") + if payload.get("type") in ("refresh", "device_management"): + logger.warning(f"Authentication candidate has invalid token type: {payload.get('type')}") + continue + sub = payload.get("sub") or payload.get("id") or payload.get("user_id") or payload.get("email") + tenant_id = payload.get("tenant_id") + + if sub is None: + logger.warning("Authentication candidate is missing a subject claim") + continue + + if jti and TokenBlacklist.is_blacklisted(jti): + logger.warning("AUTH DEBUG: Token blacklisted") + continue + + session_id = payload.get("session_id") + if session_id: + from app.modules.auth.repositories.session_repository import SessionRepository + session_repo = SessionRepository(db) + session = session_repo.get_by_id(session_id) + if not session or session.status != "ACTIVE": + if session and session.status == "REVOKED": + if session.revoked_by == "reauth": + raise HTTPException(status_code=401, detail="You have logged in from another tab on this device. Please refresh.") + else: + raise HTTPException(status_code=401, detail="Your session was remotely logged out from another device.") + raise HTTPException(status_code=401, detail="Your session has expired.") + + from datetime import timedelta + now = datetime.now(timezone.utc) + last_activity = session.last_activity + if last_activity and last_activity.tzinfo is None: + last_activity = last_activity.replace(tzinfo=timezone.utc) + if not last_activity or now - last_activity > timedelta(minutes=5): + session_repo.update_last_activity(session) + + user = None + try: + user_id = int(sub) + user = ( + db.query(User) + .options( + selectinload(User.role) + .selectinload(Role.role_accesses) + .selectinload(RoleAccess.access) + ) + .filter(User.id == user_id) + .first() + ) + except (ValueError, TypeError): + user = ( + db.query(User) + .options( + selectinload(User.role) + .selectinload(Role.role_accesses) + .selectinload(RoleAccess.access) + ) + .filter(User.email == str(sub)) + .first() + ) + + if not user: + logger.warning("AUTH DEBUG: User sub not found in database!") + continue + + if getattr(user, "is_deleted", False) or not getattr(user, "is_active", True): + logger.warning("AUTH DEBUG: User is deleted or inactive") + continue + + if user.tenant_id and tenant_id and str(user.tenant_id) != str(tenant_id): + logger.warning("AUTH DEBUG: Tenant mismatch user.tenant_id vs token") + continue + + tenant = getattr(user, "tenant", None) + if tenant is not None and ( + not getattr(tenant, "is_active", True) + or getattr(tenant, "is_deleted", False) + ): + logger.warning("Tenant %s is not active", user.tenant_id) + raise HTTPException( + status_code=403, + detail="This workspace has been suspended. Contact support.", + ) + + _enforce_subscription_access(db, request, user) + + saas_permissions = payload.get("saas_permissions", []) + setattr(user, "saas_permissions", set(saas_permissions)) + saas_subscription = payload.get("saas_subscription") + _validate_saas_subscription(saas_subscription) + _check_missing_subscription_claim(db, user, saas_subscription) + setattr(user, "saas_subscription", saas_subscription) + + set_context( + user.tenant_id, + is_super=bool(getattr(user, "is_superadmin", False)), + ) + + _attach_resolved_access(db, user) + return user + + except JWTError as err: + logger.warning("Authentication candidate JWT decoding failed: %s", type(err).__name__) + continue + + logger.warning("AUTH DEBUG: All %d token candidates failed authentication!", len(token_candidates)) + raise HTTPException(status_code=401, detail="Authentication required") diff --git a/app/middleware/csrf.py b/app/middleware/csrf.py new file mode 100644 index 0000000..80bd8d7 --- /dev/null +++ b/app/middleware/csrf.py @@ -0,0 +1,58 @@ +import uuid +from starlette.middleware.base import BaseHTTPMiddleware +from fastapi import Request +from fastapi.responses import JSONResponse +from app.core.settings import settings + +class CSRFMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + csrf_cookie = request.cookies.get("csrf_token") + csrf_header = request.headers.get("x-csrf-token") or request.headers.get("X-CSRF-Token") + + needs_validation = ( + request.method in ["POST", "PUT", "PATCH", "DELETE"] and + not request.url.path.startswith("/api/auth/") and + not request.url.path.startswith("/api/sso/") and + not request.url.path == "/api/me/tutorial/finish" and + not "/signing/ocr" in request.url.path and + not "webhook" in request.url.path + ) + + if needs_validation and not settings.DISABLE_CSRF: + if not csrf_cookie or not csrf_header or csrf_cookie != csrf_header: + import logging + logger = logging.getLogger(__name__) + logger.warning( + f"CSRF Failure on {request.url.path}: " + f"cookie_present={bool(csrf_cookie)}, " + f"header_present={bool(csrf_header)}, " + f"match={csrf_cookie == csrf_header}" + ) + + response = JSONResponse( + status_code=403, + content={"detail": "CSRF verification failed"} + ) + if not csrf_cookie: + self._set_csrf_cookie(response) + return response + + response = await call_next(request) + + if not csrf_cookie: + self._set_csrf_cookie(response) + + return response + + def _set_csrf_cookie(self, response): + """Helper to set the CSRF cookie with appropriate security flags.""" + is_prod = settings.APP_ENV == "production" + response.set_cookie( + key="csrf_token", + value=str(uuid.uuid4()), + httponly=False, + samesite="none" if is_prod else "lax", + secure=is_prod, + domain=settings.COOKIE_DOMAIN if is_prod else None, + path="/" + ) \ No newline at end of file diff --git a/app/middleware/ip_middleware.py b/app/middleware/ip_middleware.py new file mode 100644 index 0000000..ffdf738 --- /dev/null +++ b/app/middleware/ip_middleware.py @@ -0,0 +1,16 @@ +from starlette.middleware.base import BaseHTTPMiddleware +from fastapi import Request +from app.core.context import client_ip_ctx_var, user_agent_ctx_var + +class IPMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + x_forwarded_for = request.headers.get("X-Forwarded-For") + if x_forwarded_for: + ip = x_forwarded_for.split(",")[0].strip() + else: + ip = request.client.host if request.client else None + + client_ip_ctx_var.set(ip) + user_agent_ctx_var.set(request.headers.get("User-Agent")) + response = await call_next(request) + return response \ No newline at end of file diff --git a/app/middleware/rbac.py b/app/middleware/rbac.py new file mode 100644 index 0000000..fe9659f --- /dev/null +++ b/app/middleware/rbac.py @@ -0,0 +1,54 @@ +from fastapi import Depends, HTTPException +from sqlalchemy.orm import Session + +from app.db.database import get_db +from app.middleware.auth import get_current_user +from app.modules.drive.models.drive_model import DriveACL + + +ROLE_HIERARCHY = { + "owner": 4, + "editor": 3, + "commenter": 2, + "viewer": 1, +} + + +def require_role(resource_type: str, minimum_role: str): + def checker( + resource_id: int, + db: Session = Depends(get_db), + user=Depends(get_current_user), + ): + + acl = ( + db.query(DriveACL) + .filter( + DriveACL.resource_type == resource_type, + DriveACL.resource_id == resource_id, + DriveACL.subject_type == "user", + DriveACL.subject_id == user.id, + ) + .first() + ) + + if not acl: + raise HTTPException(status_code=403, detail="Access denied") + + user_level = ROLE_HIERARCHY.get(acl.role) + min_level = ROLE_HIERARCHY.get(minimum_role) + + if user_level is None or min_level is None: + import logging + logging.getLogger(__name__).error( + "rbac: unknown role encountered — acl.role=%r, minimum_role=%r", + acl.role, minimum_role, + ) + raise HTTPException(status_code=403, detail="Access denied") + + if user_level < min_level: + raise HTTPException(status_code=403, detail="Insufficient permission") + + return True + + return checker diff --git a/app/middleware/security_headers.py b/app/middleware/security_headers.py new file mode 100644 index 0000000..d3b1ee6 --- /dev/null +++ b/app/middleware/security_headers.py @@ -0,0 +1,15 @@ +from starlette.middleware.base import BaseHTTPMiddleware +from fastapi import Request + + +class SecurityHeadersMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + response = await call_next(request) + + response.headers["X-Content-Type-Options"] = "nosniff" + response.headers["X-Frame-Options"] = "DENY" + response.headers["X-XSS-Protection"] = "1; mode=block" + response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin" + response.headers["Content-Security-Policy"] = "default-src 'self'" + + return response diff --git a/app/middleware/tenant.py b/app/middleware/tenant.py new file mode 100644 index 0000000..21ad6c8 --- /dev/null +++ b/app/middleware/tenant.py @@ -0,0 +1,111 @@ +from typing import Optional +from uuid import UUID +from fastapi import Request, Depends, HTTPException, status +from sqlalchemy.orm import Session + +from app.db.database import get_db +from app.modules.auth.models.user_model import User +from app.middleware.auth import get_current_user +from app.modules.tenant.models.tenant_model import Tenant + + +def get_tenant_from_header( + request: Request, db: Session = Depends(get_db) +) -> Optional[Tenant]: + """ + Extracts X-Tenant-ID from the request headers and validates it. + Used for public routes like registration where the user is not authenticated yet. + """ + tenant_id_str = request.headers.get("X-Tenant-ID") + if not tenant_id_str: + default_tenant = db.query(Tenant).filter(Tenant.slug == "default").first() + return default_tenant + + try: + tenant_id = UUID(tenant_id_str) + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid X-Tenant-ID format" + ) + + tenant = db.query(Tenant).filter(Tenant.id == tenant_id).first() + if not tenant: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Tenant not found" + ) + + if not tenant.is_active: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, detail="Tenant is currently inactive" + ) + + return tenant + + +def get_tenant_id(user: User = Depends(get_current_user)) -> Optional[UUID]: + """ + Extracts tenant_id from the authenticated user context. + Returns None for superadmins. + """ + return user.tenant_id + + +def is_superadmin(user: User = Depends(get_current_user)) -> bool: + """ + Checks whether the user is a superadmin. + + Reads the explicit `is_superadmin` flag, and nothing else. + + It used to also accept `tenant_id IS NULL`. That made the *absence* of + tenant context a grant of authority: any bug that dropped the tenant did not + deny, it escalated. The fallback was kept for exactly one release so that + reverting the B1.0 migration could not lock the operators out mid-deploy; + that release has now shipped and this is its removal. + + **Deploy order matters.** This change assumes `users.is_superadmin` exists + and has been backfilled — that is migration `b1_0_explicit_superadmin`, + which sets the flag for every active user with no tenant. Ship this without + that migration and every superadmin loses access. `scripts/b1_preflight.py` + reports anyone still relying on the old signal before you find out the hard + way. + + A user who loses tenant context now has no tenant and no privilege, which is + the whole point. + """ + return bool(getattr(user, "is_superadmin", False)) + + +def require_superadmin(user: User = Depends(get_current_user)) -> User: + """ + Dependency that enforces superadmin access. + Returns the user if superadmin, raises HTTP 403 otherwise. + """ + if not is_superadmin(user): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Superadmin access required in the Document Vault", + ) + return user + + +def require_tenant(request: Request, user: User = Depends(get_current_user)) -> UUID: + """ + Dependency that enforces tenant context. + Superadmins can provide X-Tenant-ID to simulate tenant context. + """ + if user.tenant_id is None: + tenant_id_str = request.headers.get("X-Tenant-ID") + if not tenant_id_str: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Superadmins must provide X-Tenant-ID header to perform this operation", + ) + try: + return UUID(tenant_id_str) + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid X-Tenant-ID format", + ) + + return user.tenant_id diff --git a/app/middleware/tenant_context_middleware.py b/app/middleware/tenant_context_middleware.py new file mode 100644 index 0000000..65ebc40 --- /dev/null +++ b/app/middleware/tenant_context_middleware.py @@ -0,0 +1,133 @@ +""" +B1.4 — establish the tenant context for every request. + +This is the wiring the rest of B1 was waiting on, and its absence was the single +thing standing between "isolation is built" and "isolation works". The ORM +listener (`app/core/tenant_filter.py`) and the RLS session variables +(`app/core/rls.py`) both read `current_tenant_id()`. Both were built, tested and +correct. Nothing in the application ever set it — the only callers of +`scoped_to()` in the repository were the tests, which supply their own context +and therefore proved the machinery while never exercising the wiring. + +The consequence was quiet, because the flag defaults off: every request ran with +no context, both layers stayed inert, and the application behaved exactly as it +always had. Turning `TENANT_FILTER_ENABLED` on would have failed closed on every +query in the system. + +**Why middleware and not a dependency.** `get_current_user` is a `def`, so +FastAPI runs it in a threadpool, and a `ContextVar` set inside a worker thread +does not propagate back to the request's task — the write is simply lost at the +thread boundary. Middleware runs in the request's own task, so what it sets is +visible to every dependency and handler that follows, including the sync ones +(the context is *copied into* the threadpool, which is all a reader needs). + +**Why raw ASGI and not `BaseHTTPMiddleware`.** `BaseHTTPMiddleware` runs the +downstream app in a separate task and pumps the response through a queue, which +breaks streaming: this application serves file downloads through +`StreamingResponse` and has a Server-Sent Events endpoint on +`text/event-stream`, and SSE under that middleware stops arriving incrementally. +A plain ASGI callable adds no task, no queue and no buffering — it sets a +context variable and gets out of the way. + +**Why the token and not the database.** Reading the user here would mean a query +per request before any route is chosen, and that query would itself need the +context it is trying to establish. The token is signature-verified before its +claims are read, and `get_current_user` independently checks the tenant claim +against the loaded row, rejecting any mismatch — so a forged claim fails there +rather than being trusted here. +""" + +import logging +import uuid + +from jose import JWTError, jwt +from starlette.datastructures import Headers +from starlette.types import ASGIApp, Message, Receive, Scope, Send + +from app.core.request_cache import reset_request_cache +from app.core.scope import reset_scope_cache +from app.core.settings import settings +from app.core.tenant_context import reset_context, set_context + +logger = logging.getLogger(__name__) + +_COOKIE_NAMES = ("docqube_access_token", "access_token") + + +def _token_from(headers: Headers) -> str | None: + authorization = headers.get("authorization") or "" + if authorization.lower().startswith("bearer "): + candidate = authorization[7:].strip() + if candidate: + return candidate + + cookie_header = headers.get("cookie") + if not cookie_header: + return None + for part in cookie_header.split(";"): + name, _, value = part.strip().partition("=") + if name in _COOKIE_NAMES and value: + return value + return None + + +def _context_from_token(token: str) -> tuple[uuid.UUID | None, bool] | None: + """The tenant and privilege a verified token asserts, or None if unusable.""" + try: + claims = jwt.decode(token, settings.APP_SECRET, algorithms=[settings.ALGORITHM]) + except JWTError: + return None + + if claims.get("type") in ("refresh", "device_management"): + return None + + tenant_id = None + raw = claims.get("tenant_id") + if raw: + try: + tenant_id = uuid.UUID(str(raw)) + except (ValueError, AttributeError, TypeError): + logger.warning("Token carried an unparseable tenant_id") + + is_super = bool(claims.get("is_superadmin", False)) + + if tenant_id is None and not is_super: + return None + return tenant_id, is_super + + +class TenantContextMiddleware: + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] not in ("http", "websocket"): + await self.app(scope, receive, send) + return + + reset_scope_cache() + reset_request_cache() + + tokens = None + raw_token = _token_from(Headers(scope=scope)) + if raw_token: + resolved = _context_from_token(raw_token) + if resolved is not None: + tenant_id, is_super = resolved + tokens = set_context(tenant_id, is_super=is_super) + + try: + await self.app(scope, receive, send) + finally: + if tokens is not None: + try: + reset_context(tokens) + except ValueError: + pass + + +def install(app: ASGIApp) -> None: + app.add_middleware(TenantContextMiddleware) + + +__all__ = ["TenantContextMiddleware", "install"] diff --git a/app/modules/activity_logs/__init__.py b/app/modules/activity_logs/__init__.py new file mode 100644 index 0000000..4ed6c2e --- /dev/null +++ b/app/modules/activity_logs/__init__.py @@ -0,0 +1,3 @@ +from app.modules.activity_logs.service import log_drive_event, log_event, log_user_event + +__all__ = ["log_event", "log_drive_event", "log_user_event"] diff --git a/app/modules/activity_logs/constants.py b/app/modules/activity_logs/constants.py new file mode 100644 index 0000000..4685c9b --- /dev/null +++ b/app/modules/activity_logs/constants.py @@ -0,0 +1,67 @@ +from enum import Enum + + +class ActivityLogModule(str, Enum): + DRIVE = "drive" + EDITOR = "editor" + CONVERSION = "conversion" + SIGNING = "signing" + COLLABORATION = "collaboration" + ACCESS = "access" + TENANT = "tenant" + + +class ActivityLogStatus(str, Enum): + SUCCESS = "success" + FAILURE = "failure" + + +class ActivityLogTargetType(str, Enum): + DOCUMENT = "document" + FOLDER = "folder" + VERSION = "version" + SESSION = "session" + TENANT = "tenant" + SIGNING_REQUEST = "signing_request" + ROLE = "role" + USER = "user" + ORG_UNIT = "org_unit" + GROUP = "group" + + +class ActivityLogAction(str, Enum): + FOLDER_CREATED = "folder_created" + FOLDER_DELETED = "folder_deleted" + DOCUMENT_UPLOADED = "document_uploaded" + DOCUMENT_DELETED = "document_deleted" + DOCUMENT_MOVED = "document_moved" + DOCUMENT_SHARED = "document_shared" + ACCESS_REVOKED = "access_revoked" + DOCUMENT_EDITED = "document_edited" + CONVERSION_COMPLETED = "conversion_completed" + DOCUMENT_SIGNED = "document_signed" + DOCUMENT_EMAILED = "document_emailed" + SIGN_REQUEST_SENT = "sign_request_sent" + CONFIG_SMTP_UPDATED = "config_smtp_updated" + CONFIG_STORAGE_UPDATED = "config_storage_updated" + CONFIG_COMPRESSION_UPDATED = "config_compression_updated" + CONFIG_SIGNING_UPDATED = "config_signing_updated" + CONTACT_ADDED = "contact_added" + CONTACT_UPDATED = "contact_updated" + CONTACT_DELETED = "contact_deleted" + DEVICE_LOGOUT = "device_logout" + DEVICE_LOGIN = "device_login" + ROLE_CREATED = "role_created" + USER_DEACTIVATED = "user_deactivated" + SIGN_REMINDER_SENT = "sign_reminder_sent" + + ROLE_GRANTED = "role_granted" + ROLE_REVOKED = "role_revoked" + ROLE_EXPIRED = "role_expired" + ORG_UNIT_CREATED = "org_unit_created" + ORG_UNIT_RENAMED = "org_unit_renamed" + ORG_UNIT_MOVED = "org_unit_moved" + ORG_UNIT_DELETED = "org_unit_deleted" + GROUP_CREATED = "group_created" + GROUP_DELETED = "group_deleted" + DOCUMENT_SIGNATURE_BROKEN = "document_signature_broken" diff --git a/app/modules/activity_logs/models/activity_log.py b/app/modules/activity_logs/models/activity_log.py new file mode 100644 index 0000000..dd65dbc --- /dev/null +++ b/app/modules/activity_logs/models/activity_log.py @@ -0,0 +1,50 @@ +import uuid +from datetime import datetime +from typing import Optional + +from sqlalchemy import DateTime, ForeignKey, Index, Integer, String, func, text +from sqlalchemy.dialects.postgresql import JSONB, UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.db.database import Base + + +class ActivityLog(Base): + __tablename__ = "activity_logs" + __table_args__ = ( + Index("ix_activity_logs_tenant_created_at", "tenant_id", "created_at"), + Index("ix_activity_logs_user_created_at", "user_id", "created_at"), + Index("ix_activity_logs_module_action", "module", "action"), + Index("ix_activity_logs_target_id", "target_id"), + Index( + "ix_activity_logs_access_timeline", + "tenant_id", + "created_at", + postgresql_where=text("module = 'access'"), + ), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + tenant_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("tenants.id"), nullable=False + ) + user_id: Mapped[Optional[int]] = mapped_column( + ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + user_name: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + user_email: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + module: Mapped[str] = mapped_column(String(50), nullable=False) + action: Mapped[str] = mapped_column(String(100), nullable=False) + target_id: Mapped[str] = mapped_column(String(255), nullable=False) + target_type: Mapped[str] = mapped_column(String(50), nullable=False) + target_name: Mapped[Optional[str]] = mapped_column(String(512), nullable=True) + event_metadata: Mapped[dict] = mapped_column("metadata", JSONB, nullable=False) + ip_address: Mapped[Optional[str]] = mapped_column(String(64), nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + request_id: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + status: Mapped[str] = mapped_column(String(20), nullable=False) + + tenant = relationship("Tenant") + user = relationship("User", foreign_keys=[user_id]) diff --git a/app/modules/activity_logs/routes.py b/app/modules/activity_logs/routes.py new file mode 100644 index 0000000..a775b13 --- /dev/null +++ b/app/modules/activity_logs/routes.py @@ -0,0 +1,112 @@ +from datetime import datetime +from typing import Optional +from uuid import UUID + +from fastapi import APIRouter, Depends, Query +from sqlalchemy.orm import Session + +from app.db.database import get_db +from app.middleware.auth import get_current_user +from app.middleware.tenant import require_tenant +from app.modules.activity_logs.schemas import ActivityLogUserOption, PaginatedActivityLogsOut +from app.modules.activity_logs.service import list_activity_logs, search_activity_log_users +from app.modules.auth.dependencies.access_dependency import require_access +from app.modules.auth.models.user_model import User + +router = APIRouter(prefix="/api/activity-logs", tags=["Activity Logs"]) + + +@router.get( + "/me", + response_model=PaginatedActivityLogsOut, +) +def get_my_activity_logs( + page: int = Query(1, ge=1), + page_size: int = Query(20, ge=1, le=100), + module: Optional[str] = Query(None), + action: Optional[str] = Query(None), + status: Optional[str] = Query(None), + target_id: Optional[str] = Query(None), + target_name: Optional[str] = Query(None), + target_type: Optional[str] = Query(None), + date_from: Optional[datetime] = Query(None), + date_to: Optional[datetime] = Query(None), + search: Optional[str] = Query(None), + tenant_id: UUID = Depends(require_tenant), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + return list_activity_logs( + db=db, + tenant_id=tenant_id, + page=page, + page_size=page_size, + user_id=current_user.id, + module=module, + action=action, + status=status, + target_id=target_id, + target_name=target_name, + target_type=target_type, + date_from=date_from, + date_to=date_to, + search=search, + ) + +@router.get( + "", + response_model=PaginatedActivityLogsOut, + dependencies=[require_access("admin.logs.read")], +) +def get_activity_logs( + page: int = Query(1, ge=1), + page_size: int = Query(20, ge=1, le=100), + user_id: Optional[int] = Query(None), + module: Optional[str] = Query(None), + action: Optional[str] = Query(None), + status: Optional[str] = Query(None), + target_id: Optional[str] = Query(None), + target_name: Optional[str] = Query(None), + target_type: Optional[str] = Query(None), + date_from: Optional[datetime] = Query(None), + date_to: Optional[datetime] = Query(None), + search: Optional[str] = Query(None), + tenant_id: UUID = Depends(require_tenant), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + return list_activity_logs( + db=db, + tenant_id=tenant_id, + page=page, + page_size=page_size, + user_id=user_id, + module=module, + action=action, + status=status, + target_id=target_id, + target_name=target_name, + target_type=target_type, + date_from=date_from, + date_to=date_to, + search=search, + ) + +@router.get( + "/users/search", + response_model=list[ActivityLogUserOption], + dependencies=[require_access("admin.logs.read")], +) +def search_activity_log_users_route( + q: str = Query(..., min_length=1), + limit: int = Query(10, ge=1, le=50), + tenant_id: UUID = Depends(require_tenant), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + return search_activity_log_users( + db=db, + tenant_id=tenant_id, + query=q, + limit=limit, + ) diff --git a/app/modules/activity_logs/schemas.py b/app/modules/activity_logs/schemas.py new file mode 100644 index 0000000..2090092 --- /dev/null +++ b/app/modules/activity_logs/schemas.py @@ -0,0 +1,227 @@ +from datetime import datetime +from typing import Any +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, EmailStr, Field, field_serializer, model_validator + +from app.modules.activity_logs.constants import ( + ActivityLogAction, + ActivityLogModule, + ActivityLogStatus, + ActivityLogTargetType, +) + + +class FolderCreatedDeletedMetadata(BaseModel): + parent_folder_id: int | None + + +class DocumentUploadedMetadata(BaseModel): + file_size: int = Field(ge=0) + file_type: str + folder_id: int + + +class DocumentDeletedMetadata(BaseModel): + folder_id: int + + +class DocumentMovedMetadata(BaseModel): + from_folder_id: int + to_folder_id: int + from_folder: str + to_folder: str + + +class DocumentSharedMetadata(BaseModel): + shared_with: EmailStr + permission: str + entity_type: str + + +class AccessRevokedMetadata(BaseModel): + revoked_from: EmailStr + + +class DocumentEditedMetadata(BaseModel): + version_number: int = Field(ge=1) + user_agent: str | None = None + + + +class ConversionCompletedMetadata(BaseModel): + from_: str = Field(alias="from") + to: str + + model_config = ConfigDict(populate_by_name=True) + + +class DocumentSignedMetadata(BaseModel): + signed_by: EmailStr + signed_at: datetime + + +class DocumentEmailedMetadata(BaseModel): + to_emails: list[str] + cc_emails: list[str] = Field(default_factory=list) + + +class SignRequestSentMetadata(BaseModel): + recipients: list[str] + total_signers: int + provider: str + + +class ConfigUpdatedMetadata(BaseModel): + updated_by: str | None = None + config_type: str + + +class ContactActionMetadata(BaseModel): + contact_email: str + contact_name: str | None = None + + +class DeviceLogoutMetadata(BaseModel): + device_name: str + ip_address: str | None = None + + +class DeviceLoginMetadata(BaseModel): + pass + + +class RoleCreatedMetadata(BaseModel): + role_name: str + + +class UserDeactivatedMetadata(BaseModel): + deactivated_user_id: int + deactivated_user_email: str + + +class SignReminderMetadata(BaseModel): + recipient_email: str + request_id: str + + +ACTION_METADATA_MODELS = { + ActivityLogAction.FOLDER_CREATED: FolderCreatedDeletedMetadata, + ActivityLogAction.FOLDER_DELETED: FolderCreatedDeletedMetadata, + ActivityLogAction.DOCUMENT_UPLOADED: DocumentUploadedMetadata, + ActivityLogAction.DOCUMENT_DELETED: DocumentDeletedMetadata, + ActivityLogAction.DOCUMENT_MOVED: DocumentMovedMetadata, + ActivityLogAction.DOCUMENT_SHARED: DocumentSharedMetadata, + ActivityLogAction.ACCESS_REVOKED: AccessRevokedMetadata, + ActivityLogAction.DOCUMENT_EDITED: DocumentEditedMetadata, + ActivityLogAction.DOCUMENT_SIGNATURE_BROKEN: DocumentEditedMetadata, + ActivityLogAction.CONVERSION_COMPLETED: ConversionCompletedMetadata, + ActivityLogAction.DOCUMENT_SIGNED: DocumentSignedMetadata, + ActivityLogAction.DOCUMENT_EMAILED: DocumentEmailedMetadata, + ActivityLogAction.SIGN_REQUEST_SENT: SignRequestSentMetadata, + ActivityLogAction.CONFIG_SMTP_UPDATED: ConfigUpdatedMetadata, + ActivityLogAction.CONFIG_STORAGE_UPDATED: ConfigUpdatedMetadata, + ActivityLogAction.CONFIG_COMPRESSION_UPDATED: ConfigUpdatedMetadata, + ActivityLogAction.CONFIG_SIGNING_UPDATED: ConfigUpdatedMetadata, + ActivityLogAction.CONTACT_ADDED: ContactActionMetadata, + ActivityLogAction.CONTACT_UPDATED: ContactActionMetadata, + ActivityLogAction.CONTACT_DELETED: ContactActionMetadata, + ActivityLogAction.DEVICE_LOGOUT: DeviceLogoutMetadata, + ActivityLogAction.DEVICE_LOGIN: DeviceLoginMetadata, + ActivityLogAction.ROLE_CREATED: RoleCreatedMetadata, + ActivityLogAction.USER_DEACTIVATED: UserDeactivatedMetadata, + ActivityLogAction.SIGN_REMINDER_SENT: SignReminderMetadata, +} + +ACTION_MODULES = { + ActivityLogAction.FOLDER_CREATED: ActivityLogModule.DRIVE, + ActivityLogAction.FOLDER_DELETED: ActivityLogModule.DRIVE, + ActivityLogAction.DOCUMENT_UPLOADED: ActivityLogModule.DRIVE, + ActivityLogAction.DOCUMENT_DELETED: ActivityLogModule.DRIVE, + ActivityLogAction.DOCUMENT_MOVED: ActivityLogModule.DRIVE, + ActivityLogAction.DOCUMENT_SHARED: ActivityLogModule.DRIVE, + ActivityLogAction.ACCESS_REVOKED: ActivityLogModule.DRIVE, + ActivityLogAction.DOCUMENT_EDITED: ActivityLogModule.EDITOR, + ActivityLogAction.DOCUMENT_SIGNATURE_BROKEN: ActivityLogModule.EDITOR, + ActivityLogAction.CONVERSION_COMPLETED: ActivityLogModule.CONVERSION, + ActivityLogAction.DOCUMENT_SIGNED: ActivityLogModule.SIGNING, + ActivityLogAction.DOCUMENT_EMAILED: ActivityLogModule.DRIVE, + ActivityLogAction.SIGN_REQUEST_SENT: ActivityLogModule.SIGNING, + ActivityLogAction.CONFIG_SMTP_UPDATED: ActivityLogModule.TENANT, + ActivityLogAction.CONFIG_STORAGE_UPDATED: ActivityLogModule.TENANT, + ActivityLogAction.CONFIG_COMPRESSION_UPDATED: ActivityLogModule.TENANT, + ActivityLogAction.CONFIG_SIGNING_UPDATED: ActivityLogModule.TENANT, + ActivityLogAction.CONTACT_ADDED: ActivityLogModule.TENANT, + ActivityLogAction.CONTACT_UPDATED: ActivityLogModule.TENANT, + ActivityLogAction.CONTACT_DELETED: ActivityLogModule.TENANT, + ActivityLogAction.DEVICE_LOGOUT: ActivityLogModule.TENANT, + ActivityLogAction.DEVICE_LOGIN: ActivityLogModule.TENANT, + ActivityLogAction.ROLE_CREATED: ActivityLogModule.TENANT, + ActivityLogAction.USER_DEACTIVATED: ActivityLogModule.TENANT, + ActivityLogAction.SIGN_REMINDER_SENT: ActivityLogModule.SIGNING, +} + + +class ActivityLogCreate(BaseModel): + tenant_id: UUID + user_id: int | None = None + user_name: str | None = None + user_email: EmailStr | None = None + module: ActivityLogModule + action: ActivityLogAction + target_id: str + target_type: ActivityLogTargetType + target_name: str | None = None + metadata: dict[str, Any] + ip_address: str | None = None + request_id: str | None = None + status: ActivityLogStatus + + @model_validator(mode="after") + def validate_action_contract(self) -> "ActivityLogCreate": + expected_module = ACTION_MODULES[self.action] + if self.module != expected_module: + raise ValueError( + f"Action {self.action.value} must use module {expected_module.value}" + ) + + metadata_model = ACTION_METADATA_MODELS[self.action] + validated = metadata_model.model_validate(self.metadata) + self.metadata = validated.model_dump(mode="json", by_alias=True) + return self + + +class ActivityLogOut(BaseModel): + id: int + tenant_id: UUID + user_id: int | None = None + user_name: str | None = None + user_email: str | None = None + module: str + action: str + target_id: str + target_type: str + target_name: str | None = None + metadata: dict[str, Any] + ip_address: str | None = None + created_at: datetime + request_id: str | None = None + status: str + description: str + + @field_serializer("tenant_id") + def serialize_tenant_id(self, tenant_id: UUID) -> str: + return str(tenant_id) + + +class PaginatedActivityLogsOut(BaseModel): + items: list[ActivityLogOut] + total: int + page: int + page_size: int + total_pages: int + +class ActivityLogUserOption(BaseModel): + user_id: int + user_name: str | None = None + user_email: str | None = None \ No newline at end of file diff --git a/app/modules/activity_logs/service.py b/app/modules/activity_logs/service.py new file mode 100644 index 0000000..3d8f5e5 --- /dev/null +++ b/app/modules/activity_logs/service.py @@ -0,0 +1,339 @@ +import logging +from datetime import datetime +from math import ceil +from typing import Any + +from sqlalchemy import func, or_ +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm import Session + +from app.core.context import client_ip_ctx_var +from app.db.database import SessionLocal +from app.modules.activity_logs.constants import ( + ActivityLogAction, + ActivityLogModule, + ActivityLogStatus, + ActivityLogTargetType, +) +from app.modules.activity_logs.models.activity_log import ActivityLog +from app.modules.activity_logs.schemas import ( + ActivityLogCreate, + ActivityLogOut, + ActivityLogUserOption, + PaginatedActivityLogsOut, +) + +logger = logging.getLogger(__name__) + + +def log_event( + *, + tenant_id, + user_id=None, + user_name=None, + user_email=None, + module, + action, + target_id, + target_type, + target_name=None, + metadata: dict[str, Any], + ip_address=None, + request_id=None, + status, +) -> int | None: + try: + payload = ActivityLogCreate( + tenant_id=tenant_id, + user_id=user_id, + user_name=user_name, + user_email=user_email, + module=module, + action=action, + target_id=str(target_id), + target_type=target_type, + target_name=target_name, + metadata=metadata, + ip_address=ip_address, + request_id=request_id, + status=status, + ) + except Exception: + logger.exception("Activity log validation failed") + return None + + session = SessionLocal() + try: + row = ActivityLog( + tenant_id=payload.tenant_id, + user_id=payload.user_id, + user_name=payload.user_name, + user_email=payload.user_email, + module=payload.module.value, + action=payload.action.value, + target_id=payload.target_id, + target_type=payload.target_type.value, + target_name=payload.target_name, + event_metadata=payload.metadata, + ip_address=payload.ip_address, + request_id=payload.request_id, + status=payload.status.value, + ) + session.add(row) + session.commit() + session.refresh(row) + return row.id + except SQLAlchemyError: + session.rollback() + logger.exception("Activity log insert failed") + return None + except Exception: + session.rollback() + logger.exception("Unexpected activity log failure") + return None + finally: + session.close() + + +def log_drive_event( + *, + user, + action: ActivityLogAction, + target_id, + target_type: ActivityLogTargetType, + target_name: str | None, + metadata: dict[str, Any], + status: ActivityLogStatus = ActivityLogStatus.SUCCESS, + request_id: str | None = None, +) -> int | None: + return log_event( + tenant_id=user.tenant_id, + user_id=user.id, + user_name=user.name, + user_email=user.email, + module=ActivityLogModule.DRIVE, + action=action, + target_id=target_id, + target_type=target_type, + target_name=target_name, + metadata=metadata, + ip_address=client_ip_ctx_var.get(), + request_id=request_id, + status=status, + ) + + +def log_user_event( + *, + user, + module: ActivityLogModule, + action: ActivityLogAction, + target_id, + target_type: ActivityLogTargetType, + target_name: str | None, + metadata: dict[str, Any], + status: ActivityLogStatus = ActivityLogStatus.SUCCESS, + request_id: str | None = None, +) -> int | None: + return log_event( + tenant_id=user.tenant_id, + user_id=user.id, + user_name=user.name, + user_email=user.email, + module=module, + action=action, + target_id=target_id, + target_type=target_type, + target_name=target_name, + metadata=metadata, + ip_address=client_ip_ctx_var.get(), + request_id=request_id, + status=status, + ) + + +def build_description(action: str, target_name: str | None, metadata: dict[str, Any]) -> str: + quoted_target = f"'{target_name}'" if target_name else "item" + + if action == ActivityLogAction.DOCUMENT_UPLOADED.value: + return f"Document {quoted_target} uploaded" + if action == ActivityLogAction.DOCUMENT_DELETED.value: + return f"Document {quoted_target} deleted" + if action == ActivityLogAction.FOLDER_CREATED.value: + return f"Folder {quoted_target} created" + if action == ActivityLogAction.FOLDER_DELETED.value: + return f"Folder {quoted_target} deleted" + if action == ActivityLogAction.DOCUMENT_MOVED.value: + return ( + f"Document moved from '{metadata.get('from_folder', '')}' " + f"to '{metadata.get('to_folder', '')}'" + ) + if action == ActivityLogAction.DOCUMENT_SHARED.value: + return f"Document {quoted_target} shared with {metadata.get('shared_with', 'recipient')}" + if action == ActivityLogAction.ACCESS_REVOKED.value: + return f"Access revoked from {metadata.get('revoked_from', 'recipient')} for {quoted_target}" + if action == ActivityLogAction.DOCUMENT_EDITED.value: + return ( + f"Document {quoted_target} edited" + f" (version {metadata.get('version_number', 'unknown')})" + ) + if action == ActivityLogAction.CONVERSION_COMPLETED.value: + return ( + f"Document {quoted_target} converted from " + f"{metadata.get('from', 'unknown')} to {metadata.get('to', 'unknown')}" + ) + if action == ActivityLogAction.DOCUMENT_SIGNED.value: + return f"Document signed by {metadata.get('signed_by', 'unknown signer')}" + if action == ActivityLogAction.DOCUMENT_EMAILED.value: + to_emails = metadata.get('to_emails', []) + return f"Document {quoted_target} emailed to {', '.join(to_emails)}" + if action == ActivityLogAction.SIGN_REQUEST_SENT.value: + return f"Sign request sent for {quoted_target} via {metadata.get('provider', 'unknown')}" + if action == ActivityLogAction.CONFIG_SMTP_UPDATED.value: + return "SMTP configuration updated" + if action == ActivityLogAction.CONFIG_STORAGE_UPDATED.value: + return "Storage configuration updated" + if action == ActivityLogAction.CONFIG_COMPRESSION_UPDATED.value: + return "Compression configuration updated" + if action == ActivityLogAction.CONFIG_SIGNING_UPDATED.value: + return "Signing configuration updated" + if action == ActivityLogAction.CONTACT_ADDED.value: + return f"Contact {metadata.get('contact_email', '')} added to address book" + if action == ActivityLogAction.CONTACT_UPDATED.value: + return f"Contact {metadata.get('contact_email', '')} updated in address book" + if action == ActivityLogAction.CONTACT_DELETED.value: + return f"Contact {metadata.get('contact_email', '')} deleted from address book" + if action == ActivityLogAction.DEVICE_LOGOUT.value: + return f"Device {metadata.get('device_name', 'unknown')} logged out" + if action == ActivityLogAction.ROLE_CREATED.value: + return f"Role {metadata.get('role_name', 'unknown')} created" + if action == ActivityLogAction.USER_DEACTIVATED.value: + return f"User {metadata.get('deactivated_user_email', 'unknown')} deactivated" + if action == ActivityLogAction.SIGN_REMINDER_SENT.value: + return f"Sign reminder sent to {metadata.get('recipient_email', 'unknown')}" + + return f"{action.replace('_', ' ').capitalize()} for {quoted_target}" + + +def list_activity_logs( + *, + db: Session, + tenant_id, + page: int, + page_size: int, + user_id: int | None = None, + module: str | None = None, + action: str | None = None, + status: str | None = None, + target_id: str | None = None, + target_name: str | None = None, + target_type: str | None = None, + date_from: datetime | None = None, + date_to: datetime | None = None, + search: str | None = None, +) -> PaginatedActivityLogsOut: + query = db.query(ActivityLog).filter(ActivityLog.tenant_id == tenant_id) + + if user_id is not None: + query = query.filter(ActivityLog.user_id == user_id) + if module: + query = query.filter(ActivityLog.module == module) + if action: + query = query.filter(ActivityLog.action == action) + if status: + query = query.filter(ActivityLog.status == status) + if target_id: + query = query.filter(ActivityLog.target_id == str(target_id)) + if target_name: + query = query.filter(ActivityLog.target_name.ilike(f"%{target_name}%")) + if target_type: + query = query.filter(ActivityLog.target_type == target_type) + if date_from: + query = query.filter(ActivityLog.created_at >= date_from) + if date_to: + query = query.filter(ActivityLog.created_at <= date_to) + if search and search.strip(): + term = f"%{search.strip()}%" + query = query.filter( + or_( + ActivityLog.user_name.ilike(term), + ActivityLog.target_name.ilike(term), + ) + ) + + total = query.count() + items = ( + query.order_by(ActivityLog.created_at.desc(), ActivityLog.id.desc()) + .offset((page - 1) * page_size) + .limit(page_size) + .all() + ) + + rows = [ + ActivityLogOut( + id=row.id, + tenant_id=row.tenant_id, + user_id=row.user_id, + user_name=row.user_name, + user_email=row.user_email, + module=row.module, + action=row.action, + target_id=row.target_id, + target_type=row.target_type, + target_name=row.target_name, + metadata=row.event_metadata, + ip_address=row.ip_address, + created_at=row.created_at, + request_id=row.request_id, + status=row.status, + description=build_description(row.action, row.target_name, row.event_metadata), + ) + for row in items + ] + + return PaginatedActivityLogsOut( + items=rows, + total=total, + page=page, + page_size=page_size, + total_pages=ceil(total / page_size) if total else 0, + ) + +def search_activity_log_users( + *, + db: Session, + tenant_id, + query: str, + limit: int, +) -> list[ActivityLogUserOption]: + term = query.strip() + if not term: + return [] + + rows = ( + db.query( + ActivityLog.user_id, + func.max(ActivityLog.user_name).label("user_name"), + func.max(ActivityLog.user_email).label("user_email"), + ) + .filter( + ActivityLog.tenant_id == tenant_id, + ActivityLog.user_id.isnot(None), + or_( + ActivityLog.user_name.ilike(f"%{term}%"), + ActivityLog.user_email.ilike(f"%{term}%"), + ), + ) + .group_by(ActivityLog.user_id) + .order_by(func.max(ActivityLog.created_at).desc()) + .limit(limit) + .all() + ) + return [ + ActivityLogUserOption( + user_id=row.user_id, + user_name=row.user_name, + user_email=row.user_email, + ) + for row in rows + ] diff --git a/app/modules/auth/controllers/auth_controller.py b/app/modules/auth/controllers/auth_controller.py new file mode 100644 index 0000000..f02ab72 --- /dev/null +++ b/app/modules/auth/controllers/auth_controller.py @@ -0,0 +1,66 @@ +from sqlalchemy.orm import Session +from app.modules.auth.models.user_model import User +from app.modules.auth.services.auth_service import AuthService, UserService +from typing import Any, Optional + + +class AuthController: + @staticmethod + def register_user(payload: Any, tenant: Any, db: Session): + auth_service = AuthService(db) + return auth_service.register(payload, tenant) + + @staticmethod + def login_user(form_data: Any, db: Session, request: Optional[Any] = None): + auth_service = AuthService(db) + return auth_service.login(form_data.username, form_data.password, request) + + @staticmethod + def google_login(payload: Any, db: Session, request: Optional[Any] = None): + auth_service = AuthService(db) + return auth_service.google_login(payload, request) + + @staticmethod + def forgot_password(payload: Any, db: Session): + auth_service = AuthService(db) + return auth_service.forgot_password(payload) + + @staticmethod + def reset_password(payload: Any, db: Session): + auth_service = AuthService(db) + return auth_service.reset_password(payload) + + +class UserController: + @staticmethod + def update_profile( + user: User, + name: Optional[str], + db: Session, + designation: Optional[str] = None, + preferred_language: Optional[str] = None, + ): + user_service = UserService(db) + return user_service.update_profile( + user, + name=name, + designation=designation, + preferred_language=preferred_language, + ) + + @staticmethod + def change_password( + user: User, current_password: str, new_password: str, db: Session + ): + user_service = UserService(db) + return user_service.change_password(user, current_password, new_password) + + @staticmethod + def get_user_stats(user: User, db: Session): + user_service = UserService(db) + return user_service.get_user_stats(user) + + @staticmethod + def delete_account(user: User, db: Session): + user_service = UserService(db) + return user_service.delete_account(user) diff --git a/app/modules/auth/controllers/role_controller.py b/app/modules/auth/controllers/role_controller.py new file mode 100644 index 0000000..4dee9c2 --- /dev/null +++ b/app/modules/auth/controllers/role_controller.py @@ -0,0 +1,78 @@ +""" +Role Controller - handles role management API logic. +""" + +from sqlalchemy.orm import Session +from typing import List, Optional +from uuid import UUID + +from app.modules.auth.models.role_model import Role +from app.modules.auth.schemas.role_schema import ( + RoleCreate, + RoleUpdate, + RoleOut, + RoleWithAccessesOut, + RolePaginatedOut, +) +from app.modules.auth.services.role_service import RoleService + + +class RoleController: + """Controller for role management endpoints.""" + + @staticmethod + def create_role(db: Session, role_data: RoleCreate) -> RoleOut: + """Create a new role.""" + role = RoleService.create_role(db, role_data) + return RoleOut.model_validate(role) + + @staticmethod + def get_all_roles(db: Session, tenant_id: Optional[UUID] = None) -> List[RoleOut]: + """Get all roles.""" + roles = RoleService.get_all_roles(db, tenant_id) + return [RoleOut.model_validate(role) for role in roles] + + @staticmethod + def get_role_by_id(db: Session, role_id: UUID) -> RoleOut: + """Get a role by ID.""" + role = RoleService.get_role_by_id(db, role_id) + return RoleOut.model_validate(role) + + @staticmethod + def get_role_with_accesses(db: Session, role_id: UUID) -> RoleWithAccessesOut: + """Get a role with all its accesses.""" + role = RoleService.get_role_with_accesses(db, role_id) + return RoleWithAccessesOut.model_validate(role) + + @staticmethod + def update_role(db: Session, role_id: UUID, role_data: RoleUpdate) -> RoleOut: + """Update a role.""" + role = RoleService.update_role(db, role_id, role_data) + return RoleOut.model_validate(role) + + @staticmethod + def delete_role(db: Session, role_id: UUID): + """Delete a role.""" + return RoleService.delete_role(db, role_id) + + @staticmethod + def get_roles_paginated( + db: Session, + tenant_id: Optional[UUID] = None, + page: int = 1, + page_size: int = 10, + search: Optional[str] = None, + ) -> RolePaginatedOut: + """Get paginated list of roles.""" + return RoleService.get_roles_paginated( + db=db, + tenant_id=tenant_id, + page=page, + page_size=page_size, + search=search, + ) + + @staticmethod + def assign_role_to_user(db: Session, user_id: int, role_id: UUID): + """Assign a role to a user.""" + return RoleService.assign_role_to_user(db, user_id, role_id) diff --git a/app/modules/auth/dependencies/__init__.py b/app/modules/auth/dependencies/__init__.py new file mode 100644 index 0000000..1c5a202 --- /dev/null +++ b/app/modules/auth/dependencies/__init__.py @@ -0,0 +1,3 @@ +from .access_dependency import require_access, require_any_access, require_all_access + +__all__ = ["require_access", "require_any_access", "require_all_access"] diff --git a/app/modules/auth/dependencies/access_dependency.py b/app/modules/auth/dependencies/access_dependency.py new file mode 100644 index 0000000..ff44baf --- /dev/null +++ b/app/modules/auth/dependencies/access_dependency.py @@ -0,0 +1,91 @@ +""" +Access control dependencies for FastAPI routes. + +These dependencies are used in route definitions to require specific access codes. + +Example: + @router.post("/projects", dependencies=[require_access("project.create")]) + def create_project(user: User = Depends(get_current_user)): + ... +""" + +from typing import List +from fastapi import Depends, HTTPException +from sqlalchemy.orm import Session + +from app.db.database import get_db +from app.middleware.auth import get_current_user +from app.modules.auth.services.permission_service import PermissionService +from app.modules.auth.models.user_model import User + + +def require_access(code: str): + """ + Dependency to require a specific access code. + + Usage in routes: + @router.post("/path", dependencies=[require_access("some.code")]) + def endpoint(...): + ... + + Args: + code: The required access code (e.g. "project.create") + + Returns: + FastAPI Depends wrapper that checks access + """ + + async def check_access( + user: User = Depends(get_current_user), db: Session = Depends(get_db) + ) -> None: + PermissionService(db).require_access(user, code) + + return Depends(check_access) + + +def require_any_access(*codes: str): + """ + Dependency to require at least one of the provided access codes. + + Usage in routes: + @router.post("/path", dependencies=[require_any_access("code1", "code2")]) + def endpoint(...): + ... + + Args: + *codes: One or more access codes (user needs at least one) + + Returns: + FastAPI Depends wrapper that checks access + """ + + async def check_access( + user: User = Depends(get_current_user), db: Session = Depends(get_db) + ) -> None: + PermissionService(db).require_any_access(user, list(codes)) + + return Depends(check_access) + + +def require_all_access(*codes: str): + """ + Dependency to require all of the provided access codes. + + Usage in routes: + @router.post("/path", dependencies=[require_all_access("code1", "code2")]) + def endpoint(...): + ... + + Args: + *codes: One or more access codes (user must have all) + + Returns: + FastAPI Depends wrapper that checks access + """ + + async def check_access( + user: User = Depends(get_current_user), db: Session = Depends(get_db) + ) -> None: + PermissionService(db).require_all_access(user, list(codes)) + + return Depends(check_access) diff --git a/app/modules/auth/middleware/hmac.py b/app/modules/auth/middleware/hmac.py new file mode 100644 index 0000000..f0f88f3 --- /dev/null +++ b/app/modules/auth/middleware/hmac.py @@ -0,0 +1,33 @@ +import hmac +import hashlib +import logging +from fastapi import Request, HTTPException, status +from app.core.settings import settings + +logger = logging.getLogger(__name__) + +async def verify_saas_hmac(request: Request): + """ + Verify HMAC signature from SaaS. + + The signature is calculated as: + HMAC-SHA256(secret, canonical_string) + + where canonical_string is usually user_id=...&email=...&tenant_id=...×tamp=... + """ + signature = request.headers.get("X-Signature") + if not signature: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Missing HMAC signature" + ) + + if not settings.SAAS_TRUST_SECRET: + logger.error("SAAS_TRUST_SECRET is not configured") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Server configuration error" + ) + + + return True diff --git a/app/modules/auth/models/access_model.py b/app/modules/auth/models/access_model.py new file mode 100644 index 0000000..f67a995 --- /dev/null +++ b/app/modules/auth/models/access_model.py @@ -0,0 +1,42 @@ +import uuid +from sqlalchemy import String, DateTime, func, ForeignKey, Index +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship +from typing import Optional +from app.db.database import Base + + +class Access(Base): + __tablename__ = "accesses" + __table_args__ = ( + Index("ix_access_code", "access_code"), + Index("ix_category", "category"), + ) + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + + access_code: Mapped[str] = mapped_column( + String(255), unique=True, nullable=False, index=True + ) + + category: Mapped[str] = mapped_column(String(100), nullable=False, index=True) + + name: Mapped[str] = mapped_column(String(255), nullable=False) + + parent_id: Mapped[Optional[uuid.UUID]] = mapped_column( + ForeignKey("accesses.id", ondelete="CASCADE"), nullable=True, index=True + ) + + created_at: Mapped[DateTime] = mapped_column( + DateTime(timezone=True), server_default=func.now() + ) + + parent = relationship("Access", remote_side=[id], backref="children") + role_accesses = relationship( + "RoleAccess", back_populates="access", cascade="all, delete-orphan" + ) + + def __repr__(self): + return f"" \ No newline at end of file diff --git a/app/modules/auth/models/device_model.py b/app/modules/auth/models/device_model.py new file mode 100644 index 0000000..a7440cc --- /dev/null +++ b/app/modules/auth/models/device_model.py @@ -0,0 +1,43 @@ +import uuid +from datetime import datetime +from sqlalchemy import String, DateTime, ForeignKey, Integer, Boolean, Float, func, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column, relationship +from typing import Optional +from app.db.database import Base + + +class Device(Base): + __tablename__ = "devices" + __table_args__ = ( + UniqueConstraint("user_id", "fingerprint", name="uq_device_user_fingerprint"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + + user_id: Mapped[int] = mapped_column( + Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True + ) + user = relationship("User", backref="devices") + + fingerprint: Mapped[str] = mapped_column(String(255), index=True, nullable=False) + + browser: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + os: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + device_type: Mapped[Optional[str]] = mapped_column(String(50), nullable=True) + user_agent: Mapped[Optional[str]] = mapped_column(String(512), nullable=True) + + ip: Mapped[Optional[str]] = mapped_column(String(50), nullable=True) + country: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + state: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + city: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + latitude: Mapped[Optional[float]] = mapped_column(Float, nullable=True) + longitude: Mapped[Optional[float]] = mapped_column(Float, nullable=True) + + first_login: Mapped[DateTime] = mapped_column(DateTime(timezone=True), default=func.now()) + last_login: Mapped[DateTime] = mapped_column(DateTime(timezone=True), default=func.now()) + last_ip: Mapped[Optional[str]] = mapped_column(String(50), nullable=True) + + is_blocked: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) + + created_at: Mapped[DateTime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + updated_at: Mapped[DateTime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) diff --git a/app/modules/auth/models/role_access_model.py b/app/modules/auth/models/role_access_model.py new file mode 100644 index 0000000..38dd793 --- /dev/null +++ b/app/modules/auth/models/role_access_model.py @@ -0,0 +1,21 @@ +from sqlalchemy import ForeignKey, PrimaryKeyConstraint +from sqlalchemy.orm import Mapped, mapped_column, relationship +from app.db.database import Base +from sqlalchemy.dialects.postgresql import UUID +import uuid + +class RoleAccess(Base): + __tablename__ = "role_accesses" + __table_args__ = ( + PrimaryKeyConstraint("role_id", "access_id", name="pk_role_accesses"), + ) + + role_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("roles.id", ondelete="CASCADE"), nullable=False + ) + + access_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("accesses.id", ondelete="CASCADE"), nullable=False, index=True) + + role = relationship("Role", back_populates="role_accesses") + access = relationship("Access", back_populates="role_accesses") \ No newline at end of file diff --git a/app/modules/auth/models/role_model.py b/app/modules/auth/models/role_model.py new file mode 100644 index 0000000..3116738 --- /dev/null +++ b/app/modules/auth/models/role_model.py @@ -0,0 +1,45 @@ +import uuid +from sqlalchemy import String, DateTime, ForeignKey, UniqueConstraint, func, Boolean +from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.dialects.postgresql import UUID +from typing import Optional, List +from app.db.database import Base + + +class Role(Base): + __tablename__ = "roles" + __table_args__ = ( + UniqueConstraint("tenant_id", "name", name="uq_role_tenant_name"), + ) + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + + tenant_id: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), + ForeignKey("tenants.id", ondelete="CASCADE"), + nullable=True, + index=True, + ) + + name: Mapped[str] = mapped_column(String(100), nullable=False) + + description: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + is_default: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + is_system: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + + created_at: Mapped[DateTime] = mapped_column( + DateTime(timezone=True), server_default=func.now() + ) + + users = relationship("User", back_populates="role") + role_accesses = relationship( + "RoleAccess", back_populates="role", cascade="all, delete-orphan" + ) + tenant = relationship("Tenant", back_populates="roles") + + @property + def accesses(self): + """Return the list of Access objects for this role (used by Pydantic RoleWithAccessesOut).""" + return [ra.access for ra in self.role_accesses if ra.access] diff --git a/app/modules/auth/models/saas_models.py b/app/modules/auth/models/saas_models.py new file mode 100644 index 0000000..74a7b26 --- /dev/null +++ b/app/modules/auth/models/saas_models.py @@ -0,0 +1,44 @@ +import uuid +from sqlalchemy import Column, String, DateTime, func, ForeignKey, JSON +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import relationship +from app.db.database import Base + +class SaaSUserMapping(Base): + __tablename__ = "saas_user_mappings" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, index=True) + saas_user_id = Column(String, nullable=False, unique=True, index=True) + docqube_user_id = Column(ForeignKey("users.id"), nullable=False) + metadata_ = Column("metadata", JSON, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), onupdate=func.now(), server_default=func.now()) + + user = relationship("User", backref="saas_mapping") + +class SaaSTenantMapping(Base): + __tablename__ = "saas_tenant_mappings" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, index=True) + saas_tenant_id = Column(String, unique=True, nullable=False, index=True) + docqube_tenant_id = Column(UUID(as_uuid=True), ForeignKey("tenants.id"), nullable=False) + metadata_ = Column("metadata", JSON, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + tenant = relationship("Tenant") + +class SaaSRoleMapping(Base): + __tablename__ = "saas_role_mappings" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, index=True) + saas_role_id = Column(String, unique=True, nullable=False, index=True) + docqube_role_id = Column(UUID(as_uuid=True), ForeignKey("roles.id", ondelete="CASCADE"), nullable=False) + metadata_ = Column("metadata", JSON, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + role = relationship("Role", backref="saas_mapping") diff --git a/app/modules/auth/models/session_model.py b/app/modules/auth/models/session_model.py new file mode 100644 index 0000000..a1603a4 --- /dev/null +++ b/app/modules/auth/models/session_model.py @@ -0,0 +1,35 @@ +from datetime import datetime +from sqlalchemy import String, DateTime, ForeignKey, Integer, func +from sqlalchemy.orm import Mapped, mapped_column, relationship +from typing import Optional +from app.db.database import Base +from app.modules.auth.models.device_model import Device + + +class Session(Base): + __tablename__ = "sessions" + + id: Mapped[int] = mapped_column(primary_key=True) + + user_id: Mapped[int] = mapped_column( + Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True + ) + user = relationship("User", backref="auth_sessions", foreign_keys=[user_id]) + + device_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("devices.id", ondelete="SET NULL"), nullable=True, index=True + ) + device = relationship("Device", backref="sessions") + + refresh_token_hash: Mapped[str] = mapped_column(String(255), nullable=False) + + status: Mapped[str] = mapped_column(String(50), default="ACTIVE", nullable=False) + + created_at: Mapped[DateTime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + last_activity: Mapped[DateTime] = mapped_column(DateTime(timezone=True), default=func.now()) + expires_at: Mapped[DateTime] = mapped_column(DateTime(timezone=True), nullable=False) + + revoked_at: Mapped[Optional[DateTime]] = mapped_column(DateTime(timezone=True), nullable=True) + revoked_by: Mapped[Optional[str]] = mapped_column(String(50), nullable=True) + revoked_by_user_id: Mapped[Optional[int]] = mapped_column(Integer, ForeignKey("users.id", ondelete="SET NULL"), nullable=True) + reason: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) diff --git a/app/modules/auth/models/user_model.py b/app/modules/auth/models/user_model.py new file mode 100644 index 0000000..2c3dd44 --- /dev/null +++ b/app/modules/auth/models/user_model.py @@ -0,0 +1,226 @@ +import uuid +from datetime import datetime +from sqlalchemy import String, DateTime, func, ForeignKey, UniqueConstraint, Integer, Boolean, Index, text +from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.dialects.postgresql import UUID +from typing import Optional +from app.db.database import Base + + +class User(Base): + __tablename__ = "users" + __table_args__ = ( + UniqueConstraint("tenant_id", "email", name="uq_user_tenant_email"), + Index( + "ix_users_is_superadmin", + "is_superadmin", + postgresql_where=text("is_superadmin"), + ), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + + tenant_id: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), ForeignKey("tenants.id"), nullable=True, index=True + ) + + tenant = relationship("Tenant", back_populates="users", foreign_keys=[tenant_id]) + + is_superadmin: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=False, server_default="false" + ) + + @property + def tenant_envelope_limit(self) -> Optional[int]: + if self.tenant: + return self.tenant.envelope_limit + return None + + @property + def tenant_envelopes_used(self) -> Optional[int]: + if self.tenant: + return self.tenant.envelopes_used + return None + + role_id: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), ForeignKey("roles.id", ondelete="SET NULL"), nullable=True, index=True + ) + + role = relationship("Role", back_populates="users") + + name: Mapped[str] = mapped_column(String(255), nullable=False) + + designation: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + preferred_language: Mapped[str] = mapped_column(String(16), nullable=False, default="en") + + email: Mapped[str] = mapped_column(String(255), index=True, nullable=False) + + preferred_language: Mapped[str] = mapped_column(String(16), nullable=False, default="en", server_default="en") + + password_hash: Mapped[str] = mapped_column(String(255), nullable=False) + + subscription: Mapped[str] = mapped_column(String(50), default="free") + is_active: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + terms_accepted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + has_completed_tutorial: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + has_completed_docs_tutorial: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + is_deleted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + deleted_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True) + deleted_by_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + + created_at: Mapped[DateTime] = mapped_column( + DateTime(timezone=True), server_default=func.now() + ) + + encrypted_ai_api_key: Mapped[Optional[str]] = mapped_column( + String(512), nullable=True + ) + + smtp_user: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + + mail_from: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + + encrypted_smtp_password: Mapped[Optional[str]] = mapped_column( + String(512), nullable=True + ) + + projects = relationship( + "Project", back_populates="owner", cascade="all, delete-orphan" + ) + + drive_folders = relationship( + "DriveFolder", back_populates="owner", cascade="all, delete-orphan" + ) + + drive_files = relationship( + "DriveFile", back_populates="owner", cascade="all, delete-orphan" + ) + + project_versions = relationship( + "ProjectVersion", back_populates="creator", cascade="all, delete-orphan" + ) + + drive_file_versions = relationship( + "DriveFileVersion", back_populates="creator", cascade="all, delete-orphan" + ) + + drive_comments = relationship( + "DriveComment", back_populates="author", cascade="all, delete-orphan" + ) + + drive_activities = relationship( + "DriveActivity", back_populates="actor", cascade="all, delete-orphan" + ) + + user_files = relationship( + "UserFile", back_populates="user", cascade="all, delete-orphan" + ) + + storage_usage = relationship( + "UserStorageUsage", + back_populates="user", + uselist=False, + cascade="all, delete-orphan", + ) + + notifications = relationship( + "Notification", + back_populates="user", + cascade="all, delete-orphan", + lazy="dynamic", + ) + chat_token_quota = relationship( + "ChatTokenQuota", + back_populates="user", + uselist=False, + cascade="all, delete-orphan", + ) + deleted_by = relationship("User", remote_side=[id], foreign_keys=[deleted_by_id]) + + @property + def role_name(self) -> Optional[str]: + return self.role.name if self.role else None + + @property + def access_codes(self) -> list[str]: + """ + Every access code this user holds. + + **Prefers what `get_current_user` resolved.** This property used to be an + independent second answer to "what may this user do", computed from the + single legacy `users.role_id` and nothing else — so a user holding a role + through `user_roles` had authority that `require_access` honoured and the + product would not show them. `/api/me/profile` reports this list and the + frontend drives every route guard, menu entry and button from it, so an + incomplete answer here is indistinguishable from having no permission. + + `_resolve_access_codes` is attached by `get_current_user` and calls + `PermissionService`, the one correct resolver: legacy role, `user_roles` + grants (live and in-tenant only), group grants, SaaS claims, and + descendant expansion. It is a **closure rather than a computed list** so + that requests which never ask do not pay for the answer; the result is + memoised per request, so asking twice is free. + + The fallback below is the old computation, kept for objects that never + passed through authentication — fixtures, background tasks, a detached + row. It is deliberately not an error: raising here would turn "no session + attached" into a 500 on paths that only wanted a display name. + """ + resolver = getattr(self, "_resolve_access_codes", None) + if resolver is not None: + return list(resolver()) + + codes = set() + if self.role: + codes.update([ra.access.access_code for ra in self.role.role_accesses if ra.access]) + + saas_permissions = getattr(self, "saas_permissions", set()) + codes.update(saas_permissions) + + return list(codes) + + @property + def role_refs(self) -> list: + """ + Every role this user holds, as resolved assignment dicts. + + Named `role_refs` rather than `roles` on purpose: `roles` would sit one + typo away from a relationship name in a class that already has `role`, + and a plain attribute shadowing a mapper attribute fails at import time + in a way that is tedious to diagnose. `UserOut` exposes it as `roles`. + + Resolved through a closure attached by `get_current_user`, for the same + reason as `access_codes`: only the responses that report roles should pay + for reading them. + + Off that path the primary role is synthesised, so a detached object still + describes itself honestly rather than claiming no roles at all. + """ + resolver = getattr(self, "_resolve_role_refs", None) + if resolver is not None: + return list(resolver()) + + if self.role is None: + return [] + return [ + { + "grant_id": None, + "role_id": self.role.id, + "role_name": self.role.name, + "source": "primary", + "org_unit_id": None, + "org_unit_name": None, + "group_id": None, + "group_name": None, + "expires_at": None, + "assigned_by_id": None, + "created_at": None, + "is_expired": False, + } + ] + + @property + def subscription_details(self): + return getattr(self, "saas_subscription", None) diff --git a/app/modules/auth/models/user_role_model.py b/app/modules/auth/models/user_role_model.py new file mode 100644 index 0000000..79b5397 --- /dev/null +++ b/app/modules/auth/models/user_role_model.py @@ -0,0 +1,160 @@ +""" +C3 — a role held *somewhere*. + +`User.role_id` gives every user exactly one role, held across the whole tenant. +That is the thing scope cannot be built on: until a user can hold the same role +in two places, or two roles in one place, "scoped access" has nowhere to live. + +This is the join table the base uses (`docqube_platform_backend/app/models/rbac.py`), +with one difference: it points at DocQube's existing `roles`, so the access-code +vocabulary is unchanged and all 227 endpoints keep working exactly as they do +today. + + org_unit_id IS NULL -> tenant-wide, the grant everyone has today + org_unit_id = X -> this role, within X and everything beneath it + +The null case is what makes the migration a no-op: the backfill turns each +`User.role_id` into one row with a null unit, which is precisely the authority +that user already had. + +**Both mechanisms are honoured for one release.** `User.role_id` is not dropped +here. That is the same transition shape as `users.is_superadmin`, for the same +reason — a rollback must not be able to remove everyone's permissions — and it +ends the same way, with a preflight script and a deliberate removal. +""" + +import uuid +from datetime import datetime, timezone +from typing import Optional + +from sqlalchemy import DateTime, ForeignKey, Index, UniqueConstraint, func +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.db.database import Base + +import app.modules.org.models.group_model # noqa: F401 + + +class UserRole(Base): + __tablename__ = "user_roles" + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + tenant_id: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), + ForeignKey("tenants.id", ondelete="CASCADE"), + nullable=True, + index=True, + ) + """ + The tenant this grant belongs to. + + Present so isolation is **structural** rather than remembered: the ORM + listener scopes any model carrying this column, and the RLS policy compares + it against the session variable. Without it the table sat outside both + layers, guarded only by whatever predicate each call site wrote — the same + shape of gap that let any holder of `superadmin.tenant.delete` delete any + tenant. + + Nullable, because a superadmin has no tenant and neither do their grants. + """ + + user_id: Mapped[Optional[int]] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), nullable=True, index=True + ) + group_id: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), + ForeignKey("access_groups.id", ondelete="CASCADE"), + nullable=True, + index=True, + ) + """ + The principal: exactly one of `user_id` or `group_id`. + + Enforced by a CHECK constraint, not by the application. "Exactly one of two + nullable columns" survives in the schema and rots in code — one new call + site that sets both, and the resolver counts the grant twice. + + `user_id` became nullable in C5.1. That weakened an invariant every prior + query relied on, so all four readers were reviewed before the migration: + `core/scope.py`, `auth/services/permission_service.py`, + `org/routes/org_routes.py` and `org/services/access_maintenance.py`. + """ + role_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("roles.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + org_unit_id: Mapped[Optional[uuid.UUID]] = mapped_column( + UUID(as_uuid=True), + ForeignKey("org_units.id", ondelete="CASCADE"), + nullable=True, + index=True, + ) + assigned_by_id: Mapped[Optional[int]] = mapped_column( + ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + + expires_at: Mapped[Optional[datetime]] = mapped_column( + DateTime(timezone=True), nullable=True + ) + """ + When this grant stops applying. Null means permanent. + + Enforced in the resolution query, not in Python — an expired grant must be + invisible to *every* reader, including any future one that queries this + table directly. Filtering after the fact is how one caller ends up honouring + a dead grant. + + Expired rows are kept rather than swept: they are the history that answers + "who had access in March". + """ + + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + + role = relationship("Role", foreign_keys=[role_id]) + user = relationship("User", foreign_keys=[user_id]) + + __table_args__ = ( + UniqueConstraint( + "user_id", "role_id", "org_unit_id", name="uq_user_roles_scoped" + ), + Index( + "uq_user_roles_tenant_wide", + "user_id", + "role_id", + unique=True, + postgresql_where=(org_unit_id.is_(None)), + ), + Index("ix_user_roles_lookup", "user_id", "role_id"), + Index( + "uq_user_roles_group_scoped", + "group_id", + "role_id", + "org_unit_id", + unique=True, + postgresql_where=(group_id.isnot(None)), + ), + Index( + "uq_user_roles_group_tenant_wide", + "group_id", + "role_id", + unique=True, + postgresql_where=(org_unit_id.is_(None) & group_id.isnot(None)), + ), + Index( + "ix_user_roles_expires_at", + "expires_at", + postgresql_where=(expires_at.isnot(None)), + ), + Index("ix_user_roles_assigned_by_id", "assigned_by_id"), + ) + + def __repr__(self) -> str: # pragma: no cover - debugging aid + where = self.org_unit_id or "tenant-wide" + return f"" diff --git a/app/modules/auth/repositories/device_repository.py b/app/modules/auth/repositories/device_repository.py new file mode 100644 index 0000000..139f9f5 --- /dev/null +++ b/app/modules/auth/repositories/device_repository.py @@ -0,0 +1,45 @@ +from typing import List, Optional +from sqlalchemy.orm import Session +from app.modules.auth.models.device_model import Device +from datetime import datetime, timezone + +class DeviceRepository: + def __init__(self, db: Session): + self.db = db + + def get_by_id(self, device_id: int) -> Optional[Device]: + return self.db.query(Device).filter(Device.id == device_id).first() + + def get_by_fingerprint(self, user_id: int, fingerprint: str) -> Optional[Device]: + return self.db.query(Device).filter( + Device.user_id == user_id, + Device.fingerprint == fingerprint + ).first() + + def get_user_devices(self, user_id: int) -> List[Device]: + from app.modules.auth.models.session_model import Session as AuthSession + return self.db.query(Device).join(AuthSession, Device.id == AuthSession.device_id).filter( + Device.user_id == user_id, + AuthSession.status == "ACTIVE", + AuthSession.expires_at > datetime.now(timezone.utc) + ).order_by(Device.last_login.desc()).distinct().all() + + def create(self, device: Device) -> Device: + self.db.add(device) + self.db.flush() + self.db.refresh(device) + return device + + def update_login_activity(self, device: Device, ip: str) -> Device: + device.last_login = datetime.now(timezone.utc) + if ip: + device.last_ip = ip + self.db.flush() + self.db.refresh(device) + return device + + def block_device(self, device: Device) -> Device: + device.is_blocked = True + self.db.flush() + self.db.refresh(device) + return device diff --git a/app/modules/auth/repositories/session_repository.py b/app/modules/auth/repositories/session_repository.py new file mode 100644 index 0000000..e81b93e --- /dev/null +++ b/app/modules/auth/repositories/session_repository.py @@ -0,0 +1,72 @@ +from typing import List, Optional +from sqlalchemy.orm import Session +from app.modules.auth.models.session_model import Session as AuthSession +from datetime import datetime, timezone + +class SessionRepository: + def __init__(self, db: Session): + self.db = db + + def get_by_id(self, session_id: int) -> Optional[AuthSession]: + return self.db.query(AuthSession).filter(AuthSession.id == session_id).first() + + def get_active_sessions_for_user(self, user_id: int) -> List[AuthSession]: + return self.db.query(AuthSession).filter( + AuthSession.user_id == user_id, + AuthSession.status == "ACTIVE", + AuthSession.expires_at > datetime.now(timezone.utc) + ).all() + + def count_active_sessions(self, user_id: int) -> int: + return self.db.query(AuthSession).filter( + AuthSession.user_id == user_id, + AuthSession.status == "ACTIVE", + AuthSession.expires_at > datetime.now(timezone.utc) + ).count() + + def create(self, session: AuthSession) -> AuthSession: + self.db.add(session) + self.db.flush() + self.db.refresh(session) + return session + + def update_last_activity(self, session: AuthSession) -> AuthSession: + session.last_activity = datetime.now(timezone.utc) + return session + + def revoke(self, session: AuthSession, by: str = "user", revoked_by_user_id: Optional[int] = None, reason: Optional[str] = None) -> AuthSession: + session.status = "REVOKED" + session.revoked_at = datetime.now(timezone.utc) + session.revoked_by = by + if revoked_by_user_id: + session.revoked_by_user_id = revoked_by_user_id + if reason: + session.reason = reason + self.db.flush() + self.db.refresh(session) + + from app.db.redis import redis_cache + redis_cache.set(f"session_revoked:{session.id}", "1", ttl=600) + import json + if redis_cache.client: + redis_cache.client.publish(f"user_events:{session.user_id}", json.dumps({"type": "SESSION_REVOKED", "session_id": session.id})) + + return session + + def revoke_all_for_device(self, device_id: int, by: str = "user") -> None: + self.db.query(AuthSession).filter( + AuthSession.device_id == device_id, + AuthSession.status == "ACTIVE" + ).update({ + "status": "REVOKED", + "revoked_at": datetime.now(timezone.utc), + "revoked_by": by + }) + + from app.modules.auth.models.device_model import Device + device = self.db.query(Device).filter(Device.id == device_id).first() + if device: + from app.db.redis import redis_cache + import json + if redis_cache.client: + redis_cache.client.publish(f"user_events:{device.user_id}", json.dumps({"type": "DEVICE_REVOKED", "device_id": device.id})) diff --git a/app/modules/auth/repositories/user_repository.py b/app/modules/auth/repositories/user_repository.py new file mode 100644 index 0000000..c67b9ec --- /dev/null +++ b/app/modules/auth/repositories/user_repository.py @@ -0,0 +1,43 @@ +from typing import Optional, List +from sqlalchemy.orm import Session +from app.modules.auth.models.user_model import User + +class UserRepository: + def __init__(self, db: Session): + self.db = db + + def get_by_email(self, email: str) -> Optional[User]: + return self.db.query(User).filter(User.email == email.lower().strip()).first() + + def get_by_id(self, user_id: int) -> Optional[User]: + return self.db.query(User).filter(User.id == user_id).first() + + def create(self, user: User) -> User: + self.db.add(user) + self.db.flush() + self.db.refresh(user) + return user + + def search_users( + self, query: str, limit: int = 10, tenant_id=None + ) -> List[User]: + """ + Search users by name or email. + + `tenant_id` is required in practice: without it this returns users from + every tenant, including their email addresses and permission sets. The + parameter defaults to None only so a superadmin caller can search across + tenants deliberately — callers that serve tenant users must always pass + the caller's own tenant. + """ + q = self.db.query(User).filter( + User.is_deleted.is_(False), + (User.name.ilike(f"%{query}%")) | (User.email.ilike(f"%{query}%")), + ) + if tenant_id is not None: + q = q.filter(User.tenant_id == tenant_id) + return q.limit(limit).all() + + def get_users_by_ids(self, ids: List[int]) -> List[User]: + return self.db.query(User).filter(User.id.in_(ids), User.is_deleted.is_(False)).all() + diff --git a/app/modules/auth/routes/access_routes.py b/app/modules/auth/routes/access_routes.py new file mode 100644 index 0000000..b0639c1 --- /dev/null +++ b/app/modules/auth/routes/access_routes.py @@ -0,0 +1,76 @@ +""" +Access control routes. + +Endpoints for retrieving user's access codes and managing permissions. +Used by frontend to determine what UI elements to show. +""" + +from fastapi import APIRouter, Depends +from sqlalchemy.orm import Session + +from app.db.database import get_db +from app.middleware.auth import get_current_user +from app.modules.auth.models.user_model import User +from app.modules.auth.services.permission_service import PermissionService + +from typing import List + +from app.modules.auth.schemas.access_schema import AccessOut, RoleAccessesOut + +router = APIRouter(prefix="/api/access", tags=["Access"]) + + +@router.get("/get", response_model=RoleAccessesOut) +def get_user_accesses( + user: User = Depends(get_current_user), + db: Session = Depends(get_db), +): + """ + Get current user's access codes. + + This endpoint is called by the frontend to determine which UI elements + to show/hide based on the user's permissions. + + Returns: + { + "accesses": ["project.create", "drive.file.upload", ...], + "role": "superadmin" | "tenant_admin" | null, + "roles": ["tenant_admin", "approver"] + } + + `role` is the **primary** role and is unchanged. `roles` is every role the + caller holds — primary, granted, and through a group — because a user may + hold several and a single name cannot say so. + """ + perm_service = PermissionService(db) + accesses = sorted(list(perm_service.user_access_codes(user))) + + seen: list[str] = [] + for assignment in user.role_refs: + name = assignment.get("role_name") + if name and name not in seen: + seen.append(name) + + return { + "accesses": accesses, + "role": user.role.name if user.role else None, + "roles": seen, + "tenant_id": str(user.tenant_id) if user.tenant_id else None, + } + + +@router.get("/all", response_model=List[AccessOut]) +def get_all_system_accesses( + user: User = Depends(get_current_user), + db: Session = Depends(get_db), +): + """ + Get all available access codes in the platform. + Typically used for role management UI. + """ + from app.modules.auth.dependencies.access_dependency import require_access + + perm_service = PermissionService(db) + perm_service.require_any_access(user, ["superadmin.role.read", "admin.role.read"]) + + return perm_service.get_all_system_accesses() diff --git a/app/modules/auth/routes/admin_dashboard_routes.py b/app/modules/auth/routes/admin_dashboard_routes.py new file mode 100644 index 0000000..db2eb89 --- /dev/null +++ b/app/modules/auth/routes/admin_dashboard_routes.py @@ -0,0 +1,43 @@ +from typing import Literal + +from fastapi import APIRouter, Depends, Query +from sqlalchemy.orm import Session + +from app.db.database import get_db +from app.middleware.auth import get_current_user +from app.modules.auth.dependencies.access_dependency import require_access +from app.modules.auth.models.user_model import User +from app.modules.auth.schemas.admin_dashboard_schema import ( + AdminDashboardMetricsOut, + MetricTrendOnlyOut, +) +from app.modules.auth.services.admin_dashboard_service import AdminDashboardService + +router = APIRouter(prefix="/api/admin", tags=["Admin Dashboard"]) + + +@router.get( + "/dashboard-metrics", + response_model=AdminDashboardMetricsOut, + dependencies=[require_access("admin.user.read")], +) +def get_dashboard_metrics( + granularity: Literal["weekly", "monthly", "yearly"] = Query("weekly"), + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db), +): + return AdminDashboardService(db).get_tenant_metrics(current_user, granularity) + + +@router.get( + "/dashboard-metrics/trends/{metric}", + response_model=MetricTrendOnlyOut, + dependencies=[require_access("admin.user.read")], +) +def get_dashboard_metric_trend( + metric: Literal["signatures", "uploads"], + granularity: Literal["weekly", "monthly", "yearly"] = Query("weekly"), + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db), +): + return AdminDashboardService(db).get_trend_data(current_user, metric, granularity) diff --git a/app/modules/auth/routes/admin_user_routes.py b/app/modules/auth/routes/admin_user_routes.py new file mode 100644 index 0000000..474738a --- /dev/null +++ b/app/modules/auth/routes/admin_user_routes.py @@ -0,0 +1,655 @@ +from fastapi import APIRouter, Depends, Query, HTTPException, status +from sqlalchemy import func +from sqlalchemy.orm import Session, joinedload +from typing import Optional, List +from uuid import UUID +from pydantic import BaseModel, Field, field_validator +from datetime import datetime +import re +import logging + +from app.db.database import get_db +from app.core.scope import ScopeService +from app.core.settings import settings +from app.middleware.auth import get_current_user +from app.modules.auth.models.user_model import User +from app.modules.auth.models.role_model import Role +from app.modules.auth.models.role_access_model import RoleAccess +from app.modules.auth.schemas.auth_schema import UserOut +from app.modules.auth.schemas.role_schema import ( + AddUserRoleIn, + EffectiveAccessOut, + RoleAssignmentOut, +) +from app.modules.auth.services.permission_service import PermissionService +from app.modules.auth.services.user_role_service import UserRoleReader, invalidate_for_user +from app.modules.auth.dependencies.access_dependency import require_access +from app.modules.auth.services.privilege import holds_superadmin_access +from app.modules.configuration.services.system_configuration_service import ( + SystemConfigurationService, +) +from app.modules.chat.models.chat_usage_model import ChatTokenQuota + +from app.modules.configuration.schemas.system_configuration_schema import ( + SystemLimitsOut, +) + +from app.core.security import get_password_hash +from app.core.schemas import MessageOut + +router = APIRouter(prefix="/api/admin/users", tags=["Admin Users"]) +logger = logging.getLogger(__name__) + + +def _is_superadmin(user: User) -> bool: + """A user is treated as a superadmin if they hold any `superadmin.*` access.""" + return holds_superadmin_access(user) + + +def _resolve_effective_tenant_id( + current_user: User, requested_tenant_id: Optional[UUID] +) -> Optional[UUID]: + """ + Tenant admins are always scoped to their own tenant. + Superadmins may pass a tenant_id to narrow the scope, or None for all tenants. + """ + if _is_superadmin(current_user): + return requested_tenant_id + return current_user.tenant_id + + +@router.get( + "/summary/limits", + response_model=SystemLimitsOut, + dependencies=[require_access("admin.user.read")], +) +def get_user_management_limits(db: Session = Depends(get_db)): + """Get global user limits snapshot for administration UI.""" + return SystemConfigurationService(db).get_limits_snapshot() + + +class UpdateUserIn(BaseModel): + name: Optional[str] = None + designation: Optional[str] = Field(None, max_length=100) + is_active: Optional[bool] = None + role_id: Optional[UUID] = None + tenant_id: Optional[UUID] = None + password: Optional[str] = Field(None, min_length=12) + + @field_validator("password") + @classmethod + def password_complexity(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + if not re.search(r"[A-Z]", v): + raise ValueError("Password must contain at least one uppercase letter") + if not re.search(r"[0-9]", v): + raise ValueError("Password must contain at least one number") + if not re.search(r"[^A-Za-z0-9]", v): + raise ValueError("Password must contain at least one special character") + return v + + +class UserListOut(BaseModel): + id: int + name: str + designation: Optional[str] = None + email: str + is_active: bool + tenant_id: Optional[UUID] = None + tenant_name: Optional[str] = None + role_id: Optional[UUID] = None + role: Optional[str] = None + subscription: Optional[str] = None + is_deleted: bool = False + deleted_at: Optional[datetime] = None + chat_credit_daily_limit: Optional[float] = None + chat_credits_used_today: Optional[float] = None + tenant_chat_token_limit: Optional[int] = None + tenant_total_credits_allocated: Optional[float] = None + + roles: List[RoleAssignmentOut] = [] + """ + Every role this user holds, primary first. + + **Additive.** `role` and `role_id` above are retained and still mean the + primary role, so every existing consumer of this shape is unaffected. + """ + role_count: int = 0 + + class Config: + from_attributes = True + + +class PaginatedUsersOut(BaseModel): + items: List[UserListOut] + total: int + page: int + page_size: int + total_pages: int + + +from app.modules.chat.repositories.chat_repository import ChatbotRepository + +def _get_single_user_allocated_quota(user: User, db: Session) -> float: + if not user.tenant_id: + return 0.0 + repo = ChatbotRepository(db) + return repo.get_tenant_total_user_quota(user.tenant_id) + + +def _to_user_list_out(user: User, tenant_total_credits_allocated: float, assignments: list) -> UserListOut: + return UserListOut( + id=user.id, + name=user.name, + designation=user.designation, + email=user.email, + is_active=getattr(user, "is_active", True), + tenant_id=user.tenant_id, + tenant_name=user.tenant.name if user.tenant else None, + role_id=user.role_id, + role=user.role.name if user.role else None, + subscription=user.subscription, + is_deleted=getattr(user, "is_deleted", False), + deleted_at=getattr(user, "deleted_at", None), + chat_credit_daily_limit=getattr(user, "chat_token_quota", None).daily_credit_limit if getattr(user, "chat_token_quota", None) else None, + chat_credits_used_today=round((getattr(user, "chat_token_quota", None).credits_used_today / 1000.0), 2) if getattr(user, "chat_token_quota", None) else 0, + tenant_chat_token_limit=user.tenant.chat_token_daily_limit if user.tenant else None, + tenant_total_credits_allocated=tenant_total_credits_allocated, + roles=[RoleAssignmentOut(**a) for a in assignments], + role_count=len(assignments), + ) + + +@router.get( + "", + response_model=PaginatedUsersOut, + dependencies=[require_access("admin.user.read")], +) +def list_all_users( + page: int = Query(1, ge=1), + page_size: int = Query(20, ge=1, le=100), + search: Optional[str] = Query(None), + tenant_id: Optional[UUID] = Query(None), + include_deleted: bool = Query(False), + only_deleted: bool = Query(False), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """List all users with pagination, search, and optional tenant filter.""" + query = db.query(User).options( + joinedload(User.role), + joinedload(User.tenant), + joinedload(User.chat_token_quota), + ) + + if settings.ORG_SCOPE_ENABLED: + visible = ScopeService(db).visible_user_ids(current_user, "admin.user.read") + if visible is not None: + query = query.filter(User.id.in_(visible)) + + effective_tenant_id = _resolve_effective_tenant_id(current_user, tenant_id) + if effective_tenant_id: + query = query.filter(User.tenant_id == effective_tenant_id) + + if only_deleted: + query = query.filter(User.is_deleted == True) + elif not include_deleted: + query = query.filter(User.is_deleted == False) + + if search and search.strip(): + term = f"%{search.strip()}%" + query = query.filter(User.name.ilike(term) | User.email.ilike(term)) + + total = query.count() + offset = (page - 1) * page_size + users = query.offset(offset).limit(page_size).all() + total_pages = (total + page_size - 1) // page_size if total > 0 else 0 + + by_user = UserRoleReader(db).assignments_for_users(users) + + # Batch compute tenant allocated quotas in 1 single SQL query for all returned users + tenant_ids = {u.tenant_id for u in users if u.tenant_id} + tenant_quota_map = {} + if tenant_ids: + rows = ( + db.query(User.tenant_id, func.sum(ChatTokenQuota.daily_credit_limit)) + .join(ChatTokenQuota, User.id == ChatTokenQuota.user_id) + .filter(User.tenant_id.in_(tenant_ids), User.is_deleted == False) + .group_by(User.tenant_id) + .all() + ) + tenant_quota_map = {r[0]: float(r[1] or 0.0) for r in rows} + + return PaginatedUsersOut( + items=[_to_user_list_out(u, tenant_quota_map.get(u.tenant_id, 0.0), by_user.get(u.id, [])) for u in users], + total=total, + page=page, + page_size=page_size, + total_pages=total_pages, + ) + + +@router.get( + "/{user_id}", + response_model=UserListOut, + dependencies=[require_access("admin.user.read")], +) +def get_user( + user_id: int, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user) +): + """Get a single user by ID.""" + query = ( + db.query(User) + .options(joinedload(User.role), joinedload(User.tenant), joinedload(User.chat_token_quota)) + .filter(User.id == user_id) + ) + if not _is_superadmin(current_user) and current_user.tenant_id: + query = query.filter(User.tenant_id == current_user.tenant_id) + user = query.first() + if not user: + raise HTTPException(status_code=404, detail="User not found") + + if settings.ORG_SCOPE_ENABLED and not ScopeService(db).can_access_user( + current_user, user.id, "admin.user.read" + ): + raise HTTPException(status_code=404, detail="User not found") + + return _to_user_list_out( + user, + _get_single_user_allocated_quota(user, db), + UserRoleReader(db).assignments(user) + ) + + +@router.patch( + "/{user_id}", + response_model=UserListOut, + dependencies=[require_access("admin.user.update")], +) +def update_user( + user_id: int, + payload: UpdateUserIn, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """Update a user's name, active status, role, tenant, or password.""" + query = ( + db.query(User) + .options(joinedload(User.role), joinedload(User.tenant), joinedload(User.chat_token_quota)) + .filter(User.id == user_id) + ) + if not _is_superadmin(current_user) and current_user.tenant_id: + query = query.filter(User.tenant_id == current_user.tenant_id) + user = query.first() + if not user: + raise HTTPException(status_code=404, detail="User not found") + if getattr(user, "is_deleted", False): + raise HTTPException(status_code=400, detail="Cannot update a deleted user") + + if payload.name is not None: + user.name = payload.name + if payload.designation is not None: + user.designation = (payload.designation or "").strip() or None + was_active = user.is_active + if payload.is_active is not None: + if payload.is_active == False and user.id == current_user.id: + raise HTTPException(status_code=400, detail="You cannot deactivate your own account") + user.is_active = payload.is_active + previous_role = user.role + role_changed = False + new_role = None + if payload.role_id is not None: + role = db.query(Role).filter(Role.id == payload.role_id).first() + if not role: + raise HTTPException(status_code=404, detail="Role not found") + role_changed = user.role_id != role.id + new_role = role + user.role_id = role.id + if payload.tenant_id is not None: + user.tenant_id = payload.tenant_id + if payload.password is not None: + user.password_hash = get_password_hash(payload.password) + + db.flush() + db.refresh(user) + db.refresh(user) + + if role_changed: + from app.modules.org.services.access_audit import record_primary_role_change + + record_primary_role_change(db, current_user, user, previous_role, new_role) + invalidate_for_user(user.id) + + user_out = ( + db.query(User) + .options(joinedload(User.role), joinedload(User.tenant)) + .filter(User.id == user_id) + .first() + ) + + if was_active and payload.is_active is False: + try: + from app.modules.activity_logs.service import log_event + from app.modules.activity_logs.constants import ActivityLogModule, ActivityLogAction, ActivityLogStatus, ActivityLogTargetType + if user_out.tenant_id: + log_event( + tenant_id=user_out.tenant_id, + user_id=current_user.id, + user_email=current_user.email, + module=ActivityLogModule.TENANT, + action=ActivityLogAction.USER_DEACTIVATED, + target_id=str(user_out.id), + target_type=ActivityLogTargetType.USER, + metadata={ + "deactivated_user_id": user_out.id, + "deactivated_user_email": user_out.email + }, + status=ActivityLogStatus.SUCCESS + ) + except Exception as e: + import logging + logging.getLogger(__name__).error(f"Failed to log USER_DEACTIVATED: {e}") + + return _to_user_list_out( + user_out, + _get_single_user_allocated_quota(user_out, db), + UserRoleReader(db).assignments(user_out) + ) + + +@router.delete( + "/{user_id}", + dependencies=[require_access("admin.user.delete")], + response_model=MessageOut) +def soft_delete_user( + user_id: int, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + query = db.query(User).filter(User.id == user_id) + if not _is_superadmin(current_user) and current_user.tenant_id: + query = query.filter(User.tenant_id == current_user.tenant_id) + user = query.first() + if not user: + raise HTTPException(status_code=404, detail="User not found") + if user.id == current_user.id: + raise HTTPException(status_code=400, detail="You cannot delete your own account") + if user.is_deleted: + return {"message": "User already deleted"} + + user.is_deleted = True + user.is_active = False + user.deleted_at = datetime.utcnow() + user.deleted_by_id = current_user.id + + try: + from app.modules.activity_logs.service import log_event + from app.modules.activity_logs.constants import ActivityLogModule, ActivityLogAction, ActivityLogStatus, ActivityLogTargetType + if user.tenant_id: + log_event( + tenant_id=user.tenant_id, + user_id=current_user.id, + user_email=current_user.email, + module=ActivityLogModule.TENANT, + action=ActivityLogAction.USER_DEACTIVATED, + target_id=str(user.id), + target_type=ActivityLogTargetType.USER, + metadata={ + "deactivated_user_id": user.id, + "deactivated_user_email": user.email + }, + status=ActivityLogStatus.SUCCESS + ) + except Exception as e: + import logging + logging.getLogger(__name__).error(f"Failed to log USER_DEACTIVATED: {e}") + + return {"detail": "User deleted successfully"} + + +@router.post( + "/{user_id}/restore", + response_model=UserListOut, + dependencies=[require_access("admin.user.update")], +) +def restore_user( + user_id: int, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + query = ( + db.query(User) + .options(joinedload(User.role), joinedload(User.tenant)) + .filter(User.id == user_id) + ) + if not _is_superadmin(current_user) and current_user.tenant_id: + query = query.filter(User.tenant_id == current_user.tenant_id) + user = query.first() + if not user: + raise HTTPException(status_code=404, detail="User not found") + if not user.is_deleted: + raise HTTPException(status_code=400, detail="User is not deleted") + + SystemConfigurationService(db).ensure_can_create_user(tenant_id=user.tenant_id) + + user.is_deleted = False + user.is_active = True + user.deleted_at = None + user.deleted_by_id = None + db.flush() + db.refresh(user) + return _to_user_list_out( + user, + _get_single_user_allocated_quota(user, db), + UserRoleReader(db).assignments(user) + ) + + +@router.post( + "/bulk/distribute-credits-equally", + dependencies=[require_access("admin.user.update")], +) +def distribute_credits_equally( + tenant_id: Optional[UUID] = Query(None), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """Distribute the tenant's total credit limit equally among all active users.""" + from app.modules.chat.services.chat_service import ChatbotService + from app.modules.tenant.models.tenant_model import Tenant + + effective_tenant_id = _resolve_effective_tenant_id(current_user, tenant_id) + if not effective_tenant_id: + raise HTTPException(status_code=400, detail="Tenant ID is required for bulk distribution.") + + tenant = db.query(Tenant).filter(Tenant.id == effective_tenant_id).first() + if not tenant: + raise HTTPException(status_code=404, detail="Tenant not found") + + if tenant.chat_token_daily_limit == -1: + raise HTTPException(status_code=400, detail="Cannot distribute equally when Tenant has Unlimited quota.") + + active_users = db.query(User).filter( + User.tenant_id == effective_tenant_id, + User.is_active == True, + User.is_deleted == False + ).all() + + if not active_users: + return {"message": "No active users to distribute credits to."} + + total_tenant_credits = tenant.chat_token_daily_limit / 1000.0 + equal_quota = total_tenant_credits / len(active_users) + + logger.info(f"Bulk distributing {total_tenant_credits} Cr among {len(active_users)} users. Quota per user: {equal_quota}") + + chat_service = ChatbotService(db) + for user in active_users: + logger.info(f"Updating user {user.id} ({user.email}) to {equal_quota} Cr") + chat_service.repo.update_user_quota(user.id, equal_quota) + + return { + "message": f"Successfully distributed {total_tenant_credits} Cr among {len(active_users)} users. Each user now has {equal_quota} Cr.", + "quota_per_user": equal_quota + } + + +def _visible_user_or_404(db: Session, current_user: User, user_id: int) -> User: + """ + The target user, or 404 — never 403. + + A 403 confirms the user exists, which is the leak the cross-tenant probes + were written to catch. Applies the tenant predicate and, when + `ORG_SCOPE_ENABLED`, the org-scope check, exactly as the read endpoints + above do. + """ + query = ( + db.query(User) + .options(joinedload(User.role), joinedload(User.tenant)) + .filter(User.id == user_id) + ) + if not _is_superadmin(current_user) and current_user.tenant_id: + query = query.filter(User.tenant_id == current_user.tenant_id) + user = query.first() + if not user: + raise HTTPException(status_code=404, detail="User not found") + + if settings.ORG_SCOPE_ENABLED and not ScopeService(db).can_access_user( + current_user, user.id, "admin.user.read" + ): + raise HTTPException(status_code=404, detail="User not found") + return user + + +@router.get( + "/{user_id}/roles", + response_model=List[RoleAssignmentOut], + dependencies=[require_access("admin.role.read")], +) +def list_user_roles( + user_id: int, + include_expired: bool = Query( + False, + description=( + "Include grants that have lapsed. They confer nothing; they are " + "shown so an administrator can see that a temporary grant ended " + "rather than wondering where it went." + ), + ), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """Every role this user holds: primary, granted, and through a group.""" + user = _visible_user_or_404(db, current_user, user_id) + assignments = UserRoleReader(db).assignments(user, include_expired=include_expired) + return [RoleAssignmentOut(**a) for a in assignments] + + +@router.post( + "/{user_id}/roles", + response_model=RoleAssignmentOut, + status_code=status.HTTP_201_CREATED, + dependencies=[require_access("admin.role.assign")], +) +def add_user_role( + user_id: int, + payload: AddUserRoleIn, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """ + Grant this user an additional role. + + `org_unit_id` omitted means **tenant-wide**, which is the authority a + primary role already carries and the case the organisation tree could not + express. + + Does **not** touch `users.role_id`. The primary role is left exactly as it + was, so nothing an administrator has already set is silently replaced. + """ + from app.modules.org.services.grant_service import GrantService + + user = _visible_user_or_404(db, current_user, user_id) + + row = GrantService(db).grant( + current_user, + role_id=payload.role_id, + user_id=user.id, + org_unit_id=payload.org_unit_id, + expires_at=payload.expires_at, + ) + + for assignment in UserRoleReader(db).assignments(user): + if assignment["grant_id"] == row.id: + return RoleAssignmentOut(**assignment) + + raise HTTPException( + status_code=500, detail="The grant was written but did not resolve." + ) + + +@router.delete( + "/{user_id}/roles/{grant_id}", + response_model=MessageOut, + dependencies=[require_access("admin.role.assign")], +) +def remove_user_role( + user_id: int, + grant_id: UUID, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """ + Revoke one granted role. + + The **primary** role is not revocable here — it has no `grant_id`, and it is + changed through `PATCH /api/admin/users/{id}`. Offering a Remove button that + silently did nothing to it would be the worst of both. + + A grant reaching this user **through a group** is likewise not revocable + here: it belongs to the group, and removing it from one member would mean + editing the group for everybody. + """ + from app.modules.org.services.grant_service import GrantService + + _visible_user_or_404(db, current_user, user_id) + + row = UserRoleReader(db).grant_row(current_user.tenant_id, grant_id) + if row is None or row.user_id != user_id: + raise HTTPException(status_code=404, detail="Grant not found") + + GrantService(db).revoke(current_user, grant_id) + return {"message": "Role removed"} + + +@router.get( + "/{user_id}/effective-access", + response_model=List[EffectiveAccessOut], + dependencies=[require_access("admin.user.read")], +) +def user_effective_access( + user_id: int, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """ + Every access code this user holds, and which role confers each one. + + "Why can this person do that" answered itself while a user had one role. + With several it does not, and the alternative is reading the database — so + this exists before the interface that makes several roles easy to create. + + Descendant codes are attributed to the role holding their parent, because + that is the row an administrator would edit to take the code away. + """ + user = _visible_user_or_404(db, current_user, user_id) + + attributions = PermissionService(db).granted_by(user) + + return [ + EffectiveAccessOut(access_code=code, granted_by=sources) + for code, sources in sorted(attributions.items()) + ] diff --git a/app/modules/auth/routes/auth_routes.py b/app/modules/auth/routes/auth_routes.py new file mode 100644 index 0000000..0c33ba0 --- /dev/null +++ b/app/modules/auth/routes/auth_routes.py @@ -0,0 +1,499 @@ +from fastapi import APIRouter, Depends, Response, Request, HTTPException +from fastapi.security import OAuth2PasswordRequestForm +from sqlalchemy.orm import Session +from app.db.database import get_db +from app.modules.auth.controllers.auth_controller import AuthController, UserController +from app.modules.auth.repositories.user_repository import UserRepository +from app.modules.auth.schemas.auth_schema import ( + RegisterIn, + Token, + UserOut, + GoogleLoginIn, + UpdateProfileIn, + ChangePasswordIn, + RegisterOut, + ForgotPasswordIn, + ResetPasswordIn, +) +from app.middleware.auth import get_current_user, oauth2_scheme +from app.core.token_blacklist import TokenBlacklist +from app.modules.auth.models.user_model import User +from app.modules.tenant.models.tenant_model import Tenant +from app.middleware.tenant import get_tenant_from_header +from app.core.settings import settings +from jose import jwt +from app.modules.auth.schemas.access_schema import EditorTokenOut +from app.core.schemas import MessageOut, StatusMessageOut + +router = APIRouter(prefix="/auth", tags=["Auth"]) + +@router.post("/register", response_model=RegisterOut) +def register( + request: Request, + payload: RegisterIn, + db: Session = Depends(get_db), + tenant: Tenant = Depends(get_tenant_from_header), +): + return AuthController.register_user(payload, tenant, db) + +@router.post("/login", response_model=Token) +def login( + request: Request, + response: Response, + form_data: OAuth2PasswordRequestForm = Depends(), + db: Session = Depends(get_db), +): + token_data = AuthController.login_user(form_data, db, request) + + response.set_cookie( + key="docqube_access_token", + value=token_data["access_token"], + httponly=True, + secure=settings.APP_ENV == "production", + samesite="none" if settings.APP_ENV == "production" else "lax", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + max_age=3600, + path="/", + ) + + response.set_cookie( + key="docqube_refresh_token", + value=token_data["refresh_token"], + httponly=True, + secure=settings.APP_ENV == "production", + samesite="none" if settings.APP_ENV == "production" else "lax", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + max_age=7 * 24 * 3600, + path="/api/auth/refresh", + ) + + response.set_cookie( + key="docqube_has_session", + value="true", + httponly=False, + secure=settings.APP_ENV == "production", + samesite="none" if settings.APP_ENV == "production" else "lax", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + max_age=7 * 24 * 3600, + path="/", + ) + + if hasattr(request.state, "new_device_id"): + response.set_cookie( + key="docqube_device_id", + value=request.state.new_device_id, + httponly=True, + secure=settings.APP_ENV == "production", + samesite="none" if settings.APP_ENV == "production" else "lax", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + max_age=365 * 24 * 3600, + path="/", + ) + + return token_data + +@router.post("/google", response_model=Token) +def google_login( + request: Request, response: Response, payload: GoogleLoginIn, db: Session = Depends(get_db) +): + token_data = AuthController.google_login(payload, db, request) + + response.set_cookie( + key="docqube_access_token", + value=token_data["access_token"], + httponly=True, + secure=settings.APP_ENV == "production", + samesite="none" if settings.APP_ENV == "production" else "lax", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + max_age=3600, + path="/", + ) + + response.set_cookie( + key="docqube_refresh_token", + value=token_data["refresh_token"], + httponly=True, + secure=settings.APP_ENV == "production", + samesite="none" if settings.APP_ENV == "production" else "lax", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + max_age=7 * 24 * 3600, + path="/api/auth/refresh", + ) + + response.set_cookie( + key="docqube_has_session", + value="true", + httponly=False, + secure=settings.APP_ENV == "production", + samesite="none" if settings.APP_ENV == "production" else "lax", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + max_age=7 * 24 * 3600, + path="/", + ) + + if hasattr(request.state, "new_device_id"): + response.set_cookie( + key="docqube_device_id", + value=request.state.new_device_id, + httponly=True, + secure=settings.APP_ENV == "production", + samesite="none" if settings.APP_ENV == "production" else "lax", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + max_age=365 * 24 * 3600, + path="/", + ) + + return token_data + +@router.post("/forgot-password", response_model=MessageOut) +def forgot_password( + request: Request, payload: ForgotPasswordIn, db: Session = Depends(get_db) +): + return AuthController.forgot_password(payload, db) + +@router.post("/reset-password") +def reset_password( + request: Request, payload: ResetPasswordIn, db: Session = Depends(get_db) +): + return AuthController.reset_password(payload, db) + +@router.get("/me", response_model=UserOut) +@router.get("/verify", response_model=UserOut) +def verify_auth(user: User = Depends(get_current_user)): + return user + +@router.post("/accept-terms", response_model=StatusMessageOut) +def accept_terms(user: User = Depends(get_current_user), db: Session = Depends(get_db)): + if not user.terms_accepted: + user.terms_accepted = True + return {"status": "success", "message": "Terms accepted successfully"} + +@router.post("/refresh", response_model=Token) +def refresh_token(request: Request, response: Response, db: Session = Depends(get_db)): + refresh_token = request.cookies.get("docqube_refresh_token") + if not refresh_token: + raise HTTPException(status_code=401, detail="Refresh token missing") + + try: + payload = jwt.decode( + refresh_token, settings.APP_SECRET, algorithms=[settings.ALGORITHM] + ) + user_id = payload.get("sub") + token_type = payload.get("type") + + if not user_id or token_type != "refresh": + raise HTTPException(status_code=401, detail="Invalid refresh token") + + session_id = payload.get("session_id") + if session_id: + from app.modules.auth.repositories.session_repository import SessionRepository + from app.core.security import verify_password + session_repo = SessionRepository(db) + session = session_repo.get_by_id(session_id) + if not session or session.status != "ACTIVE": + if session and session.status == "REVOKED": + if session.revoked_by == "reauth": + raise HTTPException(status_code=401, detail="You have logged in from another tab on this device. Please refresh.") + else: + raise HTTPException(status_code=401, detail="Your session was remotely logged out from another device.") + raise HTTPException(status_code=401, detail="Your session has expired.") + + if not session.refresh_token_hash or session.refresh_token_hash == "pending" or not verify_password(refresh_token, session.refresh_token_hash): + raise HTTPException(status_code=401, detail="Invalid refresh token hash") + + from datetime import timedelta, datetime, timezone + now = datetime.now(timezone.utc) + last_activity = session.last_activity + if last_activity and last_activity.tzinfo is None: + last_activity = last_activity.replace(tzinfo=timezone.utc) + if not last_activity or now - last_activity > timedelta(minutes=5): + session_repo.update_last_activity(session) + + user_repo = UserRepository(db) + user = user_repo.get_by_id(int(user_id)) + if not user or user.is_deleted or (hasattr(user, "is_active") and not user.is_active): + raise HTTPException(status_code=401, detail="User not found") + + from app.core.security import create_access_token + + payload_access = { + "sub": str(user.id), + "tenant_id": str(user.tenant_id) if user.tenant_id else None, + } + if session_id: + payload_access["session_id"] = session_id + + new_access_token = create_access_token(payload_access) + + response.set_cookie( + key="docqube_access_token", + value=new_access_token, + httponly=True, + secure=settings.APP_ENV == "production", + samesite="none" if settings.APP_ENV == "production" else "lax", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + max_age=3600, + path="/", + ) + + response.set_cookie( + key="docqube_has_session", + value="true", + httponly=False, + secure=settings.APP_ENV == "production", + samesite="none" if settings.APP_ENV == "production" else "lax", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + max_age=7 * 24 * 3600, + path="/", + ) + + return { + "access_token": new_access_token, + "refresh_token": refresh_token, + "token_type": "bearer", + } + + except HTTPException as he: + raise he + except Exception as e: + raise HTTPException(status_code=401, detail="Invalid or expired refresh token") + +@router.get("/editor-token", response_model=EditorTokenOut) +def get_editor_token(current_user: User = Depends(get_current_user)): + from app.core.security import create_access_token + token = create_access_token( + { + "sub": str(current_user.id), + "tenant_id": str(current_user.tenant_id) if current_user.tenant_id else None, + } + ) + return {"token": token} + + +@router.post("/logout") +def logout( + response: Response, + token: str = Depends(oauth2_scheme), + request: Request = None, + db: Session = Depends(get_db) +): + auth_token = token + if not auth_token and request: + auth_token = request.cookies.get("docqube_access_token") + + if auth_token: + try: + unverified_payload = jwt.get_unverified_claims(auth_token) + jti = unverified_payload.get("jti") + exp = unverified_payload.get("exp") + + if jti: + ttl = 3600 + if exp: + from datetime import datetime + + now = datetime.utcnow().timestamp() + ttl = max(1, int(exp - now)) + + TokenBlacklist.add(jti, expires_in=ttl) + + session_id = unverified_payload.get("session_id") + if session_id: + from app.modules.auth.services.auth_service import AuthService + AuthService(db).logout(session_id) + except Exception: + pass + + response.delete_cookie( + "docqube_access_token", + path="/", + samesite="none" if settings.APP_ENV == "production" else "lax", + secure=settings.APP_ENV == "production", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + ) + response.delete_cookie( + "docqube_refresh_token", + path="/api/auth/refresh", + samesite="none" if settings.APP_ENV == "production" else "lax", + secure=settings.APP_ENV == "production", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + ) + response.delete_cookie( + "docqube_has_session", + path="/", + samesite="none" if settings.APP_ENV == "production" else "lax", + secure=settings.APP_ENV == "production", + domain=settings.COOKIE_DOMAIN if settings.APP_ENV == "production" else None, + ) + return {"status": "success", "message": "Logged out"} + +from pydantic import BaseModel +class ResolveDeviceLimitIn(BaseModel): + logout_device_id: int + +@router.post("/resolve-device-limit") +def resolve_device_limit( + payload: ResolveDeviceLimitIn, + request: Request, + db: Session = Depends(get_db) +): + auth_header = request.headers.get("Authorization") + if not auth_header or not auth_header.startswith("Bearer "): + raise HTTPException(status_code=401, detail="Authentication required") + + token = auth_header.split(" ")[1] + + try: + payload_jwt = jwt.decode(token, settings.APP_SECRET, algorithms=[settings.ALGORITHM]) + jti = payload_jwt.get("jti") + if jti and TokenBlacklist.is_blacklisted(jti): + raise HTTPException(status_code=401, detail="Token blacklisted") + except HTTPException as he: + raise he + except Exception: + raise HTTPException(status_code=401, detail="Invalid token") + + if payload_jwt.get("type") != "device_management": + raise HTTPException(status_code=403, detail="Invalid token type") + + user_id = int(payload_jwt.get("sub")) + + from app.modules.auth.repositories.device_repository import DeviceRepository + from app.modules.auth.repositories.session_repository import SessionRepository + + device_repo = DeviceRepository(db) + device = device_repo.get_by_id(payload.logout_device_id) + + if not device or device.user_id != user_id: + raise HTTPException(status_code=404, detail="Device not found") + + session_repo = SessionRepository(db) + session_repo.revoke_all_for_device(device.id, by="user") + + return {"status": "success", "message": "Device logged out successfully. You can now login."} + +from typing import List +from app.modules.auth.schemas.auth_schema import DeviceOut + +@router.get("/devices", response_model=List[DeviceOut]) +def get_user_devices( + request: Request, + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + from app.modules.auth.repositories.device_repository import DeviceRepository + from app.modules.auth.utils.device_utils import generate_device_fingerprint + + device_repo = DeviceRepository(db) + devices = device_repo.get_user_devices(current_user.id) + + current_fingerprint = generate_device_fingerprint(request) + + results = [] + for d in devices: + results.append({ + "id": d.id, + "browser": d.browser, + "os": d.os, + "device_type": d.device_type, + "city": d.city, + "country": d.country, + "state": d.state, + "last_login": d.last_login, + "is_current_device": d.fingerprint == current_fingerprint + }) + results.sort(key=lambda x: not x["is_current_device"]) + return results + +@router.delete("/devices/{device_id}") +def logout_device( + device_id: int, + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + from app.modules.auth.repositories.device_repository import DeviceRepository + from app.modules.auth.repositories.session_repository import SessionRepository + + device_repo = DeviceRepository(db) + device = device_repo.get_by_id(device_id) + + if not device or device.user_id != current_user.id: + raise HTTPException(status_code=404, detail="Device not found") + + session_repo = SessionRepository(db) + session_repo.revoke_all_for_device(device.id, by="user") + + try: + from app.modules.activity_logs.service import log_event + from app.modules.activity_logs.constants import ActivityLogModule, ActivityLogAction, ActivityLogStatus, ActivityLogTargetType + log_event( + tenant_id=current_user.tenant_id, + user_id=current_user.id, + user_email=current_user.email, + module=ActivityLogModule.TENANT, + action=ActivityLogAction.DEVICE_LOGOUT, + target_id=str(device.id), + target_type=ActivityLogTargetType.SESSION, + metadata={ + "device_name": device.device_name, + "ip_address": device.ip_address, + }, + status=ActivityLogStatus.SUCCESS + ) + except Exception as e: + import logging + logging.getLogger(__name__).error(f"Failed to log device logout: {e}") + + return {"status": "success", "message": "Logged out from device successfully"} + +from fastapi.responses import StreamingResponse +import asyncio + +@router.get("/session/events") +async def session_events( + request: Request, + current_user: User = Depends(get_current_user) +): + """ + SSE Endpoint for real-time session and device revocation events. + Frontend can listen to this to instantly log out when their session is revoked. + """ + from app.core.redis import redis_pubsub + import json + + async def event_generator(): + if not redis_pubsub._client: + await redis_pubsub.connect() + + if not redis_pubsub._client: + return + + pubsub = redis_pubsub._client.pubsub() + channel = f"user_events:{current_user.id}" + await pubsub.subscribe(channel) + + try: + while True: + if await request.is_disconnected(): + break + + message = await pubsub.get_message(ignore_subscribe_messages=True, timeout=1.0) + if message and message["type"] == "message": + data = message["data"] + if isinstance(data, bytes): + data = data.decode('utf-8') + yield f"data: {data}\n\n" + + await asyncio.sleep(0.1) + except asyncio.CancelledError: + pass + except Exception: + pass + finally: + try: + await pubsub.unsubscribe(channel) + await pubsub.aclose() + except Exception: + pass + + return StreamingResponse(event_generator(), media_type="text/event-stream") \ No newline at end of file diff --git a/app/modules/auth/routes/me_routes.py b/app/modules/auth/routes/me_routes.py new file mode 100644 index 0000000..c7da55c --- /dev/null +++ b/app/modules/auth/routes/me_routes.py @@ -0,0 +1,213 @@ +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy.orm import Session +from typing import Optional +import hmac +import hashlib + +from app.db.database import get_db +from app.modules.auth.schemas.access_schema import ( + ApiKeyStatusOut, + MeStatsOut, + StorageNonceOut, +) +from app.modules.auth.controllers.auth_controller import UserController +from app.modules.auth.schemas.auth_schema import ( + UserOut, + UpdateProfileIn, + ChangePasswordIn, + UserSMTPCredentialsOut, + UserSMTPCredentialsPatchIn, +) +from app.middleware.auth import get_current_user +from app.modules.auth.models.user_model import User +from app.core.settings import settings +from app.core.crypto import encrypt_data +from app.core.mail import _resolve_smtp_config +from pydantic import BaseModel +from app.core.schemas import StatusOut + +class TutorialCompleteSchema(BaseModel): + key: str = "dashboard" + +router = APIRouter(prefix="/me", tags=["User Profile"]) + +@router.get("/profile", response_model=UserOut) +def get_profile(user: User = Depends(get_current_user)): + return user + +@router.get("/storage-nonce", response_model=StorageNonceOut) +def get_storage_nonce(user: User = Depends(get_current_user)): + """ + Returns a deterministic, user-specific nonce for client-side encryption. + Combined with APP_SECRET to ensure it's not guessing-prone. + """ + nonce = hmac.new( + settings.APP_SECRET.encode(), + f"user_storage_{user.id}_{user.email}".encode(), + hashlib.sha256 + ).hexdigest() + return {"nonce": nonce} + +@router.patch("/profile", response_model=UserOut) +def update_profile( + payload: UpdateProfileIn, + user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + return UserController.update_profile( + user, + payload.name, + db, + designation=payload.designation, + preferred_language=payload.preferred_language, + ) + +@router.post("/password") +def change_password( + payload: ChangePasswordIn, + user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + return UserController.change_password(user, payload.current_password, payload.new_password, db) + +@router.get("/stats", response_model=MeStatsOut) +def get_stats( + user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + return UserController.get_user_stats(user, db) + +@router.delete("/account") +def delete_account( + user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + return UserController.delete_account(user, db) + +@router.post("/api-key", response_model=StatusOut) +def update_api_key( + payload: dict, + user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + key = payload.get("key") + if key and str(key).strip(): + from app.core.crypto import encrypt_data + user.encrypted_ai_api_key = encrypt_data(str(key).strip()) + else: + user.encrypted_ai_api_key = None + return {"status": "success"} + +@router.get("/api-key/status", response_model=ApiKeyStatusOut) +def get_api_key_status(user: User = Depends(get_current_user)): + return {"has_key": bool(user.encrypted_ai_api_key)} + + +@router.get("/smtp-credentials", response_model=UserSMTPCredentialsOut) +def get_smtp_credentials( + user: User = Depends(get_current_user), + db: Session = Depends(get_db), +): + smtp_config = _resolve_smtp_config( + db=db, + tenant_id=user.tenant_id, + user_id=user.id, + ) + return UserSMTPCredentialsOut( + configured=bool(user.smtp_user and user.encrypted_smtp_password), + source=smtp_config["source"], + smtp_host=smtp_config["smtp_host"], + smtp_port=smtp_config["smtp_port"], + smtp_user=smtp_config["smtp_user"], + mail_from=smtp_config["mail_from"], + has_password=bool(user.encrypted_smtp_password), + ) + + +@router.patch("/smtp-credentials", response_model=UserSMTPCredentialsOut) +def update_smtp_credentials( + payload: UserSMTPCredentialsPatchIn, + user: User = Depends(get_current_user), + db: Session = Depends(get_db), +): + if payload.smtp_user is not None: + cleaned_user = payload.smtp_user.strip() + user.smtp_user = cleaned_user or None + + if payload.mail_from is not None: + cleaned_mail_from = str(payload.mail_from).strip() + user.mail_from = cleaned_mail_from or None + + if payload.smtp_password is not None: + cleaned_password = payload.smtp_password.strip() + user.encrypted_smtp_password = ( + encrypt_data(cleaned_password) if cleaned_password else None + ) + + db.add(user) + db.flush() + db.refresh(user) + + smtp_config = _resolve_smtp_config( + db=db, + tenant_id=user.tenant_id, + user_id=user.id, + ) + return UserSMTPCredentialsOut( + configured=bool(user.smtp_user and user.encrypted_smtp_password), + source=smtp_config["source"], + smtp_host=smtp_config["smtp_host"], + smtp_port=smtp_config["smtp_port"], + smtp_user=smtp_config["smtp_user"], + mail_from=smtp_config["mail_from"], + has_password=bool(user.encrypted_smtp_password), + ) + +@router.post("/tutorial-complete", response_model=UserOut) +def complete_tutorial( + payload: TutorialCompleteSchema, + user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + tutorial_key = payload.key + print(f"DEBUG: Tutorial completion request for user {user.email}, key: {tutorial_key}") + + if tutorial_key == "docs": + user.has_completed_docs_tutorial = True + print(f"DEBUG: Setting has_completed_docs_tutorial = True") + else: + user.has_completed_tutorial = True + print(f"DEBUG: Setting has_completed_tutorial = True") + + db.add(user) +@router.post("/tutorial/finish", response_model=UserOut) +def finish_tutorial( + payload: Optional[TutorialCompleteSchema] = None, + user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + """ + A specific, robust endpoint to mark tutorials as finished. + Automatically handles both dashboard and docs depending on the key. + """ + key = payload.key if payload else "dashboard" + if key == "docs": + user.has_completed_docs_tutorial = True + elif key == "dashboard": + user.has_completed_tutorial = True + elif key == "all": + user.has_completed_tutorial = True + user.has_completed_docs_tutorial = True + + try: + db.add(user) + db.flush() + db.refresh(user) + except HTTPException: + raise + except Exception as e: + db.rollback() + raise HTTPException(status_code=500, detail="Database persistence failed") + raise HTTPException(status_code=500, detail="Database persistence failed") + + return user diff --git a/app/modules/auth/routes/role_routes.py b/app/modules/auth/routes/role_routes.py new file mode 100644 index 0000000..65a63aa --- /dev/null +++ b/app/modules/auth/routes/role_routes.py @@ -0,0 +1,344 @@ +from app.modules.auth.schemas.access_schema import RoleAssignedOut +""" +Role Management Routes - RBAC endpoints for creating, updating, deleting roles. +""" + +from fastapi import APIRouter, Depends, Query, HTTPException, status +from sqlalchemy.orm import Session +from typing import Optional, List +from uuid import UUID +from pydantic import BaseModel + +from app.db.database import get_db +from app.middleware.auth import get_current_user +from app.modules.auth.models.user_model import User +from app.modules.auth.controllers.role_controller import RoleController +from app.modules.auth.schemas.role_schema import ( + RoleCreate, + RoleUpdate, + RoleOut, + RoleWithAccessesOut, + RolePaginatedOut, +) +from app.modules.auth.dependencies.access_dependency import require_access +from app.modules.auth.services.privilege import holds_superadmin_access +from app.modules.auth.services.user_role_service import UserRoleReader +from app.core.schemas import MessageOut + + +class AssignUserRoleIn(BaseModel): + user_id: int + role_id: UUID + + +router = APIRouter(prefix="/api/roles", tags=["Roles"]) + + +def _is_superadmin(current_user: User) -> bool: + """True when the caller holds any superadmin access code.""" + return holds_superadmin_access(current_user) + + +def _assert_role_in_callers_tenant(role, current_user: User, db: Session = None, allow_system_roles: bool = False) -> None: + """ + Refuse to act on a role belonging to another tenant. + + `RoleController.get_role_by_id` looks a role up by primary key alone, so + without this every by-id route was reachable across tenants — a tenant + administrator could read, rename and delete another tenant's roles by + guessing or observing a UUID. Raises 404 rather than 403 so the response + does not confirm that the role exists. + """ + if _is_superadmin(current_user): + return + if role.tenant_id == current_user.tenant_id: + return + + if allow_system_roles and getattr(role, "is_system", False) and current_user.tenant_id and db: + from app.modules.billing.models.plan_model import TenantSubscription, PlanRole + sub = db.query(TenantSubscription).filter( + TenantSubscription.tenant_id == current_user.tenant_id, + TenantSubscription.status == 'active' + ).order_by(TenantSubscription.created_at.desc()).first() + if sub and sub.plan_id: + has_role = db.query(PlanRole).filter( + PlanRole.plan_id == sub.plan_id, + PlanRole.role_id == role.id + ).first() + if has_role: + return + + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Role not found" + ) + + +@router.post( + "/", response_model=RoleOut, dependencies=[require_access("admin.role.create")] +) +def create_role( + payload: RoleCreate, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """ + Create a new role. + + Required access: admin.role.create | superadmin.role.create + + - **role_name**: Name of the role + - **description**: Optional description + - **access_ids**: List of access IDs to assign + - **tenant_id**: Tenant ID (for tenant-specific roles) + """ + if not payload.tenant_id and current_user.tenant_id: + payload.tenant_id = current_user.tenant_id + + result = RoleController.create_role(db, payload) + try: + from app.modules.activity_logs.service import log_event + from app.modules.activity_logs.constants import ActivityLogModule, ActivityLogAction, ActivityLogStatus, ActivityLogTargetType + if payload.tenant_id: + log_event( + tenant_id=payload.tenant_id, + user_id=current_user.id, + user_email=current_user.email, + module=ActivityLogModule.TENANT, + action=ActivityLogAction.ROLE_CREATED, + target_id=str(result.id), + target_type=ActivityLogTargetType.ROLE, + metadata={ + "role_name": payload.role_name + }, + status=ActivityLogStatus.SUCCESS + ) + except Exception as e: + import logging + logging.getLogger(__name__).error(f"Failed to log ROLE_CREATED: {e}") + + return result + + +@router.get( + "/", response_model=List[RoleOut], dependencies=[require_access("admin.role.read")] +) +def get_all_roles( + tenant_id: Optional[UUID] = Query(None), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """ + Get all roles. + + Required access: admin.role.read | superadmin.role.read + + Superadmins see roles across all tenants by default, or a single + tenant when `?tenant_id=` is provided. Tenant admins are + always scoped to their own tenant regardless of the query param. + """ + effective_tenant_id = ( + tenant_id if _is_superadmin(current_user) else current_user.tenant_id + ) + return RoleController.get_all_roles(db, effective_tenant_id) + + +@router.get( + "/{role_id}", + response_model=RoleOut, + dependencies=[require_access("admin.role.read")], +) +def get_role( + role_id: UUID, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """ + Get a specific role by ID. + + Required access: admin.role.read | superadmin.role.read + """ + role = RoleController.get_role_by_id(db, role_id) + _assert_role_in_callers_tenant(role, current_user, db, allow_system_roles=True) + return role + + +@router.get( + "/{role_id}/details", + response_model=RoleWithAccessesOut, + dependencies=[require_access("admin.role.read")], +) +def get_role_with_accesses( + role_id: UUID, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """ + Get a role with all assigned access codes. + + Required access: admin.role.read | superadmin.role.read + + Also reports how many principals hold it, so the editing screen can say what + a change affects. Counted here and not on the list endpoints, which render + up to a hundred roles. + """ + _assert_role_in_callers_tenant( + RoleController.get_role_by_id(db, role_id), current_user, db, allow_system_roles=True + ) + role = RoleController.get_role_with_accesses(db, role_id) + out = RoleWithAccessesOut.model_validate(role) + out.assigned_user_count = UserRoleReader(db).users_holding_role(role_id) + return out + + +@router.patch( + "/{role_id}", + response_model=RoleOut, + dependencies=[require_access("admin.role.update")], +) +def update_role( + role_id: UUID, + payload: RoleUpdate, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """ + Update a role's details and accesses. + + Required access: admin.role.update | superadmin.role.update + + - **role_name**: New role name (optional) + - **description**: New description (optional) + - **access_ids**: New list of access IDs (optional) + """ + role = RoleController.get_role_by_id(db, role_id) + _assert_role_in_callers_tenant(role, current_user) + if role.is_default and current_user.tenant_id: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Default roles cannot be modified by tenant admins." + ) + return RoleController.update_role(db, role_id, payload) + + +@router.delete("/{role_id}", dependencies=[require_access("admin.role.update")], response_model=MessageOut) +def delete_role( + role_id: UUID, + force: bool = Query( + False, + description=( + "Delete even though the role is still held. Required once anybody " + "holds it, so the removal of their authority is a decision rather " + "than a side effect." + ), + ), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """ + Delete a role. + + Required access: admin.role.update | superadmin.role.delete + + **Refuses with 409 while the role is still held**, unless `?force=true`. + + Deleting a role has always taken authority away silently: `user_roles.role_id` + is `ON DELETE CASCADE`, so every grant of it disappears, and `users.role_id` + is `ON DELETE SET NULL`, so every holder loses their primary role. Neither + leaves a trace and neither was ever counted for the operator. With one role + per user that was survivable; with roles granted in several places it is + not, because the operator can no longer hold the affected set in their head. + + The cascade behaviour is unchanged. Only the silence is. + """ + role = RoleController.get_role_by_id(db, role_id) + _assert_role_in_callers_tenant(role, current_user) + if role.is_default and current_user.tenant_id: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Default roles cannot be deleted by tenant admins." + ) + + if not force: + holders = UserRoleReader(db).users_holding_role(role_id) + if holders: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + f"This role cannot be deleted because there are currently {holders} " + f"active {'assignment' if holders == 1 else 'assignments'} of it. " + "Please reassign or remove them first." + ), + ) + + return RoleController.delete_role(db, role_id) + + +@router.get( + "/tenant/{tenant_id}", + response_model=List[RoleOut], + dependencies=[require_access("admin.role.read")], +) +def get_roles_by_tenant( + tenant_id: UUID, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """ + Get all roles for a specific tenant. + + The tenant is named by the caller, so a tenant administrator must not be + able to name somebody else's. + """ + if not _is_superadmin(current_user) and tenant_id != current_user.tenant_id: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Tenant not found" + ) + return RoleController.get_all_roles(db, tenant_id) + + +@router.post( + "/assign-user", + dependencies=[require_access("admin.role.assign")], + response_model=RoleAssignedOut) +def assign_role_to_user( + payload: AssignUserRoleIn, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """Assign a role to a user.""" + return RoleController.assign_role_to_user(db, payload.user_id, payload.role_id) + + +@router.get( + "", + response_model=RolePaginatedOut, + dependencies=[require_access("admin.role.read")], +) +def list_roles_paginated( + page: int = Query(1, ge=1), + page_size: int = Query(10, ge=1, le=100), + search: Optional[str] = Query(None), + tenant_id: Optional[UUID] = Query(None), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """ + Get paginated list of roles with optional search. + + Required access: admin.role.read | superadmin.role.read + + Superadmins see roles across all tenants by default, or a single + tenant when `?tenant_id=` is provided. Tenant admins are + always scoped to their own tenant regardless of the query param. + """ + effective_tenant_id = ( + tenant_id if _is_superadmin(current_user) else current_user.tenant_id + ) + + return RoleController.get_roles_paginated( + db=db, + tenant_id=effective_tenant_id, + page=page, + page_size=page_size, + search=search, + ) diff --git a/app/modules/auth/routes/user_routes.py b/app/modules/auth/routes/user_routes.py new file mode 100644 index 0000000..0bdb13c --- /dev/null +++ b/app/modules/auth/routes/user_routes.py @@ -0,0 +1,33 @@ +from fastapi import APIRouter, Depends, Query +from sqlalchemy.orm import Session +from typing import List + +from app.db.database import get_db +from app.modules.auth.models.user_model import User +from app.modules.auth.repositories.user_repository import UserRepository +from app.modules.auth.schemas.auth_schema import UserOut +from app.modules.auth.services.user_role_service import UserRoleReader, attach_role_refs +from app.middleware.auth import get_current_user + +router = APIRouter(prefix="/users", tags=["Users"]) + +@router.get("/search", response_model=List[UserOut]) +def search_users( + q: str = Query(..., min_length=1), + limit: int = Query(10, ge=1, le=50), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user) +): + """ + Search users by name or email, within the caller's own tenant. + + Previously unscoped: any authenticated user could enumerate every user of + every tenant, complete with email addresses, roles and access codes. A + caller with no tenant is a superadmin under today's model and still + searches across tenants. + """ + user_repo = UserRepository(db) + users = user_repo.search_users(q, limit, tenant_id=current_user.tenant_id) + + attach_role_refs(UserRoleReader(db), users) + return users diff --git a/app/modules/auth/schemas/access_schema.py b/app/modules/auth/schemas/access_schema.py new file mode 100644 index 0000000..a42d09d --- /dev/null +++ b/app/modules/auth/schemas/access_schema.py @@ -0,0 +1,84 @@ +""" +Response schemas for the access (RBAC catalogue) and self-service endpoints. + +Every shape here was taken from a recorded response in +`tests/characterization/snapshots`, not from the ORM models — the drive module +showed how far the hand-written schemas had drifted from what the endpoints +actually return, and a `response_model` that omits a field silently deletes it +from the payload. +""" + +from typing import List, Optional +from uuid import UUID + +from pydantic import BaseModel + + +class AccessOut(BaseModel): + """One row of the access catalogue. 66 exist.""" + + id: UUID + name: str + access_code: str + category: str + parent_id: Optional[UUID] = None + + class Config: + from_attributes = True + + +class RoleAccessesOut(BaseModel): + """`GET /api/access/get` — the caller's roles and the codes they carry.""" + + role: Optional[str] = None + """The **primary** role. Retained and unchanged; `roles` is the full set.""" + + roles: List[str] = [] + """ + Every role the caller holds, primary first. + + Additive, and a plain list of names rather than the richer + `RoleAssignmentOut` used elsewhere: this endpoint exists so the frontend can + decide what to render, and scope and expiry do not change that decision. + `/api/me/profile` carries the detailed shape. + """ + + tenant_id: Optional[str] = None + accesses: List[str] = [] + + +class ApiKeyStatusOut(BaseModel): + has_key: bool + + +class StorageNonceOut(BaseModel): + nonce: str + + +class MeStatsOut(BaseModel): + """ + `GET /api/me/stats`. The camelCase keys are the existing contract — the + frontend reads them as they are, so they are pinned rather than tidied. + Renaming them is a frontend change, not a schema change. + """ + + totalProjects: int = 0 + totalFiles: int = 0 + subscription: Optional[str] = None + joined: Optional[str] = None + + +class EditorTokenOut(BaseModel): + token: str + + +class CeleryHealthOut(BaseModel): + status: str + + +class RoleAssignedOut(BaseModel): + """`POST /api/roles/assign-user`.""" + + message: str + user_id: int + role_id: str diff --git a/app/modules/auth/schemas/admin_dashboard_schema.py b/app/modules/auth/schemas/admin_dashboard_schema.py new file mode 100644 index 0000000..f8b3ab0 --- /dev/null +++ b/app/modules/auth/schemas/admin_dashboard_schema.py @@ -0,0 +1,57 @@ +from pydantic import BaseModel + + +class MetricTrendPointOut(BaseModel): + date: str + value: int + + +class MetricCountOut(BaseModel): + total: int + recent: int + trend: list[MetricTrendPointOut] + + +class TotalMetricOut(BaseModel): + total: int + recent_added: int + recent_deleted: int + + +class ShareMetricOut(BaseModel): + internal: int + external: int + + +class StorageMetricOut(BaseModel): + used_bytes: int + quota_bytes: int + usage_percentage: float + used_formatted: str + quota_formatted: str + + +class ConversionMetricOut(BaseModel): + total: int + completed: int + processing: int + failed: int + completion_rate: float + + +class SimpleMetricOut(BaseModel): + total: int + + +class AdminDashboardMetricsOut(BaseModel): + signatures: MetricCountOut | None = None + uploads: MetricCountOut + users: TotalMetricOut + shares: ShareMetricOut + storage: StorageMetricOut + conversions: ConversionMetricOut | None = None + edits: SimpleMetricOut + + +class MetricTrendOnlyOut(BaseModel): + trend: list[MetricTrendPointOut] diff --git a/app/modules/auth/schemas/auth_schema.py b/app/modules/auth/schemas/auth_schema.py new file mode 100644 index 0000000..10a4188 --- /dev/null +++ b/app/modules/auth/schemas/auth_schema.py @@ -0,0 +1,182 @@ +from pydantic import BaseModel, EmailStr, ConfigDict, Field, field_validator +from typing import Optional +from typing import Literal +from datetime import datetime +import re + + +class Token(BaseModel): + access_token: str + refresh_token: Optional[str] = None + token_type: str = "bearer" + + +class TokenData(BaseModel): + email: Optional[str] = None + + +class RegisterIn(BaseModel): + email: EmailStr + password: str = Field(..., min_length=12) + name: str = Field(..., min_length=2) + designation: Optional[str] = Field(None, max_length=100) + + @field_validator("password") + @classmethod + def password_complexity(cls, v: str) -> str: + if not re.search(r"[A-Z]", v): + raise ValueError("Password must contain at least one uppercase letter") + if not re.search(r"[0-9]", v): + raise ValueError("Password must contain at least one number") + if not re.search(r"[^A-Za-z0-9]", v): + raise ValueError("Password must contain at least one special character") + return v + + +class LoginIn(BaseModel): + email: Optional[EmailStr] = None + username: Optional[str] = None + password: str + + +from uuid import UUID + +from app.modules.auth.schemas.role_schema import RoleAssignmentOut + + +class UserOut(BaseModel): + id: int + email: EmailStr + name: str + designation: Optional[str] = None + preferred_language: str = "en" + subscription: Optional[str] = "free" + created_at: Optional[datetime] = None + tenant_id: Optional[UUID] = None + is_active: Optional[bool] = True + terms_accepted: bool = False + has_completed_tutorial: bool = False + has_completed_docs_tutorial: bool = False + + role: Optional[str] = Field(None, validation_alias="role_name") + role_id: Optional[UUID] = None + roles: list[RoleAssignmentOut] = Field( + default_factory=list, validation_alias="role_refs" + ) + """ + Every role the user holds — primary, granted, and through a group. + + **Additive.** `role` and `role_id` are retained and unchanged, because they + are read by the frontend, by the admin screens and by anything else already + consuming this shape. `role` remains the *primary* role specifically, which + is what the existing dropdown edits. + """ + accesses: list[str] = Field(default_factory=list, validation_alias="access_codes") + subscription_details: Optional[dict] = None + tenant_envelope_limit: Optional[int] = None + tenant_envelopes_used: Optional[int] = None + + model_config = ConfigDict(from_attributes=True, populate_by_name=True) + + +class GoogleLoginIn(BaseModel): + token: Optional[str] = None + credential: Optional[str] = None + tenant_id: Optional[str] = None + + +class UpdateProfileIn(BaseModel): + name: Optional[str] = Field(None, min_length=2, max_length=255) + designation: Optional[str] = Field(None, max_length=100) + preferred_language: Optional[str] = Field(None, min_length=2, max_length=16) + + @field_validator("preferred_language") + @classmethod + def normalize_preferred_language(cls, value: Optional[str]) -> Optional[str]: + if value is None: + return None + normalized = value.strip().lower() + if not normalized: + raise ValueError("Preferred language cannot be empty") + if not re.fullmatch(r"[a-z]{2,3}(?:-[a-z0-9]{2,8})?", normalized): + raise ValueError("Preferred language must be a valid language code") + return normalized + + +class ChangePasswordIn(BaseModel): + current_password: str + new_password: str = Field(..., min_length=12) + + @field_validator("new_password") + @classmethod + def password_complexity(cls, v: str) -> str: + if not re.search(r"[A-Z]", v): + raise ValueError("Password must contain at least one uppercase letter") + if not re.search(r"[0-9]", v): + raise ValueError("Password must contain at least one number") + if not re.search(r"[^A-Za-z0-9]", v): + raise ValueError("Password must contain at least one special character") + return v + + +class UserSMTPCredentialsPatchIn(BaseModel): + smtp_user: Optional[str] = Field(None, max_length=255) + mail_from: Optional[EmailStr] = None + smtp_password: Optional[str] = Field( + None, + description="If omitted during update, existing password is preserved. Send empty string to clear it.", + ) + + +class UserSMTPCredentialsOut(BaseModel): + configured: bool + source: Literal["user", "tenant", "fallback"] + smtp_host: Optional[str] = None + smtp_port: int = 587 + smtp_user: Optional[str] = None + mail_from: Optional[str] = None + has_password: bool = False + + +class UpgradeIn(BaseModel): + plan: str + + +class RegisterOut(BaseModel): + message: str + user_id: Optional[int] = None + email: Optional[EmailStr] = None + tenant_id: Optional[UUID] = None + + +class ForgotPasswordIn(BaseModel): + email: EmailStr + + +class ResetPasswordIn(BaseModel): + token: str + new_password: str = Field(..., min_length=12) + + @field_validator("new_password") + @classmethod + def password_complexity(cls, v: str) -> str: + if not re.search(r"[A-Z]", v): + raise ValueError("Password must contain at least one uppercase letter") + if not re.search(r"[0-9]", v): + raise ValueError("Password must contain at least one number") + if not re.search(r"[^A-Za-z0-9]", v): + raise ValueError("Password must contain at least one special character") + return v + +class DeviceOut(BaseModel): + id: int + browser: Optional[str] = None + os: Optional[str] = None + device_type: Optional[str] = None + city: Optional[str] = None + country: Optional[str] = None + state: Optional[str] = None + last_login: Optional[datetime] = None + is_current_device: bool = False + + model_config = ConfigDict(from_attributes=True) diff --git a/app/modules/auth/schemas/role_schema.py b/app/modules/auth/schemas/role_schema.py new file mode 100644 index 0000000..77cfaad --- /dev/null +++ b/app/modules/auth/schemas/role_schema.py @@ -0,0 +1,141 @@ +from pydantic import BaseModel, Field, ConfigDict +from typing import Literal, Optional, List +from datetime import datetime +from uuid import UUID + + +class RoleAccessOut(BaseModel): + id: UUID + name: str + access_code: str + category: str + parent_id: Optional[UUID] = None + + model_config = ConfigDict(from_attributes=True) + + +class RoleCreate(BaseModel): + role_name: str = Field(..., max_length=100) + description: Optional[str] = None + tenant_id: Optional[UUID] = None + access_ids: Optional[List[UUID]] = [] + is_default: Optional[bool] = False + + +class RoleUpdate(BaseModel): + role_name: Optional[str] = None + description: Optional[str] = None + access_ids: Optional[List[UUID]] = None + is_default: Optional[bool] = None + + +class RoleOut(BaseModel): + id: UUID + role_name: str = Field(..., alias="name") + description: Optional[str] = None + tenant_id: Optional[UUID] = None + is_default: bool = False + is_system: bool = False + created_at: Optional[datetime] = None + + model_config = ConfigDict(from_attributes=True, populate_by_name=True) + + +class RoleWithAccessesOut(RoleOut): + accesses: List[RoleAccessOut] = [] + assigned_user_count: Optional[int] = None + """ + How many principals this role reaches, or null when it was not counted. + + Added to the *details* response and deliberately not to `RoleOut`: the list + endpoints render up to a hundred roles and counting each one would be an + N+1 nobody asked for. It exists so that deleting a role can say what it is + about to do — `user_roles.role_id` is `ON DELETE CASCADE` and + `users.role_id` is `ON DELETE SET NULL`, so a silent delete removes + authority from people the operator was not thinking about. + """ + + +class RoleAssignmentOut(BaseModel): + """ + One role a user holds, and how they hold it. + + `source` is the field that keeps the interface honest: + + "primary" the legacy `users.role_id`. Changed through + `PATCH /api/admin/users/{id}`, **not** revocable as a grant, + and while it is set it cannot be narrowed by anything (see + `ScopeService._resolve`) + "grant" a `user_roles` row naming this user. Added and revoked freely + "group" a `user_roles` row naming a group this user belongs to. It + confers codes; it is removed by editing the group + + Rendering the three identically would imply that removing any of them is the + same action. It is not, and that is exactly where a permissions screen + misleads the person using it. + """ + + grant_id: Optional[UUID] = None + role_id: UUID + role_name: str + source: Literal["primary", "grant", "group"] + org_unit_id: Optional[UUID] = None + org_unit_name: Optional[str] = None + group_id: Optional[UUID] = None + group_name: Optional[str] = None + expires_at: Optional[datetime] = None + assigned_by_id: Optional[int] = None + created_at: Optional[datetime] = None + is_expired: bool = False + + model_config = ConfigDict(from_attributes=True) + + +class AddUserRoleIn(BaseModel): + """ + Add one role to one user. + + `org_unit_id` omitted or null means **tenant-wide** — the authority every + user already has through their primary role. That default is the point of + this endpoint: the organisation tree could only ever grant *scoped to a + unit*, so "Alice is an Editor and an Approver, everywhere" was not + expressible anywhere in the product. + """ + + role_id: UUID + org_unit_id: Optional[UUID] = None + expires_at: Optional[datetime] = None + + +class AccessAttributionOut(BaseModel): + """Which role conferred one access code, and at what scope.""" + + role_id: Optional[UUID] = None + role_name: Optional[str] = None + source: str + org_unit_id: Optional[UUID] = None + org_unit_name: Optional[str] = None + expires_at: Optional[datetime] = None + + +class EffectiveAccessOut(BaseModel): + """ + One access code the user holds, with every reason they hold it. + + Answers "why can this person do that". With one role per user the question + answered itself. With several it does not, and support cannot read the + database. + """ + + access_code: str + granted_by: List[AccessAttributionOut] + + +class RolePaginatedOut(BaseModel): + items: List[RoleOut] + total: int + page: int + page_size: int + total_pages: int + + model_config = ConfigDict(from_attributes=True) diff --git a/app/modules/auth/services/admin_dashboard_service.py b/app/modules/auth/services/admin_dashboard_service.py new file mode 100644 index 0000000..2168cd5 --- /dev/null +++ b/app/modules/auth/services/admin_dashboard_service.py @@ -0,0 +1,557 @@ +from datetime import datetime, timedelta, timezone +from uuid import UUID + +from fastapi import HTTPException +from sqlalchemy import func +from sqlalchemy.orm import Session + +from app.modules.activity_logs.constants import ActivityLogAction, ActivityLogStatus +from app.modules.activity_logs.models.activity_log import ActivityLog +from app.modules.auth.models.user_model import User +from app.modules.auth.services.permission_service import PermissionService +from app.modules.documents.models.document_model import Project +from app.modules.drive.models.drive_model import DriveACL, DriveFile, DriveFolder, DriveShare +from app.modules.signing.models.signing_request import SigningRequest +from app.modules.storage.services.storage_service import StorageService +from app.modules.auth.services.privilege import holds_superadmin_access + +CONVERSION_JOBS_ACCESS_CODE = "metrics:conversion_jobs" +SIGNATURES_TREND_ACCESS_CODE = "metrics:signatures_trend" + + +class AdminDashboardService: + def __init__(self, db: Session): + self.db = db + + def _is_tenant_admin(self, current_user: User) -> bool: + return bool(current_user.tenant_id) and not holds_superadmin_access( + current_user + ) + + def get_tenant_metrics(self, current_user: User, granularity: str = "weekly") -> dict: + tenant_id = current_user.tenant_id + if not tenant_id: + raise HTTPException( + status_code=403, + detail="Tenant-scoped admin metrics are only available for tenant users.", + ) + + # 1. Try Redis cache lookup (tenant + granularity isolated) + cache_key = None + try: + from app.db.redis import redis_cache + from app.modules.drive.constants import CACHE_ADMIN_METRICS, CACHE_ADMIN_METRICS_TTL_SECONDS + cache_key = CACHE_ADMIN_METRICS.format(tenant_id=str(tenant_id), granularity=granularity) + cached = redis_cache.get(cache_key) + if cached: + import json + return json.loads(cached) + except Exception: + pass + + recent_since = datetime.now(timezone.utc) - timedelta(days=7) + trend_config = self._get_trend_config(granularity) + permission_service = PermissionService(self.db) + is_tenant_admin = self._is_tenant_admin(current_user) + can_view_conversions = is_tenant_admin or permission_service.has_access( + current_user, CONVERSION_JOBS_ACCESS_CODE + ) + can_view_signatures = is_tenant_admin or permission_service.has_access( + current_user, SIGNATURES_TREND_ACCESS_CODE + ) + + metrics = { + "signatures": { + "total": self._count_signatures(tenant_id), + "recent": self._count_recent_signatures(tenant_id, recent_since), + "trend": self._get_signature_trend(tenant_id, trend_config), + } + if can_view_signatures + else None, + "uploads": { + "total": self._count_uploads(tenant_id), + "recent": self._count_recent_uploads(tenant_id, recent_since), + "trend": self._get_upload_trend(tenant_id, trend_config), + }, + "users": { + "total": self._count_total_users(tenant_id), + "recent_added": self._count_recent_users_added(tenant_id, recent_since), + "recent_deleted": self._count_recent_users_deleted(tenant_id, recent_since), + }, + "shares": { + "internal": self._count_internal_shares(tenant_id), + "external": self._count_external_shares(tenant_id), + }, + "storage": self._get_storage_metrics(current_user), + "conversions": self._get_conversion_metrics(tenant_id) + if can_view_conversions + else None, + "edits": { + "total": self._count_recent_edits(tenant_id, recent_since), + }, + } + + # 2. Store in Redis + if cache_key: + try: + import json + redis_cache.set(cache_key, json.dumps(metrics, default=str), ex=CACHE_ADMIN_METRICS_TTL_SECONDS) + except Exception: + pass + + return metrics + + def _count_signatures(self, tenant_id: UUID) -> int: + return ( + self.db.query(SigningRequest) + .filter( + SigningRequest.tenant_id == tenant_id, + SigningRequest.status == "completed", + ) + .count() + ) + + def _count_recent_signatures(self, tenant_id: UUID, recent_since: datetime) -> int: + return ( + self.db.query(SigningRequest) + .filter( + SigningRequest.tenant_id == tenant_id, + SigningRequest.status == "completed", + SigningRequest.completed_at >= recent_since, + ) + .count() + ) + + def _count_uploads(self, tenant_id: UUID) -> int: + return self.db.query(DriveFile).filter(DriveFile.tenant_id == tenant_id).count() + + def _count_recent_uploads(self, tenant_id: UUID, recent_since: datetime) -> int: + return ( + self.db.query(DriveFile) + .filter( + DriveFile.tenant_id == tenant_id, + DriveFile.created_at >= recent_since, + ) + .count() + ) + + def _count_total_users(self, tenant_id: UUID) -> int: + return ( + self.db.query(User) + .filter( + User.tenant_id == tenant_id, + User.is_deleted.is_(False), + ) + .count() + ) + + def _count_recent_users_added(self, tenant_id: UUID, recent_since: datetime) -> int: + return ( + self.db.query(User) + .filter( + User.tenant_id == tenant_id, + User.created_at >= recent_since, + ) + .count() + ) + + def _count_recent_users_deleted(self, tenant_id: UUID, recent_since: datetime) -> int: + return ( + self.db.query(User) + .filter( + User.tenant_id == tenant_id, + User.is_deleted.is_(True), + User.deleted_at.isnot(None), + User.deleted_at >= recent_since, + ) + .count() + ) + + def _count_recent_edits(self, tenant_id: UUID, recent_since: datetime) -> int: + return ( + self.db.query(ActivityLog) + .filter( + ActivityLog.tenant_id == tenant_id, + ActivityLog.action == ActivityLogAction.DOCUMENT_EDITED.value, + ActivityLog.status == ActivityLogStatus.SUCCESS.value, + ActivityLog.created_at >= recent_since, + ) + .count() + ) + + def _get_signature_trend(self, tenant_id: UUID, trend_config: dict) -> list[dict]: + if trend_config["mode"] == "rolling_weeks": + return self._build_rolling_week_trend( + self.db.query( + SigningRequest.completed_at.label("event_at"), + ) + .filter( + SigningRequest.tenant_id == tenant_id, + SigningRequest.status == "completed", + SigningRequest.completed_at.isnot(None), + ), + SigningRequest.completed_at, + trend_config["periods"], + ) + + if trend_config["mode"] == "calendar_year_months": + period_bucket = func.date_trunc("month", SigningRequest.completed_at) + return self._build_current_year_month_trend( + self.db.query( + period_bucket.label("period"), + func.count(SigningRequest.id).label("value"), + ) + .filter( + SigningRequest.tenant_id == tenant_id, + SigningRequest.status == "completed", + SigningRequest.completed_at.isnot(None), + ), + SigningRequest.completed_at, + period_bucket, + ) + + period_bucket = func.date_trunc(trend_config["bucket"], SigningRequest.completed_at) + return self._build_period_trend( + self.db.query( + period_bucket.label("period"), + func.count(SigningRequest.id).label("value"), + ) + .filter( + SigningRequest.tenant_id == tenant_id, + SigningRequest.status == "completed", + SigningRequest.completed_at.isnot(None), + ), + SigningRequest.completed_at, + period_bucket, + trend_config, + ) + + def _get_upload_trend(self, tenant_id: UUID, trend_config: dict) -> list[dict]: + if trend_config["mode"] == "rolling_weeks": + return self._build_rolling_week_trend( + self.db.query( + DriveFile.created_at.label("event_at"), + ) + .filter(DriveFile.tenant_id == tenant_id), + DriveFile.created_at, + trend_config["periods"], + ) + + if trend_config["mode"] == "calendar_year_months": + period_bucket = func.date_trunc("month", DriveFile.created_at) + return self._build_current_year_month_trend( + self.db.query( + period_bucket.label("period"), + func.count(DriveFile.id).label("value"), + ) + .filter(DriveFile.tenant_id == tenant_id), + DriveFile.created_at, + period_bucket, + ) + + period_bucket = func.date_trunc(trend_config["bucket"], DriveFile.created_at) + return self._build_period_trend( + self.db.query( + period_bucket.label("period"), + func.count(DriveFile.id).label("value"), + ) + .filter(DriveFile.tenant_id == tenant_id), + DriveFile.created_at, + period_bucket, + trend_config, + ) + + def get_trend_data(self, current_user: User, metric: str, granularity: str = "weekly") -> dict: + tenant_id = current_user.tenant_id + if not tenant_id: + raise HTTPException( + status_code=403, + detail="Tenant-scoped admin metrics are only available for tenant users.", + ) + + trend_config = self._get_trend_config(granularity) + if metric == "signatures": + if not self._is_tenant_admin(current_user): + PermissionService(self.db).require_access( + current_user, SIGNATURES_TREND_ACCESS_CODE + ) + return {"trend": self._get_signature_trend(tenant_id, trend_config)} + if metric == "uploads": + return {"trend": self._get_upload_trend(tenant_id, trend_config)} + + raise HTTPException(status_code=404, detail="Unsupported metric trend requested.") + + def _get_trend_config(self, granularity: str) -> dict: + normalized = granularity.lower() + if normalized == "monthly": + return {"mode": "rolling_weeks", "periods": 5} + if normalized == "yearly": + return {"mode": "calendar_year_months"} + return {"mode": "weekly_days", "bucket": "day", "periods": 7} + + def _build_period_trend(self, query, date_column, period_bucket, trend_config: dict) -> list[dict]: + current_period_start = self._get_period_start(datetime.now(timezone.utc), trend_config["bucket"]) + period_starts = [] + for offset in reversed(range(trend_config["periods"])): + period_starts.append(self._shift_period_start(current_period_start, trend_config["bucket"], -offset)) + + start_datetime = period_starts[0] + end_datetime = self._shift_period_start(current_period_start, trend_config["bucket"], 1) + rows = ( + query.filter( + date_column >= start_datetime, + date_column < end_datetime, + ) + .group_by(period_bucket) + .all() + ) + + counts_by_period = { + row.period.date().isoformat(): row.value + for row in rows + if row.period is not None + } + + return [ + { + "date": period_start.date().isoformat(), + "value": counts_by_period.get(period_start.date().isoformat(), 0), + } + for period_start in period_starts + ] + + def _build_rolling_week_trend(self, query, date_column, periods: int) -> list[dict]: + now = datetime.now(timezone.utc) + today_start = datetime(now.year, now.month, now.day, tzinfo=timezone.utc) + earliest_period_start = today_start - timedelta(days=(periods * 7) - 1) + end_datetime = today_start + timedelta(days=1) + + rows = ( + query.filter( + date_column >= earliest_period_start, + date_column < end_datetime, + ) + .all() + ) + + counts_by_week: dict[int, int] = {} + for row in rows: + if row.event_at is None: + continue + week_index = ((row.event_at.date() - earliest_period_start.date()).days // 7) + 1 + counts_by_week[week_index] = counts_by_week.get(week_index, 0) + 1 + + return [ + { + "date": (earliest_period_start + timedelta(days=(week_index - 1) * 7)).date().isoformat(), + "value": counts_by_week.get(week_index, 0), + } + for week_index in range(1, periods + 1) + ] + + def _build_current_year_month_trend(self, query, date_column, period_bucket) -> list[dict]: + now = datetime.now(timezone.utc) + year_start = datetime(now.year, 1, 1, tzinfo=timezone.utc) + next_year_start = datetime(now.year + 1, 1, 1, tzinfo=timezone.utc) + + rows = ( + query.filter( + date_column >= year_start, + date_column < next_year_start, + ) + .group_by(period_bucket) + .all() + ) + + counts_by_month = { + row.period.date().isoformat(): row.value + for row in rows + if row.period is not None + } + + return [ + { + "date": datetime(now.year, month, 1, tzinfo=timezone.utc).date().isoformat(), + "value": counts_by_month.get( + datetime(now.year, month, 1, tzinfo=timezone.utc).date().isoformat(), + 0, + ), + } + for month in range(1, 13) + ] + + def _get_period_start(self, value: datetime, bucket: str) -> datetime: + if bucket == "year": + return datetime(value.year, 1, 1, tzinfo=timezone.utc) + if bucket == "month": + return datetime(value.year, value.month, 1, tzinfo=timezone.utc) + return datetime(value.year, value.month, value.day, tzinfo=timezone.utc) + + def _shift_period_start(self, value: datetime, bucket: str, step: int) -> datetime: + if bucket == "year": + return datetime(value.year + step, 1, 1, tzinfo=timezone.utc) + + if bucket == "month": + month_index = (value.month - 1) + step + year = value.year + (month_index // 12) + month = (month_index % 12) + 1 + return datetime(year, month, 1, tzinfo=timezone.utc) + + return value + timedelta(days=step) + + def _get_storage_metrics(self, current_user: User) -> dict: + usage = StorageService(self.db).get_storage_usage(current_user) + return { + "used_bytes": int(usage["total_bytes"]), + "quota_bytes": int(usage["quota_bytes"]), + "usage_percentage": float(usage["usage_percentage"]), + "used_formatted": usage["total_formatted"], + "quota_formatted": usage["quota_formatted"], + } + + def _get_conversion_metrics(self, tenant_id: UUID) -> dict: + total = self.db.query(Project).filter(Project.tenant_id == tenant_id).count() + completed = ( + self.db.query(Project) + .filter( + Project.tenant_id == tenant_id, + Project.status == "completed", + ) + .count() + ) + processing = ( + self.db.query(Project) + .filter( + Project.tenant_id == tenant_id, + Project.status == "processing", + ) + .count() + ) + failed = ( + self.db.query(Project) + .filter( + Project.tenant_id == tenant_id, + Project.status == "failed", + ) + .count() + ) + completion_rate = round((completed / total) * 100, 1) if total > 0 else 0.0 + return { + "total": total, + "completed": completed, + "processing": processing, + "failed": failed, + "completion_rate": completion_rate, + } + + def _count_internal_shares(self, tenant_id: UUID) -> int: + return ( + self._count_internal_file_shares(tenant_id) + + self._count_internal_folder_shares(tenant_id) + + self._count_internal_document_shares(tenant_id) + ) + + def _count_external_shares(self, tenant_id: UUID) -> int: + return ( + self._count_external_file_shares(tenant_id) + + self._count_external_folder_shares(tenant_id) + + self._count_external_document_shares(tenant_id) + ) + + def _count_internal_file_shares(self, tenant_id: UUID) -> int: + return ( + self.db.query(DriveACL) + .join( + DriveFile, + (DriveACL.resource_type == "file") & (DriveACL.resource_id == DriveFile.id), + ) + .join(User, DriveACL.subject_id == User.id) + .filter( + DriveFile.tenant_id == tenant_id, + User.tenant_id == tenant_id, + DriveACL.role != "owner", + ) + .count() + ) + + def _count_internal_folder_shares(self, tenant_id: UUID) -> int: + return ( + self.db.query(DriveACL) + .join( + DriveFolder, + (DriveACL.resource_type == "folder") + & (DriveACL.resource_id == DriveFolder.id), + ) + .join(User, DriveACL.subject_id == User.id) + .filter( + DriveFolder.tenant_id == tenant_id, + User.tenant_id == tenant_id, + DriveACL.role != "owner", + ) + .count() + ) + + def _count_internal_document_shares(self, tenant_id: UUID) -> int: + return ( + self.db.query(DriveACL) + .join( + Project, + (DriveACL.resource_type == "document") + & (DriveACL.resource_id == Project.id), + ) + .join(User, DriveACL.subject_id == User.id) + .filter( + Project.tenant_id == tenant_id, + User.tenant_id == tenant_id, + DriveACL.role != "owner", + ) + .count() + ) + + def _count_external_file_shares(self, tenant_id: UUID) -> int: + return ( + self.db.query(DriveShare) + .join( + DriveFile, + (DriveShare.resource_type == "file") + & (DriveShare.resource_id == DriveFile.id), + ) + .filter( + DriveFile.tenant_id == tenant_id, + DriveShare.invited_email.isnot(None), + DriveShare.is_active == True, + ) + .count() + ) + + def _count_external_folder_shares(self, tenant_id: UUID) -> int: + return ( + self.db.query(DriveShare) + .join( + DriveFolder, + (DriveShare.resource_type == "folder") + & (DriveShare.resource_id == DriveFolder.id), + ) + .filter( + DriveFolder.tenant_id == tenant_id, + DriveShare.invited_email.isnot(None), + DriveShare.is_active == True, + ) + .count() + ) + + def _count_external_document_shares(self, tenant_id: UUID) -> int: + return ( + self.db.query(DriveShare) + .join( + Project, + (DriveShare.resource_type == "document") + & (DriveShare.resource_id == Project.id), + ) + .filter( + Project.tenant_id == tenant_id, + DriveShare.invited_email.isnot(None), + DriveShare.is_active == True, + ) + .count() + ) diff --git a/app/modules/auth/services/auth_service.py b/app/modules/auth/services/auth_service.py new file mode 100644 index 0000000..3e4372f --- /dev/null +++ b/app/modules/auth/services/auth_service.py @@ -0,0 +1,528 @@ +import os +import re +import uuid +import logging +from typing import Dict, Any, List, Optional +from fastapi import HTTPException +from sqlalchemy.orm import Session +from google.oauth2 import id_token +from google.auth.transport import requests + +from app.core.settings import settings +from app.db.redis import redis_cache +from app.core.mail import send_email +from app.modules.auth.models.user_model import User +from app.modules.auth.repositories.user_repository import UserRepository +from app.core.security import ( + get_password_hash, + verify_password, + create_access_token, + create_refresh_token, +) +from app.modules.auth.schemas.auth_schema import ( + RegisterIn, + GoogleLoginIn, + UpdateProfileIn, + ChangePasswordIn, + ForgotPasswordIn, + ResetPasswordIn, +) +from app.modules.configuration.services.system_configuration_service import ( + SystemConfigurationService, +) + +logger = logging.getLogger(__name__) + + +class AuthService: + def __init__(self, db: Session): + self.db = db + self.user_repo = UserRepository(db) + + def register(self, payload: RegisterIn, tenant: Any = None) -> dict: + if self.user_repo.get_by_email(payload.email): + raise HTTPException(status_code=400, detail="Email already registered") + + SystemConfigurationService(self.db).ensure_can_create_user( + tenant_id=tenant.id if tenant else None + ) + + user = User( + name=payload.name.strip(), + designation=(payload.designation or "").strip() or None, + email=payload.email.lower().strip(), + password_hash=get_password_hash(payload.password), + subscription="free", + tenant_id=tenant.id if tenant else None, + ) + new_user = self.user_repo.create(user) + + from app.modules.drive.services.drive_service import DriveService + + drive_service = DriveService(self.db) + drive_service.get_or_create_root_folder(new_user) + + from app.modules.storage.models.storage_model import UserStorageUsage + + storage_usage = UserStorageUsage(user_id=new_user.id) + self.db.add(storage_usage) + + try: + from datetime import datetime + + send_email( + subject="Welcome to DocQube!", + recipient=new_user.email, + template_name="welcome.html", + template_context={ + "name": new_user.name, + "login_url": f"{settings.FRONTEND_URL}/login", + "year": datetime.utcnow().year, + }, + db=self.db, + tenant_id=new_user.tenant_id, + ) + except Exception as e: + logger.error(f"Failed to send welcome email for {new_user.email}: {e}") + + return { + "message": "Account created successfully", + "user_id": new_user.id, + "email": new_user.email, + "tenant_id": new_user.tenant_id, + } + + def logout(self, session_id: int): + from app.modules.auth.repositories.session_repository import SessionRepository + session_repo = SessionRepository(self.db) + session = session_repo.get_by_id(session_id) + if session and session.status == "ACTIVE": + session_repo.revoke(session, by="user") + + def _handle_device_login(self, user_id: int, request: Any) -> int: + from app.modules.auth.utils.device_utils import get_client_ip, parse_user_agent, generate_device_fingerprint, get_location_from_ip + from app.modules.auth.repositories.device_repository import DeviceRepository + from app.modules.auth.repositories.session_repository import SessionRepository + from app.modules.auth.models.device_model import Device + from app.modules.auth.models.session_model import Session as AuthSession + from datetime import datetime, timezone, timedelta + + device_repo = DeviceRepository(self.db) + session_repo = SessionRepository(self.db) + + fingerprint = generate_device_fingerprint(request) + ip_addr = get_client_ip(request) + user_agent_str = request.headers.get("User-Agent", "") + browser, os_name, device_type = parse_user_agent(user_agent_str) + + device = device_repo.get_by_fingerprint(user_id, fingerprint) + if device: + if device.is_blocked: + raise HTTPException(status_code=403, detail="Device is blocked") + device_repo.update_login_activity(device, ip_addr) + session_repo.revoke_all_for_device(device.id, by="reauth") + from sqlalchemy import text + self.db.execute(text("SELECT pg_advisory_xact_lock(:lock_id)"), {"lock_id": user_id}) + + limit = settings.MAX_ACTIVE_DEVICES + active_count = session_repo.count_active_sessions(user_id) + if active_count >= limit: + active_devices = device_repo.get_user_devices(user_id) + devices_data = [{ + "id": d.id, + "os": d.os, + "browser": d.browser, + "device_type": d.device_type, + "city": d.city, + "country": d.country, + "state": d.state, + "last_login": d.last_login.isoformat() if d.last_login else None + } for d in active_devices] + + raise HTTPException(status_code=403, detail={ + "error": "DEVICE_LIMIT_REACHED", + "devices": devices_data + }) + + if not device: + country, state, city, lat, lon = get_location_from_ip(ip_addr) + + new_device = Device( + user_id=user_id, + fingerprint=fingerprint, + browser=browser, + os=os_name, + device_type=device_type, + user_agent=user_agent_str, + ip=ip_addr, + country=country, + state=state, + city=city, + latitude=lat, + longitude=lon, + last_ip=ip_addr + ) + device = device_repo.create(new_device) + + new_session = AuthSession( + user_id=user_id, + device_id=device.id, + refresh_token_hash="pending", + status="ACTIVE", + expires_at=datetime.now(timezone.utc) + timedelta(days=7) + ) + new_session = session_repo.create(new_session) + return new_session.id + + def login(self, username: str, password: str, request: Optional[Any] = None) -> Dict[str, str]: + user = self.user_repo.get_by_email(username) + + generic_error = HTTPException( + status_code=401, detail="Invalid email or password" + ) + + if ( + not user + or user.is_deleted + or (hasattr(user, "is_active") and not user.is_active) + ): + raise HTTPException(status_code=401, detail="User not found or account is inactive") + + if user.password_hash == "SAAS_MANAGED_ACCOUNT_DO_NOT_USE_PASSWORD": + raise HTTPException( + status_code=403, + detail="This account is managed by SSO. Please log in through the main application portal." + ) + + if not verify_password(password, user.password_hash): + raise HTTPException(status_code=401, detail="Incorrect email or password") + + if user.tenant_id: + from app.modules.tenant.repositories.tenant_repository import TenantRepository + tenant_repo = TenantRepository(self.db) + tenant = tenant_repo.get_by_id(user.tenant_id) + if not tenant or not tenant.is_active or tenant.is_deleted: + raise HTTPException(status_code=403, detail="Your organization is currently inactive. Please contact support.") + + try: + session_id = None + if request: + session_id = self._handle_device_login(user.id, request) + except HTTPException as e: + if isinstance(e.detail, dict) and e.detail.get("error") == "DEVICE_LIMIT_REACHED": + temp_payload = {"sub": str(user.id), "type": "device_management"} + from datetime import timedelta + temp_token = create_access_token(temp_payload, expires_delta=timedelta(minutes=10)) + e.detail["temp_token"] = temp_token + raise e + + payload_access = { + "sub": str(user.id), + "tenant_id": str(user.tenant_id) if user.tenant_id else None, + "is_superadmin": bool(getattr(user, "is_superadmin", False)), + } + if session_id: + payload_access["session_id"] = session_id + + payload_refresh = {"sub": str(user.id)} + if session_id: + payload_refresh["session_id"] = session_id + + token = create_access_token(payload_access) + refresh = create_refresh_token(payload_refresh) + + if session_id: + from app.core.security import get_password_hash + from app.modules.auth.repositories.session_repository import SessionRepository + session_repo = SessionRepository(self.db) + session = session_repo.get_by_id(session_id) + if session: + session.refresh_token_hash = get_password_hash(refresh) + + if user.tenant_id: + try: + from app.modules.activity_logs.service import log_event + from app.modules.activity_logs.constants import ActivityLogModule, ActivityLogAction, ActivityLogStatus, ActivityLogTargetType + log_event( + tenant_id=user.tenant_id, + user_id=user.id, + user_email=user.email, + module=ActivityLogModule.TENANT, + action=ActivityLogAction.DEVICE_LOGIN, + target_id=str(session_id) if session_id else str(user.id), + target_type=ActivityLogTargetType.SESSION, + metadata={}, + status=ActivityLogStatus.SUCCESS + ) + except Exception as e: + import logging + logging.getLogger(__name__).error(f"Failed to log login event: {e}") + + return {"access_token": token, "refresh_token": refresh, "token_type": "bearer"} + + + def google_login(self, payload: GoogleLoginIn, request: Optional[Any] = None) -> Dict[str, str]: + client_id = settings.GOOGLE_CLIENT_ID + token_to_verify = payload.token or payload.credential + + if not token_to_verify: + raise HTTPException(status_code=400, detail="Google token missing") + + try: + idinfo = id_token.verify_oauth2_token( + token_to_verify, requests.Request(), client_id, clock_skew_in_seconds=60 + ) + except HTTPException: + raise + except Exception as e: + logger.error(f"Google Auth Error: {e}", exc_info=True) + raise HTTPException(status_code=401, detail="Google authentication failed") + + email = idinfo["email"] + raw_name = idinfo.get("name", email.split("@")[0]) + name = re.sub(r"[^\w\s-]", "", raw_name).strip() + + user = self.user_repo.get_by_email(email) + + if user and user.tenant_id: + from app.modules.tenant.repositories.tenant_repository import TenantRepository + tenant_repo = TenantRepository(self.db) + tenant = tenant_repo.get_by_id(user.tenant_id) + if not tenant or not tenant.is_active or tenant.is_deleted: + raise HTTPException(status_code=403, detail="Your organization is currently inactive. Please contact support.") + + if not user: + tenant_uuid = None + if payload.tenant_id: + try: + tenant_uuid = uuid.UUID(str(payload.tenant_id)) + except Exception: + tenant_uuid = None + + SystemConfigurationService(self.db).ensure_can_create_user( + tenant_id=tenant_uuid + ) + user = User( + name=name, + email=email, + password_hash=get_password_hash(os.urandom(24).hex()), + tenant_id=payload.tenant_id, + ) + user = self.user_repo.create(user) + + from app.modules.drive.services.drive_service import DriveService + + drive_service = DriveService(self.db) + drive_service.get_or_create_root_folder(user) + + from app.modules.storage.models.storage_model import UserStorageUsage + + storage_usage = UserStorageUsage(user_id=user.id) + self.db.add(storage_usage) + + try: + from datetime import datetime + + send_email( + subject="Welcome to DocQube!", + recipient=user.email, + template_name="welcome.html", + template_context={ + "name": user.name, + "login_url": f"{settings.FRONTEND_URL}/login", + "year": datetime.utcnow().year, + }, + db=self.db, + tenant_id=user.tenant_id, + ) + except Exception as e: + logger.error( + f"Failed to send welcome email for google user {user.email}: {e}" + ) + + try: + session_id = None + if request: + session_id = self._handle_device_login(user.id, request) + except HTTPException as e: + if isinstance(e.detail, dict) and e.detail.get("error") == "DEVICE_LIMIT_REACHED": + temp_payload = {"sub": str(user.id), "type": "device_management"} + from datetime import timedelta + temp_token = create_access_token(temp_payload, expires_delta=timedelta(minutes=10)) + e.detail["temp_token"] = temp_token + raise e + + payload_access = { + "sub": str(user.id), + "tenant_id": str(user.tenant_id) if user.tenant_id else None, + "is_superadmin": bool(getattr(user, "is_superadmin", False)), + } + if session_id: + payload_access["session_id"] = session_id + + payload_refresh = {"sub": str(user.id)} + if session_id: + payload_refresh["session_id"] = session_id + + token = create_access_token(payload_access) + refresh = create_refresh_token(payload_refresh) + + if session_id: + from app.core.security import get_password_hash + from app.modules.auth.repositories.session_repository import SessionRepository + session_repo = SessionRepository(self.db) + session = session_repo.get_by_id(session_id) + if session: + session.refresh_token_hash = get_password_hash(refresh) + + if user.tenant_id: + try: + from app.modules.activity_logs.service import log_event + from app.modules.activity_logs.constants import ActivityLogModule, ActivityLogAction, ActivityLogStatus, ActivityLogTargetType + log_event( + tenant_id=user.tenant_id, + user_id=user.id, + user_email=user.email, + module=ActivityLogModule.TENANT, + action=ActivityLogAction.DEVICE_LOGIN, + target_id=str(session_id) if session_id else str(user.id), + target_type=ActivityLogTargetType.SESSION, + metadata={}, + status=ActivityLogStatus.SUCCESS + ) + except Exception as e: + import logging + logging.getLogger(__name__).error(f"Failed to log google login event: {e}") + + return {"access_token": token, "refresh_token": refresh, "token_type": "bearer"} + + def forgot_password(self, payload: ForgotPasswordIn) -> Dict[str, str]: + email = payload.email.lower().strip() + user = self.user_repo.get_by_email(email) + + res = { + "message": "If this email is registered, you will receive a password reset link." + } + + if not user: + logger.info(f"Forgot password request for non-existent email: {email}") + return res + if user.is_deleted or (hasattr(user, "is_active") and not user.is_active): + logger.info(f"Forgot password requested for inactive/deleted user: {email}") + return res + + logger.info(f"Found user {user.id} for forgot password: {email}") + + reset_token = uuid.uuid4().hex + redis_cache.set(f"reset_token:{reset_token}", user.id, ttl=3600) + + reset_link = f"{settings.FRONTEND_URL}/reset-password?token={reset_token}" + + subject = "DocQube - Reset Your Password" + body = f"Hello {user.name},\n\nYou requested a password reset. Use the link below to set a new password:\n\n{reset_link}\n\nThis link expires in 1 hour." + html_body = f""" +
    +

    Reset Your Password

    +

    Hello {user.name},

    +

    You requested a password reset for your DocQube account. Click the button below to continue:

    + +

    Or copy and paste this link:
    {reset_link}

    +

    This link will expire in 1 hour. If you did not request this, please ignore this email.

    +
    + """ + + logger.info(f"Attempting to send reset email to {email}") + send_email( + subject, + email, + body, + html_body, + db=self.db, + tenant_id=user.tenant_id, + ) + logger.info(f"Reset email process completed for {email}") + + return res + + def reset_password(self, payload: ResetPasswordIn) -> Dict[str, str]: + user_id = redis_cache.get(f"reset_token:{payload.token}") + if not user_id: + raise HTTPException( + status_code=400, detail="Invalid or expired reset token" + ) + + user = self.user_repo.get_by_id(user_id) + if not user: + raise HTTPException(status_code=404, detail="User not found") + + user.password_hash = get_password_hash(payload.new_password) + + redis_cache.delete(f"reset_token:{payload.token}") + + return {"message": "Password has been successfully reset."} + + +class UserService: + def __init__(self, db: Session): + self.db = db + self.user_repo = UserRepository(db) + + def update_profile( + self, + user: User, + name: Optional[str] = None, + designation: Optional[str] = None, + preferred_language: Optional[str] = None, + ) -> User: + if name is None and designation is None and preferred_language is None: + raise HTTPException( + status_code=400, + detail="At least one profile field must be provided", + ) + if name is not None: + user.name = name.strip() + if designation is not None: + user.designation = designation.strip() or None + if preferred_language is not None: + user.preferred_language = preferred_language.strip().lower() or "en" + self.db.flush() + self.db.refresh(user) + return user + + def change_password(self, user: User, current_password: str, new_password: str): + if not verify_password(current_password, user.password_hash): + raise HTTPException(status_code=400, detail="Incorrect current password") + + user.password_hash = get_password_hash(new_password) + return {"message": "Password updated successfully"} + + def get_user_stats(self, user: User) -> Dict[str, Any]: + """ + Fetch real-time statistics for the user profile dashboard. + """ + from app.modules.documents.models.document_model import Project + from app.modules.drive.models.drive_model import DriveFile + + total_projects = ( + self.db.query(Project).filter(Project.user_id == user.id).count() + ) + total_files = ( + self.db.query(DriveFile).filter(DriveFile.owner_id == user.id).count() + ) + + return { + "totalProjects": total_projects, + "totalFiles": total_files, + "subscription": user.subscription, + "joined": user.created_at.isoformat() if user.created_at else None, + } + + def delete_account(self, user: User): + user.is_deleted = True + user.is_active = False + from datetime import datetime + + user.deleted_at = datetime.utcnow() + return {"message": "Account deleted successfully"} diff --git a/app/modules/auth/services/permission_service.py b/app/modules/auth/services/permission_service.py new file mode 100644 index 0000000..3377786 --- /dev/null +++ b/app/modules/auth/services/permission_service.py @@ -0,0 +1,386 @@ +from sqlalchemy import func, or_, select +from typing import Set, Optional, List +from fastapi import HTTPException +from sqlalchemy.orm import Session +from app.core.request_cache import request_cached +from app.modules.auth.models.user_model import User +from app.modules.auth.models.access_model import Access + + +class PermissionService: + """ + Centralized permission engine for checking access codes. + + Handles: + - Retrieving user's access codes from their role + - Checking if user has specific access code(s) + - Enforcing access restrictions (raises HTTP 403 on denial) + + **This is the only correct answer to "what may this user do".** There used to + be a second one — the `User.access_codes` property — which read the single + legacy `users.role_id` and nothing else. Because `/api/me/profile` reported + that answer while `require_access` used this one, a user holding a role + through `user_roles` had authority the product would not show them: no menu + entry, no route, no button. `User.access_codes` now defers to whatever + `get_current_user` resolved through here, and + `tests/probes/test_multi_role.py` asserts the two agree. + """ + + def __init__(self, db: Session): + self.db = db + + def user_access_codes(self, user: Optional[User]) -> Set[str]: + """ + Get all access codes for a user. + Includes both local role-based permissions and SaaS-assigned permissions. + + Memoised for the life of the request: `require_access` asks once per + dependency and `get_current_user` asks once more, and the answer cannot + change beneath a half-finished operation. + """ + if not user: + return set() + + return request_cached( + f"access_codes:{user.id}", lambda: self._resolve_codes(user) + ) + + def _resolve_codes(self, user: User) -> Set[str]: + all_access_codes = set() + + saas_permissions = getattr(user, "saas_permissions", set()) + all_access_codes.update(saas_permissions) + + direct_ids: Set[object] = set() + direct_codes: Set[str] = set() + + if user.role: + for ra in user.role.role_accesses: + if ra.access: + direct_ids.add(ra.access.id) + direct_codes.add(ra.access.access_code) + + for access in self._accesses_from_scoped_roles(user): + direct_ids.add(access.id) + direct_codes.add(access.access_code) + + if direct_ids: + all_access_codes.update( + self._expand_with_descendants(direct_ids, direct_codes) + ) + + if any(code.startswith("document.conversion") for code in all_access_codes): + all_access_codes.add("document.conversion") + + return all_access_codes + + def _group_ids_subquery(self, user): + """ + The groups this user belongs to, as a **subquery** rather than a round trip. + + Filtered by tenant on both sides: a membership row naming another + tenant's group must confer nothing, and `user_access_groups` carries its + own `tenant_id` precisely so a forged or stale row cannot reach across. + + A subquery and not a `list` because this used to be one extra SELECT on + every permission check, and permission checks are the hottest path in + the application. + """ + from app.modules.org.models.group_model import AccessGroup, UserAccessGroup + + return ( + select(UserAccessGroup.group_id) + .join(AccessGroup, AccessGroup.id == UserAccessGroup.group_id) + .where( + UserAccessGroup.user_id == user.id, + UserAccessGroup.tenant_id == user.tenant_id, + AccessGroup.tenant_id == user.tenant_id, + ) + .scalar_subquery() + ) + + def _group_ids_for(self, user) -> list: + """Groups the user belongs to. Retained for callers that want the ids.""" + from app.modules.org.models.group_model import AccessGroup, UserAccessGroup + + rows = ( + self.db.query(UserAccessGroup.group_id) + .join(AccessGroup, AccessGroup.id == UserAccessGroup.group_id) + .filter( + UserAccessGroup.user_id == user.id, + UserAccessGroup.tenant_id == user.tenant_id, + AccessGroup.tenant_id == user.tenant_id, + ) + .all() + ) + return [r[0] for r in rows] + + def _accesses_from_scoped_roles(self, user: User) -> list: + """ + Every `Access` reachable through this user's `user_roles` rows. + + Deliberately ignores `org_unit_id`: holding a role in one team still + means holding its codes. Narrowing to a subtree is `ScopeService`'s job, + and conflating the two here would mean an endpoint that has not been + converted to ask about scope silently refuses people who legitimately + hold the permission somewhere. + + **The three predicates below are not optional.** This query originally + filtered on the principal alone, which meant the coarse gate disagreed + with `ScopeService` in two ways that both fail open: + + - an **expired** grant still passed `require_access`, so temporary + elevation did not expire on any endpoint that had not been converted + to ask about scope — which is almost all of them; + - a grant naming **another tenant's role** conferred that role's codes. + This is the defect the C-series found in `ScopeService._resolve` and + fixed there; it lived on here for a release. + + They are expressed in SQL rather than filtered afterwards for the same + reason `ScopeService` gives: an expired grant must be invisible to every + reader, and a post-filter is how one caller ends up honouring a dead + grant. + """ + from app.modules.auth.models.role_access_model import RoleAccess + from app.modules.auth.models.role_model import Role + from app.modules.auth.models.user_role_model import UserRole + + return ( + self.db.query(Access) + .join(RoleAccess, RoleAccess.access_id == Access.id) + .join(UserRole, UserRole.role_id == RoleAccess.role_id) + .join(Role, Role.id == UserRole.role_id) + .filter( + or_( + UserRole.user_id == user.id, + UserRole.group_id.in_(self._group_ids_subquery(user)), + ), + or_( + Role.tenant_id == user.tenant_id, + Role.tenant_id.is_(None), + ), + or_( + UserRole.expires_at.is_(None), + UserRole.expires_at > func.now(), + ), + ) + .distinct() + .all() + ) + + def _access_tree(self): + """ + `(code_by_id, children_by_parent)` for the whole catalogue. + + One query, memoised per request. The catalogue is the same for every + user, so resolving a page of twenty users used to read this table twenty + times. + """ + + def _load(): + rows = self.db.query( + Access.id, Access.parent_id, Access.access_code + ).all() + code_by_id = {row.id: row.access_code for row in rows} + children_by_parent: dict = {} + for row in rows: + if row.parent_id is None: + continue + children_by_parent.setdefault(row.parent_id, set()).add(row.id) + return code_by_id, children_by_parent + + return request_cached("access_tree", _load) + + def _expand_with_descendants( + self, direct_ids: Set[object], direct_codes: Set[str] + ) -> Set[str]: + """ + Expand assigned access codes with all descendants so parent access grants + full module access. + """ + code_by_id, children_by_parent = self._access_tree() + + visited = set(direct_ids) + stack = list(direct_ids) + + while stack: + current = stack.pop() + for child_id in children_by_parent.get(current, set()): + if child_id in visited: + continue + visited.add(child_id) + stack.append(child_id) + + expanded_codes = {code_by_id[access_id] for access_id in visited if access_id in code_by_id} + return direct_codes.union(expanded_codes) + + def has_access(self, user: Optional[User], code: str) -> bool: + """ + Check if user has a specific access code. + + Args: + user: Current user + code: Access code to check (e.g. "project.create") + + Returns: + True if user has the access code, False otherwise + """ + return code in self.user_access_codes(user) + + def require_access(self, user: Optional[User], code: str) -> None: + """ + Assert user has a specific access code. + Raises HTTPException 403 if not authorized. + + Args: + user: Current user + code: Required access code + + Raises: + HTTPException: 403 if user not authenticated or lacks access + """ + if not user: + raise HTTPException(status_code=403, detail="Authentication required") + + if not self.has_access(user, code): + raise HTTPException( + status_code=403, detail=f"Access denied: insufficient permissions" + ) + + def require_any_access(self, user: Optional[User], codes: List[str]) -> None: + """ + Assert user has at least one of the provided access codes. + Raises HTTPException 403 if none match. + + Args: + user: Current user + codes: List of access codes (user needs at least one) + + Raises: + HTTPException: 403 if user lacks all codes + """ + if not user: + raise HTTPException(status_code=403, detail="Authentication required") + + if not any(self.has_access(user, code) for code in codes): + raise HTTPException( + status_code=403, detail="Access denied: insufficient permissions" + ) + + def require_all_access(self, user: Optional[User], codes: List[str]) -> None: + """ + Assert user has ALL of the provided access codes. + Raises HTTPException 403 if any are missing. + + Args: + user: Current user + codes: List of access codes (user must have all) + + Raises: + HTTPException: 403 if user lacks any code + """ + if not user: + raise HTTPException(status_code=403, detail="Authentication required") + + missing = [code for code in codes if not self.has_access(user, code)] + if missing: + raise HTTPException( + status_code=403, detail="Access denied: insufficient permissions" + ) + + def get_all_system_accesses(self) -> List[dict]: + """ + Get all defined access codes in the system grouped by category. + """ + accesses = self.db.query(Access).all() + return [ + { + "id": a.id, + "access_code": a.access_code, + "category": a.category, + "name": a.name, + "parent_id": a.parent_id, + } + for a in accesses + ] + + def granted_by(self, user: Optional[User]) -> dict: + """ + `{access_code: [attribution, ...]}` — which role conferred each code. + + The answer to "why can this person do that", which + `/api/admin/users/{id}/effective-access` exposes. With one role per user + the question answered itself; with several it does not, and support + cannot read the database. + + Attribution names the role, the scope, and whether it arrived through + the primary role or a grant. Descendant expansion is attributed to the + role that holds the parent code, because that is the row an + administrator would edit to take it away. + """ + if not user: + return {} + + attributions: dict = {} + + def _add(code: str, entry: dict) -> None: + existing = attributions.setdefault(code, []) + if entry not in existing: + existing.append(entry) + + from app.modules.auth.services.user_role_service import UserRoleReader + + for assignment in UserRoleReader(self.db).assignments(user): + direct_ids = set() + direct_codes = set() + for access in self._accesses_of_role(assignment["role_id"]): + direct_ids.add(access.id) + direct_codes.add(access.access_code) + if not direct_ids: + continue + entry = { + "role_id": str(assignment["role_id"]), + "role_name": assignment["role_name"], + "source": assignment["source"], + "org_unit_id": ( + str(assignment["org_unit_id"]) + if assignment["org_unit_id"] + else None + ), + "org_unit_name": assignment["org_unit_name"], + "expires_at": ( + assignment["expires_at"].isoformat() + if assignment["expires_at"] + else None + ), + } + for code in self._expand_with_descendants(direct_ids, direct_codes): + _add(code, entry) + + for code in getattr(user, "saas_permissions", set()) or set(): + _add( + code, + { + "role_id": None, + "role_name": "Subscription platform", + "source": "saas", + "org_unit_id": None, + "org_unit_name": None, + "expires_at": None, + }, + ) + + return attributions + + def _accesses_of_role(self, role_id) -> list: + from app.modules.auth.models.role_access_model import RoleAccess + + def _load(): + return ( + self.db.query(Access) + .join(RoleAccess, RoleAccess.access_id == Access.id) + .filter(RoleAccess.role_id == role_id) + .all() + ) + + return request_cached(f"role_accesses:{role_id}", _load) diff --git a/app/modules/auth/services/privilege.py b/app/modules/auth/services/privilege.py new file mode 100644 index 0000000..56f84c0 --- /dev/null +++ b/app/modules/auth/services/privilege.py @@ -0,0 +1,54 @@ +""" +One answer to "is this caller a superadmin", for the routes that branch on it. + +There were five copies of this predicate — in `role_routes`, `admin_user_routes`, +`tenant_routes`, `chat_controller` and `admin_dashboard_service` — each written +as: + + any(acc.startswith("superadmin.") for acc in user.access_codes) + +Identical, and all five wrong in the same way, because `User.access_codes` read +only the legacy `users.role_id`. A user whose superadmin role arrived through a +`user_roles` grant therefore **passed `require_access` and was then treated as a +tenant administrator by the same request**. Getting through the door and then +being scoped as somebody else is the worst failure shape available: it looks like +a data bug, not an authorisation one. + +The property is fixed at its source (`get_current_user` resolves through +`PermissionService`), so these five call sites are now correct. They are +collapsed into one function anyway, because five copies of a security predicate +is five chances for the sixth to be written differently. + +**`users.is_superadmin` is deliberately not consulted here, and the name says so.** +DocQube has two superadmin predicates because it has two questions: + + app.middleware.tenant.is_superadmin(user) the B1 privilege flag — + may this session bypass + the tenant filter and RLS + holds_superadmin_access(user) does this caller hold a + `superadmin.*` code — may + they administer across + tenants through the API + +Two functions both called `is_superadmin` with different semantics is a trap, so +this one is not called that. Folding the flag in here would silently widen five +existing authorisation decisions — a tenant-less operator holding no +`superadmin.*` code would begin passing `_assert_role_in_callers_tenant`, which +today refuses them. That widening may well be desirable; it is a deliberate +product change with its own probe, not a side effect of de-duplicating a helper. +""" + +from __future__ import annotations + +SUPERADMIN_PREFIX = "superadmin." + + +def holds_superadmin_access(user) -> bool: + """True when the caller holds any `superadmin.*` access code.""" + if user is None: + return False + codes = getattr(user, "access_codes", None) or [] + return any(str(code).startswith(SUPERADMIN_PREFIX) for code in codes) + + +__all__ = ["holds_superadmin_access", "SUPERADMIN_PREFIX"] diff --git a/app/modules/auth/services/role_service.py b/app/modules/auth/services/role_service.py new file mode 100644 index 0000000..1708421 --- /dev/null +++ b/app/modules/auth/services/role_service.py @@ -0,0 +1,186 @@ +from sqlalchemy.orm import Session +from sqlalchemy import or_, cast, String +from fastapi import HTTPException, status +from app.modules.auth.models.role_model import Role +from app.modules.auth.models.role_access_model import RoleAccess +from app.modules.auth.models.access_model import Access +from app.modules.auth.models.user_model import User +from app.modules.auth.schemas.role_schema import ( + RoleCreate, + RoleUpdate, + RoleOut, + RolePaginatedOut, +) +from typing import List, Optional +from uuid import UUID + + +class RoleService: + @staticmethod + def create_role(db: Session, role_data: RoleCreate) -> Role: + existing = ( + db.query(Role) + .filter( + Role.name == role_data.role_name, Role.tenant_id == role_data.tenant_id + ) + .first() + ) + if existing: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Role name already exists for this tenant", + ) + role = Role( + name=role_data.role_name, + description=role_data.description, + tenant_id=role_data.tenant_id, + is_default=role_data.is_default or False, + ) + db.add(role) + db.flush() + db.refresh(role) + + if role_data.access_ids: + RoleService.assign_accesses(db, role.id, role_data.access_ids) + + return role + + @staticmethod + def assign_accesses(db: Session, role_id: UUID, access_ids: List[UUID]): + db.query(RoleAccess).filter(RoleAccess.role_id == role_id).delete() + for aid in access_ids: + access = db.query(Access).filter(Access.id == aid).first() + if not access: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Access {aid} not found", + ) + ra = RoleAccess(role_id=role_id, access_id=aid) + db.add(ra) + + @staticmethod + def get_all_roles(db: Session, tenant_id: Optional[UUID] = None): + query = db.query(Role) + if tenant_id is not None: + from app.modules.billing.models.plan_model import TenantSubscription, PlanRole + sub = db.query(TenantSubscription).filter( + TenantSubscription.tenant_id == tenant_id, + TenantSubscription.status == 'active' + ).order_by(TenantSubscription.created_at.desc()).first() + if sub and sub.plan_id: + plan_roles = db.query(PlanRole).filter(PlanRole.plan_id == sub.plan_id).all() + plan_role_ids = [pr.role_id for pr in plan_roles] + query = query.filter(or_(Role.tenant_id == tenant_id, Role.id.in_(plan_role_ids))) + else: + query = query.filter(Role.tenant_id == tenant_id) + return query.all() + + @staticmethod + def get_role_by_id(db: Session, role_id: UUID) -> Role: + role = db.query(Role).filter(Role.id == role_id).first() + if not role: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Role not found" + ) + return role + + @staticmethod + def update_role(db: Session, role_id: UUID, role_data: RoleUpdate) -> Role: + role = RoleService.get_role_by_id(db, role_id) + if getattr(role, "is_system", False): + raise HTTPException(status_code=400, detail="Cannot edit a system role.") + update_dict = role_data.model_dump(exclude_unset=True) + if "role_name" in update_dict: + name = update_dict.pop("role_name") + if name and name != role.name: + conflict = ( + db.query(Role) + .filter(Role.name == name, Role.tenant_id == role.tenant_id) + .first() + ) + if conflict: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Role name already exists", + ) + role.name = name + if "description" in update_dict: + role.description = update_dict.get("description") + if "is_default" in update_dict: + role.is_default = update_dict.get("is_default") + if "access_ids" in update_dict: + access_ids = update_dict.get("access_ids") + if access_ids is not None: + RoleService.assign_accesses(db, role_id, access_ids) + db.flush() + db.refresh(role) + return role + + @staticmethod + def delete_role(db: Session, role_id: UUID): + role = RoleService.get_role_by_id(db, role_id) + if getattr(role, "is_system", False): + raise HTTPException(status_code=400, detail="Cannot delete a system role.") + db.delete(role) + return {"message": "Role deleted successfully"} + + @staticmethod + def get_roles_paginated( + db: Session, + tenant_id: Optional[UUID] = None, + page: int = 1, + page_size: int = 10, + search: Optional[str] = None, + ): + query = db.query(Role) + if tenant_id is not None: + from app.modules.billing.models.plan_model import TenantSubscription, PlanRole + sub = db.query(TenantSubscription).filter( + TenantSubscription.tenant_id == tenant_id, + TenantSubscription.status == 'active' + ).order_by(TenantSubscription.created_at.desc()).first() + if sub and sub.plan_id: + plan_roles = db.query(PlanRole).filter(PlanRole.plan_id == sub.plan_id).all() + plan_role_ids = [pr.role_id for pr in plan_roles] + query = query.filter(or_(Role.tenant_id == tenant_id, Role.id.in_(plan_role_ids))) + else: + query = query.filter(Role.tenant_id == tenant_id) + if search and search.strip(): + search_term = search.strip() + query = query.filter(Role.name.ilike(f"%{search_term}%")) + total = query.count() + offset = (page - 1) * page_size + roles = query.offset(offset).limit(page_size).all() + total_pages = (total + page_size - 1) // page_size if total > 0 else 0 + return { + "items": [r for r in roles], + "total": total, + "page": page, + "page_size": page_size, + "total_pages": total_pages, + } + + @staticmethod + def get_role_with_accesses(db: Session, role_id: UUID) -> Role: + """Get a role with all its access assignments.""" + return RoleService.get_role_by_id(db, role_id) + + @staticmethod + def assign_role_to_user(db: Session, user_id: int, role_id: UUID): + """Assign a role to a specific user.""" + from fastapi import HTTPException, status + + user = db.query(User).filter(User.id == user_id).first() + if not user: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="User not found" + ) + role = RoleService.get_role_by_id(db, role_id) + user.role_id = role.id + db.flush() + db.refresh(user) + return { + "message": "Role assigned successfully", + "user_id": user_id, + "role_id": str(role_id), + } diff --git a/app/modules/auth/services/user_role_service.py b/app/modules/auth/services/user_role_service.py new file mode 100644 index 0000000..7bb8416 --- /dev/null +++ b/app/modules/auth/services/user_role_service.py @@ -0,0 +1,389 @@ +""" +Every role a user actually holds, resolved and named. + +`users.role_id` gives a user one role. `user_roles` gives them any number, at a +scope, optionally expiring, optionally through a group. Both mechanisms are live +and are **unioned, never intersected** — that is the compatibility rule the +C-series migration set, and it is why turning any of this on cannot remove +somebody's access. + +This module answers "which roles", where `PermissionService` answers "which +codes". They are separate because they are asked by different callers for +different reasons: the gate needs codes, and the *interface* needs roles, so an +administrator can see that Alice is both an Editor and an Approver rather than +inferring it from forty-eight access codes. + +**`source` is the field that keeps the UI honest.** + + "primary" the legacy `users.role_id` — changed through PATCH /admin/users, + not revocable as a grant, and (see `ScopeService._resolve`) it + cannot be narrowed by anything while it is still set + "grant" a `user_roles` row — added and revoked freely + "group" a `user_roles` row naming a group this user belongs to; it + confers codes, and it is revoked by editing the group, not the + user + +Rendering all three as one undifferentiated list would imply that removing any +of them is the same action. It is not, and the difference is exactly where a +permissions UI misleads its operator. +""" + +from __future__ import annotations + +import uuid +from typing import Any, Iterable, Optional + +from sqlalchemy import func, or_, select +from sqlalchemy.orm import Session + +from app.core.request_cache import request_cached + + +class UserRoleReader: + """Read-only resolution of role assignments. No writes, no side effects.""" + + def __init__(self, db: Session): + self.db = db + + + def assignments(self, user, include_expired: bool = False) -> list[dict]: + """ + Primary role first, then grants, then group-derived grants. + + Ordered deliberately: the primary role is the one an operator changes + through the existing dropdown, so it belongs at the top of any list that + also offers "remove". + + Memoised per request unless `include_expired` is set — the expired view + is an administrative read, not a hot path, and caching two shapes under + one key is how a stale answer gets served to the wrong caller. + """ + if user is None: + return [] + if include_expired: + return self._resolve(user, include_expired=True) + return request_cached( + f"role_assignments:{user.id}", lambda: self._resolve(user) + ) + + def _resolve(self, user, include_expired: bool = False) -> list[dict]: + out: list[dict] = [] + + primary = getattr(user, "role", None) + if primary is not None: + out.append( + { + "grant_id": None, + "role_id": primary.id, + "role_name": primary.name, + "source": "primary", + "org_unit_id": None, + "org_unit_name": None, + "group_id": None, + "group_name": None, + "expires_at": None, + "assigned_by_id": None, + "created_at": None, + "is_expired": False, + } + ) + + out.extend(self._grants(user, include_expired=include_expired)) + return out + + def _grants(self, user, include_expired: bool = False) -> list[dict]: + from app.modules.auth.models.role_model import Role + from app.modules.auth.models.user_role_model import UserRole + from app.modules.org.models.group_model import AccessGroup, UserAccessGroup + from app.modules.org.models.org_model import OrgUnit + + my_groups = ( + select(UserAccessGroup.group_id) + .join(AccessGroup, AccessGroup.id == UserAccessGroup.group_id) + .where( + UserAccessGroup.user_id == user.id, + UserAccessGroup.tenant_id == user.tenant_id, + AccessGroup.tenant_id == user.tenant_id, + ) + .scalar_subquery() + ) + + query = ( + self.db.query( + UserRole.id, + UserRole.role_id, + Role.name, + UserRole.org_unit_id, + OrgUnit.name, + UserRole.group_id, + AccessGroup.name, + UserRole.expires_at, + UserRole.assigned_by_id, + UserRole.created_at, + ) + .join(Role, Role.id == UserRole.role_id) + .outerjoin(OrgUnit, OrgUnit.id == UserRole.org_unit_id) + .outerjoin(AccessGroup, AccessGroup.id == UserRole.group_id) + .filter( + or_( + UserRole.user_id == user.id, + UserRole.group_id.in_(my_groups), + ), + or_(Role.tenant_id == user.tenant_id, Role.tenant_id.is_(None)), + ) + ) + if not include_expired: + query = query.filter( + or_(UserRole.expires_at.is_(None), UserRole.expires_at > func.now()) + ) + + rows = query.order_by(UserRole.created_at).all() + return [self._row_to_assignment(row) for row in rows] + + @staticmethod + def _row_to_assignment(row) -> dict: + from datetime import datetime, timezone + + ( + grant_id, + role_id, + role_name, + org_unit_id, + org_unit_name, + group_id, + group_name, + expires_at, + assigned_by_id, + created_at, + ) = row + + expired = False + if expires_at is not None: + reference = expires_at + if reference.tzinfo is None: + reference = reference.replace(tzinfo=timezone.utc) + expired = reference <= datetime.now(timezone.utc) + + return { + "grant_id": grant_id, + "role_id": role_id, + "role_name": role_name, + "source": "group" if group_id is not None else "grant", + "org_unit_id": org_unit_id, + "org_unit_name": org_unit_name, + "group_id": group_id, + "group_name": group_name, + "expires_at": expires_at, + "assigned_by_id": assigned_by_id, + "created_at": created_at, + "is_expired": expired, + } + + + def assignments_for_users( + self, users: Iterable[Any] + ) -> dict[int, list[dict]]: + """ + The same resolution for a page of users, in **two** queries rather than + two per user. + + `/api/admin/users` renders up to a hundred rows. Resolving each one + through `assignments()` would be a textbook N+1 that a seeded test + database with three users cannot see and a real tenant feels immediately, + which is precisely the failure `tests/characterization/test_query_budget.py` + exists to catch — so this is written batched from the start. + """ + from app.modules.auth.models.role_model import Role + from app.modules.auth.models.user_model import User + from app.modules.auth.models.user_role_model import UserRole + from app.modules.org.models.group_model import AccessGroup, UserAccessGroup + from app.modules.org.models.org_model import OrgUnit + + users = list(users) + result: dict[int, list[dict]] = {} + for user in users: + primary = getattr(user, "role", None) + entries = [] + if primary is not None: + entries.append( + { + "grant_id": None, + "role_id": primary.id, + "role_name": primary.name, + "source": "primary", + "org_unit_id": None, + "org_unit_name": None, + "group_id": None, + "group_name": None, + "expires_at": None, + "assigned_by_id": None, + "created_at": None, + "is_expired": False, + } + ) + result[user.id] = entries + + user_ids = [u.id for u in users] + if not user_ids: + return result + + rows = ( + self.db.query( + UserRole.user_id, + UserRole.id, + UserRole.role_id, + Role.name, + UserRole.org_unit_id, + OrgUnit.name, + UserRole.expires_at, + UserRole.assigned_by_id, + UserRole.created_at, + ) + .join(Role, Role.id == UserRole.role_id) + .join(User, User.id == UserRole.user_id) + .outerjoin(OrgUnit, OrgUnit.id == UserRole.org_unit_id) + .filter( + UserRole.user_id.in_(user_ids), + or_(Role.tenant_id == User.tenant_id, Role.tenant_id.is_(None)), + or_(UserRole.expires_at.is_(None), UserRole.expires_at > func.now()), + ) + .order_by(UserRole.created_at) + .all() + ) + for row in rows: + user_id = row[0] + if user_id not in result: + continue + result[user_id].append( + self._row_to_assignment( + (row[1], row[2], row[3], row[4], row[5], None, None, row[6], row[7], row[8]) + ) + ) + + group_rows = ( + self.db.query( + UserAccessGroup.user_id, + UserRole.id, + UserRole.role_id, + Role.name, + UserRole.org_unit_id, + OrgUnit.name, + UserRole.group_id, + AccessGroup.name, + UserRole.expires_at, + UserRole.assigned_by_id, + UserRole.created_at, + ) + .join(UserRole, UserRole.group_id == UserAccessGroup.group_id) + .join(Role, Role.id == UserRole.role_id) + .join(AccessGroup, AccessGroup.id == UserRole.group_id) + .join(User, User.id == UserAccessGroup.user_id) + .outerjoin(OrgUnit, OrgUnit.id == UserRole.org_unit_id) + .filter( + UserAccessGroup.user_id.in_(user_ids), + UserAccessGroup.tenant_id == User.tenant_id, + AccessGroup.tenant_id == User.tenant_id, + or_(Role.tenant_id == User.tenant_id, Role.tenant_id.is_(None)), + or_(UserRole.expires_at.is_(None), UserRole.expires_at > func.now()), + ) + .order_by(UserRole.created_at) + .all() + ) + for row in group_rows: + user_id = row[0] + if user_id not in result: + continue + result[user_id].append(self._row_to_assignment(tuple(row[1:]))) + + return result + + + def grant_row(self, tenant_id, grant_id: uuid.UUID): + """One `user_roles` row, refusing to reach outside the caller's tenant.""" + from app.modules.auth.models.role_model import Role + from app.modules.auth.models.user_role_model import UserRole + + return ( + self.db.query(UserRole) + .join(Role, Role.id == UserRole.role_id) + .filter( + UserRole.id == grant_id, + or_(Role.tenant_id == tenant_id, Role.tenant_id.is_(None)), + ) + .first() + ) + + def users_holding_role(self, role_id) -> int: + """ + How many users a role reaches, counting both mechanisms. + + Used to make role deletion say what it is about to do. Deleting a role + cascades its grants away (`user_roles.role_id ON DELETE CASCADE`) and + nulls the primary (`users.role_id ON DELETE SET NULL`), so a silent + delete can remove authority from people nobody was thinking about. + """ + from app.modules.auth.models.user_model import User + from app.modules.auth.models.user_role_model import UserRole + + primary = ( + self.db.query(func.count(User.id)) + .filter(User.role_id == role_id, User.is_deleted.is_(False)) + .scalar() + or 0 + ) + granted = ( + self.db.query(func.count(func.distinct(UserRole.user_id))) + .filter(UserRole.role_id == role_id, UserRole.user_id.isnot(None)) + .scalar() + or 0 + ) + groups = ( + self.db.query(func.count(func.distinct(UserRole.group_id))) + .filter(UserRole.role_id == role_id, UserRole.group_id.isnot(None)) + .scalar() + or 0 + ) + return int(primary) + int(granted) + int(groups) + + +def attach_role_refs(reader: "UserRoleReader", users: Iterable[Any]) -> None: + """ + Give a list of user rows their resolved roles, in two queries for the batch. + + `UserOut.roles` is declared as "every role this user holds". Any endpoint + that serialises a *different* user than the caller has to fill it in, or the + field means one thing on `/api/me/profile` and something narrower elsewhere + — which is precisely the split between the gate and the response that this + work exists to close, reintroduced one endpoint at a time. + + Sets the same `_resolve_role_refs` closure `get_current_user` attaches, so + `User.role_refs` needs no knowledge of where its answer came from. + """ + users = list(users) + if not users: + return + by_user = reader.assignments_for_users(users) + for row in users: + assignments = by_user.get(row.id, []) + setattr(row, "_resolve_role_refs", lambda captured=assignments: captured) + + +def invalidate_for_user(user_id: Optional[int]) -> None: + """ + Forget this request's memoised answers for one user. + + Called after a grant or a revoke. Without it, a POST that adds a role and + then returns the user's roles in the same response serves the pre-grant + answer — which reads as "the button did nothing". + """ + from app.core.request_cache import invalidate_request_cache + + if user_id is None: + invalidate_request_cache("role_assignments:") + invalidate_request_cache("access_codes:") + return + invalidate_request_cache(f"role_assignments:{user_id}") + invalidate_request_cache(f"access_codes:{user_id}") + + +__all__ = ["UserRoleReader", "attach_role_refs", "invalidate_for_user"] diff --git a/app/modules/auth/sso_router.py b/app/modules/auth/sso_router.py new file mode 100644 index 0000000..d35ab0a --- /dev/null +++ b/app/modules/auth/sso_router.py @@ -0,0 +1,356 @@ +from fastapi import APIRouter, Depends, HTTPException, status, Request, Response +from sqlalchemy.orm import Session +import time +import logging +import hmac +import hashlib + +from app.db.database import get_db +from app.core.settings import settings +from app.services.saas_service import SaaSService +from app.modules.tenant.models.tenant_model import Tenant +from app.modules.auth.models.saas_models import SaaSTenantMapping + +router = APIRouter(prefix="/sso", tags=["SSO"]) +logger = logging.getLogger(__name__) + +@router.post("/login") +async def sso_login( + request: Request, + response: Response, + db: Session = Depends(get_db) +): + """ + Endpoint for SaaS to initiate a login session for a user. + Verified via HMAC signature. + """ + try: + data = await read_json_body(request, what="SSO login body") + + user_id = data.get("user_id") + email = data.get("email") + tenant_id = data.get("tenant_id") + timestamp = data.get("timestamp") + signature = request.headers.get("X-Signature") + + if not all([user_id, email, timestamp, signature]): + raise HTTPException(status_code=400, detail="Missing required SSO parameters") + + now = int(time.time() * 1000) + req_timestamp = int(timestamp) + if abs(now - req_timestamp) > 5 * 60 * 1000: + raise HTTPException(status_code=401, detail="Invalid or stale timestamp") + + import json + canonical_string = json.dumps(data, sort_keys=True, separators=(",", ":")) + + if not SaaSService.verify_signature(signature, canonical_string): + raise HTTPException(status_code=401, detail="Invalid signature") + + saas_data = { + "id": user_id, + "email": email, + "name": data.get("first_name") or email.split('@')[0], + "company_id": tenant_id, + "tenant_name": data.get("tenant_name"), + "role": data.get("role"), + "role_id": data.get("role_id"), + "is_superadmin": data.get("is_superadmin"), + "metadata": { + "permissions": data.get("permissions", []), + "subscription": data.get("subscription"), + }, + } + + user, mapping = SaaSService.ensure_saas_user(db, saas_data) + logger.info(f"SSO: User {email} mapped to local user ID {user.id}") + + permissions = data.get("permissions", []) + if mapping: + mapping.metadata_ = { + "permissions": permissions, + "subscription": data.get("subscription"), + } + + is_user_superadmin = bool( + getattr(user, "is_superadmin", False) + or data.get("is_superadmin") + or data.get("role") == "superadmin" + or (isinstance(data.get("permissions"), list) and "superadmin.main.view" in data.get("permissions")) + ) + if is_user_superadmin and not user.is_superadmin: + user.is_superadmin = True + db.flush() + db.refresh(user) + + access_token = create_access_token( + { + "sub": str(user.id), + "tenant_id": str(user.tenant_id) if user.tenant_id else None, + "is_superadmin": is_user_superadmin, + "saas_permissions": permissions, + "saas_subscription": data.get("subscription"), + } + ) + refresh_token = create_refresh_token( + { + "sub": str(user.id), + } + ) + + is_prod = settings.APP_ENV == "production" + + response.set_cookie( + key="docqube_access_token", + value=access_token, + httponly=True, + secure=is_prod, + samesite="lax", + path="/", + max_age=settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60 + ) + + response.set_cookie( + key="docqube_refresh_token", + value=refresh_token, + httponly=True, + secure=is_prod, + samesite="lax", + path="/", + max_age=7 * 24 * 3600 + ) + + return { + "status": "success", + "message": "SSO Login successful", + "access_token": access_token, + "refresh_token": refresh_token, + "user": { + "id": user.id, + "email": user.email, + "name": user.name, + "permissions": permissions + } + } + + except HTTPException as he: + raise he + except Exception as e: + logger.error("SSO Login Failed", exc_info=True) + raise HTTPException(status_code=500, detail="Internal Server Error") + +@router.post("/sync-permissions") +def sync_permissions(request: Request, db: Session = Depends(get_db)): + """ + Return all permission nodes defined in DocQube to the SaaS platform. + """ + signature = request.headers.get("X-SaaS-Signature") or request.headers.get("X-Signature") + if not signature: + raise HTTPException(status_code=401, detail="Missing signature") + + payload_body = "{}" + expected_signature = hmac.new( + settings.SAAS_TRUST_SECRET.encode("utf-8"), + payload_body.encode("utf-8"), + hashlib.sha256 + ).hexdigest() + + if not hmac.compare_digest(signature, expected_signature): + raise HTTPException(status_code=401, detail="Invalid signature") + + try: + from app.modules.auth.models.access_model import Access + all_nodes = db.query(Access).all() + node_map = {str(node.id): node.access_code for node in all_nodes} + + permissions = [] + for node in all_nodes: + parent_code = node_map.get(str(node.parent_id)) if node.parent_id else None + permissions.append({ + "permission_code": node.access_code, + "name": node.name, + "category": node.category, + "parent_code": parent_code + }) + return {"status": "success", "permissions": permissions} + except HTTPException: + raise + except Exception: + logger.exception("Failed to build the permission list for SaaS") + raise HTTPException(status_code=500, detail="Internal Server Error") + +@router.post("/provision") +async def provision_tenant(request: Request, db: Session = Depends(get_db)): + """ + Receive tenant provisioning/update events from SaaS and sync the mapped + DocQube tenant, including max_users_allowed. + """ + raw_body = await request.body() + signature = request.headers.get("X-SaaS-Signature") or request.headers.get("X-Signature") + if not signature: + raise HTTPException(status_code=401, detail="Missing signature") + + expected_signature = hmac.new( + settings.SAAS_TRUST_SECRET.encode("utf-8"), + raw_body, + hashlib.sha256, + ).hexdigest() + if not hmac.compare_digest(signature, expected_signature): + raise HTTPException(status_code=401, detail="Invalid signature") + + payload = await read_json_body(request, what="webhook payload") + event_type = payload.get("event_type") + data = payload.get("data", {}) or {} + + saas_tenant_id = data.get("tenant_id") + tenant_name = data.get("tenant_name") or "SaaS Tenant" + max_users_allowed = data.get("max_users_allowed") + is_active = data.get("is_active") + plan_code = data.get("plan_code") or data.get("plan") + subscription = data.get("subscription") or {} + + try: + if event_type in {"TENANT_PROVISION_REQUESTED", "TENANT_UPDATED", "TENANT_STATUS_CHANGED"}: + if not saas_tenant_id: + raise HTTPException(status_code=400, detail="Missing tenant_id") + tenant = SaaSService.ensure_saas_tenant( + db, + saas_tenant_id=saas_tenant_id, + name=tenant_name, + max_users_allowed=max_users_allowed, + is_active=is_active, + ) + _sync_subscription(db, tenant, plan_code, subscription) + + return { + "status": "success", + "message": f"{event_type} synced", + "tenant_id": str(tenant.id), + "max_users_allowed": tenant.max_users_allowed, + } + + if event_type == "TENANT_DEPROVISION_REQUESTED": + mapping = ( + db.query(SaaSTenantMapping) + .filter(SaaSTenantMapping.saas_tenant_id == str(saas_tenant_id)) + .first() + ) + if mapping: + tenant = db.query(Tenant).filter(Tenant.id == mapping.docqube_tenant_id).first() + if tenant: + tenant.is_active = False + db.flush() + db.refresh(tenant) + return {"status": "success", "message": "Tenant deprovisioned"} + + if event_type == "USER_PROVISION_REQUESTED": + user, _ = SaaSService.provision_user(db, data) + return {"status": "success", "message": "User provisioned", "user_id": str(user.id) if user else None} + + if event_type == "USER_DEPROVISION_REQUESTED": + SaaSService.deprovision_user(db, data) + return {"status": "success", "message": "User deprovisioned"} + + if event_type == "ROLE_PROVISION_REQUESTED": + role, _ = SaaSService.provision_role(db, data) + return {"status": "success", "message": "Role provisioned", "role_id": str(role.id) if role else None} + + if event_type == "ROLE_DEPROVISION_REQUESTED": + SaaSService.deprovision_role(db, data) + return {"status": "success", "message": "Role deprovisioned"} + + if event_type == "PLAN_PROVISION_REQUESTED": + plan = SaaSService.ensure_saas_plan(db, data) + db.commit() + return {"status": "success", "message": "Plan provisioned", "plan_id": str(plan.id)} + + if event_type == "PLAN_UPDATED": + plan = SaaSService.update_saas_plan(db, data) + db.commit() + return {"status": "success", "message": "Plan updated", "plan_id": str(plan.id)} + + if event_type == "PLAN_DEPROVISION_REQUESTED": + SaaSService.deprovision_plan(db, data) + db.commit() + return {"status": "success", "message": "Plan deprovisioned"} + + return {"status": "ignored", "message": f"Unhandled event type: {event_type}"} + except HTTPException: + raise + except Exception as e: + logger.exception("Provisioning webhook failed for event %s", event_type) + raise HTTPException(status_code=500, detail=f"Provisioning error: {str(e)}") + +from app.core.security import create_access_token, create_refresh_token +from app.core.request_body import read_json_body + + +def _sync_subscription(db, tenant, plan_code, subscription: dict) -> None: + """ + Record what the platform says about this tenant's plan. + + **Unknown plan codes are logged and ignored, not created.** Auto-creating a + plan from a webhook would let the other system invent products here — with + no price, no limits, and no one having decided what it includes. A tenant + keeps its current plan until somebody adds the new one deliberately. + + Nothing here is fatal. A provisioning webhook that fails because the plan + name changed would block the tenant from being created at all, which is a + worse outcome than a tenant whose plan is briefly stale. + """ + import logging + + logger = logging.getLogger(__name__) + + from app.modules.billing.models.plan_model import Plan, TenantSubscription + + if not plan_code and not subscription: + return + + row = ( + db.query(TenantSubscription) + .filter(TenantSubscription.tenant_id == tenant.id) + .first() + ) + + plan = None + if plan_code: + plan = db.query(Plan).filter(Plan.code == str(plan_code)).first() + if plan is None: + logger.warning( + "SaaS webhook named plan %r, which does not exist here; leaving " + "tenant %s on its current plan", + plan_code, + tenant.id, + ) + + if row is None: + if plan is None: + return + row = TenantSubscription(tenant_id=tenant.id, plan_id=plan.id) + db.add(row) + elif plan is not None: + row.plan_id = plan.id + + status = subscription.get("status") + if status: + mapped = { + "ACTIVE": "active", + "TRIAL": "trialing", + "TRIALING": "trialing", + "PAST_DUE": "past_due", + "CANCELLED": "cancelled", + "CANCELED": "cancelled", + "EXPIRED": "expired", + }.get(str(status).upper()) + if mapped: + row.status = mapped + else: + logger.warning("SaaS webhook sent unknown status %r; leaving as-is", status) + + external_ref = subscription.get("id") or subscription.get("external_ref") + if external_ref: + row.external_ref = str(external_ref) + + db.flush() + diff --git a/app/modules/auth/utils/device_utils.py b/app/modules/auth/utils/device_utils.py new file mode 100644 index 0000000..a76b041 --- /dev/null +++ b/app/modules/auth/utils/device_utils.py @@ -0,0 +1,105 @@ +import hashlib +import uuid +from fastapi import Request +import re +import requests +import logging + +logger = logging.getLogger(__name__) + +def get_client_ip(request: Request) -> str: + client_host = request.client.host if request.client else "0.0.0.0" + + is_trusted = client_host.startswith("10.") or client_host.startswith("172.") or client_host.startswith("192.168.") or client_host == "127.0.0.1" + + if is_trusted: + x_forwarded_for = request.headers.get("X-Forwarded-For") + if x_forwarded_for: + return x_forwarded_for.split(",")[0].strip() + + return client_host + +def parse_user_agent(user_agent: str): + try: + from user_agents import parse + user_agent_parsed = parse(user_agent) + + browser = user_agent_parsed.browser.family + os = user_agent_parsed.os.family + + device_type = "Desktop" + if user_agent_parsed.is_mobile: + device_type = "Mobile" + elif user_agent_parsed.is_tablet: + device_type = "Tablet" + + return browser, os, device_type + except ImportError: + browser = "Unknown" + os = "Unknown" + device_type = "Desktop" + + ua_lower = user_agent.lower() + + if "mobile" in ua_lower or "android" in ua_lower or "iphone" in ua_lower: + device_type = "Mobile" + if "tablet" in ua_lower or "ipad" in ua_lower: + device_type = "Tablet" + + if "windows" in ua_lower: + os = "Windows" + elif "mac os" in ua_lower or "macos" in ua_lower: + os = "macOS" + elif "android" in ua_lower: + os = "Android" + elif "iphone" in ua_lower or "ipad" in ua_lower: + os = "iOS" + elif "linux" in ua_lower: + os = "Linux" + + if "edg" in ua_lower: + browser = "Edge" + elif "opr" in ua_lower or "opera" in ua_lower: + browser = "Opera" + elif "chrome" in ua_lower: + browser = "Chrome" + elif "safari" in ua_lower: + browser = "Safari" + elif "firefox" in ua_lower: + browser = "Firefox" + + return browser, os, device_type + +def generate_device_fingerprint(request: Request) -> str: + device_id = request.cookies.get("docqube_device_id") + if device_id: + return device_id + + new_device_id = str(uuid.uuid4()) + request.state.new_device_id = new_device_id + return new_device_id + +def get_location_from_ip(ip: str): + """ + Fetch approximate location from IP using free ip-api.com. + Returns (country, state, city, lat, lon) + """ + if not ip or ip in ("127.0.0.1", "0.0.0.0", "localhost"): + return "Unknown", "Unknown", "Unknown", None, None + + try: + response = requests.get(f"http://ip-api.com/json/{ip}", timeout=3) + if response.status_code == 200: + data = response.json() + if data.get("status") == "success": + return ( + data.get("country", "Unknown"), + data.get("regionName", "Unknown"), + data.get("city", "Unknown"), + data.get("lat"), + data.get("lon") + ) + except Exception as e: + logger.warning(f"Failed to fetch location for IP") + + return "Unknown", "Unknown", "Unknown", None, None diff --git a/app/modules/billing/__init__.py b/app/modules/billing/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/modules/billing/models/__init__.py b/app/modules/billing/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/modules/billing/models/plan_model.py b/app/modules/billing/models/plan_model.py new file mode 100644 index 0000000..685aa72 --- /dev/null +++ b/app/modules/billing/models/plan_model.py @@ -0,0 +1,285 @@ +""" +D1 — plans, and what a tenant is entitled to. + +The gap this closes: `src/features/Landing/Pricing/index.tsx` advertises four +priced plans — Starter, Professional, Elite, Custom — and the backend had never +heard of a plan. Meanwhile `tenants` carried four limit columns set per tenant +by hand, so two customers on the same advertised plan could silently have +different limits and nothing would notice. + +So this is not "add subscriptions". It is: **give the limits that already exist +something to be derived from.** + +Four tables: + +- `Plan` what is sold +- `PlanLimit` what it includes +- `TenantSubscription` who is on what +- `TenantLimitOverride` the negotiated exception + +**DocQube does not compute a price.** `price_display` is a string for the +pricing page, and `external_ref` on the subscription is whatever the system that +takes the money calls this. Two systems that both own the money is a +reconciliation problem you cannot test your way out of; one owns it, and this +one owns *entitlement*. +""" + +import uuid +from datetime import datetime, timezone +from typing import Optional + +from sqlalchemy import ( + BigInteger, + Boolean, + DateTime, + ForeignKey, + Index, + Integer, + Numeric, + String, + Text, + UniqueConstraint, + func, +) +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.db.database import Base + + +def _utcnow() -> datetime: + return datetime.now(timezone.utc) + + +class Plan(Base): + """ + Something a customer can be on. + + Not tenant-scoped: a plan is the catalogue, shared by everyone, the same way + `accesses` is. That also keeps it out of `TENANT_TABLES` — it has no + `tenant_id`, so neither the query listener nor RLS applies, which is correct + for a catalogue and would be wrong for anything else here. + """ + + __tablename__ = "plans" + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + code: Mapped[str] = mapped_column(String(50), nullable=False, index=True) + name: Mapped[str] = mapped_column(String(255), nullable=False) + description: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + price_amount: Mapped[Optional[float]] = mapped_column(Numeric(10, 2), nullable=True) + currency: Mapped[Optional[str]] = mapped_column(String(10), nullable=True) + interval: Mapped[Optional[str]] = mapped_column(String(20), nullable=True, default="monthly") + + grace_period_days: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + notify_days_before_expiry: Mapped[int] = mapped_column(Integer, nullable=False, default=7) + + is_public: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + """Whether it appears on the pricing page. A bespoke plan is not public.""" + + is_popular: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + """Whether this is marked as the 'Popular' plan on the pricing page. Only one can be popular.""" + + sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + + saas_plan_id: Mapped[Optional[str]] = mapped_column( + String(255), nullable=True, index=True, unique=True + ) + """The UUID of the corresponding SubscriptionPlan in the SaaS platform. + NULL for plans that were created directly in DocQube. + Used to upsert plan data when a webhook event arrives from SaaS. + """ + + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), onupdate=_utcnow, nullable=False + ) + + limits: Mapped[list["PlanLimit"]] = relationship( + "PlanLimit", back_populates="plan", cascade="all, delete-orphan" + ) + roles: Mapped[list["PlanRole"]] = relationship( + "PlanRole", back_populates="plan", cascade="all, delete-orphan" + ) + + __table_args__ = (UniqueConstraint("code", name="plans_code_key"),) + + def __repr__(self) -> str: # pragma: no cover - debugging aid + return f"" + + +class PlanRole(Base): + """ + Mapping between a Plan and a System Role. + """ + __tablename__ = "plan_roles" + + plan_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("plans.id", ondelete="CASCADE"), primary_key=True + ) + role_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("roles.id", ondelete="CASCADE"), primary_key=True + ) + + plan = relationship("Plan", back_populates="roles") + + +class PlanLimit(Base): + """ + One limit a plan includes. + + A row, not a column. The four that exist today became four columns on + `tenants`, and a fifth would have been a fifth — which is how a schema turns + into a changelog. Adding a limit is now a seed row. + + `value` is a signed 64-bit integer and **-1 means unlimited**, matching the + convention `tenants.envelope_limit` already uses. Null would have been the + tidier spelling and would collide with "not set", which is a different + thing. + """ + + __tablename__ = "plan_limits" + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + plan_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("plans.id", ondelete="CASCADE"), nullable=False, index=True + ) + key: Mapped[str] = mapped_column(String(50), nullable=False) + value: Mapped[int] = mapped_column(BigInteger, nullable=False) + + plan: Mapped["Plan"] = relationship("Plan", back_populates="limits") + + __table_args__ = ( + UniqueConstraint("plan_id", "key", name="uq_plan_limits_plan_key"), + ) + + +class TenantSubscription(Base): + """ + Which plan a tenant is on, and what state that is in. + + **`status` is not `tenant.is_active`.** One is commercial state, set by + billing; the other is suspension, set by an operator. Merging them tells a + customer the wrong thing about why they cannot get in, which is the worst + kind of support call. + """ + + __tablename__ = "tenant_subscriptions" + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + tenant_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("tenants.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + plan_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("plans.id", ondelete="RESTRICT"), nullable=False, index=True + ) + status: Mapped[str] = mapped_column(String(20), nullable=False, default="active") + """active | trialing | past_due | cancelled | expired.""" + + started_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + current_period_end: Mapped[Optional[datetime]] = mapped_column( + DateTime(timezone=True), nullable=True + ) + cancel_at_period_end: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=False + ) + + external_ref: Mapped[Optional[str]] = mapped_column(String(255), nullable=True, index=True) + """ + Whatever the system that bills calls this — a Stripe id, a platform + subscription id. DocQube never interprets it. It exists so that when the two + systems disagree, support has something to compare. + """ + + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), onupdate=_utcnow, nullable=False + ) + + plan: Mapped["Plan"] = relationship("Plan") + + __table_args__ = ( + UniqueConstraint("tenant_id", name="uq_tenant_subscriptions_tenant"), + Index("ix_tenant_subscriptions_status", "status"), + ) + + +class TenantLimitOverride(Base): + """ + The negotiated exception: "Elite, but 500 GB, because the contract says so". + + Deliberately a separate row rather than an edit to the tenant's copy of a + plan limit. Editing loses the fact that it *was* an exception, and that is + exactly what somebody needs to know at renewal. + """ + + __tablename__ = "tenant_limit_overrides" + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + tenant_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("tenants.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + key: Mapped[str] = mapped_column(String(50), nullable=False) + value: Mapped[int] = mapped_column(BigInteger, nullable=False) + note: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + """Why. An exception with no reason recorded becomes folklore.""" + + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + + __table_args__ = ( + UniqueConstraint("tenant_id", "key", name="uq_tenant_limit_overrides"), + ) + + +class PlanPriceHistory(Base): + """ + Tracks pricing changes over time for a plan. + """ + __tablename__ = "plan_price_history" + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + plan_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("plans.id", ondelete="CASCADE"), nullable=False, index=True + ) + old_price: Mapped[Optional[float]] = mapped_column(Numeric(10, 2), nullable=True) + new_price: Mapped[Optional[float]] = mapped_column(Numeric(10, 2), nullable=True) + currency: Mapped[Optional[str]] = mapped_column(String(10), nullable=True) + + changed_by: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + """User ID who made the change.""" + + changed_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + + plan: Mapped["Plan"] = relationship("Plan") + + def __repr__(self) -> str: + return f"" + + +__all__ = ["Plan", "PlanLimit", "TenantSubscription", "TenantLimitOverride", "PlanPriceHistory"] diff --git a/app/modules/billing/routes/__init__.py b/app/modules/billing/routes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/modules/billing/routes/billing_routes.py b/app/modules/billing/routes/billing_routes.py new file mode 100644 index 0000000..d735cf9 --- /dev/null +++ b/app/modules/billing/routes/billing_routes.py @@ -0,0 +1,535 @@ +""" +D4 — the plans, and what a tenant is entitled to. + +Three audiences, three levels of access: + +- **anyone**, including a signed-out visitor on the pricing page — the public + catalogue. This is the endpoint that lets the landing page stop hardcoding + what it sells; +- **any signed-in user** — their own plan and their own limits. Not privileged + information: it is what they bought; +- **a superadmin** — change a tenant's plan, set an override. + +Changing a plan is `superadmin.tenant.update`, not a tenant-level code. A +customer moving themselves onto Elite would be a pricing decision made by the +person who benefits from it. +""" + +import uuid +from typing import List, Optional + +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy.orm import Session + +from app.db.database import get_db +from app.middleware.auth import get_current_user +from app.middleware.tenant import require_superadmin +from app.modules.auth.dependencies.access_dependency import require_access +from app.modules.auth.models.user_model import User +from app.modules.billing.schemas.billing_schema import ( + EntitlementOut, + PlanOut, + CreatePlanIn, + UpdatePlanIn, + PlanPriceHistoryOut, + SetOverrideIn, + SetPlanIn, + SubscriptionOut, +) +from app.modules.billing.services.entitlement_service import EntitlementService + +router = APIRouter(prefix="/billing", tags=["Billing"]) + + +def _plan_out(db: Session, plan) -> dict: + from app.modules.billing.models.plan_model import PlanLimit, PlanRole + from app.modules.auth.models.role_model import Role + from app.modules.auth.models.role_access_model import RoleAccess + from sqlalchemy.orm import selectinload + + limits = { + row.key: row.value + for row in db.query(PlanLimit).filter(PlanLimit.plan_id == plan.id) + } + + roles_out = [] + plan_roles = db.query(PlanRole).filter(PlanRole.plan_id == plan.id).all() + if plan_roles: + role_ids = [pr.role_id for pr in plan_roles] + roles = db.query(Role).options(selectinload(Role.role_accesses).selectinload(RoleAccess.access)).filter(Role.id.in_(role_ids)).all() + for r in roles: + roles_out.append({ + "id": r.id, + "name": r.name, + "description": r.description, + "access_codes": [ra.access.access_code for ra in r.role_accesses if ra.access] + }) + + return { + "id": plan.id, + "code": plan.code, + "name": plan.name, + "description": plan.description, + "price_amount": float(plan.price_amount) if plan.price_amount is not None else None, + "currency": plan.currency, + "interval": plan.interval, + "grace_period_days": plan.grace_period_days, + "notify_days_before_expiry": plan.notify_days_before_expiry, + "is_public": plan.is_public, + "is_popular": plan.is_popular, + "sort_order": plan.sort_order, + "limits": limits, + "roles": roles_out, + } + + +@router.get("/plans", response_model=List[PlanOut]) +def list_plans(db: Session = Depends(get_db)): + """ + The public catalogue. + + **Deliberately unauthenticated.** It is what the pricing page shows to a + visitor who has not signed up yet, and requiring a token would defeat the + purpose. It exposes only what is already printed on that page — names, + prices and limits — and no tenant is named. + + This is the endpoint that closes the original gap: the page advertised four + priced plans and the backend had never heard of a plan. + """ + from app.modules.billing.models.plan_model import Plan + + plans = ( + db.query(Plan) + .filter(Plan.is_public.is_(True)) + .order_by(Plan.sort_order, Plan.name) + .all() + ) + return [_plan_out(db, plan) for plan in plans] + + +@router.get( + "/plans/admin", + response_model=List[PlanOut], + dependencies=[require_access("superadmin.tenant.read"), Depends(require_superadmin)] +) +def list_plans_admin(db: Session = Depends(get_db)): + """ + Returns all plans, including inactive (is_public=False) ones. + Only accessible by superadmins. + """ + from app.modules.billing.models.plan_model import Plan + + plans = ( + db.query(Plan) + .order_by(Plan.sort_order, Plan.name) + .all() + ) + return [_plan_out(db, plan) for plan in plans] + + +@router.get("/me", response_model=EntitlementOut) +def my_entitlement( + db: Session = Depends(get_db), user: User = Depends(get_current_user) +): + """ + What this caller's workspace is entitled to. + + Not gated on an admin code: a user asking what their own workspace includes + is asking what they bought. + """ + service = EntitlementService(db) + tenant = getattr(user, "tenant", None) + subscription = service.subscription(tenant) + plan = subscription.plan if subscription is not None else None + + return { + "plan": _plan_out(db, plan) if plan is not None else None, + "status": subscription.status if subscription is not None else None, + "current_period_end": ( + subscription.current_period_end if subscription is not None else None + ), + "cancel_at_period_end": ( + subscription.cancel_at_period_end if subscription is not None else False + ), + "subscription_active": service.is_active(tenant), + "access_level": service.access_level(tenant), + "limits": service.all_limits(tenant), + } + + +@router.get( + "/tenants/{tenant_id}/subscription", + response_model=SubscriptionOut, + dependencies=[require_access("superadmin.tenant.read"), Depends(require_superadmin)], +) +def get_tenant_subscription( + tenant_id: uuid.UUID, + db: Session = Depends(get_db), + user: User = Depends(get_current_user), +): + from app.modules.billing.models.plan_model import TenantSubscription + + row = ( + db.query(TenantSubscription) + .filter(TenantSubscription.tenant_id == tenant_id) + .first() + ) + if row is None: + raise HTTPException(status_code=404, detail="No subscription for that tenant") + return row + + +@router.put( + "/tenants/{tenant_id}/subscription", + response_model=SubscriptionOut, + dependencies=[require_access("superadmin.tenant.update"), Depends(require_superadmin)], +) +def set_tenant_plan( + tenant_id: uuid.UUID, + payload: SetPlanIn, + db: Session = Depends(get_db), + user: User = Depends(get_current_user), +): + """ + Move a tenant onto a plan. + + A superadmin decision. A tenant admin moving themselves onto Elite would be + a pricing decision made by the person who benefits from it. + """ + from app.modules.billing.models.plan_model import Plan, TenantSubscription + + plan = db.query(Plan).filter(Plan.code == payload.plan_code).first() + if plan is None: + raise HTTPException(status_code=404, detail="No such plan") + + row = ( + db.query(TenantSubscription) + .filter(TenantSubscription.tenant_id == tenant_id) + .first() + ) + if row is None: + row = TenantSubscription(tenant_id=tenant_id, plan_id=plan.id) + db.add(row) + else: + row.plan_id = plan.id + + if payload.status is not None: + row.status = payload.status + if payload.external_ref is not None: + row.external_ref = payload.external_ref + if payload.current_period_end is not None: + row.current_period_end = payload.current_period_end + + # Auto-assign the first plan role to the tenant's first user + from app.modules.billing.models.plan_model import PlanRole + from app.modules.auth.models.user_model import User + + plan_roles = db.query(PlanRole).filter(PlanRole.plan_id == plan.id).all() + if plan_roles: + first_plan_role_id = plan_roles[0].role_id + # Find the first user in this tenant (usually the owner) + tenant_user = db.query(User).filter(User.tenant_id == tenant_id).order_by(User.id.asc()).first() + if tenant_user: + tenant_user.role_id = first_plan_role_id + + db.flush() + db.refresh(row) + return row + + +@router.put( + "/tenants/{tenant_id}/overrides", + response_model=EntitlementOut, + dependencies=[require_access("superadmin.tenant.update"), Depends(require_superadmin)], +) +def set_override( + tenant_id: uuid.UUID, + payload: SetOverrideIn, + db: Session = Depends(get_db), + user: User = Depends(get_current_user), +): + """ + The negotiated exception — "Elite, but 500 GB, because the contract says so". + + Stored as its own row rather than by editing the plan, so the plan stays + honest and the exception stays visible at renewal. `note` is not optional in + spirit: an exception with no reason recorded becomes folklore. + """ + from app.modules.billing.models.plan_model import TenantLimitOverride + from app.modules.billing.services.entitlement_service import DEFAULTS + from app.modules.tenant.models.tenant_model import Tenant + + if payload.key not in DEFAULTS: + raise HTTPException( + status_code=400, + detail=f"Unknown limit. Known limits: {sorted(DEFAULTS)}.", + ) + + tenant = db.query(Tenant).filter(Tenant.id == tenant_id).first() + if tenant is None: + raise HTTPException(status_code=404, detail="Tenant not found") + + row = ( + db.query(TenantLimitOverride) + .filter( + TenantLimitOverride.tenant_id == tenant_id, + TenantLimitOverride.key == payload.key, + ) + .first() + ) + if row is None: + row = TenantLimitOverride( + tenant_id=tenant_id, key=payload.key, value=payload.value, note=payload.note + ) + db.add(row) + else: + row.value = payload.value + row.note = payload.note + db.flush() + + service = EntitlementService(db) + subscription = service.subscription(tenant) + plan = subscription.plan if subscription is not None else None + return { + "plan": _plan_out(db, plan) if plan is not None else None, + "status": subscription.status if subscription is not None else None, + "current_period_end": ( + subscription.current_period_end if subscription is not None else None + ), + "cancel_at_period_end": ( + subscription.cancel_at_period_end if subscription is not None else False + ), + "subscription_active": service.is_active(tenant), + "access_level": service.access_level(tenant), + "limits": service.all_limits(tenant), + } + + +@router.post( + "/plans", + response_model=PlanOut, + dependencies=[require_access("superadmin.tenant.update"), Depends(require_superadmin)], +) +def create_plan( + payload: CreatePlanIn, + db: Session = Depends(get_db), + user: User = Depends(get_current_user), +): + from app.modules.billing.models.plan_model import Plan, PlanLimit, PlanPriceHistory + + plan = db.query(Plan).filter(Plan.code == payload.code).first() + if plan is not None: + raise HTTPException(status_code=400, detail="Plan code already exists") + + plan = Plan( + code=payload.code, + name=payload.name, + description=payload.description, + price_amount=payload.price_amount, + currency=payload.currency, + interval=payload.interval, + grace_period_days=payload.grace_period_days, + notify_days_before_expiry=payload.notify_days_before_expiry, + is_public=payload.is_public, + is_popular=payload.is_popular, + sort_order=payload.sort_order, + ) + db.add(plan) + + if payload.is_popular: + db.query(Plan).filter(Plan.is_popular == True).update({"is_popular": False}) + + db.flush() + + if payload.roles: + from app.modules.auth.models.role_model import Role + from app.modules.auth.models.role_access_model import RoleAccess + from app.modules.auth.models.access_model import Access + from app.modules.billing.models.plan_model import PlanRole + for role_data in payload.roles: + new_role = Role( + name=role_data.name, + description=role_data.description, + is_system=True, + ) + db.add(new_role) + db.flush() + db.add(PlanRole(plan_id=plan.id, role_id=new_role.id)) + for access_code in role_data.access_codes: + acc = db.query(Access).filter(Access.access_code == access_code).first() + if acc: + db.add(RoleAccess(role_id=new_role.id, access_id=acc.id)) + + for key, value in payload.limits.items(): + db.add(PlanLimit(plan_id=plan.id, key=key, value=value)) + + # Log to price history + if payload.price_amount is not None: + history = PlanPriceHistory( + plan_id=plan.id, + old_price=None, + new_price=payload.price_amount, + currency=payload.currency, + changed_by=user.id + ) + db.add(history) + + db.commit() + db.refresh(plan) + return _plan_out(db, plan) + + +@router.put( + "/plans/{plan_id}", + response_model=PlanOut, + dependencies=[require_access("superadmin.tenant.update"), Depends(require_superadmin)], +) +def update_plan( + plan_id: uuid.UUID, + payload: UpdatePlanIn, + db: Session = Depends(get_db), + user: User = Depends(get_current_user), +): + from app.modules.billing.models.plan_model import Plan, PlanLimit, PlanPriceHistory + + plan = db.query(Plan).filter(Plan.id == plan_id).first() + if plan is None: + raise HTTPException(status_code=404, detail="Plan not found") + + old_price = plan.price_amount + old_currency = plan.currency + + update_data = payload.model_dump(exclude_unset=True) + for field, value in update_data.items(): + if field in ["limits", "roles"]: + continue + setattr(plan, field, value) + + if payload.is_popular is True: + db.query(Plan).filter(Plan.id != plan.id, Plan.is_popular == True).update({"is_popular": False}) + + + if payload.limits is not None: + db.query(PlanLimit).filter(PlanLimit.plan_id == plan.id).delete() + for key, value in payload.limits.items(): + db.add(PlanLimit(plan_id=plan.id, key=key, value=value)) + + if payload.roles is not None: + from app.modules.auth.models.role_model import Role + from app.modules.auth.models.role_access_model import RoleAccess + from app.modules.auth.models.access_model import Access + from app.modules.billing.models.plan_model import PlanRole + + existing_plan_roles = db.query(PlanRole).filter(PlanRole.plan_id == plan.id).all() + existing_roles = { + r.name: r + for r in db.query(Role).filter(Role.id.in_([pr.role_id for pr in existing_plan_roles])).all() + } if existing_plan_roles else {} + + processed_role_names = set() + + for role_data in payload.roles: + processed_role_names.add(role_data.name) + if role_data.name in existing_roles: + existing_role = existing_roles[role_data.name] + existing_role.description = role_data.description + + db.query(RoleAccess).filter(RoleAccess.role_id == existing_role.id).delete() + for access_code in role_data.access_codes: + acc = db.query(Access).filter(Access.access_code == access_code).first() + if acc: + db.add(RoleAccess(role_id=existing_role.id, access_id=acc.id)) + else: + new_role = Role( + name=role_data.name, + description=role_data.description, + is_system=True, + ) + db.add(new_role) + db.flush() + db.add(PlanRole(plan_id=plan.id, role_id=new_role.id)) + for access_code in role_data.access_codes: + acc = db.query(Access).filter(Access.access_code == access_code).first() + if acc: + db.add(RoleAccess(role_id=new_role.id, access_id=acc.id)) + + for name, role in existing_roles.items(): + if name not in processed_role_names: + db.query(PlanRole).filter(PlanRole.plan_id == plan.id, PlanRole.role_id == role.id).delete() + db.delete(role) + + # Log to price history if price or currency changed + if plan.price_amount != old_price or plan.currency != old_currency: + history = PlanPriceHistory( + plan_id=plan.id, + old_price=old_price, + new_price=plan.price_amount, + currency=plan.currency, + changed_by=user.id + ) + db.add(history) + + db.commit() + db.refresh(plan) + return _plan_out(db, plan) + + +@router.delete( + "/plans/{plan_id}", + dependencies=[require_access("superadmin.tenant.update"), Depends(require_superadmin)], +) +def delete_plan( + plan_id: uuid.UUID, + db: Session = Depends(get_db), + user: User = Depends(get_current_user), +): + from app.modules.billing.models.plan_model import Plan + + plan = db.query(Plan).filter(Plan.id == plan_id).first() + if plan is None: + raise HTTPException(status_code=404, detail="Plan not found") + + try: + db.delete(plan) + db.commit() + except Exception as e: + db.rollback() + raise HTTPException(status_code=400, detail="Cannot delete plan in use by tenants") + + return {"status": "ok"} + + +@router.get( + "/plans/{plan_id}/price-history", + response_model=List[PlanPriceHistoryOut], + dependencies=[require_access("superadmin.tenant.read"), Depends(require_superadmin)], +) +def get_plan_price_history( + plan_id: uuid.UUID, + db: Session = Depends(get_db), + user: User = Depends(get_current_user), +): + from app.modules.billing.models.plan_model import PlanPriceHistory + + history = ( + db.query(PlanPriceHistory) + .filter(PlanPriceHistory.plan_id == plan_id) + .order_by(PlanPriceHistory.changed_at.desc()) + .all() + ) + return history + + +@router.post( + "/plans/reorder", + dependencies=[require_access("superadmin.tenant.update"), Depends(require_superadmin)], +) +def reorder_plans( + plan_ids: List[uuid.UUID], + db: Session = Depends(get_db), +): + from app.modules.billing.models.plan_model import Plan + for index, plan_id in enumerate(plan_ids): + db.query(Plan).filter(Plan.id == plan_id).update({"sort_order": index + 1}) + db.commit() + return {"message": "Plans reordered successfully"} diff --git a/app/modules/billing/schemas/__init__.py b/app/modules/billing/schemas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/modules/billing/schemas/billing_schema.py b/app/modules/billing/schemas/billing_schema.py new file mode 100644 index 0000000..0a33715 --- /dev/null +++ b/app/modules/billing/schemas/billing_schema.py @@ -0,0 +1,146 @@ +"""D4 — the billing contract.""" + +import uuid +from datetime import datetime +from typing import Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field + + +class PlanRoleIn(BaseModel): + name: str + description: Optional[str] = None + access_codes: List[str] = Field(default_factory=list) + + +class PlanRoleOut(BaseModel): + id: uuid.UUID + name: str + description: Optional[str] = None + access_codes: List[str] + + +class PlanOut(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: uuid.UUID + code: str + name: str + description: Optional[str] = None + currency: Optional[str] = None + is_public: bool + is_popular: bool + sort_order: int + interval: Optional[str] = "monthly" + price_amount: Optional[float] = None + grace_period_days: int = 0 + notify_days_before_expiry: int = 7 + limits: Dict[str, int] + """ + Every limit the plan includes, keyed by name. A dict rather than named + fields, so adding a limit is a seed row rather than a schema change and a + frontend deploy. + """ + roles: List[PlanRoleOut] = Field(default_factory=list) + + +class CreatePlanIn(BaseModel): + code: str + name: str + description: Optional[str] = None + price_amount: Optional[float] = None + currency: Optional[str] = None + interval: Optional[str] = "monthly" + is_public: bool = True + is_popular: bool = False + sort_order: int = 0 + grace_period_days: int = 0 + notify_days_before_expiry: int = 7 + limits: Dict[str, int] = {} + roles: List[PlanRoleIn] = Field(default_factory=list) + + +class UpdatePlanIn(BaseModel): + name: Optional[str] = None + description: Optional[str] = None + price_amount: Optional[float] = None + currency: Optional[str] = None + interval: Optional[str] = None + is_public: Optional[bool] = None + is_popular: Optional[bool] = None + sort_order: Optional[int] = None + grace_period_days: Optional[int] = None + notify_days_before_expiry: Optional[int] = None + limits: Optional[Dict[str, int]] = None + roles: Optional[List[PlanRoleIn]] = None + + +class PlanPriceHistoryOut(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: uuid.UUID + plan_id: uuid.UUID + old_price: Optional[float] = None + new_price: Optional[float] = None + currency: Optional[str] = None + changed_by: Optional[int] = None + changed_at: datetime + + +class SubscriptionOut(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: uuid.UUID + tenant_id: uuid.UUID + plan_id: uuid.UUID + status: str + started_at: datetime + current_period_end: Optional[datetime] = None + cancel_at_period_end: bool + external_ref: Optional[str] = None + + +class EntitlementOut(BaseModel): + """What a workspace is entitled to, resolved.""" + + plan: Optional[PlanOut] = None + status: Optional[str] = None + current_period_end: Optional[datetime] = None + cancel_at_period_end: bool = False + + subscription_active: bool + """ + Commercial state only. Deliberately says nothing about whether the tenant is + suspended — that is an operator's act, and merging the two tells a customer + the wrong thing about why they cannot get in. + """ + + access_level: str + """ + `full` or `read_only` — the same value the middleware enforces on. + + Reported rather than left to be inferred. Without it a client has to + reconstruct the rule from `status` and `subscription_active`, which means + the decision about what a lapsed customer may do exists twice, in two + languages, and drifts the first time the rule gains a case. The middleware + already computes this to decide whether to answer 402; this is that answer. + """ + + limits: Dict[str, int] + """Resolved: override, else plan, else default. `-1` means unlimited.""" + + +class SetPlanIn(BaseModel): + plan_code: str + status: Optional[str] = None + external_ref: Optional[str] = None + current_period_end: Optional[datetime] = None + + +class SetOverrideIn(BaseModel): + key: str + value: int = Field(description="-1 means unlimited") + note: Optional[str] = Field( + default=None, + description="Why. An exception with no reason recorded becomes folklore.", + ) diff --git a/app/modules/billing/services/__init__.py b/app/modules/billing/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/modules/billing/services/billing_tasks.py b/app/modules/billing/services/billing_tasks.py new file mode 100644 index 0000000..c8f03d0 --- /dev/null +++ b/app/modules/billing/services/billing_tasks.py @@ -0,0 +1,62 @@ +import logging +from datetime import datetime, timezone, timedelta +from sqlalchemy.orm import Session +from app.db.database import SessionLocal +from app.modules.billing.models.plan_model import TenantSubscription, Plan +# Assuming you have a notification service, e.g. from app.modules.notifications... + +logger = logging.getLogger(__name__) + +def evaluate_subscriptions(): + """ + Evaluates all active subscriptions and updates their statuses or sends notifications + based on expiry and grace periods. + This should be run daily via a cron job or Celery beat. + """ + db: Session = SessionLocal() + try: + now = datetime.now(timezone.utc) + + subscriptions = db.query(TenantSubscription).filter( + TenantSubscription.status.in_(["active", "trialing", "grace", "past_due"]) + ).all() + + for sub in subscriptions: + plan = sub.plan + if not sub.current_period_end: + continue + + days_until_expiry = (sub.current_period_end - now).days + + if sub.status in ["active", "trialing"]: + if 0 < days_until_expiry <= plan.notify_days_before_expiry: + # TODO: Trigger SUBSCRIPTION_EXPIRING_SOON notification + logger.info(f"Subscription {sub.id} expiring in {days_until_expiry} days. Sending notification.") + + elif days_until_expiry <= 0: + # Expired. Move to grace period or past_due + if plan.grace_period_days > 0: + sub.status = "grace" + logger.info(f"Subscription {sub.id} expired. Moving to grace period.") + # TODO: Trigger SUBSCRIPTION_GRACE_PERIOD notification + else: + sub.status = "past_due" + logger.info(f"Subscription {sub.id} expired. Moving to past_due.") + # TODO: Trigger SUBSCRIPTION_EXPIRED notification + + elif sub.status == "grace": + days_since_expiry = -days_until_expiry + if days_since_expiry > plan.grace_period_days: + sub.status = "past_due" + logger.info(f"Subscription {sub.id} grace period ended. Moving to past_due.") + # TODO: Trigger SUBSCRIPTION_EXPIRED notification + else: + # Still in grace period, maybe send another reminder? + pass + + db.commit() + except Exception as e: + logger.error(f"Error evaluating subscriptions: {e}") + db.rollback() + finally: + db.close() diff --git a/app/modules/billing/services/entitlement_service.py b/app/modules/billing/services/entitlement_service.py new file mode 100644 index 0000000..152e8b5 --- /dev/null +++ b/app/modules/billing/services/entitlement_service.py @@ -0,0 +1,235 @@ +""" +D2 — what is this tenant entitled to? + +One function that every enforcement point asks, instead of four columns that +four places read. + + limit(tenant, key) = + TenantLimitOverride the negotiated exception + else PlanLimit what was actually sold + else the tenant column the transition fallback, one release only + else DEFAULTS the built-in floor + +**It fails open, and that is deliberate.** Everywhere else in this codebase the +answer is fail closed — the tenant filter denies without context, the scope +resolver returns an empty set, `require_access` refuses. Those are about +*authority*, where the risk is showing somebody another tenant's data. + +This is about *capacity*, where the risk is refusing a customer their own. A +tenant with no subscription row is an operational mistake — a failed backfill, a +half-finished signup — and locking a paying customer out of their own documents +is a worse answer to that mistake than serving them. Different question, +different default. + +The one thing it will not do is guess at a key it does not know. An unknown key +raises rather than returning zero, because "limit of zero" and "I have never +heard of that limit" are different, and only one of them should silently switch a +feature off. +""" + +from __future__ import annotations + +from contextvars import ContextVar +from typing import Optional + +from sqlalchemy.orm import Session + +CAPACITY_DEFAULTS: dict[str, int] = { + "storage_bytes": 1024 ** 3, + "seats": -1, + "chat_tokens_daily": 1_000_000, + "envelopes": -1, +} +""" +How much of something. **Fails open**: a resolution failure serves the default +rather than zero, because the risk here is refusing a customer their own data. +""" + +FEATURE_DEFAULTS: dict[str, int] = { +} +""" +Whether something is switched on at all. **Fails closed**: 0 unless a plan says +otherwise. + +Separated from capacity because the two want opposite defaults and sharing a +dict would silently give features the capacity behaviour. On a resolution +failure, "serve them a gigabyte" is generous; "grant them the Elite-only feature" +is giving the product away. Splitting now costs nothing; splitting after the +first feature flag ships means auditing every call site. +""" + +DEFAULTS: dict[str, int] = {**CAPACITY_DEFAULTS, **FEATURE_DEFAULTS} + +LEGACY_COLUMNS: dict[str, str] = { + "storage_bytes": "storage_quota_bytes", + "seats": "max_users_allowed", + "chat_tokens_daily": "chat_token_daily_limit", + "envelopes": "envelope_limit", +} + +UNLIMITED = -1 + +_cache: ContextVar[Optional[dict]] = ContextVar("entitlement_cache", default=None) + + +def reset_entitlement_cache() -> None: + _cache.set({}) + + +def _cached(key: str, compute): + store = _cache.get() + if store is None: + return compute() + if key not in store: + store[key] = compute() + return store[key] + + +class EntitlementService: + def __init__(self, db: Session): + self.db = db + + + def limit(self, tenant, key: str) -> int: + """The effective limit for *key*. `-1` means unlimited.""" + if key not in DEFAULTS: + raise KeyError( + f"Unknown entitlement key {key!r}. Known keys: " + f"{sorted(DEFAULTS)}. A typo must not read as a limit of zero." + ) + if tenant is None: + return DEFAULTS[key] + + return _cached(f"limit:{tenant.id}:{key}", lambda: self._resolve(tenant, key)) + + def is_unlimited(self, tenant, key: str) -> bool: + return self.limit(tenant, key) == UNLIMITED + + def within(self, tenant, key: str, used: int) -> bool: + """Whether *used* is inside the limit. Unlimited is always inside.""" + allowed = self.limit(tenant, key) + return allowed == UNLIMITED or used <= allowed + + def all_limits(self, tenant) -> dict[str, int]: + """Every known limit, for a settings screen or an API response.""" + return {key: self.limit(tenant, key) for key in DEFAULTS} + + + def subscription(self, tenant): + from app.modules.billing.models.plan_model import TenantSubscription + + if tenant is None: + return None + return ( + self.db.query(TenantSubscription) + .filter(TenantSubscription.tenant_id == tenant.id) + .first() + ) + + def is_active(self, tenant) -> bool: + """ + Whether the *subscription* is in good standing. + + Deliberately says nothing about `tenant.is_active`, which is suspension — + an operator's act, not a commercial state. Merging them tells a customer + the wrong thing about why they cannot get in, and "we paid, why are we + locked out?" is the support call that follows. + + A tenant with no subscription row is treated as active, for the same + reason the limits fall back rather than failing closed. + """ + row = self.subscription(tenant) + if row is None: + return True + return row.status in {"active", "trialing"} + + def plan(self, tenant): + row = self.subscription(tenant) + return row.plan if row is not None else None + + def access_level(self, tenant) -> str: + """ + What a lapsed subscription still permits: `full`, `read_only` or `none`. + + Ported from the base, which is also how mature systems behave: an + expired customer is not locked out of their own documents, they are + stopped from creating more. Locking them out is hostile, makes export + impossible, and is bad for renewals — the customer most likely to come + back is the one who can still see what they would be coming back to. + + A tenant with no subscription row gets `full`, for the same reason the + limits fall back rather than failing closed. + """ + row = self.subscription(tenant) + if row is None: + return "full" + if row.status in {"active", "trialing", "grace"}: + return "full" + if row.status in {"past_due"}: + return "read_only" + return "read_only" + + + def _resolve(self, tenant, key: str) -> int: + from app.modules.billing.models.plan_model import ( + PlanLimit, + TenantLimitOverride, + TenantSubscription, + ) + + override = ( + self.db.query(TenantLimitOverride.value) + .filter( + TenantLimitOverride.tenant_id == tenant.id, + TenantLimitOverride.key == key, + ) + .first() + ) + if override is not None: + return int(override[0]) + + row = ( + self.db.query(PlanLimit.value) + .join(TenantSubscription, TenantSubscription.plan_id == PlanLimit.plan_id) + .filter( + TenantSubscription.tenant_id == tenant.id, + PlanLimit.key == key, + ) + .first() + ) + if row is not None: + return int(row[0]) + + column = LEGACY_COLUMNS.get(key) + if column is not None: + value = getattr(tenant, column, None) + if value is not None: + return int(value) + + return DEFAULTS[key] + + def has_feature(self, tenant, key: str) -> bool: + """ + Whether a feature is switched on for this tenant. + + Separate from `limit()` so the caller cannot accidentally read a + feature with capacity semantics — `limit(tenant, 'feature_sso') > 0` + would work today and would quietly grant the feature the moment + resolution failed. + """ + if key not in FEATURE_DEFAULTS: + raise KeyError( + f"Unknown feature {key!r}. Known features: {sorted(FEATURE_DEFAULTS)}." + ) + return self.limit(tenant, key) != 0 + + +__all__ = [ + "EntitlementService", + "DEFAULTS", + "CAPACITY_DEFAULTS", + "FEATURE_DEFAULTS", + "LEGACY_COLUMNS", + "UNLIMITED", + "reset_entitlement_cache", +] diff --git a/app/modules/chat/README.md b/app/modules/chat/README.md new file mode 100644 index 0000000..9d16a83 --- /dev/null +++ b/app/modules/chat/README.md @@ -0,0 +1,41 @@ +# Chatbot Module + +This module implements a robust, conversational RAG (Retrieval-Augmented Generation) chatbot for documents, integrated into the DocQube application. + +## 📂 Structure + +- **`engine/`**: The core RAG orchestration logic (using LlamaIndex and custom adapters). + - **`chatbot_engine.py`**: Main orchestrator handling intent, memory, and path selection. + - **`intent.py`**: Heuristic and LLM-based intent classification. + - **`retrieval.py`**: Qdrant-based retrieval pipeline. + - **`rewrite.py`**: Context-aware query rewriting. + - **`prompts.py`**: Expertly engineered RAG and intent prompts. + - **`formatter.py`**: Dynamic response formatting logic. +- **`services/`**: Integration services. + - **`chat_service.py`**: FastAPI service layer managing session engines and streaming. +- **`models/`**: SQLAlchemy and Pydantic models for chat data. +- **`routes/`**: FastAPI endpoint definitions. + +## 🏗️ Architecture + +``` +User Query → Intent Detection → Query Rewrite → Retrieval → LLM (DeepSeek) → Streaming Response +``` + +- **LLM**: DeepSeek Chat (via OpenAI-compatible adapter). +- **Vector DB**: Qdrant (integrated via LlamaIndex/Custom pipeline). +- **Embeddings**: `BAAI/bge-base-en-v1.5` for semantic search. + +## ⚙️ Configuration + +- **DeepSeek API**: Requires `DEEPSEEK_API_KEY` in `.env`. +- **Vector Store**: Qdrant must be running (configured via `docker-compose`). +- **Persistence**: Engine indices are cached per-document for performance. + +## 🚀 Endpoints + +- `POST /api/chat/upload`: Upload a document for analysis. +- `POST /api/chat/chat`: Synchronous chat response. +- `GET /api/chat/stream`: Streaming chat response (Server-Sent Events). +- `GET /api/chat/documents`: List uploaded documents. +- `DELETE /api/chat/documents/{id}`: Delete a document and its indices. diff --git a/app/modules/chat/controllers/chat_controller.py b/app/modules/chat/controllers/chat_controller.py new file mode 100644 index 0000000..c1bef4a --- /dev/null +++ b/app/modules/chat/controllers/chat_controller.py @@ -0,0 +1,173 @@ +from fastapi import UploadFile, HTTPException +from fastapi.responses import StreamingResponse +from sqlalchemy.orm import Session +from typing import Any + +from app.modules.chat.services.chat_service import ChatbotService +from app.modules.chat.schemas.chat_schema import ChatRequest +from app.modules.auth.models.user_model import User +from app.modules.auth.services import privilege + +import logging + +logger = logging.getLogger(__name__) + + +class ChatbotController: + + @staticmethod + def _get_user(current_user: Any, db: Session) -> User: + user_id = getattr(current_user, "id", None) or current_user.get("id") + + if not user_id: + raise HTTPException(400, "User ID not found") + + if isinstance(current_user, dict): + user = db.query(User).filter(User.id == user_id).first() + else: + user = current_user + + if not user: + raise HTTPException(404, "User not found") + + return user + + @staticmethod + async def upload_document(file: UploadFile, current_user: Any, db: Session): + try: + user = ChatbotController._get_user(current_user, db) + service = ChatbotService(db) + + return await service.upload_document(file, user) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Upload error: {e}", exc_info=True) + raise HTTPException(500, "Upload failed") + + @staticmethod + async def list_documents(current_user: Any, db: Session): + try: + user = ChatbotController._get_user(current_user, db) + service = ChatbotService(db) + + return service.list_documents(user) + + except HTTPException: + raise + except Exception as e: + logger.error(f"List documents error: {e}") + raise HTTPException(500, "Failed to fetch documents") + + @staticmethod + async def ingest_drive_file(file_id: int, current_user: Any, db: Session): + try: + service = ChatbotService(db) + return await service.ingest_drive_file(file_id, current_user) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Drive ingest error: {e}", exc_info=True) + raise HTTPException(500, "Failed to ingest drive file") + + @staticmethod + async def chat(request: ChatRequest, current_user: Any, db: Session): + try: + service = ChatbotService(db) + return await service.chat(current_user, request) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Chat error: {e}", exc_info=True) + raise HTTPException(500, "Chat failed") + + @staticmethod + async def chat_stream(request: ChatRequest, current_user: Any, db: Session): + try: + service = ChatbotService(db) + user = ChatbotController._get_user(current_user, db) + user_id = user.id + tenant_id = user.tenant_id + + return StreamingResponse( + service.chat_stream(user_id, tenant_id, request), + media_type="text/plain", + headers={ + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", + }, + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Streaming error: {e}", exc_info=True) + raise HTTPException(500, "Streaming failed") + + @staticmethod + async def delete_document(doc_id: str, current_user: Any, db: Session): + try: + user = ChatbotController._get_user(current_user, db) + service = ChatbotService(db) + + return service.delete_document(user, doc_id) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Delete error: {e}", exc_info=True) + raise HTTPException(500, "Delete failed") + + @staticmethod + async def get_history(document_id: str, current_user: Any, db: Session): + try: + user = ChatbotController._get_user(current_user, db) + service = ChatbotService(db) + + return service.get_session_history(user.id, user.tenant_id, document_id) + + except HTTPException: + raise + except Exception as e: + logger.error(f"History fetch error: {e}") + raise HTTPException(500, "Failed to fetch history") + + @staticmethod + async def get_usage(current_user: Any, db: Session): + try: + user = ChatbotController._get_user(current_user, db) + service = ChatbotService(db) + + return service.get_user_usage(user.id) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Usage fetch error: {e}") + raise HTTPException(500, "Failed to fetch token usage") + + @staticmethod + async def update_user_quota(user_id: int, daily_limit: int, current_user: Any, db: Session): + try: + admin = ChatbotController._get_user(current_user, db) + target_user = db.query(User).filter(User.id == user_id).first() + is_superadmin = privilege.holds_superadmin_access(admin) + + if not target_user: + raise HTTPException(404, "User not found") + + if not is_superadmin and target_user.tenant_id != admin.tenant_id: + raise HTTPException(403, "Not authorized to update this user's quota") + + service = ChatbotService(db) + return service.update_user_quota(user_id, daily_limit) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Quota update error: {e}") + raise HTTPException(500, "Failed to update user quota") \ No newline at end of file diff --git a/app/modules/chat/engine/__init__.py b/app/modules/chat/engine/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/modules/chat/engine/chatbot_engine.py b/app/modules/chat/engine/chatbot_engine.py new file mode 100644 index 0000000..fba13f3 --- /dev/null +++ b/app/modules/chat/engine/chatbot_engine.py @@ -0,0 +1,692 @@ +"""Conversation orchestration engine — integrated into docQube chat module.""" + +from __future__ import annotations + +import os +import re +from collections import defaultdict, deque +from dataclasses import dataclass +from datetime import datetime, timezone +from threading import Lock +from typing import Any, Protocol +import uuid +import logging + +from app.modules.chat.engine.formatter import FormatDecision, FormatStyle, FormattingEngine +from app.modules.chat.engine.intent import Intent, IntentClassifier +from app.modules.chat.engine.prompts import ANSWER_PROMPT, GROUNDED_SYSTEM_PROMPT, SUMMARY_PROMPT +from app.modules.chat.engine.retrieval import RetrievalConfig, RetrievalPipeline +from app.modules.chat.engine.rewrite import QueryRewriter +from app.modules.chat.services.redis_memory_service import RedisMemoryService + + +logger = logging.getLogger(__name__) + + +class SupportsComplete(Protocol): + def complete(self, prompt: str, system_prompt: str | None = None) -> tuple[str, dict[str, int]]: + ... + + def raw_llm(self) -> Any | None: + ... + + +class OpenRouterAdapter: + """Adapter for OpenRouter API (supports many models including DeepSeek, Mistral, etc.).""" + + def __init__( + self, + model: str, + api_key: str, + api_base: str = "https://openrouter.ai/api/v1", + temperature: float = 0.0, + ): + try: + from openai import OpenAI + except Exception as exc: + raise RuntimeError("Install openai package to use OpenRouter.") from exc + + self._client = OpenAI(api_key=api_key, base_url=api_base) + self._model = model + self._temperature = temperature + + def complete(self, prompt: str, system_prompt: str | None = None) -> tuple[str, dict[str, int]]: + messages: list[dict[str, str]] = [] + if system_prompt: + messages.append({"role": "system", "content": system_prompt}) + messages.append({"role": "user", "content": prompt}) + + response = self._client.chat.completions.create( + model=self._model, + messages=messages, + temperature=self._temperature, + ) + content = response.choices[0].message.content + usage = { + "prompt_tokens": response.usage.prompt_tokens, + "completion_tokens": response.usage.completion_tokens, + "total_tokens": response.usage.total_tokens, + } + return str(content or ""), usage + + def raw_llm(self) -> None: + return None + + +class OpenAILlamaIndexAdapter: + """Adapter around LlamaIndex OpenAI LLM for prompt completion.""" + + def __init__( + self, + model: str = "gpt-4o-mini", + temperature: float = 0.0, + api_base: str | None = None, + api_key: str | None = None, + ): + try: + from llama_index.llms.openai import OpenAI + except Exception as exc: + raise RuntimeError( + "Install llama-index-llms-openai and set OPENAI_API_KEY for production mode." + ) from exc + + kwargs: dict[str, Any] = { + "model": model, + "temperature": temperature, + } + if api_base: + kwargs["api_base"] = api_base + if api_key: + kwargs["api_key"] = api_key + + self._llm = OpenAI(**kwargs) + + def complete(self, prompt: str, system_prompt: str | None = None) -> tuple[str, dict[str, int]]: + usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + + if system_prompt: + try: + from llama_index.core.llms import ChatMessage + + response = self._llm.chat( + [ + ChatMessage(role="system", content=system_prompt), + ChatMessage(role="user", content=prompt), + ] + ) + content = getattr(getattr(response, "message", None), "content", "") + + raw = getattr(response, "raw", {}) + if hasattr(raw, "usage"): + usage = { + "prompt_tokens": raw.usage.prompt_tokens, + "completion_tokens": raw.usage.completion_tokens, + "total_tokens": raw.usage.total_tokens, + } + + if content: + return str(content), usage + except Exception: + pass + + response = self._llm.complete(prompt) + text = getattr(response, "text", "") + + raw = getattr(response, "raw", {}) + if hasattr(raw, "usage"): + usage = { + "prompt_tokens": raw.usage.prompt_tokens, + "completion_tokens": raw.usage.completion_tokens, + "total_tokens": raw.usage.total_tokens, + } + + return str(text), usage + + def raw_llm(self) -> Any: + return self._llm + + +class DeepSeekAdapter: + """Adapter for DeepSeek API (OpenAI-compatible).""" + + def __init__( + self, + model: str = "deepseek-chat", + api_key: str | None = None, + api_base: str = "https://api.deepseek.com/v1", + temperature: float = 0.3, + ): + try: + from openai import OpenAI + except Exception as exc: + raise RuntimeError("Install openai package to use DeepSeek API.") from exc + + self._client = OpenAI(api_key=api_key, base_url=api_base) + self._model = model + self._temperature = temperature + + def complete(self, prompt: str, system_prompt: str | None = None) -> tuple[str, dict[str, int]]: + messages: list[dict[str, str]] = [] + if system_prompt: + messages.append({"role": "system", "content": system_prompt}) + messages.append({"role": "user", "content": prompt}) + + logger.info(f"[DeepSeek] Calling {self._model} (temp={self._temperature})") + logger.debug(f"[DeepSeek] Prompt: {prompt[:100]}...") + + response = self._client.chat.completions.create( + model=self._model, + messages=messages, + temperature=self._temperature, + ) + content = response.choices[0].message.content + usage = { + "prompt_tokens": response.usage.prompt_tokens, + "completion_tokens": response.usage.completion_tokens, + "total_tokens": response.usage.total_tokens, + } + return str(content or ""), usage + + def raw_llm(self) -> None: + return None + + +@dataclass +class ConversationTurn: + user_message: str + assistant_message: str + intent: str + rewritten_query: str + timestamp_utc: str + + +class ConversationMemoryStore: + """Stores recent N interactions per session (conversation buffer).""" + + def __init__(self, max_turns: int = 12): + self.max_turns = max_turns + self._store: dict[str, deque[ConversationTurn]] = defaultdict( + lambda: deque(maxlen=self.max_turns) + ) + self._lock = Lock() + + def add_turn( + self, + session_id: str, + user_message: str, + assistant_message: str, + intent: str, + rewritten_query: str, + ) -> None: + turn = ConversationTurn( + user_message=user_message, + assistant_message=assistant_message, + intent=intent, + rewritten_query=rewritten_query, + timestamp_utc=datetime.now(timezone.utc).isoformat(), + ) + with self._lock: + self._store[session_id].append(turn) + + def get_turns(self, session_id: str) -> list[ConversationTurn]: + with self._lock: + return list(self._store.get(session_id, [])) + + def get_chat_history_messages(self, session_id: str) -> list[dict[str, str]]: + turns = self.get_turns(session_id) + messages: list[dict[str, str]] = [] + for turn in turns: + messages.append({"role": "user", "content": turn.user_message}) + messages.append({"role": "assistant", "content": turn.assistant_message}) + return messages + + def get_last_question(self, session_id: str) -> str | None: + turns = self.get_turns(session_id) + if not turns: + return None + return turns[-1].user_message + + def get_last_answer(self, session_id: str) -> str | None: + turns = self.get_turns(session_id) + if not turns: + return None + return turns[-1].assistant_message + + def resolve_memory_query(self, session_id: str, message: str) -> str: + msg = message.lower() + turns = self.get_turns(session_id) + + if not turns: + return "No previous conversation found for this session." + + if "previous questions" in msg or "all questions" in msg or "list questions" in msg: + qs = [f"{i+1}. **{t.user_message}**" for i, t in enumerate(turns[-5:])] + return "### Previous Questions\n" + "\n".join(qs) + + if "previous question" in msg or "what did i ask" in msg or "last question" in msg: + return f"### Last Question\nYou recently asked: **{turns[-1].user_message}**" + + if "previous answers" in msg or "all answers" in msg or "list answers" in msg: + as_list = [f"{i+1}. {t.assistant_message}" for i, t in enumerate(turns[-5:])] + return "### Previous Answers\n" + "\n".join(as_list) + + if "previous answer" in msg or "last answer" in msg or "what did you answer" in msg: + return f"### Last Answer\nMy previous response was: {turns[-1].assistant_message}" + + if "history" in msg or "all" in msg or "everything" in msg: + formatted = ["### Chat History"] + for idx, turn in enumerate(turns[-5:], start=1): + formatted.append( + f"{idx}. Q: **{turn.user_message}**\nA: {turn.assistant_message}" + ) + return "\n".join(formatted) + + return "Memory query received, but I could not map it to a previous question or answer." + + def clear_session(self, session_id: str) -> None: + with self._lock: + self._store.pop(session_id, None) + + def clear_all(self) -> None: + with self._lock: + self._store.clear() + + +@dataclass +class ChatbotConfig: + memory_turns: int = 12 + not_found_response: str = "Not found in document." + + +@dataclass +class ChatResult: + session_id: str + response: str + intent: str + rewritten_query: str + format_style: str + sources: list[dict[str, Any]] + usage: dict[str, int] = None + + +class ChatbotEngine: + """End-to-end conversational RAG pipeline with intent, rewrite, memory and grounding.""" + + def __init__( + self, + retrieval: RetrievalPipeline, + llm: SupportsComplete, + config: ChatbotConfig | None = None, + memory: ConversationMemoryStore | None = None, + intent_classifier: IntentClassifier | None = None, + rewriter: QueryRewriter | None = None, + formatter: FormattingEngine | None = None, + ): + self.retrieval = retrieval + self.llm = llm + self.config = config or ChatbotConfig() + self.memory = memory or ConversationMemoryStore(max_turns=self.config.memory_turns) + self.intent_classifier = intent_classifier or IntentClassifier(llm=self.llm) + self.rewriter = rewriter or QueryRewriter(llm=self.llm) + self.formatter = formatter or FormattingEngine() + self._session_formats: dict[str, FormatStyle] = {} + + def chat(self, session_id: str, message: str, repo: Any = None, db_session_id: int | None = None, user_id: int | None = None) -> ChatResult: + if repo and db_session_id: + try: + repo.save_message(session_id=db_session_id, role="user", content=message) + + RedisMemoryService.add_message(db_session_id, { + "id": str(uuid.uuid4()), + "role": "user", + "content": message, + "timestamp": datetime.now(timezone.utc).isoformat() + }) + except Exception as e: + logger.warning(f"Failed to persist user message: {e}") + + history = self.memory.get_chat_history_messages(session_id) + + total_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + + intent_decision, intent_usage = self.intent_classifier.classify(message=message, chat_history=history) + self._accumulate_usage(total_usage, intent_usage) + + if repo and user_id and intent_usage: + repo.update_quota_usage(user_id, intent_usage.get("total_tokens", 0)) + + + logger.info(f"[ChatbotEngine] Intent: {intent_decision.intent} (conf: {intent_decision.confidence})") + + if intent_decision.intent == Intent.MEMORY_QUERY: + memory_answer = self.memory.resolve_memory_query(session_id=session_id, message=message) + self.memory.add_turn( + session_id=session_id, + user_message=message, + assistant_message=memory_answer, + intent=intent_decision.intent.value, + rewritten_query=message, + ) + return ChatResult( + session_id=session_id, + response=memory_answer, + intent=intent_decision.intent.value, + rewritten_query=message, + format_style=FormatStyle.PARAGRAPH.value, + sources=[], + usage=total_usage + ) + + if intent_decision.intent == Intent.GREETING: + greeting_resp = "### Welcome to docQube\nHello! I am your **DocQube assistant**. How can I help you with your **documents** today?" + self.memory.add_turn( + session_id=session_id, + user_message=message, + assistant_message=greeting_resp, + intent=intent_decision.intent.value, + rewritten_query=message, + ) + return ChatResult( + session_id=session_id, + response=greeting_resp, + intent=intent_decision.intent.value, + rewritten_query=message, + format_style=FormatStyle.PARAGRAPH.value, + sources=[], + usage=total_usage + ) + + if intent_decision.intent == Intent.CHITCHAT: + chitchat_resp = "### Assistant Information\nI'm here to help you **analyze your documents**. Please ask a question related to the **uploaded content**!" + self.memory.add_turn( + session_id=session_id, + user_message=message, + assistant_message=chitchat_resp, + intent=intent_decision.intent.value, + rewritten_query=message, + ) + return ChatResult( + session_id=session_id, + response=chitchat_resp, + intent=intent_decision.intent.value, + rewritten_query=message, + format_style=FormatStyle.PARAGRAPH.value, + sources=[], + usage=total_usage + ) + + last_q = self.memory.get_last_question(session_id) + last_a = self.memory.get_last_answer(session_id) + + rewrite_result, rewrite_usage = self.rewriter.rewrite( + message=message, + intent=intent_decision.intent, + chat_history=history, + last_question=last_q, + last_answer=last_a, + ) + self._accumulate_usage(total_usage, rewrite_usage) + + if repo and user_id and rewrite_usage: + repo.update_quota_usage(user_id, rewrite_usage.get("total_tokens", 0)) + + previous_style = self._session_formats.get(session_id) + format_decision = self.formatter.detect( + message=message, + intent=intent_decision.intent, + previous_style=previous_style, + ) + self._session_formats[session_id] = format_decision.style + + logger.info(f"[ChatbotEngine] Rewritten Query: {rewrite_result.rewritten_query}") + + if intent_decision.intent == Intent.SUMMARY: + response_text, sources, final_usage = self._run_summary_path( + rewritten_query=rewrite_result.rewritten_query, + history=history, + format_decision=format_decision, + ) + else: + response_text, sources, final_usage = self._run_qa_path( + rewritten_query=rewrite_result.rewritten_query, + history=history, + intent=intent_decision.intent, + format_decision=format_decision, + ) + + self._accumulate_usage(total_usage, final_usage) + + if repo and user_id and final_usage: + repo.update_quota_usage(user_id, final_usage.get("total_tokens", 0)) + + response_text = self._normalize_grounded_response(response_text) + + self.memory.add_turn( + session_id=session_id, + user_message=message, + assistant_message=response_text, + intent=intent_decision.intent.value, + rewritten_query=rewrite_result.rewritten_query, + ) + + if repo and db_session_id: + try: + repo.save_message(session_id=db_session_id, role="assistant", content=response_text) + + RedisMemoryService.add_message(db_session_id, { + "id": str(uuid.uuid4()), + "role": "assistant", + "content": response_text, + "timestamp": datetime.now(timezone.utc).isoformat() + }) + except Exception as e: + logger.warning(f"Failed to persist assistant response: {e}") + + return ChatResult( + session_id=session_id, + response=response_text, + intent=intent_decision.intent.value, + rewritten_query=rewrite_result.rewritten_query, + format_style=format_decision.style.value, + sources=sources, + usage=total_usage + ) + + def _accumulate_usage(self, target: dict[str, int], source: dict[str, int | None] | None) -> None: + if not source: + return + for key in ["prompt_tokens", "completion_tokens", "total_tokens"]: + target[key] += source.get(key, 0) or 0 + + def rebuild_indices(self) -> None: + """Rebuild vector and summary indexes after document updates.""" + self.retrieval.build() + + def clear_session(self, session_id: str) -> None: + """Clear memory and session formatting state for a single session.""" + self.memory.clear_session(session_id) + self._session_formats.pop(session_id, None) + + def clear_all_sessions(self) -> None: + """Clear memory and formatting state for all sessions.""" + self.memory.clear_all() + self._session_formats.clear() + + def _run_qa_path( + self, + rewritten_query: str, + history: list[dict[str, str]], + intent: Intent, + format_decision: FormatDecision, + ) -> tuple[str, list[dict[str, Any]], dict[str, int]]: + logger.info(f"[ChatbotEngine] Running QA path for query: {rewritten_query}") + + nodes = self.retrieval.retrieve(query=rewritten_query, top_k=self.retrieval.config.top_k) + context, sources = self.retrieval.nodes_to_context(nodes) + + if not context.strip(): + return self.config.not_found_response, [], {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + + prompt = ANSWER_PROMPT.format( + chat_history=self._compact_history(history), + intent=intent.value, + format_instruction=format_decision.instruction, + rewritten_query=rewritten_query, + context=context, + ) + response, usage = self.llm.complete(prompt=prompt, system_prompt=GROUNDED_SYSTEM_PROMPT) + if self._looks_like_not_found(response): + fallback = self._term_mention_fallback(rewritten_query, context) + if fallback: + response = fallback + return response, sources, usage + + def _run_summary_path( + self, + rewritten_query: str, + history: list[dict[str, str]], + format_decision: FormatDecision, + ) -> tuple[str, list[dict[str, Any]], dict[str, int]]: + logger.info(f"[ChatbotEngine] Running Summary path for query: {rewritten_query}") + + raw_llm = self.llm.raw_llm() + summary_query = SUMMARY_PROMPT.format( + rewritten_query=rewritten_query, + format_instruction=format_decision.instruction, + ) + + if raw_llm is not None: + try: + response = self.retrieval.summarize(rewritten_query=summary_query, llm=raw_llm) + normalized = response.strip() + if normalized: + return normalized, [], {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + except Exception: + pass + + nodes = self.retrieval.retrieve(query=rewritten_query, top_k=self.retrieval.config.top_k) + context, sources = self.retrieval.nodes_to_context(nodes) + if not context.strip(): + return self.config.not_found_response, [], {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + + prompt = ANSWER_PROMPT.format( + chat_history=self._compact_history(history), + intent=Intent.SUMMARY.value, + format_instruction=format_decision.instruction, + rewritten_query=summary_query, + context=context, + ) + response, usage = self.llm.complete(prompt=prompt, system_prompt=GROUNDED_SYSTEM_PROMPT) + return response, sources, usage + + def _normalize_grounded_response(self, response: str) -> str: + text = (response or "").strip() + if not text: + return self.config.not_found_response + + lowered = text.lower() + if "not found in document" in lowered: + return self.config.not_found_response + + uncertainty_markers = ["i am not sure", "i don't know", "cannot determine"] + if any(marker in lowered for marker in uncertainty_markers): + return self.config.not_found_response + + return text + + @staticmethod + def _looks_like_not_found(response: str) -> bool: + lowered = (response or "").strip().lower() + return not lowered or "not found in document" in lowered + + @staticmethod + def _term_mention_fallback(query: str, context: str) -> str | None: + lowered_query = query.strip().lower() + patterns = [ + r"^\s*what\s+is\s+['\"']?([a-z0-9][a-z0-9\-/\.]{1,30})['\"']?\s*\??\s*$", + r"^\s*define\s+['\"']?([a-z0-9][a-z0-9\-/\.]{1,30})['\"']?\s*\??\s*$", + r"^\s*what\s+does\s+['\"']?([a-z0-9][a-z0-9\-/\.]{1,30})['\"']?\s+stand\s+for\s*\??\s*$", + ] + term = None + for pattern in patterns: + match = re.match(pattern, lowered_query) + if match: + term = match.group(1) + break + + if not term: + return None + + term_regex = re.compile(rf"\b{re.escape(term)}\b", flags=re.IGNORECASE) + if not term_regex.search(context): + return None + + clean_context = re.sub(r"\[Source:\s+.*?\]\n?", "", context) + snippet = "" + first = term_regex.search(clean_context) + if first: + start = max(0, first.start() - 80) + end = min(len(clean_context), first.end() + 80) + snippet = clean_context[start:end].strip().replace("\n", " ") + + if snippet: + return ( + f'### Term Mention Found\n' + f'The document mentions **"{term.upper()}"** (for example: *"{snippet}"*), ' + f"but it does not explicitly define what **{term.upper()}** stands for." + ) + return ( + f'### Term Mention Found\n' + f'The document mentions **"{term.upper()}"**, but it does not explicitly define ' + f"what **{term.upper()}** stands for." + ) + + @staticmethod + def _compact_history(history: list[dict[str, str]]) -> str: + if not history: + return "(empty)" + return "\n".join(f"{m.get('role', 'unknown')}: {m.get('content', '')}" for m in history[-8:]) + + +def build_engine_from_settings( + docs_path: str, + persist_dir: str, + collection_name: str = "chatbot_docs", +) -> ChatbotEngine: + """ + Build a ChatbotEngine using the NVIDIA/DeepSeek credentials already present + in docQube's central settings. Falls back to DeepSeek if NVIDIA key is absent. + """ + from app.core.settings import settings + + embedding_provider = os.getenv("CHATBOT_EMBEDDING_PROVIDER", "bge") + vector_db = os.getenv("CHATBOT_VECTOR_DB", "qdrant") + + retrieval_cfg = RetrievalConfig( + docs_path=docs_path, + vector_db=vector_db, + persist_dir=persist_dir, + collection_name=collection_name, + embedding_provider=embedding_provider, + bge_model=getattr(settings, "EMBEDDING_MODEL", "BAAI/bge-base-en-v1.5"), + ) + + retrieval = RetrievalPipeline(config=retrieval_cfg) + + deepseek_key = getattr(settings, "DEEPSEEK_API_KEY", None) + deepseek_model = getattr(settings, "DEEPSEEK_MODEL", "deepseek-chat") + deepseek_base = getattr(settings, "DEEPSEEK_BASE_URL", "https://api.deepseek.com/v1") + + if deepseek_key: + llm_adapter: SupportsComplete = DeepSeekAdapter( + model=deepseek_model, + api_key=deepseek_key, + api_base=deepseek_base, + temperature=0.3, + ) + else: + raise RuntimeError( + "DeepSeek API key not found. Set DEEPSEEK_API_KEY in your .env file." + ) + + return ChatbotEngine(retrieval=retrieval, llm=llm_adapter) diff --git a/app/modules/chat/engine/formatter.py b/app/modules/chat/engine/formatter.py new file mode 100644 index 0000000..34d0d03 --- /dev/null +++ b/app/modules/chat/engine/formatter.py @@ -0,0 +1,80 @@ +"""Formatting request detection and instruction synthesis.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum + +from app.modules.chat.engine.intent import Intent + + +class FormatStyle(str, Enum): + PARAGRAPH = "paragraph" + BULLETS = "bullets" + SHORT = "short" + DETAILED = "detailed" + TABLE = "table" + COMPARISON = "comparison" + + +@dataclass +class FormatDecision: + style: FormatStyle + instruction: str + + +class FormattingEngine: + """Detects requested answer style without changing factual grounding.""" + + def detect(self, message: str, intent: Intent, previous_style: FormatStyle | None = None) -> FormatDecision: + text = message.lower().strip() + + if intent == Intent.COMPARISON: + return FormatDecision( + style=FormatStyle.COMPARISON, + instruction="Every response MUST start with a relevant Markdown header (###). Use a clear side-by-side comparison with similarities and differences, using **bolding** for key parameters.", + ) + + if "table" in text: + return FormatDecision( + style=FormatStyle.TABLE, + instruction="Every response MUST start with a relevant Markdown header (###). Use a markdown table with concise rows and grounded facts only. Use **bolding** for the first column or key headers.", + ) + + if "bullet" in text or "points" in text: + return FormatDecision( + style=FormatStyle.BULLETS, + instruction="Every response MUST start with a relevant Markdown header (###). Respond in bullet points only, using **bolding** for the start of each point or key facts.", + ) + + if "short" in text or "brief" in text or "concise" in text: + return FormatDecision( + style=FormatStyle.SHORT, + instruction="Respond in 2-4 concise sentences. YOU MUST still include a relevant Markdown header (###) and use **bolding** for key terms.", + ) + + if "detailed" in text or "in depth" in text or "deep" in text: + return FormatDecision( + style=FormatStyle.DETAILED, + instruction="Provide a detailed explanation with structured sections.", + ) + + direct_keywords = ["who", "when", "where", "what is", "how many", "how much", "list", "show"] + is_direct = any(text.startswith(kw) for kw in direct_keywords) or len(text.split()) <= 6 + + if is_direct and intent == Intent.QUESTION: + return FormatDecision( + style=FormatStyle.SHORT, + instruction="Respond in 1-2 concise sentences. Include a relevant Markdown header (###) and use **bolding** for key terms.", + ) + + if previous_style is not None: + return FormatDecision( + style=previous_style, + instruction=f"Continue using {previous_style.value} format unless user changes it.", + ) + + return FormatDecision( + style=FormatStyle.PARAGRAPH, + instruction="Use rich Markdown formatting. You MUST include a relevant Markdown header (###), use **bolding** for key terms, and use bullet points for lists to ensure a professional, structured layout like ChatGPT, even for short responses.", + ) diff --git a/app/modules/chat/engine/intent.py b/app/modules/chat/engine/intent.py new file mode 100644 index 0000000..1f6752a --- /dev/null +++ b/app/modules/chat/engine/intent.py @@ -0,0 +1,222 @@ +"""Intent detection layer for conversational RAG — integrated into docQube chat module.""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from enum import Enum +from typing import Protocol + +from app.modules.chat.engine.prompts import INTENT_CLASSIFIER_PROMPT + + +class SupportsComplete(Protocol): + def complete(self, prompt: str, system_prompt: str | None = None) -> tuple[str, dict[str, int]]: + ... + + +class Intent(str, Enum): + QUESTION = "QUESTION" + FOLLOW_UP = "FOLLOW_UP" + SUMMARY = "SUMMARY" + COMPARISON = "COMPARISON" + FORMAT_REQUEST = "FORMAT_REQUEST" + MEMORY_QUERY = "MEMORY_QUERY" + GREETING = "GREETING" + CHITCHAT = "CHITCHAT" + + +@dataclass +class IntentDecision: + intent: Intent + confidence: float + reason: str + + +def _compact_history(chat_history: list[dict[str, str]]) -> str: + if not chat_history: + return "(empty)" + lines: list[str] = [] + for msg in chat_history[-8:]: + role = msg.get("role", "unknown") + content = msg.get("content", "") + lines.append(f"{role}: {content}") + return "\n".join(lines) + + +class IntentClassifier: + """LLM-first intent classifier with deterministic fallback.""" + + def __init__(self, llm: SupportsComplete): + self.llm = llm + + def classify(self, message: str, chat_history: list[dict[str, str]]) -> tuple[IntentDecision, dict[str, int]]: + if self._is_date_ops_question(message): + return IntentDecision( + intent=Intent.QUESTION, + confidence=0.95, + reason="Date/operations standalone question override", + ), {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + + if self._is_direct_term_question(message): + return IntentDecision( + intent=Intent.QUESTION, + confidence=0.96, + reason="Direct term-definition style question override", + ), {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + + prompt = INTENT_CLASSIFIER_PROMPT.format( + chat_history=_compact_history(chat_history), + message=message, + ) + raw, usage = self.llm.complete(prompt) + + parsed = self._parse_json(raw) + if parsed: + intent = self._normalize_intent(parsed.get("intent", "")) + if intent is not None: + return IntentDecision( + intent=intent, + confidence=float(parsed.get("confidence", 0.7)), + reason=str(parsed.get("reason", "LLM classification")), + ), usage + + fallback_intent, reason = self._heuristic_intent(message) + return IntentDecision(intent=fallback_intent, confidence=0.51, reason=reason), usage + + @staticmethod + def _is_direct_term_question(message: str) -> bool: + text = message.strip() + lowered = text.lower() + + if any( + token in lowered + for token in [ + "previous question", + "previous answer", + "what did i ask", + "what did you answer", + "history", + "i don't understand", + "explain better", + "clarify", + ] + ): + return False + + patterns = [ + r"^\s*what\s+is\s+['\"']?[a-z0-9][a-z0-9\-/\.]{1,30}['\"']?\s*\??\s*$", + r"^\s*define\s+['\"']?[a-z0-9][a-z0-9\-/\.]{1,30}['\"']?\s*\??\s*$", + r"^\s*what\s+does\s+['\"']?[a-z0-9][a-z0-9\-/\.]{1,30}['\"']?\s+stand\s+for\s*\??\s*$", + ] + return any(re.match(pattern, lowered) for pattern in patterns) + + @staticmethod + def _is_date_ops_question(message: str) -> bool: + lowered = message.strip().lower() + + if any( + token in lowered + for token in [ + "i don't understand", + "i dont understand", + "explain better", + "clarify", + "previous question", + "previous answer", + "what do you mean", + ] + ): + return False + + has_ops_keyword = any( + token in lowered + for token in [ + "operational hours", + "operation hours", + "where is the vessel", + "where was the vessel", + "vessel on", + ] + ) + if not has_ops_keyword: + return False + + has_date = bool( + re.search( + r"(\b\d{1,2}[-/\s]?(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\b)|" + r"(\b(january|february|march|april|may|june|july|august|september|october|november|december)\b)|" + r"(\b20\d{2}\b)", + lowered, + ) + ) + return has_date + + @staticmethod + def _parse_json(raw: str) -> dict | None: + try: + return json.loads(raw) + except json.JSONDecodeError: + pass + + match = re.search(r"\{[\s\S]*\}", raw) + if not match: + return None + + try: + return json.loads(match.group(0)) + except json.JSONDecodeError: + return None + + @staticmethod + def _normalize_intent(value: str) -> Intent | None: + normalized = value.strip().upper() + try: + return Intent(normalized) + except ValueError: + return None + + @staticmethod + def _heuristic_intent(message: str) -> tuple[Intent, str]: + text = message.lower().strip() + + import logging + logger = logging.getLogger(__name__) + + if "previous question" in text or "previous answer" in text or "what did i ask" in text: + return Intent.MEMORY_QUERY, "Detected memory lookup phrase" + + summary_keywords = ["summarize the whole", "full summary", "total overview", "summarize everything"] + if any(token in text for token in summary_keywords): + return Intent.SUMMARY, "Detected explicit full summary keyword" + + if text in ["summary", "summarize", "tl;dr", "overview"]: + return Intent.SUMMARY, "Detected standalone summary keyword" + + if any(token in text for token in ["difference", "differentiate", "vs", "compare", "comparison"]): + return Intent.COMPARISON, "Detected comparison keyword" + + if any(token in text for token in ["bullet", "table", "detailed", "format"]): + return Intent.FORMAT_REQUEST, "Detected formatting keyword" + + if any( + token in text + for token in [ + "i don't understand", + "i dont understand", + "explain better", + "explain simply", + "what do you mean", + "clarify", + ] + ): + return Intent.FOLLOW_UP, "Detected follow-up clarifier phrase" + + if text in ["hi", "hello", "hey", "greetings", "good morning", "good afternoon"]: + return Intent.GREETING, "Detected simple greeting" + + if len(text) > 4 and not any(v in text for v in "aeiouy") and not text.isdigit(): + return Intent.CHITCHAT, "Detected nonsensical input (no vowels)" + + return Intent.QUESTION, "Default question classification" diff --git a/app/modules/chat/engine/prompts.py b/app/modules/chat/engine/prompts.py new file mode 100644 index 0000000..ead7109 --- /dev/null +++ b/app/modules/chat/engine/prompts.py @@ -0,0 +1,120 @@ +"""Prompt templates for the production conversational RAG chatbot.""" + +INTENT_CLASSIFIER_PROMPT = """ +You are an intent classifier for a document-grounded conversational RAG system. + +Classify the user message into one of these labels: +- QUESTION +- FOLLOW_UP +- SUMMARY +- COMPARISON +- FORMAT_REQUEST +- MEMORY_QUERY +- GREETING +- CHITCHAT + +Classification guidance: +- QUESTION: first-time factual question about document content. +- FOLLOW_UP: clarification on prior answer (e.g., "I don't understand", "explain better"). +- SUMMARY: asks to summarize full document or section. +- COMPARISON: asks differences/similarities between two concepts. +- FORMAT_REQUEST: asks to reformat output style (bullets/table/short/detailed). +- MEMORY_QUERY: asks what was previously asked/answered in this chat. +- GREETING: simple greetings like "hi", "hello", "good morning". +- CHITCHAT: random words, jokes, or non-document related talk. + +Return JSON ONLY with keys: +{{ + "intent": "