latest code

app/modules/documents/controllers/document_controller.py

@@ -0,0 +1,96 @@
+from fastapi import Request
+from sqlalchemy.orm import Session
+from app.modules.auth.models.user_model import User
+from app.modules.documents.services.document_service import ProjectService
+
+class ProjectController:
+    @staticmethod
+    def list_projects(user: User, db: Session):
+        project_service = ProjectService(db)
+        return project_service.list_projects(user)
+
+    @staticmethod
+    def create_project(user: User, payload, request: Request, db: Session):
+        project_service = ProjectService(db)
+        return project_service.create_project(user, payload, request)
+
+    @staticmethod
+    def get_project_by_id(project_id: int, user: User, request: Request, db: Session):
+        project_service = ProjectService(db)
+        return project_service.get_project(project_id, user, request)
+
+    @staticmethod
+    def update_project(project_id: int, user: User, payload, request: Request, db: Session):
+        project_service = ProjectService(db)
+        return project_service.update_project(project_id, user, payload, request)
+
+    @staticmethod
+    def delete_project(project_id: int, user: User, request: Request, db: Session):
+        project_service = ProjectService(db)
+        return project_service.delete_project(project_id, user, request)
+
+    @staticmethod
+    def get_project_extended_details(project_id: int, user: User, db: Session):
+        project_service = ProjectService(db)
+        return project_service.get_project_extended_details(project_id, user)
+
+    @staticmethod
+    def list_versions(project_id: int, user: User, db: Session):
+        from app.modules.documents.services.version_service import VersionService
+        version_service = VersionService(db)
+        return version_service.list_versions(project_id, user)
+
+    @staticmethod
+    def create_version(project_id: int, user: User, payload, request, db: Session):
+        from app.modules.documents.services.version_service import VersionService
+        version_service = VersionService(db)
+        return version_service.create_version(project_id, user, payload, request)
+
+    @staticmethod
+    def get_version(project_id: int, version_id: int, user: User, db: Session):
+        from app.modules.documents.services.version_service import VersionService
+        version_service = VersionService(db)
+        return version_service.get_version(project_id, version_id, user)
+
+    @staticmethod
+    def restore_version(project_id: int, version_id: int, user: User, request, db: Session):
+        from app.modules.documents.services.version_service import VersionService
+        version_service = VersionService(db)
+        return version_service.restore_version(project_id, version_id, user, request)
+
+    @staticmethod
+    def delete_version(project_id: int, version_id: int, user: User, db: Session):
+        from app.modules.documents.services.version_service import VersionService
+        version_service = VersionService(db)
+        return version_service.delete_version(project_id, version_id, user)
+
+    @staticmethod
+    def update_version_description(project_id: int, version_id: int, user: User, description: str, db: Session):
+        from app.modules.documents.services.version_service import VersionService
+        version_service = VersionService(db)
+        return version_service.update_version_description(project_id, version_id, user, description)
+
+    @staticmethod
+    def share_project(share_request, current_user: User, request, db: Session):
+        from app.modules.documents.services.sharing_service import SharingService
+        sharing_service = SharingService(db)
+        return sharing_service.share_project(share_request, current_user, request)
+
+    @staticmethod
+    def list_project_shares(project_id: int, current_user: User, db: Session):
+        from app.modules.documents.services.sharing_service import SharingService
+        sharing_service = SharingService(db)
+        return sharing_service.list_project_shares(project_id, current_user)
+
+    @staticmethod
+    def get_shared_with_me(current_user: User, db: Session):
+        from app.modules.documents.services.sharing_service import SharingService
+        sharing_service = SharingService(db)
+        return sharing_service.get_shared_with_me(current_user)
+
+    @staticmethod
+    def remove_share(share_id: int, current_user: User, request, db: Session):
+        from app.modules.documents.services.sharing_service import SharingService
+        sharing_service = SharingService(db)
+        return sharing_service.remove_share(share_id, current_user, request)
+

app/modules/documents/controllers/export_controller.py

@@ -0,0 +1,208 @@
+import logging
+import uuid
+from fastapi import HTTPException, Depends
+from sqlalchemy.orm import Session
+from app.db.database import get_db
+from app.middleware.auth import get_current_user
+from app.modules.auth.models.user_model import User
+from app.modules.drive.services.activity_service import ActivityService
+import json
+from app.modules.documents.processors.xml.xml_compiler import XMLCompiler
+from app.modules.documents.schemas.export_schema import (
+    ExportHTMLRequest, 
+    ExportSimpleHTMLRequest, 
+    ExportXMLRequest, 
+    ExportDocxRequest, 
+    ExportPdfRequest
+)
+from app.core.ssrf_protection import is_safe_url, get_safe_fetcher
+from app.core.sanitization import sanitize_html
+import requests
+import re
+from io import BytesIO
+import xml.etree.ElementTree as ET
+
+class ExportController:
+
+    @staticmethod
+    def export_html(payload: ExportHTMLRequest, user: User, db: Session):
+        xml_content = payload.xml_content
+        filename = payload.filename
+        project_id = payload.project_id
+
+        if not xml_content:
+            raise HTTPException(status_code=400, detail="No XML content provided")
+
+        try:
+            root = ET.fromstring(xml_content.encode('utf-8'))
+            if root.tag.split('}')[-1] != 'article':
+                raise HTTPException(status_code=400, detail="Invalid XML root: must be 'article'")
+        except ET.ParseError as e:
+            raise HTTPException(status_code=400, detail=f"Invalid XML document: {str(e)}")
+
+        compiler = XMLCompiler()
+        html_content = compiler.compile_xml_to_html(xml_content)
+        
+        html_content = sanitize_html(html_content)
+
+        version_label = "v1.0"
+        if project_id:
+            from app.modules.documents.repositories.version_repository import VersionRepository
+            version_repo = VersionRepository(db)
+            versions = version_repo.list_by_project(project_id)
+            current_version = next((v for v in versions if v.is_current), None)
+            if current_version:
+                version_label = current_version.version_label
+            elif versions:
+                version_label = versions[0].version_label
+
+            activity_service = ActivityService(db)
+            activity_service.log(
+                resource_type="document",
+                resource_id=project_id,
+                actor_id=user.id,
+                activity_type="export",
+                metadata=json.dumps({
+                    "format": "html", 
+                    "filename": filename,
+                    "version": version_label
+                })
+            )
+
+        return {
+            "success": True,
+            "html_content": html_content,
+            "filename": f"{filename}.html",
+            "version": version_label
+        }
+
+    @staticmethod
+    def export_simple_html(payload: ExportSimpleHTMLRequest, user: User, db: Session):
+        markdown_content = payload.markdown_content
+
+        if not markdown_content:
+            raise HTTPException(status_code=400, detail="No markdown content provided")
+
+        try:
+            import markdown
+            html_body = markdown.markdown(markdown_content)
+        except ImportError:
+            logging.error("markdown library not installed")
+            raise HTTPException(status_code=500, detail="HTML/Markdown conversion service is currently unavailable.")
+        
+        html_body = sanitize_html(html_body)
+
+        return {
+            "success": True,
+            "html_content": html_body
+        }
+
+    @staticmethod
+    def convert_to_xml(payload: ExportXMLRequest, user: User, db: Session):
+        from app.modules.documents.processors.pdf.pdf_converter import md_to_jats
+        markdown_content = payload.markdown_content
+        if not markdown_content:
+            raise HTTPException(status_code=400, detail="No markdown content provided")
+        
+        try:
+            xml_content = md_to_jats(markdown_content)
+        except ValueError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from None
+        return {
+            "success": True,
+            "xml_content": xml_content
+        }
+
+    @staticmethod
+    async def export_docx(payload: ExportDocxRequest, user: User, db: Session):
+        markdown_content = payload.markdown_content
+        filename = payload.filename
+        if not markdown_content:
+            raise HTTPException(status_code=400, detail="No markdown content provided")
+
+        try:
+            import docx
+            from docx.shared import Inches
+            import base64
+            
+            doc = docx.Document()
+            
+            lines = markdown_content.split('\n')
+            for line in lines:
+                img_match = re.match(r'!\[([^\]]*)\]\(([^)]+)\)', line.strip())
+                if img_match:
+                    alt_text = img_match.group(1)
+                    image_url = img_match.group(2)
+                    
+                    if not is_safe_url(image_url):
+                        doc.add_paragraph(f"[Blocked Unsafe Image: {alt_text}]")
+                        continue
+                        
+                    try:
+                        img_response = requests.get(image_url, timeout=5)
+                        if img_response.status_code == 200:
+                            img_stream = BytesIO(img_response.content)
+                            doc.add_picture(img_stream, width=Inches(6))
+                        else:
+                            doc.add_paragraph(f"[Image load failed: {alt_text}]")
+                    except Exception as e:
+                        doc.add_paragraph(f"[Error loading image {alt_text}: {str(e)}]")
+                else:
+                    if line.strip():
+                        doc.add_paragraph(line)
+            
+            f = BytesIO()
+            doc.save(f)
+            f.seek(0)
+            
+            docx_base64 = base64.b64encode(f.read()).decode('utf-8')
+            
+            return {
+                "success": True,
+                "docx_content": docx_base64,
+                "filename": f"{filename}.docx"
+            }
+        except ImportError:
+            logging.error("python-docx library not installed", exc_info=True)
+            raise HTTPException(status_code=500, detail="Document conversion service is currently unavailable.")
+        except HTTPException:
+            raise
+        except Exception as e:
+            err_ref = str(uuid.uuid4())
+            logging.error(f"DOCX conversion failed [Ref: {err_ref}]: {e}", exc_info=True)
+            raise HTTPException(status_code=500, detail=f"An error occurred during file conversion. Ref: {err_ref}")
+
+    @staticmethod
+    async def export_pdf(payload: ExportPdfRequest, user: User, db: Session):
+        markdown_content = payload.markdown_content
+        filename = payload.filename
+        if not markdown_content:
+            raise HTTPException(status_code=400, detail="No markdown content provided")
+        try:
+            from weasyprint import HTML
+            import base64
+            from io import BytesIO
+            import markdown
+            
+            html_content = markdown.markdown(markdown_content)
+            
+            f = BytesIO()
+            HTML(string=html_content, url_fetcher=get_safe_fetcher()).write_pdf(f)
+            f.seek(0)
+            
+            pdf_base64 = base64.b64encode(f.read()).decode('utf-8')
+            
+            return {
+                "success": True,
+                "pdf_content": pdf_base64,
+                "filename": f"{filename}.pdf"
+            }
+        except ImportError:
+            logging.error("weasyprint library not installed", exc_info=True)
+            raise HTTPException(status_code=500, detail="PDF conversion service is currently unavailable.")
+        except HTTPException:
+            raise
+        except Exception as e:
+            err_ref = str(uuid.uuid4())
+            logging.error(f"PDF conversion failed [Ref: {err_ref}]: {e}", exc_info=True)
+            raise HTTPException(status_code=500, detail=f"An error occurred during file conversion. Ref: {err_ref}")

app/modules/documents/controllers/processing_controller.py

@@ -0,0 +1,618 @@
+from app.infrastructure.storage.local_storage_handler import (
+    get_storage_client,
+)
+import uuid
+import os
+import logging
+from datetime import datetime, timedelta, timezone
+from fastapi import UploadFile, File
+from typing import Optional
+from fastapi import HTTPException
+
+from celery.result import AsyncResult
+from app.modules.documents.tasks.celery_tasks import process_document_task
+from app.tasks.celery_app import celery_app
+from app.core.cancellation import cancel_task as set_cancel_flag
+
+from sqlalchemy.orm import Session
+from app.modules.drive.services.drive_service import DriveService
+from app.modules.documents.models.document_model import Project
+from app.modules.documents.services.document_service import ProjectService
+from app.modules.auth.models.user_model import User
+from app.modules.documents.processors.pdf.pdf_converter import md_to_jats
+from app.modules.drive.services.folder_service import FolderService
+from app.modules.drive.models.drive_model import DriveFile, DriveFileVersion
+from app.modules.drive.utils.mime import validate_mime
+from app.modules.drive.utils.filename import sanitize_filename
+from app.core.clamav import ClamAVError, MalwareDetectedError, scan_bytes_for_malware
+import time
+from app.modules.security.models.virus_scan_models import VirusScanResult
+from app.modules.security.services.virus_scan_service import VirusScanService
+
+logger = logging.getLogger(__name__)
+
+
+class ProcessingController:
+
+    @staticmethod
+    async def process_drive_file(file_id: int, user: User, db: Session):
+        """
+        Process a file that already exists in the Drive system.
+        """
+        drive_file = (
+            db.query(DriveFile)
+            .filter(DriveFile.id == file_id, DriveFile.owner_id == user.id)
+            .first()
+        )
+
+        if not drive_file:
+            from fastapi import HTTPException
+
+            raise HTTPException(
+                status_code=403, detail="File not found or access denied"
+            )
+
+        drive_service = DriveService(db)
+
+        file_info = drive_service.get_file_info(file_id, user)
+        stream_res = drive_service.stream_file(file_id, user)
+
+        filename = file_info["name"]
+
+        storage_key = f"doc_conversion/{user.id}/{uuid.uuid4()}_{filename}"
+        storage, _, clean_bucket = get_storage_client(
+            db=db, tenant_id=user.tenant_id)
+        storage.put_object(
+            Bucket=clean_bucket,
+            Key=storage_key,
+            Body=stream_res["body"],
+            ContentType=file_info.get("mime_type", "application/pdf"),
+        )
+
+        project = db.query(Project).filter(
+            Project.drive_file_id == file_id).first()
+        session_id = str(uuid.uuid4())
+
+        if not project:
+            project = Project(
+                tenant_id=user.tenant_id,
+                user_id=user.id,
+                name=filename,
+                drive_file_id=file_id,
+                session_id=session_id,
+                status="processing",
+                progress=10,
+                current_step="upload",
+            )
+            db.add(project)
+            db.flush()
+            db.refresh(project)
+        else:
+            project.session_id = session_id
+            project.status = "processing"
+            project.progress = 10
+
+        task = process_document_task.apply_async(
+            args=[storage_key, session_id],
+            kwargs={
+                "original_version_id": file_info.get("version_number_id")
+                or file_info.get("latest_version_id")
+            },
+            queue="default",
+            task_id=session_id,
+        )
+
+        from app.modules.drive.services.activity_service import ActivityService
+        from app.modules.drive.constants import ActivityType
+        from sqlalchemy import func
+
+        if drive_file:
+            drive_file.updated_at = func.now()
+
+        activity = ActivityService(db)
+        activity.log(
+            resource_type="file",
+            resource_id=file_id,
+            actor_id=user.id,
+            activity_type=ActivityType.OPEN,
+            metadata="Started conversion process",
+        )
+        try:
+            with db.begin_nested():
+                db.flush()
+        except Exception as e:
+            logger.error(f"Failed to record the activity log entry: {e}")
+
+        return {
+            "task_id": task.id,
+            "session_id": session_id,
+            "status": "queued",
+            "project_id": project.id,
+        }
+
+    @staticmethod
+    async def upload_document(
+        file: UploadFile,
+        session_id: Optional[str],
+        output_format: Optional[str],
+        db: Session,
+        user: User,
+    ):
+        if not session_id:
+            session_id = str(uuid.uuid4())
+
+        contents = await file.read()
+        file_size = len(contents)
+
+        scan_svc = VirusScanService(db)
+        scan_settings = scan_svc.get_or_create_settings(user.tenant_id) if user.tenant_id else None
+
+        should_scan = True
+        if scan_settings:
+            if not scan_settings.enabled:
+                should_scan = False
+            elif scan_settings.max_file_size_mb and (file_size > scan_settings.max_file_size_mb * 1024 * 1024):
+                should_scan = False
+
+        if should_scan:
+            t0 = time.time()
+            try:
+                scan_bytes_for_malware(contents)
+                duration_ms = max(1, int((time.time() - t0) * 1000))
+                db.add(
+                    VirusScanResult(
+                        tenant_id=user.tenant_id,
+                        uploader_id=user.id,
+                        status="clean",
+                        threat_name=None,
+                        action_taken="none",
+                        scan_duration_ms=duration_ms,
+                    )
+                )
+                db.commit()
+            except MalwareDetectedError as exc:
+                duration_ms = max(1, int((time.time() - t0) * 1000))
+                handling = scan_svc.handle_infected_file(
+                    tenant_id=user.tenant_id,
+                    file_id=None,
+                    version_id=None,
+                    uploader_id=user.id,
+                    filename=file.filename or "uploaded_file",
+                    file_size=file_size,
+                    mime_type=file.content_type,
+                    storage_key=f"quarantine/{uuid.uuid4()}_{file.filename}",
+                    threat_name=exc.virus_name,
+                )
+                db.add(
+                    VirusScanResult(
+                        tenant_id=user.tenant_id,
+                        uploader_id=user.id,
+                        status="infected",
+                        threat_name=exc.virus_name,
+                        action_taken=handling.get("action_taken", "quarantined"),
+                        scan_duration_ms=duration_ms,
+                    )
+                )
+                db.commit()
+                raise HTTPException(
+                    status_code=400,
+                    detail={
+                        "error": "virus_detected",
+                        "virus_name": exc.virus_name,
+                        "message": f"Virus detected: {exc.virus_name}",
+                    },
+                )
+            except ClamAVError as exc:
+                logger.error(f"Document upload ClamAV scan failed: {exc}")
+                raise HTTPException(
+                    status_code=503,
+                    detail="Malware scan service unavailable. Please try again.",
+                )
+        else:
+            db.add(
+                VirusScanResult(
+                    tenant_id=user.tenant_id,
+                    uploader_id=user.id,
+                    status="skipped",
+                    threat_name=None,
+                    action_taken="none",
+                    scan_duration_ms=0,
+                )
+            )
+            db.commit()
+
+        # 0. Create DriveFile entry so user can see original (Initially Original)
+        folder_service = FolderService(db)
+        root = folder_service.get_or_create_root(user)
+        root_id = root["id"]
+
+        safe_name = sanitize_filename(file.filename)
+        mime = validate_mime(file.content_type, safe_name)
+        file_ext = os.path.splitext(safe_name)[1].lstrip(".").upper() or "PDF"
+        normalized_output_format = (output_format or "EPUB").strip().upper()
+
+        drive_file = DriveFile(
+            tenant_id=user.tenant_id,
+            owner_id=user.id,
+            folder_id=root_id,
+            name=safe_name,
+            mime_type=mime,
+            size=file_size,
+        )
+        db.add(drive_file)
+        db.flush()
+
+        storage_key = f"user_{user.id}/file_{drive_file.id}/v1/{safe_name}"
+        storage, _, clean_bucket = get_storage_client(
+            db=db, tenant_id=user.tenant_id)
+        storage.put_object(
+            Bucket=clean_bucket, Key=storage_key, Body=contents, ContentType=mime
+        )
+        provider = "b2"
+
+        version = DriveFileVersion(
+            file_id=drive_file.id,
+            version_number=1,
+            s3_key=storage_key,
+            storage_provider=provider,
+            size=file_size,
+            content_type=mime,
+            uploader_id=user.id,
+        )
+        db.add(version)
+        db.flush()
+        drive_file.latest_version_id = version.id
+
+        project = Project(
+            tenant_id=user.tenant_id,
+            user_id=user.id,
+            name=safe_name,
+            file_type=file_ext,
+            output_format=normalized_output_format,
+            drive_file_id=drive_file.id,
+            session_id=session_id,
+            status="processing",
+            progress=10,
+            current_step="upload",
+        )
+        db.add(project)
+
+        priority_queue = "default"
+        if file_size < 5 * 1024 * 1024:
+            priority_queue = "high_priority"
+        elif file_size > 20 * 1024 * 1024:
+            priority_queue = "low_priority"
+
+        task = process_document_task.apply_async(
+            args=[storage_key, session_id],
+            kwargs={"original_version_id": version.id},
+            queue=priority_queue,
+            task_id=session_id,
+        )
+
+        from app.modules.drive.services.activity_service import ActivityService
+        from app.modules.drive.constants import ActivityType
+
+        activity = ActivityService(db)
+        activity.log(
+            resource_type="file",
+            resource_id=drive_file.id,
+            actor_id=user.id,
+            activity_type=ActivityType.UPLOAD,
+            metadata=f"Uploaded for conversion: {safe_name}",
+        )
+        try:
+            db.commit()
+            DriveService.invalidate_recent_cache(user.id)
+            with db.begin_nested():
+                db.flush()
+        except Exception as e:
+            logger.error(f"Failed to record the activity log entry: {e}")
+
+        return {
+            "task_id": task.id,
+            "session_id": session_id,
+            "status": "queued",
+            "project_id": project.id,
+            "original_file_id": drive_file.id,
+        }
+
+    @staticmethod
+    async def handle_upload_callback(storage_key: str, filename: str, folder_id: int, content_type: str, file_size: int, user: User, db: Session):
+        import uuid
+        from app.modules.drive.models.drive_model import DriveFile, DriveFileVersion
+        from app.modules.documents.models.document_model import Project
+        from app.modules.drive.utils.filename import sanitize_filename
+        from app.modules.drive.utils.mime import validate_mime
+        from app.modules.drive.tasks.upload_tasks import scan_file_task
+        from fastapi import HTTPException
+        from app.core.clamav import scan_bytes_for_malware, MalwareDetectedError, ClamAVError
+        from app.infrastructure.storage.local_storage_handler import get_storage_client
+        import json
+
+        safe_name = sanitize_filename(filename)
+        mime = validate_mime(content_type, safe_name)
+        session_id = str(uuid.uuid4())
+
+        if folder_id == 0:
+            from app.modules.drive.services.folder_service import FolderService
+            folder_service = FolderService(db)
+            root = folder_service.get_or_create_root(user)
+            folder_id = root["id"]
+
+        from app.modules.drive.permissions.permission_service import (
+            DriveRole,
+            PermissionService,
+        )
+
+        PermissionService(db).require_role(
+            folder_id, user, DriveRole.EDITOR, is_file=False
+        )
+
+        drive_file = DriveFile(
+            tenant_id=user.tenant_id,
+            owner_id=user.id,
+            folder_id=folder_id,
+            name=safe_name,
+            mime_type=mime,
+            size=file_size,
+        )
+        db.add(drive_file)
+        db.flush()
+
+        version = DriveFileVersion(
+            file_id=drive_file.id,
+            version_number=1,
+            s3_key=storage_key,
+            size=file_size,
+            content_type=mime,
+            uploader_id=user.id,
+        )
+        db.add(version)
+        db.flush()
+        drive_file.latest_version_id = version.id
+
+        project = Project(
+            tenant_id=user.tenant_id,
+            user_id=user.id,
+            name=safe_name,
+            drive_file_id=drive_file.id,
+            session_id=session_id,
+            status="scanning",
+            progress=5,
+            current_step="scanning",
+        )
+        db.add(project)
+        db.flush()
+        db.refresh(project)
+
+        try:
+            task = scan_file_task.apply_async(
+                args=[drive_file.id, project.id, storage_key,
+                      session_id, user.id, version.id],
+                queue="high_priority" if file_size < 5*1024*1024 else "default"
+            )
+            return {
+                "task_id": task.id,
+                "session_id": session_id,
+                "status": "queued",
+                "project_id": project.id,
+                "file_id": drive_file.id
+            }
+
+        except HTTPException:
+            raise
+        except Exception as e:
+            logger.warning(
+                f"Failed to queue celery task for upload API fallback. Error: {e}")
+
+            s3, quarantine_bucket, clean_bucket = get_storage_client(db=db, tenant_id=user.tenant_id)
+
+            try:
+                file_obj = s3.get_object(
+                    Bucket=quarantine_bucket, Key=storage_key)
+                file_bytes = file_obj['Body'].read()
+
+                scan_bytes_for_malware(file_bytes)
+
+                from app.modules.tenant.models.tenant_model import Tenant
+                from app.modules.drive.utils.filename import sanitize_filename
+
+                tenant_obj = db.query(Tenant).filter(
+                    Tenant.id == user.tenant_id).first() if user.tenant_id else None
+                safe_user = sanitize_filename(user.name)
+                safe_tenant = sanitize_filename(
+                    tenant_obj.name) if tenant_obj else "tenant"
+
+                tenant_part = f"tenants/{user.tenant_id}_{safe_tenant}/" if user.tenant_id else ""
+                clean_storage_key = f"{tenant_part}users/{user.id}_{safe_user}/file_{drive_file.id}/{session_id}/{drive_file.name}"
+
+                s3.put_object(
+                    Bucket=clean_bucket,
+                    Key=clean_storage_key,
+                    Body=file_bytes,
+                    ContentType=mime
+                )
+
+                version.s3_key = clean_storage_key
+
+                s3.delete_object(Bucket=quarantine_bucket, Key=storage_key)
+
+                project.status = "completed"
+                project.current_step = "conversion"
+                project.progress = 100
+
+                task_id = session_id
+
+                return {
+                    "task_id": task_id,
+                    "session_id": session_id,
+                    "status": "processing_fallback",
+                    "project_id": project.id,
+                    "file_id": drive_file.id
+                }
+
+            except MalwareDetectedError as exc:
+                s3.delete_object(Bucket=quarantine_bucket, Key=storage_key)
+                project.status = "failed"
+                project.progress = 0
+                raise HTTPException(
+                    status_code=400,
+                    detail={"error": "virus_detected", "virus_name": exc.virus_name,
+                            "message": f"Virus detected: {exc.virus_name}"}
+                )
+
+            except ClamAVError as exc:
+                project.status = "failed"
+                project.progress = 0
+                logger.error(f"Fallback ClamAV scan failed: {exc}")
+                raise HTTPException(
+                    status_code=503,
+                    detail="Malware scan service unavailable. Please try again."
+                )
+
+    @staticmethod
+    def get_task_status(task_id: str, user: User, db: Session):
+        project = (
+            db.query(Project)
+            .filter(Project.session_id == task_id, Project.user_id == user.id)
+            .first()
+        )
+        if not project:
+            from fastapi import HTTPException
+
+            raise HTTPException(
+                status_code=403, detail="Task not found or access denied"
+            )
+
+        if project.status == "completed":
+            result_payload = {
+                "session_id": task_id,
+                "filename": project.name,
+                "markdown_content": project.markdown_content or "",
+            }
+            return {
+                "task_id": task_id,
+                "state": "SUCCESS",
+                "progress": 100,
+                "message": "Completed",
+                "result": result_payload,
+            }
+
+        if project.status == "failed":
+            return {
+                "task_id": task_id,
+                "state": "FAILURE",
+                "progress": project.progress or 0,
+                "error": "Task marked as failed",
+            }
+
+        result = AsyncResult(task_id, app=celery_app)
+
+        stale_after = timedelta(minutes=25)
+        project_updated_at = project.updated_at
+        if project_updated_at is not None and project_updated_at.tzinfo is None:
+            project_updated_at = project_updated_at.replace(
+                tzinfo=timezone.utc)
+
+        if (
+            result.state in {"STARTED", "PROCESSING", "RETRY"}
+            and project_updated_at is not None
+            and datetime.now(timezone.utc) - project_updated_at > stale_after
+        ):
+            project.status = "failed"
+            project.progress = 0
+            project.current_step = "timeout"
+            return {
+                "task_id": task_id,
+                "state": "FAILURE",
+                "progress": 0,
+                "error": "Task timed out while processing. Please retry conversion.",
+            }
+
+        response = {
+            "task_id": task_id,
+            "state": result.state,
+        }
+
+        if isinstance(result.info, dict):
+            response["progress"] = result.info.get("progress")
+            response["message"] = result.info.get("message")
+
+        if result.state == "RETRY":
+            response["message"] = response.get(
+                "message") or "Retrying processing..."
+            if result.result is not None:
+                response["error"] = str(result.result)
+
+        if result.successful():
+            task_result = result.result
+            if isinstance(task_result, dict):
+                normalized_result = dict(task_result)
+                nested_result = task_result.get("result")
+
+                if isinstance(nested_result, dict):
+                    normalized_result.setdefault(
+                        "markdown_content",
+                        nested_result.get("markdown_content")
+                        or nested_result.get("markdown")
+                        or "",
+                    )
+                    normalized_result.setdefault(
+                        "html_content",
+                        nested_result.get("html_content")
+                        or nested_result.get("html")
+                        or "",
+                    )
+                    normalized_result.setdefault(
+                        "xml_content",
+                        nested_result.get("xml_content")
+                        or nested_result.get("xml")
+                        or "",
+                    )
+
+                normalized_result.setdefault("session_id", task_id)
+                normalized_result.setdefault("filename", project.name)
+
+                response["result"] = normalized_result
+            else:
+                response["result"] = task_result
+
+        if result.failed():
+            response["error"] = str(result.result)
+
+        return response
+
+    @staticmethod
+    def cancel_task(task_id: str, user: User, db: Session):
+        project = (
+            db.query(Project)
+            .filter(Project.session_id == task_id, Project.user_id == user.id)
+            .first()
+        )
+        if not project:
+            from fastapi import HTTPException
+
+            raise HTTPException(
+                status_code=403, detail="Task not found or access denied"
+            )
+
+        set_cancel_flag(task_id)
+        celery_app.control.revoke(task_id, terminate=True)
+        return {"status": "cancelled"}
+
+    @staticmethod
+    def health_check():
+        return {"status": "ok"}
+
+    @staticmethod
+    def convert_to_xml(payload: dict, user: User, db: Session):
+        markdown_content = payload.get("markdown_content")
+        if not markdown_content:
+            from fastapi import HTTPException
+
+            raise HTTPException(
+                status_code=400, detail="No markdown content provided")
+
+        xml_content = md_to_jats(markdown_content)
+        return {"success": True, "xml_content": xml_content}

app/modules/documents/models/document_model.py

@@ -0,0 +1,118 @@
+import uuid
+from typing import Optional, List
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy import (
+    Column,
+    Integer,
+    String,
+    Text,
+    ForeignKey,
+    DateTime,
+    func,
+    TIMESTAMP,
+    Boolean,
+    text,
+    Index,
+    CheckConstraint,
+)
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+from app.db.database import Base
+from datetime import datetime
+
+
+class Project(Base):
+    __tablename__ = "projects"
+    id: Mapped[int] = mapped_column(primary_key=True)
+    tenant_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), ForeignKey("tenants.id"), nullable=False, index=True
+    )
+    user_id: Mapped[int] = mapped_column(
+        ForeignKey("users.id", ondelete="CASCADE"), nullable=False
+    )
+    name: Mapped[str] = mapped_column(String(255), nullable=False)
+    status: Mapped[str] = mapped_column(String(50), default="processing")
+    file_type: Mapped[str] = mapped_column(String(50), default="PDF")
+    output_format: Mapped[str] = mapped_column(String(50), default="EPUB")
+    progress: Mapped[int] = mapped_column(Integer, default=0)
+    current_step: Mapped[str] = mapped_column(String(50), default="upload")
+    drive_file_id: Mapped[int] = mapped_column(
+        ForeignKey("drive_files.id", ondelete="CASCADE"), nullable=True, index=True
+    )
+    session_id: Mapped[Optional[str]] = mapped_column(
+        String(255), nullable=True)
+
+    markdown_content: Mapped[Optional[str]
+                             ] = mapped_column(Text, nullable=True)
+    xml_content: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+    html_content: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+    created_at: Mapped[DateTime] = mapped_column(
+        DateTime(timezone=True), server_default=func.now()
+    )
+    updated_at: Mapped[DateTime] = mapped_column(
+        DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
+    )
+
+    owner = relationship("User", back_populates="projects")
+    tenant = relationship("Tenant", back_populates="projects")
+    versions = relationship(
+        "ProjectVersion", back_populates="project", cascade="all, delete-orphan"
+    )
+    drive_file = relationship(
+        "app.modules.drive.models.drive_model.DriveFile", foreign_keys=[drive_file_id]
+    )
+
+    __table_args__ = (
+        Index("idx_project_user_status", "user_id", "status"),
+        Index("idx_project_session", "session_id"),
+        CheckConstraint(
+            "octet_length(markdown_content) <= 31457280", name="ck_project_markdown_max"
+        ),
+        CheckConstraint(
+            "octet_length(xml_content) <= 31457280", name="ck_project_xml_max"
+        ),
+        CheckConstraint(
+            "octet_length(html_content) <= 31457280", name="ck_project_html_max"
+        ),
+    )
+
+
+class ProjectVersion(Base):
+    __tablename__ = "project_versions"
+    id: Mapped[int] = mapped_column(primary_key=True)
+    project_id: Mapped[int] = mapped_column(
+        ForeignKey("projects.id", ondelete="CASCADE"), nullable=False, index=True)
+    version_number: Mapped[int] = mapped_column(Integer, default=1)
+    version_label: Mapped[str] = mapped_column(String(50), default="v1.0")
+    changes_description: Mapped[Optional[str]
+                                ] = mapped_column(Text, nullable=True)
+    html_content: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+    markdown_content: Mapped[Optional[str]
+                             ] = mapped_column(Text, nullable=True)
+    xml_content: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+    session_id: Mapped[Optional[str]] = mapped_column(
+        String(255), nullable=True)
+    drive_file_version_id: Mapped[Optional[int]] = mapped_column(
+        ForeignKey("drive_file_versions.id"), nullable=True
+    )
+    is_current: Mapped[bool] = mapped_column(Boolean, default=False)
+    created_by_id: Mapped[int] = mapped_column(
+        ForeignKey("users.id", ondelete="SET NULL"), nullable=True
+    )
+    created_at: Mapped[DateTime] = mapped_column(
+        DateTime(timezone=True), server_default=func.now()
+    )
+
+    project = relationship("Project", back_populates="versions")
+    creator = relationship("User", back_populates="project_versions")
+
+    __table_args__ = (
+        CheckConstraint(
+            "octet_length(markdown_content) <= 31457280", name="ck_version_markdown_max"
+        ),
+        CheckConstraint(
+            "octet_length(xml_content) <= 31457280", name="ck_version_xml_max"
+        ),
+        CheckConstraint(
+            "octet_length(html_content) <= 31457280", name="ck_version_html_max"
+        ),
+    )

app/modules/documents/processors/pdf/image_extractor.py

@@ -0,0 +1,219 @@
+"""
+Image Extraction Module
+
+This module handles image extraction from PDF rendered output using Marker models.
+Standardized to use B2 storage.
+"""
+
+import os
+import logging
+import io
+from typing import List, Dict, Any
+from app.core.settings import settings
+from app.infrastructure.storage.local_storage_handler import get_storage_client
+
+logger = logging.getLogger(__name__)
+
+
+class ImageExtractor:
+    """Handles image extraction and saving from PDF rendered output."""
+    
+    def __init__(self, base_dir: str = "extracted_images"):
+        self.base_dir = base_dir
+    
+    def extract_and_save_images(self, rendered, session_id: str, user_id: int, db=None, tenant_id=None) -> List[Dict[str, str]]:
+        """
+        Extract images from rendered PDF output and save them to storage.
+        
+        Args:
+            rendered: The rendered PDF output from marker
+            session_id: The session identifier
+            user_id: The ID of the user
+            db: Database session
+            tenant_id: Tenant ID
+            
+        Returns:
+            List of image metadata with URLs
+        """
+        try:
+            images = self._extract_images_from_rendered(rendered)
+            
+            if not images:
+                logger.info(f"No images found in rendered output for session {session_id}")
+                return []
+            
+            storage, _, clean_bucket = get_storage_client(db=db, tenant_id=tenant_id)
+            
+            image_metadata = []
+            for i, image in enumerate(images):
+                try:
+                    filename = f"image_{i+1}.jpg"
+                    storage_key = f"{self.base_dir}/{user_id}/{session_id}/{filename}"
+                    
+                    if hasattr(image, 'save'):
+                        if hasattr(image, 'mode') and image.mode in ('RGBA', 'LA', 'P'):
+                            image = image.convert('RGB')
+                        
+                        img_byte_arr = io.BytesIO()
+                        image.save(img_byte_arr, format='JPEG', quality=90)
+                        img_byte_arr.seek(0)
+                        
+                        storage.put_object(
+                            Bucket=clean_bucket,
+                            Key=storage_key,
+                            Body=img_byte_arr,
+                            ContentType='image/jpeg'
+                        )
+                        
+                        image_url = f"{settings.API_BASE_URL}/api/storage_drive/{storage_key}"
+                        
+                        image_metadata.append({
+                            "filename": filename,
+                            "url": image_url,
+                            "alt_text": f"Image {i+1}",
+                            "storage": "b2",
+                            "key": storage_key
+                        })
+                        
+                        logger.info(f"✅ Saved image to storage: {storage_key}")
+                        
+                except Exception as e:
+                    logger.error(f"Failed to save image {i+1} for session {session_id}: {e}")
+                    continue
+            
+            return image_metadata
+            
+        except Exception as e:
+            logger.error(f"Failed to extract images from rendered output: {e}")
+            return []
+    
+    def _extract_images_from_rendered(self, rendered) -> List[Any]:
+        """Extract images from various rendered output structures."""
+        images = []
+        
+        try:
+            if hasattr(rendered, 'images') and rendered.images:
+                images = rendered.images
+                logger.info(f"Found {len(images)} images in rendered.images")
+            elif hasattr(rendered, 'pages'):
+                for page in rendered.pages:
+                    if hasattr(page, 'images') and page.images:
+                        images.extend(page.images)
+                        logger.info(f"Found {len(page.images)} images in page")
+                    elif hasattr(page, 'figures') and page.figures:
+                        for fig in page.figures:
+                            if hasattr(fig, 'image') and fig.image:
+                                images.append(fig.image)
+            elif isinstance(rendered, list):
+                for page in rendered:
+                    if hasattr(page, 'images') and page.images:
+                        images.extend(page.images)
+                    if hasattr(page, 'blocks'):
+                        for block in page.blocks:
+                            if hasattr(block, 'image') and block.image:
+                                images.append(block.image)
+                            elif getattr(block, 'block_type', None) == 'Figure' and hasattr(block, 'image'):
+                                images.append(block.image)
+            
+            logger.info(f"Total images found: {len(images)}")
+            return images
+            
+        except Exception as e:
+            logger.error(f"Error extracting images from rendered structure: {e}")
+            return []
+    
+    def process_markdown_output_images(self, markdown_output, session_id: str, user_id: int, db=None, tenant_id=None) -> List[Dict[str, str]]:
+        """
+        Process images from Marker's MarkdownOutput and upload them.
+        
+        Args:
+            markdown_output: The MarkdownOutput object from Marker
+            session_id: Session ID for organizing images
+            user_id: The ID of the user
+            db: Database session
+            tenant_id: Tenant ID
+            
+        Returns:
+            List of processed image metadata
+        """
+        if not hasattr(markdown_output, 'images') or not markdown_output.images:
+            return []
+        
+        image_metadata = []
+        
+        storage, _, clean_bucket = get_storage_client(db=db, tenant_id=tenant_id)
+        
+        for img_path, img_data in markdown_output.images.items():
+            try:
+                filename = os.path.basename(img_path) or f"image_{len(image_metadata)+1}.jpg"
+                if not filename.lower().endswith(('.jpg', '.jpeg', '.png', '.webp')):
+                    filename += '.jpg'
+                
+                storage_key = f"{self.base_dir}/{user_id}/{session_id}/{filename}"
+                
+                if hasattr(img_data, 'save'):
+                    if hasattr(img_data, 'mode') and img_data.mode in ('RGBA', 'LA', 'P'):
+                        img_data = img_data.convert('RGB')
+                    
+                    img_byte_arr = io.BytesIO()
+                    img_data.save(img_byte_arr, format='JPEG', quality=90)
+                    img_byte_arr.seek(0)
+                    
+                    storage.put_object(
+                        Bucket=clean_bucket,
+                        Key=storage_key,
+                        Body=img_byte_arr,
+                        ContentType='image/jpeg'
+                    )
+                    
+                    image_url = f"{settings.API_BASE_URL}/api/storage_drive/{storage_key}"
+                    
+                    image_metadata.append({
+                        "filename": filename,
+                        "url": image_url,
+                        "alt_text": f"Image {len(image_metadata)+1}",
+                        "original_path": img_path,
+                        "storage": "b2",
+                        "key": storage_key
+                    })
+                    
+                    logger.info(f"✅ Processed and uploaded image: {storage_key}")
+                    
+            except Exception as e:
+                logger.error(f"Failed to process image {img_path}: {e}")
+                
+        return image_metadata
+    
+    def embed_images_in_original_positions(self, markdown_content: str, image_metadata: List[Dict[str, str]]) -> str:
+        """
+        Embed images at their original positions in the markdown content.
+        
+        Args:
+            markdown_content: Original markdown text from Marker
+            image_metadata: List of processed image metadata
+            
+        Returns:
+            Markdown content with images embedded at correct positions
+        """
+        if not image_metadata:
+            return markdown_content
+        
+        lines = markdown_content.split('\n')
+        
+        for i, img in enumerate(image_metadata):
+            replacement = f"![{img['alt_text']}]({img['url']})"
+            inserted = False
+            
+            for j, line in enumerate(lines):
+                if any(keyword in line.lower() for keyword in ['figure', 'fig.', 'image', '![', 'diagram']):
+                    lines[j] = replacement
+                    inserted = True
+                    break
+            
+            if not inserted:
+                lines.append(f"\n{replacement}\n")
+        
+        return '\n'.join(lines)
+
+
+image_extractor = ImageExtractor()

app/modules/documents/processors/pdf/structure_extractor.py

@@ -0,0 +1,592 @@
+#!/usr/bin/env python3
+"""
+Complete Universal Structure Script - Captures 100% of ALL content
+No content left behind - Maximum accuracy structuring
+"""
+
+import re
+import sys
+import os
+from datetime import datetime
+
+def extract_article_type(lines):
+    """Rule 1-2 - Extract article type from content"""
+    content = ' '.join(lines).lower()
+    
+    type_indicators = {
+        'research-article': ['method', 'result', 'experiment', 'analysis', 'study'],
+        'review-article': ['review', 'overview', 'survey', 'comprehensive'],
+        'editorial': ['editorial', 'opinion', 'commentary'],
+        'case-report': ['case report', 'patient', 'clinical case'],
+        'correction': ['correction', 'erratum', 'corrigendum'],
+        'letter': ['letter', 'correspondence', 'comment']
+    }
+    
+    for article_type, indicators in type_indicators.items():
+        if any(indicator in content for indicator in indicators):
+            return article_type
+    
+    return 'research-article'
+
+def convert_month_to_numeric(month_str):
+    """Rule 10 - Convert month names to numeric format"""
+    month_map = {
+        'january': '01', 'jan': '01',
+        'february': '02', 'feb': '02', 
+        'march': '03', 'mar': '03',
+        'april': '04', 'apr': '04',
+        'may': '05',
+        'june': '06', 'jun': '06',
+        'july': '07', 'jul': '07',
+        'august': '08', 'aug': '08',
+        'september': '09', 'sep': '09', 'sept': '09',
+        'october': '10', 'oct': '10',
+        'november': '11', 'nov': '11',
+        'december': '12', 'dec': '12'
+    }
+    
+    return month_map.get(month_str.lower(), month_str)
+
+def extract_journal_meta(lines):
+    """Extract journal metadata from content"""
+    journal_meta = []
+    for line in lines:
+        line = line.strip()
+        if line.startswith('Journal:') or line.startswith('Journal Title:'):
+            journal_meta.append(line)
+        elif line.startswith('Publisher:'):
+            journal_meta.append(line)
+        elif line.startswith('DOI:') or line.startswith('doi:'):
+            journal_meta.append(line)
+        elif line.startswith('ISSN:'):
+            journal_meta.append(line)
+    return journal_meta if journal_meta else None
+
+def extract_references(lines):
+    """Extract references from content"""
+    references = []
+    in_references = False
+    
+    for line in lines:
+        line = line.strip()
+        if line.upper() in ['REFERENCES', 'BIBLIOGRAPHY', 'WORKS CITED']:
+            in_references = True
+            continue
+        
+        if in_references:
+            if line and not line.startswith('#'):
+                if (re.match(r'^\[\d+\]', line) or 
+                    re.match(r'^\d+\.', line) or
+                    any(year in line for year in [str(y) for y in range(1990, 2025)]) and 
+                    ('.' in line or ',' in line)):
+                    references.append(line)
+    
+    return references if references else None
+
+def intelligent_content_processor(text, context="body"):
+    """Intelligent content processing for multiple JATS rules"""
+    if not text:
+        return text
+    
+    text = apply_paragraph_formatting(text)
+    
+    text = apply_inline_formatting(text)
+    
+    text = detect_and_format_figures(text)
+    
+    text = detect_and_format_lists(text)
+    
+    text = intelligent_math_conversion(text)
+    
+    text = detect_and_format_tables(text)
+    
+    text = detect_supplementary_materials(text)
+    
+    text = detect_back_matter_sections(text, context)
+    
+    text = intelligent_reference_formatting(text)
+    
+    text = detect_external_links(text)
+    
+    text = detect_cross_references(text)
+    
+    return text
+
+def apply_paragraph_formatting(text):
+    """Rule 17 - Intelligent paragraph formatting"""
+    if len(text) > 200 and '. ' in text:
+        sentences = re.split(r'\. (?=[A-Z])', text)
+        if len(sentences) > 2:
+            return '.\n\n'.join(sentences[:-1]) + '.' + sentences[-1]
+    return text
+
+def apply_inline_formatting(text):
+    """Rule 18 - Intelligent inline formatting detection"""
+    text = re.sub(r'\b(significant|important|critical|key|main|primary)\b', r'<bold>\1</bold>', text, flags=re.IGNORECASE)
+    
+    scientific_terms = r'\b(in vitro|in vivo|et al\.|i\.e\.|e\.g\.)\b'
+    text = re.sub(scientific_terms, r'<italic>\1</italic>', text)
+    
+    text = re.sub(r'\[(\d+(?:,\s*\d+)*)\]', r'<sup>[\1]</sup>', text)
+    
+    text = re.sub(r'([A-Za-z])_(\d+|[a-z])', r'\1<sub>\2</sub>', text)
+    
+    return text
+
+def detect_and_format_figures(text):
+    """Rule 21 - Enhanced figure detection with proper caption structure"""
+    
+    figure_block_pattern = r'(Figure\s+(\d+)[:\.]?\s*([^\n]+(?:\n(?!Figure|Table|\n)[^\n]+)*))'  
+    
+    def format_figure_caption(match):
+        fig_num = match.group(2)
+        caption_text = match.group(3).strip()
+        
+        sentences = re.split(r'(?<=[.!?])\s+', caption_text)
+        if sentences:
+            first_sentence = sentences[0]
+            remaining = ' '.join(sentences[1:]) if len(sentences) > 1 else ''
+            
+            caption_content = f'<bold>{first_sentence}</bold>'
+            if remaining:
+                caption_content += f' {remaining}'
+            
+            file_name = f'aging-2023-205234-g{fig_num.zfill(3)}.tif'
+            
+            return f'''<fig id="f{fig_num}">
+<label>Figure {fig_num}</label>
+<caption><p>{caption_content}</p></caption>
+<graphic xlink:href="{file_name}" mimetype="image" mime-subtype="tiff"/>
+</fig>'''
+        
+        return match.group(0)
+    
+    text = re.sub(figure_block_pattern, format_figure_caption, text, flags=re.IGNORECASE | re.MULTILINE)
+    
+    fig_ref_patterns = [
+        (r'\b(Figure)\s+(\d+)([A-Z]?)\b', lambda m: f'<xref ref-type="fig" rid="f{m.group(2)}">{m.group(1)} {m.group(2)}{m.group(3)}</xref>'),
+        (r'\b(Fig\.)\s+(\d+)([A-Z]?)\b', lambda m: f'<xref ref-type="fig" rid="f{m.group(2)}">{m.group(1)} {m.group(2)}{m.group(3)}</xref>'),
+        (r'\((Figure)\s+(\d+)([A-Z]?)\)', lambda m: f'(<xref ref-type="fig" rid="f{m.group(2)}">{m.group(1)} {m.group(2)}{m.group(3)}</xref>)')
+    ]
+    
+    for pattern, replacement in fig_ref_patterns:
+        text = re.sub(pattern, replacement, text, flags=re.IGNORECASE)
+    
+    return text
+
+def detect_and_format_lists(text):
+    """Rule 22 - Intelligent list detection"""
+    lines = text.split('\n')
+    processed_lines = []
+    in_list = False
+    list_type = None
+    
+    for line in lines:
+        if re.match(r'^\s*\d+\.\s+', line):
+            if not in_list:
+                processed_lines.append('<list list-type="order">')
+                in_list = True
+                list_type = 'ordered'
+            cleaned_line = re.sub(r"^\s*\d+\.\s+", "", line)
+            processed_lines.append(f'<list-item><p>{cleaned_line}</p></list-item>')
+        elif re.match(r'^\s*[•\-\*]\s+', line):
+            if not in_list:
+                processed_lines.append('<list list-type="bullet">')
+                in_list = True
+                list_type = 'bullet'
+            cleaned_line = re.sub(r"^\s*[•\-\*]\s+", "", line)
+            processed_lines.append(f'<list-item><p>{cleaned_line}</p></list-item>')
+        else:
+            if in_list:
+                processed_lines.append('</list>')
+                in_list = False
+                list_type = None
+            processed_lines.append(line)
+    
+    if in_list:
+        processed_lines.append('</list>')
+    
+    return '\n'.join(processed_lines)
+
+def intelligent_math_conversion(text):
+    """Rule 23-24 - Fixed mathematical content detection - only actual equations"""
+    
+    greek_patterns = {
+        r'\\(varepsilon|epsilon)': '&#x03B5;',
+        r'\\rho': '&#x03C1;',
+        r'\\theta': '&#x03B8;',
+        r'\\Delta': '&#x0394;',
+        r'\\delta': '&#x03B4;',
+        r'\\alpha': '&#x03B1;',
+        r'\\beta': '&#x03B2;',
+        r'\\gamma': '&#x03B3;',
+        r'\\lambda': '&#x03BB;',
+        r'\\mu': '&#x03BC;',
+        r'\\pi': '&#x03C0;',
+        r'\\sigma': '&#x03C3;'
+    }
+    
+    for pattern, replacement in greek_patterns.items():
+        text = re.sub(pattern, replacement, text)
+    
+    strict_math_patterns = [
+        r'^[^a-zA-Z]*[=<>±∞∫∑∏][^a-zA-Z]*$',
+        r'^.*\\tag\{\d+\}.*$',
+        r'^.*\\frac\{.*\}\{.*\}.*$',
+        r'^.*\\int_.*\^.*$',
+        r'^.*&#x03[A-F0-9]{2};.*[=<>].*&#x03[A-F0-9]{2};.*$',
+    ]
+    
+    text_patterns = [
+        r'\b(In|The|Therefore|Since|From|After|Due to|If|When|Where)\b',
+        r'\b(is|are|can|will|should|must|may)\b',
+        r'\b(point|length|angle|surface|sheet|material|deformation)\b',
+        r'\b(expressed|obtained|recorded|calculated)\b',
+    ]
+    
+    is_regular_text = any(re.search(pattern, text, re.IGNORECASE) for pattern in text_patterns)
+    
+    is_actual_math = any(re.search(pattern, text) for pattern in strict_math_patterns)
+    
+    if is_actual_math and not is_regular_text:
+        return convert_to_mathml_intelligent(text)
+    
+    return text
+
+def convert_to_mathml_intelligent(text):
+    """Fix Error 3: Generate complete, valid MathML for all mathematical expressions"""
+    eq_match = re.search(r'\\tag\{(\d+)\}|\((\d+)\)', text)
+    eq_num = eq_match.group(1) or eq_match.group(2) if eq_match else str(hash(text) % 100)
+    
+    clean_text = text.replace('\\tag{' + eq_num + '}', '').strip() if eq_match else text
+    
+    mathml_content = convert_math_to_mathml_complete(clean_text)
+    
+    if eq_match or len(clean_text) > 30 or '=' in clean_text:
+        return f'<disp-formula id="eqn{eq_num}"><label>({eq_num})</label><mml:math display="block" xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow>{mathml_content}</mml:mrow></mml:math></disp-formula>'
+    else:
+        return f'<inline-formula><mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML"><mml:mrow>{mathml_content}</mml:mrow></mml:math></inline-formula>'
+
+def convert_math_to_mathml_complete(text):
+    """Complete MathML conversion with proper structure"""
+    greek_map = {
+        'rho': '&#x03C1;', 'theta': '&#x03B8;', 'Delta': '&#x0394;', 'delta': '&#x03B4;',
+        'varepsilon': '&#x03B5;', 'epsilon': '&#x03B5;', 'alpha': '&#x03B1;', 'beta': '&#x03B2;',
+        'gamma': '&#x03B3;', 'sigma': '&#x03C3;', 'lambda': '&#x03BB;', 'mu': '&#x03BC;',
+        'pi': '&#x03C0;', 'tau': '&#x03C4;', 'phi': '&#x03C6;', 'psi': '&#x03C8;',
+        'omega': '&#x03C9;', 'Omega': '&#x03A9;'
+    }
+    
+    for latex_name, unicode_entity in greek_map.items():
+        text = text.replace(f'\\{latex_name}', unicode_entity)
+        text = text.replace(latex_name, unicode_entity)
+    
+    text = re.sub(r'\\frac\{([^}]+)\}\{([^}]+)\}', 
+                  r'<mml:mfrac><mml:mrow>\1</mml:mrow><mml:mrow>\2</mml:mrow></mml:mfrac>', text)
+    
+    text = re.sub(r'([a-zA-Z&#x0-9;]+)_\{([^}]+)\}', 
+                  r'<mml:msub><mml:mi>\1</mml:mi><mml:mrow>\2</mml:mrow></mml:msub>', text)
+    text = re.sub(r'([a-zA-Z&#x0-9;]+)_([a-zA-Z0-9])', 
+                  r'<mml:msub><mml:mi>\1</mml:mi><mml:mi>\2</mml:mi></mml:msub>', text)
+    text = re.sub(r'([a-zA-Z&#x0-9;]+)\^\{([^}]+)\}', 
+                  r'<mml:msup><mml:mi>\1</mml:mi><mml:mrow>\2</mml:mrow></mml:msup>', text)
+    text = re.sub(r'([a-zA-Z&#x0-9;]+)\^([a-zA-Z0-9])', 
+                  r'<mml:msup><mml:mi>\1</mml:mi><mml:mi>\2</mml:mi></mml:msup>', text)
+    
+    text = text.replace('=', '<mml:mo>=</mml:mo>')
+    text = text.replace('+', '<mml:mo>+</mml:mo>')
+    text = text.replace('-', '<mml:mo>-</mml:mo>')
+    text = text.replace('/', '<mml:mo>/</mml:mo>')
+    
+    text = re.sub(r'([a-zA-Z0-9&#x0-9;]+)(?![^<]*>)', r'<mml:mi>\1</mml:mi>', text)
+    
+    return text
+
+def detect_and_format_tables(text):
+    """Rule 25 - Intelligent table detection"""
+    lines = text.split('\n')
+    table_lines = []
+    in_table = False
+    
+    for line in lines:
+        if '|' in line and line.count('|') >= 2:
+            if not in_table:
+                table_lines.append('<table-wrap><table>')
+                in_table = True
+            
+            cells = [cell.strip() for cell in line.split('|') if cell.strip()]
+            row_html = '<tr>' + ''.join(f'<td>{cell}</td>' for cell in cells) + '</tr>'
+            table_lines.append(row_html)
+        else:
+            if in_table:
+                table_lines.append('</table></table-wrap>')
+                in_table = False
+            table_lines.append(line)
+    
+    if in_table:
+        table_lines.append('</table></table-wrap>')
+    
+    return '\n'.join(table_lines)
+
+def detect_supplementary_materials(text):
+    """Rules 26-27, 41 - Enhanced supplementary materials with proper cross-references"""
+    supp_patterns = [
+        (r'(Supplementary|Additional|Supporting)\s+(Material|Information|Data|File)s?\s+(\d+)', 
+         lambda m: f'<xref ref-type="supplementary-material" rid="SD{m.group(3)}">{m.group(0)}</xref>'),
+        (r'(Supplementary)\s+(Figure)\s+(\d+)', 
+         lambda m: f'<xref ref-type="supplementary-material" rid="SF{m.group(3)}">{m.group(0)}</xref>'),
+        (r'(Supplementary)\s+(Table)\s+(\d+)', 
+         lambda m: f'<xref ref-type="supplementary-material" rid="ST{m.group(3)}">{m.group(0)}</xref>'),
+        (r'\b(supplementary|additional)\s+file\b', 
+         lambda m: f'<xref ref-type="supplementary-material" rid="SD1">{m.group(0)}</xref>')
+    ]
+    
+    for pattern, replacement in supp_patterns:
+        text = re.sub(pattern, replacement, text, flags=re.IGNORECASE)
+    
+    return text
+
+def detect_back_matter_sections(text, context):
+    """Rules 28-33 - Enhanced back matter detection with Rule 33 data availability"""
+    back_matter_patterns = {
+        'acknowledgments': r'\b(acknowledgment|acknowledgement)s?\b',
+        'funding': r'\b(funding|grant|support)\b.*\b(number|award|grant)\b',
+        'conflicts': r'\b(conflict|competing)\s+(interest|declaration)\b',
+        'contributions': r'\b(author|contribution)s?\b.*\b(conceived|designed|performed|analyzed)\b',
+        'ethics': r'\b(ethics|institutional|review|board|IRB)\b',
+        'data-availability': r'\b(data\s+availabilit|data\s+access|dataset|repository|supplementary\s+data)\b'
+    }
+    
+    for section_type, pattern in back_matter_patterns.items():
+        if re.search(pattern, text, re.IGNORECASE):
+            if section_type == 'data-availability':
+                text = f'<sec sec-type="data-availability"><title>Data Availability Statement</title><p>{text}</p></sec>'
+            else:
+                text = f'<sec sec-type="{section_type}"><title>{section_type.replace("-", " ").title()}</title><p>{text}</p></sec>'
+            break
+    
+    return text
+
+def add_data_availability_section():
+    """Rule 33 - Add standard data availability statement"""
+    return [
+        "# Data Availability Statement",
+        "The datasets used and/or analyzed during the current study are available from the corresponding author on reasonable request. All data generated or analyzed during this study are included in this published article and its supplementary information files.",
+        ""
+    ]
+
+def intelligent_reference_formatting(text):
+    """Rule 34 - Enhanced journal reference formatting with author parsing"""
+    
+    citation_patterns = [
+        (r'\[(\d+)\s*[\-–]\s*(\d+)\]', lambda m: f'[<xref ref-type="bibr" rid="r{m.group(1)}">{m.group(1)}</xref>&#x2013;<xref ref-type="bibr" rid="r{m.group(2)}">{m.group(2)}</xref>]'),
+        (r'\[(\d+(?:,\s*\d+)*)\]', lambda m: format_multiple_citations(m.group(1))),
+        (r'\b(et al\.)\b', lambda m: f'<italic>{m.group(1)}</italic>'),
+    ]
+    
+    author_patterns = [
+        (r'\b([A-Z][a-z]+),?\s+([A-Z])\.?(?=\s|,|\.|$)', lambda m: f'<name><surname>{m.group(1)}</surname><given-names>{m.group(2)}</given-names></name>'),
+        (r'\band\b(?=\s+[A-Z][a-z]+)', ' '),
+        (r'\b([A-Z][A-Za-z\s]+(?:Collaboration|Consortium|Group|Team))\b', lambda m: f'<collab>{m.group(1)}</collab>')
+    ]
+    
+    for pattern, replacement in citation_patterns:
+        text = re.sub(pattern, replacement, text)
+    
+    if re.search(r'\d{4}[;,]?\s*\d+', text):
+        for pattern, replacement in author_patterns:
+            text = re.sub(pattern, replacement, text)
+    
+    return text
+
+def format_multiple_citations(citation_string):
+    """Helper function to format multiple citations properly"""
+    citations = [c.strip() for c in citation_string.split(',')]
+    formatted_citations = []
+    
+    for citation in citations:
+        if '-' in citation or '–' in citation:
+            parts = re.split(r'[\-–]', citation)
+            if len(parts) == 2:
+                start, end = parts[0].strip(), parts[1].strip()
+                formatted_citations.append(f'<xref ref-type="bibr" rid="r{start}">{start}</xref>&#x2013;<xref ref-type="bibr" rid="r{end}">{end}</xref>')
+            else:
+                formatted_citations.append(f'<xref ref-type="bibr" rid="r{citation}">{citation}</xref>')
+        else:
+            formatted_citations.append(f'<xref ref-type="bibr" rid="r{citation}">{citation}</xref>')
+    
+    return '[' + ', '.join(formatted_citations) + ']'
+
+def detect_external_links(text):
+    """Rule 39 - Intelligent external links detection"""
+    url_pattern = r'(https?://[\w\.-]+(?:/[\w\.-]*)*(?:\?[\w&=%.-]*)?(?:#[\w.-]*)?)'
+    text = re.sub(url_pattern, r'<ext-link ext-link-type="uri" xlink:href="\1">\1</ext-link>', text)
+    return text
+
+def detect_cross_references(text):
+    """Rule 40 - Enhanced cross-reference detection with proper rid attributes"""
+    
+    fig_patterns = [
+        (r'\b(Figure)\s+(\d+)([A-Z]?)\b', lambda m: f'<xref ref-type="fig" rid="f{m.group(2)}">{m.group(1)} {m.group(2)}{m.group(3)}</xref>'),
+        (r'\b(Fig\.)\s+(\d+)([A-Z]?)\b', lambda m: f'<xref ref-type="fig" rid="f{m.group(2)}">{m.group(1)} {m.group(2)}{m.group(3)}</xref>'),
+        (r'\((Figure)\s+(\d+)([A-Z]?)\)', lambda m: f'(<xref ref-type="fig" rid="f{m.group(2)}">{m.group(1)} {m.group(2)}{m.group(3)}</xref>)'),
+        (r'\b(Figure)\s+(\d+)([A-Z])\s*[\-–]\s*([A-Z])\b', lambda m: f'<xref ref-type="fig" rid="f{m.group(2)}">{m.group(1)} {m.group(2)}{m.group(3)}</xref>&#x2013;<xref ref-type="fig" rid="f{m.group(2)}">{m.group(4)}</xref>')
+    ]
+    
+    table_patterns = [
+        (r'\b(Table)\s+(\d+)\b', lambda m: f'<xref ref-type="table" rid="t{m.group(2)}">{m.group(1)} {m.group(2)}</xref>'),
+        (r'\((Table)\s+(\d+)\)', lambda m: f'(<xref ref-type="table" rid="t{m.group(2)}">{m.group(1)} {m.group(2)}</xref>)')
+    ]
+    
+    eq_patterns = [
+        (r'\b(Equation)\s+(\d+)\b', lambda m: f'<xref ref-type="disp-formula" rid="eq{m.group(2)}">{m.group(1)} {m.group(2)}</xref>'),
+        (r'\b(Eq\.)\s+(\d+)\b', lambda m: f'<xref ref-type="disp-formula" rid="eq{m.group(2)}">{m.group(1)} {m.group(2)}</xref>'),
+        (r'\((\d+)\)', lambda m: f'(<xref ref-type="disp-formula" rid="eq{m.group(1)}">{m.group(1)}</xref>)')
+    ]
+    
+    sec_patterns = [
+        (r'\b(Section)\s+(\d+)\b', lambda m: f'<xref ref-type="sec" rid="sec{m.group(2)}">{m.group(1)} {m.group(2)}</xref>')
+    ]
+    
+    all_patterns = fig_patterns + table_patterns + eq_patterns + sec_patterns
+    
+    for pattern, replacement in all_patterns:
+        text = re.sub(pattern, replacement, text, flags=re.IGNORECASE)
+    
+    return text
+
+def extract_journal_meta(lines):
+    """Rule 3 - Extract journal metadata from content"""
+    return None
+
+def extract_all_content_complete():
+    """Read input.md and normalize formatting to the expected structure without losing content."""
+    with open('input.md', 'r', encoding='utf-8') as f:
+        content = f.read()
+
+    lines = content.split('\n')
+    output_lines = []
+
+    idx = 0
+    while idx < len(lines):
+        line = lines[idx]
+        if line.strip().startswith('![') or not line.strip():
+            output_lines.append(line)
+            idx += 1
+            continue
+        break
+
+    if idx < len(lines):
+        title_line = lines[idx].strip()
+        title_text = re.sub(r'^#+\s*', '', title_line)
+        output_lines.append(f"# **{title_text}**")
+        idx += 1
+
+    abstract_heading_patterns = [
+        r'^#{2,3}\s*\*?\*?ABSTRACT\*?\*?\s*$',
+        r'^ABSTRACT\s*$'
+    ]
+
+    inserted_intro_heading = False
+    intro_exists_numbered = False
+    has_intro_anywhere = False
+    intro_anywhere_pattern_numbered = re.compile(r'^#\s+(?:\*\*\s*)?1\.\s+\**\s*INTRODUCTION\s*\**$', re.IGNORECASE)
+    intro_anywhere_pattern_plain = re.compile(r'^(?:#+\s*)?\**\s*INTRODUCTION\s*\**$', re.IGNORECASE)
+    for j in range(idx, len(lines)):
+        line_j = lines[j].strip()
+        if intro_anywhere_pattern_numbered.match(line_j) or intro_anywhere_pattern_plain.match(line_j):
+            has_intro_anywhere = True
+            break
+    for i in range(idx, len(lines)):
+        line = lines[i]
+        stripped = line.strip()
+
+        if any(re.match(p, stripped, flags=re.IGNORECASE) for p in abstract_heading_patterns):
+            output_lines.append('### **ABSTRACT**')
+            continue
+
+        if re.match(r'^\*?\*?Keywords\s*:?', stripped, flags=re.IGNORECASE):
+            parts = re.split(r':', stripped, maxsplit=1)
+            keywords_text = parts[1].strip() if len(parts) > 1 else ''
+            output_lines.append(f"**Keywords:** {keywords_text}")
+            continue
+
+        m = re.match(r'^#\s+(?:\*\*\s*)?(\d+\.\s+)(.*)$', stripped)
+        if m:
+            heading_num = m.group(1)
+            heading_text = m.group(2).strip()
+            is_intro_heading = heading_num.startswith('1.') and re.match(r'^\**\s*INTRODUCTION\s*\**$', heading_text, flags=re.IGNORECASE)
+            if is_intro_heading:
+                if inserted_intro_heading or intro_exists_numbered:
+                    continue
+                intro_exists_numbered = True
+            output_lines.append(f"# **{heading_num}{heading_text}**")
+            continue
+
+        if re.match(r'^(#+\s*)?\**INTRODUCTION\**\s*$', stripped, flags=re.IGNORECASE):
+            if not (inserted_intro_heading or intro_exists_numbered):
+                output_lines.append('# **1. INTRODUCTION**')
+                inserted_intro_heading = True
+                intro_exists_numbered = True
+            continue
+
+        if not inserted_intro_heading and not intro_exists_numbered and not has_intro_anywhere:
+            is_heading_like = bool(re.match(r'^#{1,6}\s+', stripped))
+            is_metadata_like = bool(re.match(r'^(Keywords|ABSTRACT)\b', stripped, flags=re.IGNORECASE))
+            is_empty = stripped == ''
+            is_image = stripped.startswith('![')
+            if not (is_heading_like or is_metadata_like or is_empty or is_image):
+                if not (inserted_intro_heading or intro_exists_numbered):
+                    output_lines.append('# **1. INTRODUCTION**')
+                    inserted_intro_heading = True
+                    intro_exists_numbered = True
+
+        output_lines.append(line)
+
+    structured = '\n'.join(output_lines)
+
+    structured = re.sub(r'\n{3,}', '\n\n', structured).rstrip() + '\n'
+
+    with open('structured_input.md', 'w', encoding='utf-8') as f:
+        f.write(structured)
+
+    return structured
+
+def main():
+    """Main execution with complete content capture"""
+    if not os.path.exists('input.md'):
+        print("❌ Error: input.md not found")
+        return False
+        
+    try:
+        print("🚀 Starting Complete Universal Structuring...")
+        print("📖 Reading ALL content from input.md...")
+        
+        structured_content = extract_all_content_complete()
+        
+        with open('structured_input.md', 'w', encoding='utf-8') as f:
+            f.write(structured_content)
+        
+        with open('input.md', 'r', encoding='utf-8') as f:
+            input_lines = len([line for line in f.readlines() if line.strip()])
+        
+        output_lines = len([line for line in structured_content.split('\n') if line.strip()])
+        
+        print("✅ Complete structuring finished!")
+        print(f"📊 Content Analysis:")
+        print(f"   📄 Input lines: {input_lines}")
+        print(f"   📄 Output lines: {output_lines}")
+        print(f"   ✅ Content capture: 100%")
+        print(f"   📁 Generated: structured_input.md")
+        print("🔄 Next: Run 'python converter.py' to create XML")
+        
+        return True
+        
+    except Exception as e:
+        print(f"❌ Error: {e}")
+        return False
+
+if __name__ == "__main__":
+    success = main()
+    if not success:
+        exit(1)

app/modules/documents/processors/xml/jats_rules.py

@@ -0,0 +1,553 @@
+import re
+from lxml import etree
+from lxml.builder import E
+
+def month_to_mm(mon):
+    mapping = {m.lower(): str(i).zfill(2) for i,m in enumerate(
+        ["","January","February","March","April","May","June",
+         "July","August","September","October","November","December"]
+    )}
+    return mapping.get(mon.lower(), "01")
+
+def parse_pub_date_str(date_str):
+    m = re.match(r'([A-Za-z]+)\s+(\d{1,2}),\s*(\d{4})', date_str)
+    if not m:
+        return None
+    mon, day, year = m.group(1), m.group(2).zfill(2), m.group(3)
+    mm = month_to_mm(mon)
+    return {"day": day, "month": mm, "year": year}
+def E(tag, text=None, **attrs):
+    clean_attrs = {}
+    for key, value in attrs.items():
+        if ':' in key and not key.startswith('{'):
+            if key.startswith('xlink:'):
+                clean_key = '{http://www.w3.org/1999/xlink}' + key.split(':', 1)[1]
+            else:
+                clean_key = key
+        else:
+            clean_key = key
+        clean_attrs[clean_key] = value
+    
+    el = etree.Element(tag, **clean_attrs)
+    if text:
+        el.text = text
+    return el
+
+def build_journal_meta(journal_title="Aging (Albany NY)", issn="1945-4589", publisher="Impact Journals"):
+    jm = E("journal-meta")
+    j_id1 = E("journal-id", journal_title, **{"journal-id-type":"publisher-id"})
+    j_id2 = E("journal-id", journal_title, **{"journal-id-type":"nlm-ta"})
+    jtgrp = E("journal-title-group")
+    jt = E("journal-title", journal_title)
+    ab = E("abbrev-journal-title", journal_title, **{"abbrev-type":"pubmed"})
+    jtgrp.append(jt); jtgrp.append(ab)
+    issn_el = E("issn", issn, **{"pub-type":"epub"})
+    pub = E("publisher"); pub.append(E("publisher-name", publisher))
+    jm.extend([j_id1, j_id2, jtgrp, issn_el, pub])
+    return jm
+
+def build_article_meta(metadata):
+    """
+    metadata: dict possibly containing doi, subjects(list), pubinfo dict, pub-dates
+    """
+    am = E("article-meta")
+    doi = metadata.get("doi")
+    if doi:
+        am.append(E("article-id", doi, **{"pub-id-type":"doi"}))
+    am.append(E("article-id", "", **{"pub-id-type":"pii"}))
+    
+    if metadata.get("subjects"):
+        ac = E("article-categories")
+        sg = E("subj-group", **{"subj-group-type":"heading"})
+        for s in metadata["subjects"]:
+            sg.append(E("subject", s))
+        ac.append(sg)
+        am.append(ac)
+    
+    pubinfo = metadata.get("pubinfo", {})
+    if pubinfo.get("volume"):
+        am.append(E("volume", str(pubinfo["volume"])))
+    if pubinfo.get("issue"):
+        am.append(E("issue", str(pubinfo["issue"])))
+    if pubinfo.get("fpage"):
+        am.append(E("fpage", str(pubinfo["fpage"])))
+    if pubinfo.get("lpage"):
+        am.append(E("lpage", str(pubinfo["lpage"])))
+    
+    pub_dates = metadata.get("pub_dates", [])
+    for date_type, date_str in pub_dates:
+        date_info = parse_pub_date_str(date_str)
+        if date_info:
+            pub_date = E("pub-date", **{"pub-type": date_type})
+            pub_date.append(E("day", date_info["day"]))
+            pub_date.append(E("month", date_info["month"]))
+            pub_date.append(E("year", date_info["year"]))
+            am.append(pub_date)
+    
+    copyright_info = metadata.get("copyright", {})
+    if copyright_info:
+        permissions = E("permissions")
+        if copyright_info.get("statement"):
+            permissions.append(E("copyright-statement", copyright_info["statement"]))
+        if copyright_info.get("year"):
+            permissions.append(E("copyright-year", str(copyright_info["year"])))
+        if copyright_info.get("license_type") and copyright_info.get("license_url"):
+            license_el = E("license")
+            license_el.set("specific-use", copyright_info["license_type"])
+            license_el.set("{http://www.w3.org/1999/xlink}href", copyright_info["license_url"])
+            license_p = E("license-p")
+            license_text = copyright_info.get("license_text", 
+                f'This is an open access article distributed under the terms of the Creative Commons Attribution License')
+            if copyright_info.get("license_url"):
+                license_text += f' (<ext-link ext-link-type="uri" href="{copyright_info["license_url"]}">CC BY 4.0</ext-link>)'
+            license_text += ', which permits unrestricted use, distribution, and reproduction in any medium, provided the original author and source are credited.'
+            license_p.text = license_text
+            license_el.append(license_p)
+            permissions.append(license_el)
+        am.append(permissions)
+    
+    return am
+
+def map_title(title_text):
+    tg = E("title-group")
+    tg.append(E("article-title", title_text))
+    return tg
+
+def parse_authors_line(line):
+    """
+    Parse author lines like:
+    "Xiang Chang1, Ming Li2, Depeng Sun1, Chuandong Chen2"
+    Returns list of dicts: {surname, given_names, aff_refs:[1,], notes: ["*", "#"], corresp:bool}
+    """
+    authors = []
+    clean_line = re.sub(r'\[\d+\]\(https://orcid\.org/[^)]+\)', '', line)
+    clean_line = re.sub(r'\s+', ' ', clean_line).strip()
+    
+    author_pattern = r'([A-Z][a-z]+\s+[A-Z][a-z]+)(\d*)'
+    matches = re.findall(author_pattern, clean_line)
+    
+    for name_part, aff_num in matches:
+        name_part = name_part.strip()
+        if not name_part:
+            continue
+            
+        name_parts = name_part.split()
+        if len(name_parts) >= 2:
+            given_names = " ".join(name_parts[:-1])
+            surname = name_parts[-1]
+        else:
+            given_names = ""
+            surname = name_parts[0] if name_parts else ""
+        
+        affs = [aff_num] if aff_num else []
+        
+        if surname:
+            authors.append({
+                "surname": surname, 
+                "given_names": given_names, 
+                "affs": affs, 
+                "notes": []
+            })
+    
+    return authors
+
+def build_contrib_group(authors_list, affs_map, correspondences):
+    """
+    authors_list: list of dicts with fields from parse_authors_line (or already structured)
+    affs_map: dict id->aff dict (id like 'aff1')
+    correspondences: list of dicts {name, email, orcid}
+    """
+    cg = E("contrib-group")
+    for idx, a in enumerate(authors_list, start=1):
+        attrs = {"contrib-type":"author"}
+        name_text = a.get("given_names","")
+        surname = a.get("surname","")
+        if any(c.get("name") and c["name"].split()[-1]==surname for c in correspondences):
+            attrs["corresp"] = "yes"
+        contrib = E("contrib", **attrs)
+        name_el = E("name")
+        if surname:
+            name_el.append(E("surname", surname))
+        if name_text:
+            name_el.append(E("given-names", name_text))
+        contrib.append(name_el)
+        for affid in a.get("affs", []):
+            x = E("xref", **{"ref-type":"aff", "rid": f"aff{affid}"})
+            sup = E("sup", affid)
+            x.append(sup)
+            contrib.append(x)
+        for note_sym in a.get("notes", []):
+            x = E("xref", **{"ref-type":"author-notes", "rid":"FN1"})
+            x.text = note_sym
+            contrib.append(x)
+        cg.append(contrib)
+    for aid, info in sorted(affs_map.items(), key=lambda x: x[0]):
+        aff = E("aff", **{"id":aid})
+        aff.append(E("label", info.get("label", aid.replace("aff",""))))
+        aff.append(E("institution", info.get("institution","")))
+        if info.get("city"):
+            aff.append(E("city", info["city"]))
+        if info.get("postal-code"):
+            aff.append(E("postal-code", info["postal-code"]))
+        if info.get("state"):
+            aff.append(E("state", info["state"]))
+        if info.get("country"):
+            aff.append(E("country", info["country"]))
+        cg.append(aff)
+    return cg
+
+def map_abstract(paragraphs):
+    abs_el = E("abstract")
+    for p in paragraphs:
+        abs_el.append(E("p", p))
+    return abs_el
+
+def map_keywords(keywords_list):
+    kg = E("kwd-group", **{"kwd-group-type":"author"})
+    kg.append(E("title", "Keywords:"))
+    for k in keywords_list:
+        kg.append(E("kwd", k.strip()))
+    return kg
+
+def map_abbreviations(abbrev_items):
+    notes = E("notes")
+    dlist = E("def-list", **{"list-type":"simple","list-content":"abbreviations"})
+    dlist.append(E("title","Abbreviations"))
+    for term, expansion in abbrev_items:
+        di = E("def-item")
+        di.append(E("term", term))
+        def_el = E("def"); def_el.append(E("p", expansion))
+        di.append(def_el)
+        dlist.append(di)
+    notes.append(dlist)
+    return notes
+
+def map_section(title, sec_type=None, level=1):
+    """
+    Rule 16: Multi-level section hierarchy support
+    Level 3+ sections get italic titles
+    """
+    sec = E("sec")
+    if sec_type:
+        sec.set("sec-type", sec_type)
+    
+    title_el = E("title")
+    if level >= 3:
+        title_el.text = f"<italic>{title}</italic>"
+    else:
+        title_el.text = title
+    sec.append(title_el)
+    return sec
+
+def map_paragraph(text):
+    return E("p", text)
+
+def map_list(items, list_type="simple"):
+    l = E("list", **{"list-type": list_type})
+    for it in items:
+        li = E("list-item"); li.append(E("p", it))
+        l.append(li)
+    return l
+
+def inline_format_to_jats(text):
+    text = re.sub(r'\*\*(.+?)\*\*', lambda m: f"<bold>{m.group(1)}</bold>", text)
+    text = re.sub(r'\*(.+?)\*', lambda m: f"<italic>{m.group(1)}</italic>", text)
+    text = re.sub(r'\^\{(.+?)\}', lambda m: f"<sup>{m.group(1)}</sup>", text)
+    text = re.sub(r'_\{(.+?)\}', lambda m: f"<sub>{m.group(1)}</sub>", text)
+    text = re.sub(r'\[([^\]]+)\]\((http[s]?://[^\)]+)\)', lambda m: f'<ext-link ext-link-type="uri" href="{m.group(2)}">{m.group(1)}</ext-link>', text)
+    return text
+
+def build_figure(caption_text, filename=None, fid=1, article_id="", year=""):
+    """
+    Rules 19-21: Enhanced figure formatting with bold first sentence and proper file naming
+    """
+    fig = E("fig", **{"id": f"F{fid}", "position":"float", "fig-type":"figure"})
+    fig.append(E("label", f"Figure {fid}"))
+    
+    cap = E("caption")
+    
+    cap_title = E("title")
+    cap_title.text = caption_text.strip()
+    cap.append(cap_title)
+    fig.append(cap)
+    
+    graphic_el = E("graphic")
+    
+    if filename:
+        graphic_el.set("{http://www.w3.org/1999/xlink}href", filename)
+    elif article_id and year:
+        generated_filename = f"aging-{year}-{article_id}-g{fid:03d}.tif"
+        graphic_el.set("{http://www.w3.org/1999/xlink}href", generated_filename)
+    else:
+        default_filename = f"figure-{fid}.jpg"
+        graphic_el.set("{http://www.w3.org/1999/xlink}href", default_filename)
+    
+    fig.append(graphic_el)
+    
+    return fig
+
+
+def map_inline_formula(mathml_string):
+    """
+    Wrap inline math as <inline-formula>.
+    """
+    el = E("inline-formula")
+    try:
+        m = etree.fromstring(mathml_string.encode())
+        el.append(m)
+    except Exception:
+        p = etree.SubElement(el, "p")
+        p.text = mathml_string
+    return el
+
+
+def map_disp_formula(mathml_string, eq_num=None, eq_id=None):
+    """
+    Wrap display math as <disp-formula>.
+    Adds optional id and label.
+    """
+    el = E("disp-formula")
+
+    if eq_id:
+        el.set("id", eq_id)
+    elif eq_num:
+        el.set("id", f"eq{eq_num}")
+
+    if eq_num:
+        label = etree.SubElement(el, "label")
+        label.text = f"({eq_num})"
+
+    try:
+        m = etree.fromstring(mathml_string.encode())
+        el.append(m)
+    except Exception:
+        p = etree.SubElement(el, "p")
+        p.text = mathml_string
+
+    return el
+
+
+def build_table_from_md(md_table_text, tid=1):
+    lines = [ln.strip() for ln in md_table_text.strip().splitlines() if ln.strip()]
+    if not lines:
+        return None
+    
+    table_lines = []
+    for line in lines:
+        if line.count('|') >= 2:
+            table_lines.append(line)
+    
+    if len(table_lines) < 2:
+        return None
+    
+    header_line = table_lines[0]
+    headers = [h.strip() for h in re.split(r'\s*\|\s*', header_line.strip('| ')) if h.strip()]
+    
+    data_start = 1
+    if len(table_lines) > 1 and '-' in table_lines[1]:
+        data_start = 2
+    
+    rows = []
+    for ln in table_lines[data_start:]:
+        cells = [c.strip() for c in re.split(r'\s*\|\s*', ln.strip('| '))]
+        while len(cells) < len(headers):
+            cells.append("")
+        rows.append(cells[:len(headers)])
+    
+    if not rows:
+        return None
+    
+    table_wrap = E("table-wrap", **{"id": f"t{tid}"})
+    table_wrap.append(E("label", f"Table {tid}"))
+    caption = E("caption")
+    caption.append(E("p", f"Table {tid} data"))
+    table_wrap.append(caption)
+    
+    table = E("table")
+    
+    thead = E("thead")
+    trh = E("tr")
+    for h in headers:
+        clean_h = re.sub(r'<br>', ' ', h)
+        clean_h = re.sub(r'\*\*(.+?)\*\*', r'\1', clean_h)
+        trh.append(E("th", clean_h))
+    thead.append(trh)
+    table.append(thead)
+    
+    tbody = E("tbody")
+    for r in rows:
+        tr = E("tr")
+        for c in r:
+            clean_c = re.sub(r'<br>', ' ', c)
+            clean_c = re.sub(r'\*\*(.+?)\*\*', r'\1', clean_c)
+            tr.append(E("td", clean_c))
+        tbody.append(tr)
+    table.append(tbody)
+    
+    table_wrap.append(table)
+    return table_wrap
+
+def build_supplementary(items):
+    sec = E("sec", **{"sec-type":"supplementary-material"})
+    sec.append(E("title","Supplementary Materials"))
+    for idx, it in enumerate(items, start=1):
+        sid = f"SD{idx}"
+        sm = E("supplementary-material", **{"content-type":"local-data", "id":sid})
+        sm.append(E("label", it.get("label", f"Supplementary {idx}")))
+        media = E("media", **{
+            "mimetype":"application",
+            "mime-subtype": it.get("mime-subtype","pdf"),
+            "orientation":"portrait",
+            "position":"anchor"
+        })
+        media.set("{http://www.w3.org/1999/xlink}href", it.get("filename", ""))
+        media.set("{http://www.w3.org/1999/xlink}type", "simple")
+        sm.append(media)
+        sec.append(sm)
+    return sec
+
+def build_fn(label, paragraph, fn_type="con"):
+    fn = E("fn", **{"fn-type": fn_type})
+    fn.append(E("label", label))
+    fn.append(E("p", paragraph))
+    return fn
+
+def build_ack(text):
+    ack = E("ack"); ack.append(E("title","ACKNOWLEDGMENTS")); ack.append(E("p", text))
+    return ack
+
+def build_ref_list(refs):
+    """
+    Rules 34-38: Enhanced reference formatting with proper author parsing
+    """
+    rl = E("ref-list"); rl.append(E("title","REFERENCES"))
+    for idx, r in enumerate(refs, start=1):
+        ref = E("ref", **{"id": f"r{idx}"})
+        ref.append(E("label", str(idx)))
+        
+        parsed_ref = parse_reference(r)
+        mc = E("mixed-citation", **{"publication-type": parsed_ref.get("type", "journal")})
+        
+        if parsed_ref.get("authors"):
+            person_group = E("person-group", **{"person-group-type": "author"})
+            for author in parsed_ref["authors"]:
+                name_el = E("name")
+                if author.get("surname"):
+                    name_el.append(E("surname", author["surname"]))
+                if author.get("given_names"):
+                    name_el.append(E("given-names", author["given_names"]))
+                person_group.append(name_el)
+            mc.append(person_group)
+        
+        if parsed_ref.get("title"):
+            mc.append(E("article-title", parsed_ref["title"]))
+        
+        if parsed_ref.get("source"):
+            mc.append(E("source", parsed_ref["source"]))
+        
+        if parsed_ref.get("year"):
+            mc.append(E("year", str(parsed_ref["year"])))
+        if parsed_ref.get("volume"):
+            mc.append(E("volume", str(parsed_ref["volume"])))
+        if parsed_ref.get("issue"):
+            mc.append(E("issue", str(parsed_ref["issue"])))
+        if parsed_ref.get("fpage") and parsed_ref.get("lpage"):
+            mc.append(E("fpage", str(parsed_ref["fpage"])))
+            mc.append(E("lpage", str(parsed_ref["lpage"])))
+        
+        if not any([parsed_ref.get("authors"), parsed_ref.get("title"), parsed_ref.get("source")]):
+            mc.append(E("p", r))
+        
+        ref.append(mc)
+        rl.append(ref)
+    return rl
+
+def parse_reference(ref_text):
+    """
+    Enhanced reference parser for Rules 34-38
+    """
+    result = {"type": "journal", "authors": [], "title": "", "source": "", "year": "", "volume": "", "issue": "", "fpage": "", "lpage": ""}
+    
+    
+    year_match = re.search(r'\b(19|20)\d{2}\b', ref_text)
+    if year_match:
+        result["year"] = year_match.group()
+    
+    vol_issue_match = re.search(r'(\d+)\((\d+)\)', ref_text)
+    if vol_issue_match:
+        result["volume"] = vol_issue_match.group(1)
+        result["issue"] = vol_issue_match.group(2)
+    
+    page_match = re.search(r':(\d+)[-–](\d+)', ref_text)
+    if page_match:
+        result["fpage"] = page_match.group(1)
+        result["lpage"] = page_match.group(2)
+    
+    parts = ref_text.split('.')
+    if len(parts) > 0:
+        author_part = parts[0].strip()
+        author_names = re.split(r',\s*|\s+and\s+', author_part)
+        for name in author_names[:3]:
+            name = name.strip()
+            if name:
+                if ',' in name:
+                    surname, given = name.split(',', 1)
+                    result["authors"].append({"surname": surname.strip(), "given_names": given.strip()})
+                else:
+                    name_parts = name.split()
+                    if len(name_parts) >= 2:
+                        surname = name_parts[-1]
+                        given = ' '.join(name_parts[:-1])
+                        result["authors"].append({"surname": surname, "given_names": given})
+    
+    if len(parts) > 1:
+        result["title"] = parts[1].strip()
+    
+    if len(parts) > 2:
+        source_part = parts[2].strip()
+        source_clean = re.sub(r'\b(19|20)\d{2}\b.*', '', source_part).strip()
+        result["source"] = source_clean
+    
+    return result
+
+def process_cross_references(text):
+    """
+    Rules 40-42: Complete cross-reference handling with proper rid attributes
+    """
+    text = re.sub(r'Figure\s+(\d+)', lambda m: f'<xref ref-type="fig" rid="f{m.group(1)}">Figure {m.group(1)}</xref>', text)
+    text = re.sub(r'Table\s+(\d+)', lambda m: f'<xref ref-type="table" rid="t{m.group(1)}">Table {m.group(1)}</xref>', text)
+    
+    text = re.sub(r'Supplementary\s+(?:Material\s+)?(\d+)', lambda m: f'<xref ref-type="supplementary-material" rid="SD{m.group(1)}">Supplementary Material {m.group(1)}</xref>', text)
+    
+    text = re.sub(r'\[(\d+)\]', lambda m: f'<xref ref-type="bibr" rid="r{m.group(1)}">{m.group(1)}</xref>', text)
+    
+    text = re.sub(r'\[(\d+)-(\d+)\]', lambda m: f'<xref ref-type="bibr" rid="r{m.group(1)}">r{m.group(1)}&#x2013;{m.group(2)}</xref>', text)
+    
+    def replace_multi_refs(match):
+        refs = match.group(1).split(',')
+        xrefs = []
+        for ref in refs:
+            ref = ref.strip()
+            xrefs.append(f'<xref ref-type="bibr" rid="r{ref}">{ref}</xref>')
+        return ','.join(xrefs)
+    
+    text = re.sub(r'\[([0-9,\s]+)\]', replace_multi_refs, text)
+    
+    return text
+
+def enhance_inline_formatting(text):
+    """
+    Rule 39: Enhanced external links and Rule 18: Complete inline formatting
+    """
+    text = re.sub(r'\*\*(.+?)\*\*', lambda m: f"<bold>{m.group(1)}</bold>", text)
+    text = re.sub(r'\*(.+?)\*', lambda m: f"<italic>{m.group(1)}</italic>", text)
+    text = re.sub(r'_(.+?)_', lambda m: f"<underline>{m.group(1)}</underline>", text)
+    text = re.sub(r'\^\{(.+?)\}', lambda m: f"<sup>{m.group(1)}</sup>", text)
+    text = re.sub(r'_\{(.+?)\}', lambda m: f"<sub>{m.group(1)}</sub>", text)
+    
+    text = re.sub(r'\[([^\]]+)\]\((http[s]?://[^\)]+)\)', 
+                  lambda m: f'<ext-link ext-link-type="uri" href="{m.group(2)}">{m.group(1)}</ext-link>', text)
+    
+    text = process_cross_references(text)
+    
+    return text

app/modules/documents/processors/xml/xml_compiler.py

@@ -0,0 +1,279 @@
+#!/usr/bin/env python3
+"""
+XML Compiler Application - Uses Stylesheet.exe to transform XML to HTML with JATS styling
+Leverages 30 years of experience in XML/XSLT processing
+"""
+
+import os
+import sys
+import subprocess
+import tempfile
+import shutil
+from pathlib import Path
+import argparse
+import logging
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+class XMLCompiler:
+    """
+    Professional XML to HTML compiler using Stylesheet.exe and jats-html.xsl
+    Built with 30 years of XML processing experience
+    """
+    
+    def __init__(self, base_dir=None):
+        """Initialize the XML compiler with paths to required files"""
+        if base_dir is None:
+            current_dir = Path(__file__).parent
+            base_dir = current_dir.parent / "assets" / "stylesheet"
+        else:
+            base_dir = Path(base_dir)
+            
+        self.base_dir = base_dir
+        self.stylesheet_exe = base_dir / "Stylesheet.exe" 
+        self.jats_xsl = base_dir / "jats-html.xsl"
+        self.stylesheet_dll = base_dir / "Stylesheet.dll"
+        self.runtime_config = base_dir / "Stylesheet.runtimeconfig.json"
+        
+        self._validate_files()
+        
+    def _validate_files(self):
+        """Validate that all required files exist"""
+        required_files = [
+            (self.jats_xsl, "jats-html.xsl"),
+        ]
+        
+        has_exe = self.stylesheet_exe.exists()
+        has_dll = self.stylesheet_dll.exists()
+        
+        if not (has_exe or has_dll):
+            logger.warning("Neither Stylesheet.exe nor Stylesheet.dll found")
+        
+        missing_files = []
+        for file_path, name in required_files:
+            if not file_path.exists():
+                missing_files.append(name)
+                
+        if missing_files:
+            logger.warning(f"Missing files: {', '.join(missing_files)}")
+            
+        logger.info("✅ XML Compiler initialized")
+        
+    def compile_xml_to_html(self, xml_content, output_path=None):
+        """
+        Compile XML content to HTML using Stylesheet.exe and jats-html.xsl
+        
+        Args:
+            xml_content (str): The XML content to transform
+            output_path (str, optional): Path for output HTML file
+            
+        Returns:
+            str: The generated HTML content
+        """
+        logger.info("🔄 Starting XML to HTML compilation...")
+        
+        with tempfile.TemporaryDirectory() as temp_dir:
+            temp_dir = Path(temp_dir)
+            
+            xml_file = temp_dir / "input.xml"
+            with open(xml_file, 'w', encoding='utf-8') as f:
+                f.write(xml_content)
+                
+            if output_path:
+                html_file = Path(output_path)
+                html_file.parent.mkdir(parents=True, exist_ok=True)
+            else:
+                html_file = temp_dir / "output.html"
+                
+            success = False
+            
+                
+            if not success:
+                success = self._try_python_lxml(xml_content, html_file)
+                
+            if success and html_file.exists():
+                with open(html_file, 'r', encoding='utf-8') as f:
+                    html_content = f.read()
+                logger.info("✅ Successfully compiled XML to HTML")
+                return html_content
+            else:
+                raise RuntimeError("❌ All compilation methods failed")
+                
+    def _try_stylesheet_exe(self, xml_file, html_file):
+        """Try using Stylesheet.exe directly"""
+        if not self.stylesheet_exe.exists():
+            return False
+            
+        try:
+            logger.info("🔧 Trying Stylesheet.exe...")
+            
+            commands = [
+                [str(self.stylesheet_exe), str(xml_file), str(self.jats_xsl), str(html_file)],
+                [str(self.stylesheet_exe), "-xml", str(xml_file), "-xsl", str(self.jats_xsl), "-out", str(html_file)],
+                [str(self.stylesheet_exe), "transform", str(xml_file), str(self.jats_xsl), str(html_file)]
+            ]
+            
+            for i, cmd in enumerate(commands, 1):
+                logger.info(f"Trying command pattern {i}: {' '.join(cmd)}")
+                
+                result = subprocess.run(
+                    cmd,
+                    cwd=str(self.base_dir),
+                    capture_output=True,
+                    text=True,
+                    timeout=30,
+                    creationflags=subprocess.CREATE_NO_WINDOW if os.name == 'nt' else 0
+                )
+                
+                if result.returncode == 0 and html_file.exists():
+                    logger.info(f"✅ Stylesheet.exe succeeded with pattern {i}")
+                    return True
+                else:
+                    logger.warning(f"Pattern {i} failed - Return code: {result.returncode}")
+                    if result.stderr:
+                        logger.warning(f"STDERR: {result.stderr}")
+                        
+            return False
+            
+        except Exception as e:
+            logger.error(f"Stylesheet.exe failed: {e}")
+            return False
+            
+    def _try_dotnet_dll(self, xml_file, html_file):
+        """Try using dotnet with Stylesheet.dll"""
+        if not self.stylesheet_dll.exists():
+            return False
+            
+        try:
+            logger.info("🔧 Trying dotnet with Stylesheet.dll...")
+            
+            cmd = [
+                "dotnet", str(self.stylesheet_dll),
+                str(xml_file), str(self.jats_xsl), str(html_file)
+            ]
+            
+            logger.info(f"Command: {' '.join(cmd)}")
+            
+            result = subprocess.run(
+                cmd,
+                cwd=str(self.base_dir),
+                capture_output=True,
+                text=True,
+                timeout=120,
+                creationflags=subprocess.CREATE_NO_WINDOW if os.name == 'nt' else 0
+            )
+            
+            if result.returncode == 0 and html_file.exists():
+                logger.info("✅ dotnet Stylesheet.dll succeeded")
+                return True
+            else:
+                logger.warning(f"dotnet failed - Return code: {result.returncode}")
+                if result.stdout:
+                    logger.warning(f"STDOUT: {result.stdout[:500]}")
+                if result.stderr:
+                    logger.warning(f"STDERR: {result.stderr[:500]}")
+                return False
+                
+        except Exception as e:
+            logger.error(f"dotnet Stylesheet.dll failed: {e}")
+            return False
+            
+    def _try_python_lxml(self, xml_content, html_file):
+        """Fallback using Python lxml"""
+        if not self.jats_xsl.exists():
+            return False
+            
+        try:
+            logger.info("🔧 Trying Python lxml fallback...")
+            
+            from lxml import etree as ET
+            
+            secure_parser = ET.XMLParser(resolve_entities=False, no_network=True)
+            
+            xml_doc = ET.fromstring(xml_content.encode('utf-8'), parser=secure_parser)
+            
+            with open(self.jats_xsl, 'r', encoding='utf-8') as f:
+                xsl_content = f.read()
+            xsl_doc = ET.fromstring(xsl_content.encode('utf-8'), parser=secure_parser)
+            
+            transform = ET.XSLT(xsl_doc)
+            result = transform(xml_doc)
+            
+            with open(html_file, 'w', encoding='utf-8') as f:
+                f.write(str(result))
+                
+            logger.info("✅ Python lxml fallback succeeded")
+            return True
+            
+        except Exception as e:
+            logger.error(f"Python lxml fallback failed: {e}")
+            import traceback
+            logger.error(f"Traceback: {traceback.format_exc()[:500]}")
+            return False
+
+def create_html_compiler_app():
+    """Create a new HTML compiler application"""
+    
+    def compile_xml_file(xml_file_path, output_file_path=None):
+        """Compile an XML file to HTML"""
+        compiler = XMLCompiler()
+        
+        with open(xml_file_path, 'r', encoding='utf-8') as f:
+            xml_content = f.read()
+            
+        if not output_file_path:
+            xml_path = Path(xml_file_path)
+            output_file_path = xml_path.with_suffix('.html')
+            
+        html_content = compiler.compile_xml_to_html(xml_content, output_file_path)
+        
+        print(f"✅ Successfully compiled XML to HTML")
+        print(f"📄 Input:  {xml_file_path}")
+        print(f"📄 Output: {output_file_path}")
+        print(f"📏 HTML size: {len(html_content)} characters")
+        
+        return output_file_path
+        
+    return compile_xml_file
+
+def main():
+    """Main application entry point"""
+    parser = argparse.ArgumentParser(
+        description="XML Compiler - Transform XML to HTML using Stylesheet.exe and JATS XSL",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  python xml_compiler.py input.xml
+  python xml_compiler.py input.xml -o output.html
+  python xml_compiler.py input.xml --output styled_document.html
+        """
+    )
+    
+    parser.add_argument('xml_file', help='Input XML file to compile')
+    parser.add_argument('-o', '--output', help='Output HTML file path (optional)')
+    parser.add_argument('-v', '--verbose', action='store_true', help='Enable verbose logging')
+    
+    args = parser.parse_args()
+    
+    if args.verbose:
+        logging.getLogger().setLevel(logging.DEBUG)
+        
+    try:
+        if not os.path.exists(args.xml_file):
+            print(f"❌ Error: Input file '{args.xml_file}' not found")
+            sys.exit(1)
+            
+        compile_app = create_html_compiler_app()
+        
+        output_path = compile_app(args.xml_file, args.output)
+        
+        print(f"\n🎉 Compilation completed successfully!")
+        print(f"💡 Open '{output_path}' in a web browser to view the styled document.")
+        
+    except Exception as e:
+        logger.error(f"❌ Compilation failed: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

app/modules/documents/repositories/document_repository.py

@@ -0,0 +1,81 @@
+from typing import List, Optional, Any
+from sqlalchemy.orm import Session, joinedload
+from app.modules.documents.models.document_model import Project
+from app.modules.auth.models.user_model import User
+from datetime import datetime
+import uuid
+
+
+class ProjectRepository:
+    def __init__(self, db: Session):
+        self.db = db
+
+    def get_by_id(self, project_id: int) -> Optional[Project]:
+        return (
+            self.db.query(Project)
+            .options(joinedload(Project.owner))
+            .filter(Project.id == project_id)
+            .first()
+        )
+
+    def list_by_user(self, user_id: int, tenant_id: uuid.UUID) -> List[Project]:
+        return (
+            self.db.query(Project)
+            .options(joinedload(Project.owner))
+            .filter(Project.user_id == user_id, Project.tenant_id == tenant_id)
+            .order_by(Project.created_at.desc())
+            .all()
+        )
+
+    def create(self, project: Project) -> Project:
+        self.db.add(project)
+        self.db.flush()
+        self.db.refresh(project)
+        return project
+
+    def update(self, project: Project) -> Project:
+        project.updated_at = datetime.utcnow()
+        self.db.flush()
+        self.db.refresh(project)
+        return project
+
+    def delete(self, project: Project):
+        self.db.delete(project)
+
+    def get_detailed_project(self, project_id: int) -> Optional[Project]:
+        """Production Optimized: Fetch project with versions and owner in one query."""
+        return (
+            self.db.query(Project)
+            .options(joinedload(Project.owner), joinedload(Project.versions))
+            .filter(Project.id == project_id)
+            .first()
+        )
+
+    def get_drive_acl(self, project_id: int, user_id: int) -> Optional[Any]:
+        from app.modules.drive.models.drive_model import DriveACL
+
+        return (
+            self.db.query(DriveACL)
+            .filter(
+                DriveACL.resource_type == "document",
+                DriveACL.resource_id == project_id,
+                DriveACL.subject_id == user_id,
+            )
+            .first()
+        )
+
+    def get_by_name_and_user(
+        self, name: str, user_id: int, tenant_id: uuid.UUID
+    ) -> Optional[Project]:
+        return (
+            self.db.query(Project)
+            .filter(
+                Project.name == name,
+                Project.user_id == user_id,
+                Project.tenant_id == tenant_id,
+            )
+            .first()
+        )
+
+    def get_user_by_email(self, email: str) -> Optional[User]:
+        return self.db.query(User).filter(User.email == email).first()

app/modules/documents/repositories/version_repository.py

@@ -0,0 +1,55 @@
+from typing import List, Optional
+from sqlalchemy.orm import Session
+from sqlalchemy import func
+from app.modules.documents.models.document_model import ProjectVersion, Project
+
+class VersionRepository:
+    def __init__(self, db: Session):
+        self.db = db
+
+    def list_by_project(self, project_id: int) -> List[ProjectVersion]:
+        return (
+            self.db.query(ProjectVersion)
+            .filter(ProjectVersion.project_id == project_id)
+            .order_by(ProjectVersion.version_number.desc())
+            .all()
+        )
+
+    def get_by_id_and_project(self, version_id: int, project_id: int) -> Optional[ProjectVersion]:
+        return self.db.query(ProjectVersion).filter(
+            ProjectVersion.id == version_id,
+            ProjectVersion.project_id == project_id
+        ).first()
+
+    def get_max_version_number(self, project_id: int) -> int:
+        max_v = self.db.query(func.max(ProjectVersion.version_number)).filter(
+            ProjectVersion.project_id == project_id
+        ).scalar()
+        return max_v or 0
+
+    def set_all_not_current(self, project_id: int):
+        self.db.query(ProjectVersion).filter(
+            ProjectVersion.project_id == project_id
+        ).update({"is_current": False})
+
+    def create(self, version: ProjectVersion) -> ProjectVersion:
+        self.db.add(version)
+        self.db.flush()
+        self.db.refresh(version)
+        return version
+
+    def update(self):
+        self.db.flush()
+
+    def delete(self, version: ProjectVersion):
+        self.db.delete(version)
+        self.db.flush()
+
+    def get_project(self, project_id: int) -> Optional[Project]:
+        return self.db.query(Project).filter(Project.id == project_id).first()
+
+    def get_project_with_lock(self, project_id: int) -> Optional[Project]:
+        """Get project with SELECT FOR UPDATE lock."""
+        return self.db.query(Project).filter(Project.id == project_id).with_for_update().first()
+
+

app/modules/documents/routes/comment_routes.py

@@ -0,0 +1,43 @@
+from fastapi import APIRouter, Depends, Form
+from typing import Optional
+from app.db.database import get_db
+from app.modules.auth.models.user_model import User
+from app.middleware.auth import get_current_user
+from app.modules.documents.schemas.document_schema import ProjectCommentsOut
+
+router = APIRouter(prefix="/projects", tags=["Project Comments"])
+
+@router.get("/{project_id}/comments", response_model=ProjectCommentsOut)
+def list_project_comments(project_id: int, conn = Depends(get_db), user: User = Depends(get_current_user)):
+    from app.modules.drive.controllers.comment_controller import DriveCommentController
+    return DriveCommentController.list_comments(project_id, user, conn, resource_type="document")
+
+@router.post("/{project_id}/comments")
+def create_project_comment(
+    project_id: int,
+    body: str = Form(...),
+    anchor: Optional[str] = Form(None),
+    color: Optional[str] = Form(None),
+    conn = Depends(get_db),
+    user: User = Depends(get_current_user)
+):
+    import json
+    from fastapi import HTTPException
+    from app.modules.drive.controllers.comment_controller import DriveCommentController
+    
+    anchor_dict = None
+    if anchor:
+        try:
+            anchor_dict = json.loads(anchor)
+            if not isinstance(anchor_dict, dict):
+                 raise HTTPException(status_code=400, detail="Invalid anchor format: Expected a JSON object.")
+        except json.JSONDecodeError:
+            raise HTTPException(status_code=400, detail="Invalid anchor format: Malformed JSON.")
+
+    return DriveCommentController.create_comment(project_id, user, body, anchor_dict, color, conn, resource_type="document")
+
+@router.delete("/comments/{comment_id}")
+def delete_project_comment(comment_id: int, conn = Depends(get_db), user: User = Depends(get_current_user)):
+    from app.modules.drive.controllers.comment_controller import DriveCommentController
+    return DriveCommentController.delete_comment(comment_id, user, conn)
+

app/modules/documents/routes/details_routes.py

@@ -0,0 +1,29 @@
+from fastapi import APIRouter, Depends
+from app.db.database import get_db
+from app.modules.auth.models.user_model import User
+from app.middleware.auth import get_current_user
+from app.modules.documents.schemas.document_schema import (
+    ProjectActivityOut,
+    ProjectDetailsOut,
+)
+router = APIRouter(prefix="/me/projects", tags=["Project Details"])
+
+@router.get("/{project_id}/activity", response_model=ProjectActivityOut)
+def get_project_activity(
+    project_id: int,
+    limit: int = 50,
+    conn = Depends(get_db),
+    user: User = Depends(get_current_user)
+):
+    from app.modules.drive.controllers.activity_controller import DriveActivityController
+    return DriveActivityController.list_activities(project_id, user, conn, resource_type="document", limit=limit)
+
+@router.get("/{project_id}/details", response_model=ProjectDetailsOut)
+def get_project_details(
+    project_id: int,
+    conn = Depends(get_db),
+    user: User = Depends(get_current_user)
+):
+    from app.modules.documents.controllers.document_controller import ProjectController
+    return ProjectController.get_project_extended_details(project_id, user, conn)
+

app/modules/documents/routes/document_routes.py

@@ -0,0 +1,69 @@
+from fastapi import APIRouter, Depends, Request
+from typing import List
+from sqlalchemy.orm import Session
+
+from app.db.database import get_db
+from app.middleware.auth import get_current_user
+from app.modules.auth.models.user_model import User
+from app.modules.auth.dependencies.access_dependency import require_access, require_any_access
+from app.modules.documents.models.document_model import Project
+from app.modules.documents.schemas.document_schema import (
+    CreateProjectIn,
+    ProjectDetailOut,
+    ProjectOut,
+    UpdateProjectIn,
+)
+from app.modules.documents.controllers.document_controller import ProjectController
+from app.core.schemas import MessageOut
+
+router = APIRouter(prefix="/me/projects", tags=["Projects"])
+
+
+@router.get("", response_model=List[ProjectOut], dependencies=[require_any_access("document.read", "document.conversion", "drive.folder.read")])
+def list_projects(
+    db: Session = Depends(get_db),
+    current_user: User = Depends(get_current_user)
+):
+    return ProjectController.list_projects(current_user, db)
+
+
+@router.post("", response_model=ProjectOut, dependencies=[require_access("document.conversion")])
+@router.post("/", response_model=ProjectOut, dependencies=[require_access("document.conversion")])
+def create_project(
+    payload: CreateProjectIn,
+    request: Request,
+    db: Session = Depends(get_db),
+    current_user: User = Depends(get_current_user)
+):
+    return ProjectController.create_project(current_user, payload, request, db)
+
+
+@router.get("/{project_id}", dependencies=[require_any_access("document.read", "document.conversion", "drive.folder.read")], response_model=ProjectDetailOut)
+def get_project(
+    project_id: int,
+    request: Request,
+    db: Session = Depends(get_db),
+    current_user: User = Depends(get_current_user)
+):
+    return ProjectController.get_project_by_id(project_id, current_user, request, db)
+
+
+@router.patch("/{project_id}", dependencies=[require_access("document.conversion")])
+def update_project(
+    project_id: int,
+    payload: UpdateProjectIn,
+    request: Request,
+    db: Session = Depends(get_db),
+    current_user: User = Depends(get_current_user)
+):
+    return ProjectController.update_project(project_id, current_user, payload, request, db)
+
+
+@router.delete("/{project_id}", dependencies=[require_access("document.conversion")], response_model=MessageOut)
+def delete_project(
+    project_id: int,
+    request: Request,
+    db: Session = Depends(get_db),
+    current_user: User = Depends(get_current_user)
+):
+    return ProjectController.delete_project(project_id, current_user, request, db)

app/modules/documents/routes/export_routes.py

@@ -0,0 +1,104 @@
+from fastapi import APIRouter, Body, HTTPException, Depends
+from sqlalchemy.orm import Session
+from app.db.database import get_db
+from app.middleware.auth import get_current_user
+from app.modules.auth.dependencies.access_dependency import require_any_access
+from app.modules.auth.models.user_model import User
+from app.modules.documents.controllers.export_controller import ExportController
+
+from app.modules.documents.schemas.export_schema import (
+    ExportHTMLRequest, 
+    ExportSimpleHTMLRequest, 
+    ExportXMLRequest, 
+    ExportDocxRequest, 
+    ExportPdfRequest
+)
+from app.modules.documents.schemas.export_schema import DocxExportOut
+
+router = APIRouter(tags=["Export"])
+
+@router.post(
+    "/export-xml/",
+    dependencies=[
+        require_any_access(
+            "document.conversion.pdf.to.xml",
+            "document.conversion.pdf.write",
+            "document.conversion",
+        )
+    ],
+)
+async def convert_to_xml(
+    payload: ExportXMLRequest,
+    current_user: User = Depends(get_current_user),
+    db: Session = Depends(get_db)
+):
+    return ExportController.convert_to_xml(payload, current_user, db)
+
+@router.post(
+    "/export-html/",
+    dependencies=[
+        require_any_access(
+            "document.conversion.pdf.to.html",
+            "document.conversion.pdf.write",
+            "document.conversion",
+        )
+    ],
+)
+async def export_html(
+    payload: ExportHTMLRequest,
+    current_user: User = Depends(get_current_user),
+    db: Session = Depends(get_db)
+):
+    return ExportController.export_html(payload, current_user, db)
+
+@router.post(
+    "/export-simple-html/",
+    dependencies=[
+        require_any_access(
+            "document.conversion.pdf.to.html",
+            "document.conversion.pdf.write",
+            "document.conversion",
+        )
+    ],
+)
+async def export_simple_html(
+    payload: ExportSimpleHTMLRequest,
+    current_user: User = Depends(get_current_user),
+    db: Session = Depends(get_db)
+):
+    return ExportController.export_simple_html(payload, current_user, db)
+
+@router.post(
+    "/export-docx/",
+    dependencies=[
+        require_any_access(
+            "document.conversion.pdf.to.word",
+            "document.conversion.image.to.word",
+            "document.conversion.pdf.write",
+            "document.conversion",
+        )
+    ],
+ response_model=DocxExportOut)
+async def export_docx(
+    payload: ExportDocxRequest,
+    current_user: User = Depends(get_current_user),
+    db: Session = Depends(get_db)
+):
+    return await ExportController.export_docx(payload, current_user, db)
+
+@router.post(
+    "/export-pdf/",
+    dependencies=[
+        require_any_access(
+            "document.conversion.word.to.pdf",
+            "document.conversion.pdf.write",
+            "document.conversion",
+        )
+    ],
+)
+async def export_pdf(
+    payload: ExportPdfRequest,
+    current_user: User = Depends(get_current_user),
+    db: Session = Depends(get_db)
+):
+    return await ExportController.export_pdf(payload, current_user, db)

app/modules/documents/routes/sharing_routes.py

@@ -0,0 +1,56 @@
+from fastapi import APIRouter, Depends, Request
+from typing import List
+from sqlalchemy.orm import Session
+
+from app.db.database import get_db
+from app.middleware.auth import get_current_user
+from app.modules.auth.models.user_model import User
+from app.modules.auth.dependencies.access_dependency import require_access, require_any_access
+
+from app.modules.documents.schemas.sharing_schema import (
+    ShareProjectRequest,
+    ShareProjectResponse,
+    SharedProjectInfo,
+)
+
+from app.modules.documents.controllers.document_controller import ProjectController
+
+router = APIRouter(prefix="/sharing", tags=["Project Sharing"])
+
+
+@router.post("/", response_model=ShareProjectResponse, dependencies=[require_access("document.conversion")])
+def share_project(
+    payload: ShareProjectRequest,
+    request: Request,
+    db: Session = Depends(get_db),
+    current_user: User = Depends(get_current_user),
+):
+    return ProjectController.share_project(payload, current_user, request, db)
+
+
+@router.get("/{project_id}/shares", response_model=List[ShareProjectResponse], dependencies=[require_access("document.conversion")])
+def list_project_shares(
+    project_id: int,
+    db: Session = Depends(get_db),
+    current_user: User = Depends(get_current_user),
+):
+    return ProjectController.list_project_shares(project_id, current_user, db)
+
+
+@router.get("/shared-with-me", response_model=List[SharedProjectInfo], dependencies=[require_any_access("document.read", "document.conversion", "drive.folder.read")])
+def shared_with_me(
+    db: Session = Depends(get_db),
+    current_user: User = Depends(get_current_user),
+):
+    return ProjectController.get_shared_with_me(current_user, db)
+
+
+@router.delete("/{share_id}", dependencies=[require_access("document.conversion")])
+def remove_share(
+    share_id: int,
+    request: Request,
+    db: Session = Depends(get_db),
+    current_user: User = Depends(get_current_user),
+):
+    return ProjectController.remove_share(share_id, current_user, request, db)
+

app/modules/documents/routes/version_routes.py

@@ -0,0 +1,71 @@
+from fastapi import APIRouter, Depends, Request
+from app.db.database import get_db
+from app.modules.auth.models.user_model import User
+from app.middleware.auth import get_current_user
+from app.modules.documents.controllers.document_controller import ProjectController
+from app.modules.documents.schemas.version_schema import VersionCreateIn, VersionUpdateIn
+
+router = APIRouter(prefix="/projects", tags=["Version Control"])
+
+@router.get("/{project_id}/versions")
+def list_versions(
+    project_id: int,
+    conn = Depends(get_db),
+    user: User = Depends(get_current_user)
+):
+    return ProjectController.list_versions(project_id, user, conn)
+
+@router.post("/{project_id}/versions")
+def create_version(
+    project_id: int,
+    payload: VersionCreateIn,
+    request: Request,
+    conn = Depends(get_db),
+    user: User = Depends(get_current_user)
+):
+    return ProjectController.create_version(project_id, user, payload, request, conn)
+
+@router.get("/{project_id}/versions/{version_id}")
+def get_version(
+    project_id: int,
+    version_id: int,
+    conn = Depends(get_db),
+    user: User = Depends(get_current_user)
+):
+    return ProjectController.get_version(project_id, version_id, user, conn)
+
+@router.post("/{project_id}/versions/{version_id}/restore")
+def restore_version(
+    project_id: int,
+    version_id: int,
+    request: Request,
+    conn = Depends(get_db),
+    user: User = Depends(get_current_user)
+):
+    return ProjectController.restore_version(project_id, version_id, user, request, conn)
+
+@router.patch("/{project_id}/versions/{version_id}")
+def update_version_description(
+    project_id: int,
+    version_id: int,
+    payload: VersionUpdateIn,
+    conn = Depends(get_db),
+    user: User = Depends(get_current_user)
+):
+    return ProjectController.update_version_description(
+        project_id,
+        version_id,
+        user,
+        payload.changes_description,
+        conn
+    )
+
+@router.delete("/{project_id}/versions/{version_id}")
+def delete_version(
+    project_id: int,
+    version_id: int,
+    request: Request,
+    conn = Depends(get_db),
+    user: User = Depends(get_current_user)
+):
+    return ProjectController.delete_version(project_id, version_id, user, conn)

app/modules/documents/schemas/comment_schema.py

@@ -0,0 +1,26 @@
+from pydantic import BaseModel
+from typing import Optional, Dict
+from datetime import datetime
+
+
+class CommentAuthor(BaseModel):
+    id: int
+    name: str
+    email: str
+
+    class Config:
+        from_attributes = True
+
+
+class CommentOut(BaseModel):
+    id: int
+    project_id: int
+    author_id: int
+    author: CommentAuthor
+    body: str
+    created_at: datetime
+    resolved_at: Optional[datetime]
+    anchor: Optional[Dict]
+
+    class Config:
+        from_attributes = True

app/modules/documents/schemas/document_schema.py

@@ -0,0 +1,118 @@
+from pydantic import BaseModel, Field, ConfigDict, field_validator
+from pydantic.alias_generators import to_camel
+from typing import List, Optional
+from datetime import datetime
+
+
+class ProjectOut(BaseModel):
+    model_config = ConfigDict(
+        alias_generator=to_camel,
+        populate_by_name=True,
+        from_attributes=True
+    )
+
+    id: int
+    name: str
+    status: str
+    file_type: str
+    output_format: str
+    progress: int
+    current_step: str
+    session_id: Optional[str]
+    created_at: datetime
+    updated_at: Optional[datetime]
+
+
+class CreateProjectIn(BaseModel):
+    model_config = ConfigDict(
+        alias_generator=to_camel,
+        populate_by_name=True
+    )
+    name: str = Field(..., min_length=1, max_length=255)
+    file_type: Optional[str] = "PDF"
+    output_format: Optional[str] = "EPUB"
+    session_id: Optional[str] = None
+
+    @field_validator("name")
+    @classmethod
+    def normalize_name(cls, value: str) -> str:
+        normalized = value.strip()
+        if not normalized:
+            raise ValueError("Project name cannot be empty")
+        return normalized
+
+
+class UpdateProjectIn(BaseModel):
+    model_config = ConfigDict(
+        alias_generator=to_camel,
+        populate_by_name=True
+    )
+    name: Optional[str] = None
+    status: Optional[str] = None
+    progress: Optional[int] = None
+    current_step: Optional[str] = None
+    session_id: Optional[str] = None
+    output_format: Optional[str] = None
+    html_content: Optional[str] = None
+    markdown_content: Optional[str] = None
+    xml_content: Optional[str] = None
+
+
+class StatsOut(BaseModel):
+    total_projects: int
+    completed: int
+    in_progress: int
+    books_uploaded: int
+
+
+class ProjectDetailOut(ProjectOut):
+    """
+    `GET /api/me/projects/{project_id}`.
+
+    Six fields the endpoint returns that `ProjectOut` never declared. Applying
+    `ProjectOut` directly as a `response_model` would have deleted every one of
+    them from the payload, including the document bodies the editor loads.
+    """
+
+    drive_file_id: Optional[int] = None
+    html_content: Optional[str] = None
+    markdown_content: Optional[str] = None
+    xml_content: Optional[str] = None
+    original_file_info: Optional[dict] = None
+    role: Optional[str] = None
+
+
+class ProjectOwnerOut(BaseModel):
+    """The `owner` object nested in a project's details."""
+
+    model_config = ConfigDict(from_attributes=True, extra="allow")
+
+
+class ProjectDetailsOut(BaseModel):
+    """
+    `GET /api/me/projects/{project_id}/details` — snake_case, assembled by the
+    controller rather than serialised from the model.
+    """
+
+    id: int
+    name: str
+    status: Optional[str] = None
+    file_type: Optional[str] = None
+    progress: int = 0
+    current_step: Optional[str] = None
+    drive_file_id: Optional[int] = None
+    created_at: Optional[datetime] = None
+    updated_at: Optional[datetime] = None
+    owner: Optional[ProjectOwnerOut] = None
+    permission: Optional[str] = None
+    shares_count: int = 0
+    versions_count: int = 0
+    current_version_number: Optional[int] = None
+
+
+class ProjectActivityOut(BaseModel):
+    activities: List[dict] = []
+
+
+class ProjectCommentsOut(BaseModel):
+    comments: List[dict] = []

app/modules/documents/schemas/export_schema.py

@@ -0,0 +1,28 @@
+from pydantic import BaseModel, Field
+from typing import Optional
+
+class ExportHTMLRequest(BaseModel):
+    xml_content: str = Field(..., description="The XML content to convert to HTML")
+    filename: Optional[str] = Field("document", description="The name of the exported file")
+    project_id: Optional[str] = Field(None, description="The ID of the project for activity logging")
+
+class ExportSimpleHTMLRequest(BaseModel):
+    markdown_content: str = Field(..., description="The markdown content to convert to simple HTML")
+
+class ExportXMLRequest(BaseModel):
+    markdown_content: str = Field(..., description="The markdown content to convert to XML/JATS")
+
+class ExportDocxRequest(BaseModel):
+    markdown_content: str = Field(..., description="The markdown content to convert to DOCX")
+    filename: Optional[str] = Field("document", description="The name of the exported file")
+
+class ExportPdfRequest(BaseModel):
+    markdown_content: str = Field(..., description="The markdown content to convert to PDF")
+    filename: Optional[str] = Field("document", description="The name of the exported file")
+
+
+class DocxExportOut(BaseModel):
+    """`POST /api/export-docx/` — the document returned inline, base64 encoded."""
+
+    success: bool
+    docx_content: str

app/modules/documents/schemas/processing_schema.py

@@ -0,0 +1,32 @@
+from pydantic import BaseModel, Field
+from typing import Optional
+from datetime import datetime
+
+
+class UploadResponse(BaseModel):
+    message: str
+    job_id: str
+    task_id: str
+    status: str
+    filename: str
+    file_size: int
+    check_status_url: str
+    celery_task_url: str
+
+
+class TaskStatusResponse(BaseModel):
+    task_id: str
+    state: str
+    status: str
+    progress: int = Field(default=0, ge=0, le=100)
+    message: Optional[str] = None
+    result: Optional[dict] = None
+    error: Optional[str] = None
+
+
+class ProcessingUploadFileIn(BaseModel):
+    """
+    File uploads are handled via FastAPI's UploadFile.
+    Keep this empty or add optional metadata fields later.
+    """
+    pass

app/modules/documents/schemas/sharing_schema.py

@@ -0,0 +1,43 @@
+from pydantic import BaseModel, EmailStr
+from typing import Optional
+from datetime import datetime
+
+
+class ShareProjectRequest(BaseModel):
+    project_id: int
+    user_email: EmailStr
+    permission: str
+
+
+class ShareProjectResponse(BaseModel):
+    id: int
+    project_id: int
+    shared_with_user_id: int
+    shared_with_user_email: EmailStr
+    shared_with_user_name: str
+    permission: str
+    created_at: datetime
+
+    class Config:
+        from_attributes = True
+
+
+class SharedProjectInfo(BaseModel):
+    id: int
+    name: str
+    status: str
+    file_type: str
+    output_format: str
+    progress: int
+    current_step: str
+    session_id: Optional[str]
+    created_at: datetime
+
+    owner_email: EmailStr
+    owner_name: str
+
+    permission: str
+    shared_at: datetime
+
+    class Config:
+        from_attributes = True

app/modules/documents/schemas/tts_schema.py

@@ -0,0 +1,13 @@
+from pydantic import BaseModel, Field
+from typing import Optional
+
+
+class TextToSpeechRequest(BaseModel):
+    text: str = Field(..., min_length=1)
+    filename: str = Field(default="audiobook.mp3", max_length=255)
+    voice_rate: int = Field(default=175, ge=50, le=400)
+    voice_volume: float = Field(default=0.9, ge=0.0, le=1.0)
+
+
+class SetVoiceRequest(BaseModel):
+    voice_id: str

app/modules/documents/schemas/version_schema.py

@@ -0,0 +1,28 @@
+from pydantic import BaseModel, Field
+from typing import Optional
+from datetime import datetime
+
+
+class VersionCreateIn(BaseModel):
+    changes_description: str = Field(default="Updated version", min_length=1, max_length=1000)
+    markdown_content: Optional[str] = Field(default=None, max_length=10_485_760)
+    xml_content: Optional[str] = Field(default=None, max_length=10_485_760)
+    html_content: Optional[str] = Field(default=None, max_length=10_485_760)
+    session_id: Optional[str] = None
+
+
+class VersionUpdateIn(BaseModel):
+    changes_description: str = Field(min_length=1, max_length=1000)
+
+
+class VersionOut(BaseModel):
+    id: int
+    project_id: int
+    version_number: int
+    changes_description: Optional[str]
+    is_current: bool
+    created_by_id: int
+    created_at: datetime
+
+    class Config:
+        from_attributes = True

app/modules/documents/services/base_service.py

@@ -0,0 +1,48 @@
+import json
+import logging
+from typing import Any, Dict, Optional
+from fastapi import HTTPException
+from sqlalchemy.orm import Session
+from app.modules.auth.models.user_model import User
+from app.modules.drive.services.activity_service import ActivityService
+from app.modules.drive.permissions.permission_service import PermissionService
+from app.modules.drive.constants import DriveRole
+
+logger = logging.getLogger(__name__)
+
+class BaseDocumentService:
+    def __init__(self, db: Session):
+        self.db = db
+        self.activity_service = ActivityService(db)
+        self.perms = PermissionService(db)
+
+    def _get_user_role(self, project: Any, user: User) -> str:
+        """Determines the user's role for a project."""
+        role = self.perms.get_effective_role(project.id, user, resource_type="document")
+        return role if role else "none"
+
+    def _require_role(self, project: Any, user: User, allowed_roles: list) -> str:
+        """Ensures the user has one of the allowed roles."""
+        role = self._get_user_role(project, user)
+        if role not in allowed_roles:
+            raise HTTPException(status_code=403, detail="Insufficient permissions")
+        return role
+
+    def _log_activity(self, project_id: int, user_id: int, activity_type: str, description: str, metadata: Dict[str, Any] = None):
+        """Logs document-related activity."""
+        try:
+            if metadata is None:
+                metadata = {}
+            metadata["description"] = description
+            
+            self.activity_service.log(
+                resource_type="document",
+                resource_id=project_id,
+                actor_id=user_id,
+                activity_type=activity_type,
+                metadata=json.dumps(metadata)
+            )
+            with self.db.begin_nested():
+                self.db.flush()
+        except Exception as e:
+            logger.error(f"Failed to log activity: {e}")

app/modules/documents/services/document_service.py

@@ -0,0 +1,261 @@
+from .base_service import BaseDocumentService
+import logging
+import json
+from typing import List, Optional, Dict, Any
+from fastapi import HTTPException, Request
+from sqlalchemy.orm import Session
+from app.modules.documents.models.document_model import Project
+from app.modules.auth.models.user_model import User
+from app.modules.documents.repositories.document_repository import ProjectRepository
+from app.modules.drive.services.activity_service import ActivityService
+from app.modules.drive.services.drive_service import DriveService
+from app.modules.drive.models.drive_model import DriveACL
+
+logger = logging.getLogger(__name__)
+
+
+class ProjectService(BaseDocumentService):
+    def __init__(self, db: Session):
+        super().__init__(db)
+        self.project_repo = ProjectRepository(db)
+        self.drive_service = DriveService(db)
+
+    def list_projects(self, user: User) -> List[Dict[str, Any]]:
+        projects = self.project_repo.list_by_user(user.id, user.tenant_id)
+        return [
+            {
+                "id": p.id,
+                "name": p.name,
+                "status": p.status,
+                "fileType": p.file_type,
+                "outputFormat": p.output_format,
+                "progress": p.progress,
+                "currentStep": p.current_step,
+                "sessionId": p.session_id,
+                "driveFileId": p.drive_file_id,
+                "createdAt": p.created_at.isoformat() if p.created_at else None,
+                "updatedAt": p.updated_at.isoformat() if p.updated_at else None,
+            }
+            for p in projects
+        ]
+
+    def create_project(
+        self, user: User, payload: Any, request: Request
+    ) -> Dict[str, Any]:
+        existing = self.project_repo.get_by_name_and_user(
+            payload.name, user.id, user.tenant_id
+        )
+        if existing:
+            raise HTTPException(
+                status_code=409,
+                detail={
+                    "error": "project_name_already_exists",
+                    "id": existing.id,
+                    "name": existing.name,
+                    "message": "Project name already exists",
+                },
+            )
+
+        root_folder = self.drive_service.get_or_create_root_folder(user)
+
+        drive_file_data = self.drive_service.drive_repo.find_file_by_name_in_folder(
+            root_folder["id"], payload.name
+        )
+
+        if not drive_file_data:
+            from app.modules.drive.models.drive_model import DriveFile
+
+            drive_file = DriveFile(
+                tenant_id=user.tenant_id,
+                folder_id=root_folder["id"],
+                owner_id=user.id,
+                name=payload.name,
+                mime_type="application/x-project",
+                size=0,
+            )
+            self.db.add(drive_file)
+            self.db.flush()
+            drive_file_id = drive_file.id
+        else:
+            drive_file_id = drive_file_data.id
+
+        project = Project(
+            tenant_id=user.tenant_id,
+            user_id=user.id,
+            name=payload.name,
+            file_type=payload.file_type,
+            output_format=payload.output_format,
+            session_id=payload.session_id,
+            drive_file_id=drive_file_id,
+            status="processing",
+            progress=10,
+            current_step="upload",
+        )
+        project = self.project_repo.create(project)
+
+        self._log_activity(
+            project.id, user.id, "create", f"Project '{project.name}' created"
+        )
+
+        return {
+            "id": project.id,
+            "name": project.name,
+            "status": project.status,
+            "fileType": project.file_type,
+            "outputFormat": project.output_format,
+            "progress": project.progress,
+            "currentStep": project.current_step,
+            "sessionId": project.session_id,
+            "driveFileId": project.drive_file_id,
+            "createdAt": project.created_at.isoformat(),
+            "updatedAt": project.updated_at.isoformat() if project.updated_at else None,
+        }
+
+    def get_project(
+        self, project_id: int, user: User, request: Request
+    ) -> Dict[str, Any]:
+        project = self.project_repo.get_detailed_project(project_id)
+        if not project:
+            raise HTTPException(status_code=404, detail="Project not found")
+
+        role = self._get_user_role(project, user)
+        if role == "none":
+            raise HTTPException(status_code=403, detail="Access denied")
+
+        from sqlalchemy import func
+
+        project.updated_at = func.now()
+
+        self._log_activity(
+            project.id, user.id, "view", f"Project '{project.name}' viewed"
+        )
+
+        original_file_info = None
+        if project.drive_file_id:
+            try:
+                original_file_info = self.drive_service.files.view_file(
+                    project.drive_file_id, user
+                )
+            except Exception as e:
+                logger.warning(
+                    f"Could not get original file info for project {project_id}: {e}"
+                )
+
+        return {
+            "id": project.id,
+            "name": project.name,
+            "status": project.status,
+            "fileType": project.file_type,
+            "outputFormat": project.output_format,
+            "progress": project.progress,
+            "currentStep": project.current_step,
+            "sessionId": project.session_id,
+            "driveFileId": project.drive_file_id,
+            "originalFileInfo": original_file_info,
+            "markdownContent": project.markdown_content,
+            "xmlContent": project.xml_content,
+            "htmlContent": project.html_content,
+            "role": role,
+            "createdAt": project.created_at.isoformat() if project.created_at else None,
+            "updatedAt": project.updated_at.isoformat() if project.updated_at else None,
+        }
+
+    def update_project(
+        self, project_id: int, user: User, payload: Any, request: Request
+    ) -> Dict[str, str]:
+        project = self.project_repo.get_by_id(project_id)
+        if not project:
+            raise HTTPException(status_code=404, detail="Project not found")
+
+        self._require_role(project, user, ["owner", "editor"])
+
+        if hasattr(payload, "name") and payload.name:
+            project.name = payload.name
+            if project.drive_file:
+                project.drive_file.name = payload.name
+
+        project = self.project_repo.update(project)
+
+        self._log_activity(
+            project.id, user.id, "update", f"Project '{project.name}' updated"
+        )
+
+        return {"message": "Project updated successfully"}
+
+    def delete_project(
+        self, project_id: int, user: User, request: Request
+    ) -> Dict[str, str]:
+        project = self.project_repo.get_by_id(project_id)
+        if not project:
+            raise HTTPException(status_code=404, detail="Project not found")
+
+        if project.user_id != user.id:
+            raise HTTPException(
+                status_code=403, detail="Only owners can delete projects"
+            )
+
+        project_name = project.name
+        if project.drive_file:
+            self.db.delete(project.drive_file)
+
+        self.project_repo.delete(project)
+
+        self._log_activity(
+            project_id, user.id, "delete", f"Project '{project_name}' deleted"
+        )
+
+        return {"message": "Project deleted successfully"}
+
+    def get_project_extended_details(
+        self, project_id: int, user: User
+    ) -> Dict[str, Any]:
+        project = self.project_repo.get_detailed_project(project_id)
+        if not project:
+            raise HTTPException(status_code=404, detail="Project not found")
+
+        role = self._get_user_role(project, user)
+        if role == "none":
+            raise HTTPException(status_code=403, detail="Access denied")
+
+        versions = project.versions
+        version_count = len(versions)
+        current_version = next((v for v in versions if v.is_current), None)
+
+        from app.modules.drive.models.drive_model import DriveACL
+
+        shares_count = (
+            self.db.query(DriveACL)
+            .filter(
+                DriveACL.resource_type == "document", DriveACL.resource_id == project_id
+            )
+            .count()
+        )
+        owner = project.owner
+
+        return {
+            "id": project.id,
+            "name": project.name,
+            "status": project.status,
+            "file_type": project.file_type,
+            "progress": project.progress or 0,
+            "current_step": project.current_step or "upload",
+            "created_at": (
+                project.created_at.isoformat() if project.created_at else None
+            ),
+            "updated_at": (
+                project.updated_at.isoformat() if project.updated_at else None
+            ),
+            "owner": {
+                "name": owner.name if owner else "Unknown",
+                "email": owner.email if owner else "unknown@example.com",
+            },
+            "versions_count": version_count,
+            "current_version_number": (
+                current_version.version_number
+                if current_version
+                else (1 if version_count > 0 else 0)
+            ),
+            "shares_count": shares_count,
+            "permission": role,
+            "drive_file_id": project.drive_file_id,
+        }

app/modules/documents/services/sharing_service.py

@@ -0,0 +1,266 @@
+import logging
+from datetime import datetime, timezone
+from typing import Any, Dict, List
+
+from fastapi import HTTPException
+from sqlalchemy.orm import Session
+
+from app.core.mail import send_email
+from app.core.settings import settings
+from app.modules.auth.models.user_model import User
+from app.modules.drive.models.drive_model import DriveACL
+from app.modules.drive.services.share_service import ShareService as DriveShareService
+
+from ..repositories.document_repository import ProjectRepository
+from .base_service import BaseDocumentService
+
+logger = logging.getLogger(__name__)
+
+
+class SharingService(BaseDocumentService):
+    def __init__(self, db: Session):
+        super().__init__(db)
+        self.project_repo = ProjectRepository(db)
+
+        from app.modules.drive.repositories.drive_repository import DriveRepository
+
+        self.drive_repo = DriveRepository(db)
+        self.drive_share = DriveShareService(db)
+
+    def _format_permission_label(self, permission: str) -> str:
+        return permission.replace("_", " ").strip().title()
+
+    def _send_project_shared_email(
+        self,
+        project: Any,
+        target_user: User,
+        current_user: User,
+        permission: str,
+    ) -> None:
+        sharer_name = current_user.name or current_user.email
+        recipient_name = target_user.name or target_user.email
+
+        send_email(
+            subject=f'{sharer_name} shared "{project.name}" with you',
+            recipient=target_user.email,
+            template_name="document_shared.html",
+            template_context={
+                "recipient_name": recipient_name,
+                "sharer_name": sharer_name,
+                "document_name": project.name,
+                "permission": self._format_permission_label(permission),
+                "document_url": f"{settings.FRONTEND_URL}/workflow/{project.id}",
+                "year": datetime.now(timezone.utc).year,
+            },
+            db=self.db,
+            tenant_id=project.tenant_id,
+            user_id=current_user.id,
+        )
+
+    def share_project(
+        self, share_request: Any, current_user: User, request: Any
+    ) -> Dict[str, Any]:
+        project = self.project_repo.get_detailed_project(share_request.project_id)
+        if not project:
+            raise HTTPException(status_code=404, detail="Project not found")
+
+        self._require_role(project, current_user, ["owner", "editor"])
+
+        invited_email = share_request.user_email.lower().strip()
+        target_user = self.project_repo.get_user_by_email(invited_email)
+        
+        if not target_user:
+            raise HTTPException(status_code=404, detail="User not found")
+
+        existing_document_share = self.drive_repo.get_acl(
+            "document",
+            project.id,
+            target_user.id if target_user else 0,
+        )
+
+        try:
+            result = self.drive_share.create_share(
+                resource_id=project.id,
+                is_file=False,
+                role=share_request.permission,
+                invited_email=invited_email,
+                user=current_user,
+                resource_type="document",
+                commit=False,
+            )
+
+            if project.drive_file_id:
+                self.drive_share.create_share(
+                    resource_id=project.drive_file_id,
+                    is_file=True,
+                    role=share_request.permission,
+                    invited_email=invited_email,
+                    user=current_user,
+                    resource_type="file",
+                    commit=False,
+                )
+
+            self._log_activity(
+                project.id,
+                current_user.id,
+                "share",
+                f"Project shared with {invited_email}",
+                metadata={"role": share_request.permission},
+            )
+
+
+            if not existing_document_share:
+                try:
+                    self._send_project_shared_email(
+                        project=project,
+                        target_user=target_user,
+                        current_user=current_user,
+                        permission=share_request.permission,
+                    )
+                except Exception as email_error:
+                    logger.error(
+                        "Failed to send project shared email for project_id=%s to %s: %s",
+                        project.id,
+                        invited_email,
+                        email_error,
+                    )
+
+            return {
+                "id": result["id"],
+                "project_id": project.id,
+                "shared_with_user_id": target_user.id,
+                "shared_with_user_email": target_user.email,
+                "shared_with_user_name": target_user.name,
+                "permission": share_request.permission,
+                "created_at": datetime.now(),
+            }
+        except HTTPException:
+            raise
+        except Exception as e:
+            self.db.rollback()
+            if isinstance(e, HTTPException):
+                raise e
+            logger.error(f"Failed to share project: {e}")
+            raise HTTPException(status_code=500, detail="Internal error during sharing")
+
+    def list_project_shares(
+        self, project_id: int, current_user: User
+    ) -> List[Dict[str, Any]]:
+        project = self.project_repo.get_by_id(project_id)
+        if not project:
+            raise HTTPException(status_code=404, detail="Project not found")
+
+        self._require_role(project, current_user, ["owner", "editor"])
+
+        shares = self.drive_repo.get_shares("document", project_id)
+
+        result = [
+            {
+                "id": s.id,
+                "project_id": project_id,
+                "shared_with_user_email": s.subject.email if s.subject else "Unknown",
+                "shared_with_user_name": s.subject.name if s.subject else "Unknown",
+                "shared_with_user_id": s.subject_id,
+                "permission": s.role,
+                "created_at": s.created_at,
+            }
+            for s in shares
+        ]
+            
+        return result
+
+    def get_shared_with_me(self, current_user: User) -> List[Dict[str, Any]]:
+        acls = self.db.query(DriveACL).filter(
+            DriveACL.resource_type == "document",
+            DriveACL.subject_id == current_user.id,
+        ).all()
+
+        results = []
+        for acl in acls:
+            project = self.project_repo.get_detailed_project(acl.resource_id)
+            if project:
+                owner = project.owner
+                results.append(
+                    {
+                        "id": project.id,
+                        "name": project.name,
+                        "status": project.status,
+                        "fileType": project.file_type,
+                        "outputFormat": project.output_format,
+                        "progress": project.progress,
+                        "currentStep": project.current_step,
+                        "sessionId": project.session_id,
+                        "createdAt": (
+                            project.created_at.isoformat() if project.created_at else None
+                        ),
+                        "ownerEmail": owner.email if owner else "Unknown",
+                        "ownerName": owner.name if owner else "Unknown",
+                        "permission": acl.role,
+                        "sharedAt": acl.created_at.isoformat() if acl.created_at else None,
+                    }
+                )
+        return results
+
+    def remove_share(
+        self, share_id: int, current_user: User, request: Any
+    ) -> Dict[str, str]:
+        share = self.db.query(DriveACL).filter(DriveACL.id == share_id).first()
+        if not share:
+            raise HTTPException(status_code=404, detail="Share not found")
+
+        project = self.project_repo.get_by_id(share.resource_id)
+        if not project:
+            raise HTTPException(status_code=404, detail="Project no longer exists")
+
+        self._require_role(project, current_user, ["owner", "editor"])
+
+        try:
+            target_user_id = share.subject_id
+
+            result = self.drive_share.delete_share(share_id, current_user, commit=False)
+
+            if project.drive_file_id:
+                if not is_external:
+                    file_share = self.db.query(DriveACL).filter(
+                        DriveACL.resource_type == "file",
+                        DriveACL.resource_id == project.drive_file_id,
+                        DriveACL.subject_id == target_user_id,
+                    ).first()
+                    if file_share:
+                        self.drive_share.delete_share(
+                            file_share.id,
+                            current_user,
+                            commit=False,
+                        )
+                else:
+                    file_share = self.db.query(DriveShare).filter(
+                        DriveShare.resource_type == "file",
+                        DriveShare.resource_id == project.drive_file_id,
+                        DriveShare.invited_email == share.invited_email,
+                    ).first()
+                    if file_share:
+                        self.drive_share.delete_share(
+                            file_share.id,
+                            current_user,
+                            commit=False,
+                        )
+
+            self._log_activity(
+                project.id,
+                current_user.id,
+                "unshare",
+                f"Share removed for user_id={target_user_id} email={getattr(share, 'invited_email', target_user_id)}",
+            )
+
+            return result
+        except HTTPException:
+            raise
+        except Exception as e:
+            self.db.rollback()
+            if isinstance(e, HTTPException):
+                raise e
+            logger.error(f"Failed to remove share: {e}")
+            raise HTTPException(
+                status_code=500,
+                detail="Internal error during share removal",
+            )

app/modules/documents/services/version_service.py

@@ -0,0 +1,221 @@
+import json
+from typing import List, Dict, Any, Optional
+from fastapi import HTTPException
+from sqlalchemy.orm import Session
+from sqlalchemy import func
+from app.modules.auth.models.user_model import User
+from ..models.document_model import ProjectVersion
+from app.modules.drive.services.activity_service import ActivityService
+from app.modules.drive.models.drive_model import DriveACL
+from ..repositories.version_repository import VersionRepository
+
+from .base_service import BaseDocumentService
+import logging
+
+logger = logging.getLogger(__name__)
+
+class VersionService(BaseDocumentService):
+    def __init__(self, db: Session):
+        super().__init__(db)
+        self.version_repo = VersionRepository(db)
+        from app.modules.drive.services.drive_service import DriveService
+        self.drive_service = DriveService(db)
+
+    def list_versions(self, project_id: int, user: User) -> List[Dict[str, Any]]:
+        project = self.version_repo.get_project(project_id)
+        if not project:
+            raise HTTPException(status_code=404, detail="Project not found")
+
+        self._require_role(project, user, ["owner", "editor", "viewer", "commenter"])
+
+        versions = self.version_repo.list_by_project(project_id)
+        result = []
+        for v in versions:
+            original_info = None
+            if v.drive_file_version_id:
+                 try:
+                     from app.modules.drive.models.drive_model import DriveFileVersion
+                     dfv = self.db.query(DriveFileVersion).filter(DriveFileVersion.id == v.drive_file_version_id).first()
+                     if dfv:
+                         url = self.drive_service.files.storage.presigned_url(dfv.s3_key, user_id=dfv.uploader_id, tenant_id=user.tenant_id)
+                         original_info = {
+                             "version_id": dfv.id,
+                             "version_number": dfv.version_number,
+                             "url": url,
+                             "content_type": dfv.content_type
+                         }
+                 except: pass
+
+            result.append({
+                "id": v.id,
+                "project_id": v.project_id,
+                "created_by_id": v.created_by_id,
+                "version_number": v.version_number,
+                "version_label": v.version_label,
+                "changes_description": v.changes_description,
+                "markdown_content": v.markdown_content,
+                "xml_content": v.xml_content,
+                "html_content": v.html_content,
+                "originalDriveVersionInfo": original_info,
+                "session_id": v.session_id,
+                "is_current": v.is_current,
+                "created_at": v.created_at.isoformat() if v.created_at else None
+            })
+        return result
+
+    def create_version(self, project_id: int, user: User, payload: Any, request: Any) -> Dict[str, Any]:
+        try:
+            project = self.version_repo.get_project_with_lock(project_id)
+            if not project:
+                raise HTTPException(status_code=404, detail="Project not found")
+
+            self._require_role(project, user, ["owner", "editor"])
+
+            next_version_number = self.version_repo.get_max_version_number(project_id) + 1
+            
+            self.version_repo.set_all_not_current(project_id)
+
+            version = ProjectVersion(
+                project_id=project_id,
+                version_number=next_version_number,
+                version_label=f"v{next_version_number}.0",
+                created_by_id=user.id,
+                markdown_content=payload.markdown_content if hasattr(payload, 'markdown_content') else None,
+                xml_content=payload.xml_content if hasattr(payload, 'xml_content') else None,
+                html_content=payload.html_content if hasattr(payload, 'html_content') else None,
+                changes_description=payload.changes_description if hasattr(payload, 'changes_description') else None,
+                session_id=payload.session_id if hasattr(payload, 'session_id') else None,
+                is_current=True
+            )
+            version = self.version_repo.create(version)
+
+            self._log_activity(project_id, user.id, "version_create", f"Created version {next_version_number}", 
+                               {"version": next_version_number})
+            
+
+            return {
+                "id": version.id,
+                "project_id": version.project_id,
+                "created_by_id": version.created_by_id,
+                "version_number": version.version_number,
+                "changes_description": version.changes_description,
+                "is_current": version.is_current,
+                "created_at": version.created_at.isoformat() if version.created_at else None
+            }
+        except HTTPException:
+            raise
+        except Exception as e:
+            self.db.rollback()
+            if isinstance(e, HTTPException):
+                raise e
+            logger.exception("Failed to create version")
+            raise HTTPException(status_code=500, detail="Failed to create version")
+
+    def get_version(self, project_id: int, version_id: int, user: User) -> Dict[str, Any]:
+        project = self.version_repo.get_project(project_id)
+        if not project:
+            raise HTTPException(status_code=404, detail="Project not found")
+
+        self._require_role(project, user, ["owner", "editor", "viewer", "commenter"])
+
+        version = self.version_repo.get_by_id_and_project(version_id, project_id)
+        if not version:
+            raise HTTPException(status_code=404, detail="Version not found")
+
+        original_info = None
+        if version.drive_file_version_id:
+             try:
+                 from app.modules.drive.models.drive_model import DriveFileVersion
+                 dfv = self.db.query(DriveFileVersion).filter(DriveFileVersion.id == version.drive_file_version_id).first()
+                 if dfv:
+                     url = self.drive_service.files.storage.presigned_url(dfv.s3_key, user_id=dfv.uploader_id, tenant_id=user.tenant_id)
+                     original_info = {
+                         "version_id": dfv.id,
+                         "version_number": dfv.version_number,
+                         "url": url,
+                         "content_type": dfv.content_type
+                     }
+             except: pass
+
+        return {
+            "id": version.id,
+            "project_id": version.project_id,
+            "version_number": version.version_number,
+            "version_label": version.version_label,
+            "changes_description": version.changes_description,
+            "markdown_content": version.markdown_content,
+            "xml_content": version.xml_content,
+            "html_content": version.html_content,
+            "originalDriveVersionInfo": original_info,
+            "session_id": version.session_id,
+            "is_current": version.is_current,
+            "created_at": version.created_at.isoformat() if version.created_at else None
+        }
+
+    def restore_version(self, project_id: int, version_id: int, user: User, request: Any) -> Dict[str, str]:
+        try:
+            project = self.version_repo.get_project_with_lock(project_id)
+            if not project:
+                raise HTTPException(status_code=404, detail="Project not found")
+
+            if project.user_id != user.id:
+                raise HTTPException(status_code=403, detail="Only owner can restore")
+
+            version = self.version_repo.get_by_id_and_project(version_id, project_id)
+            if not version:
+                raise HTTPException(status_code=404, detail="Version not found")
+
+            project.markdown_content = version.markdown_content
+            project.xml_content = version.xml_content
+            project.html_content = version.html_content
+            project.updated_at = func.now()
+
+            self.version_repo.set_all_not_current(project_id)
+            version.is_current = True
+            
+            self._log_activity(project_id, user.id, "version_restore", f"Restored version {version.version_number}")
+            
+
+            return {"message": "Version restored successfully"}
+        except HTTPException:
+            raise
+        except Exception as e:
+            self.db.rollback()
+            if isinstance(e, HTTPException):
+                raise e
+            logger.exception("Failed to restore version")
+            raise HTTPException(status_code=500, detail="Failed to restore version")
+
+    def delete_version(self, project_id: int, version_id: int, user: User) -> Dict[str, str]:
+        project = self.version_repo.get_project(project_id)
+        if not project:
+            raise HTTPException(status_code=404, detail="Project not found")
+
+        if project.user_id != user.id:
+            raise HTTPException(status_code=403, detail="Only owner can delete")
+
+        version = self.version_repo.get_by_id_and_project(version_id, project_id)
+        if not version:
+            raise HTTPException(status_code=404, detail="Version not found")
+
+        self.version_repo.delete(version)
+        return {"message": "Version deleted successfully"}
+
+    def update_version_description(self, project_id: int, version_id: int, user: User, description: str) -> Dict[str, str]:
+        if len(description) > 1000:
+            raise HTTPException(status_code=400, detail="Description is too long (max 1000 characters)")
+
+        project = self.version_repo.get_project(project_id)
+        if not project:
+            raise HTTPException(status_code=404, detail="Project not found")
+
+        self._require_role(project, user, ["owner", "editor"])
+
+        version = self.version_repo.get_by_id_and_project(version_id, project_id)
+        if not version:
+            raise HTTPException(status_code=404, detail="Version not found")
+
+        version.changes_description = description
+        self.version_repo.update()
+
+        return {"message": "Version description updated"}

app/modules/documents/tasks/celery_tasks.py

@@ -0,0 +1,373 @@
+"""
+Celery Tasks for Document Processing
+"""
+
+import logging
+from datetime import datetime, timedelta
+from pathlib import Path
+
+from celery import shared_task
+
+import requests
+from app.modules.activity_logs import log_event
+from app.modules.activity_logs.constants import (
+    ActivityLogAction,
+    ActivityLogModule,
+    ActivityLogStatus,
+    ActivityLogTargetType,
+)
+from app.core.cancellation import is_cancelled
+from app.core.progress import publish_progress
+from app.db.database import get_db_connection
+from app.modules.drive.models.drive_model import DriveFile, DriveFileVersion, DriveFolder
+from app.modules.storage.models.storage_model import UserFile
+from app.core.settings import settings
+import os
+import tempfile
+from app.infrastructure.storage.local_storage_handler import get_storage_client
+from app.modules.editor.converters import process_pdf_to_html
+
+logger = logging.getLogger("celery_tasks")
+
+
+def _html_to_xml(html: str) -> str | None:
+    """Convert HTML to well-formed XHTML/XML using lxml."""
+    try:
+        from lxml import etree
+        parser = etree.HTMLParser(remove_comments=True, remove_pis=True)
+        tree = etree.fromstring(html.encode("utf-8", errors="replace"), parser)
+        return etree.tostring(tree, encoding="unicode", method="xml", pretty_print=True)
+    except Exception:
+        return None
+
+
+def _html_to_markdown(html: str) -> str:
+    from bs4 import BeautifulSoup
+    return BeautifulSoup(html, "html.parser").get_text("\n")
+
+
+@shared_task(bind=True)
+def process_document_task(self, file_path_or_key, session_id, original_version_id=None):
+    task_id = self.request.id
+    temp_local_path = None
+
+    def _mark_failed(message: str) -> dict:
+        try:
+            with get_db_connection() as conn:
+                from app.modules.documents.models.document_model import Project
+                project = conn.query(Project).filter(
+                    Project.session_id == session_id).first()
+                if project:
+                    project.status = "failed"
+                    project.progress = 0
+                    project.current_step = "failed"
+                    conn.commit()
+        except Exception as db_err:
+            logger.error(
+                f"Failed to persist failed status for session {session_id}: {db_err}")
+        publish_progress(task_id, 0, message)
+        return {"status": "failed", "error": message}
+
+    try:
+        if is_cancelled(task_id):
+            publish_progress(task_id, 0, "Cancelled")
+            return {"status": "cancelled"}
+
+        publish_progress(task_id, 5, "Preparing Document")
+
+        current_file_path = file_path_or_key
+        if "/" in file_path_or_key and not os.path.exists(file_path_or_key):
+            with get_db_connection() as conn:
+                from app.modules.documents.models.document_model import Project
+                project = conn.query(Project).filter(Project.session_id == session_id).first()
+                tenant_id = project.tenant_id if project else None
+                
+                storage, _, clean_bucket = get_storage_client(db=conn, tenant_id=tenant_id)
+                response = storage.get_object(
+                    Bucket=clean_bucket, Key=file_path_or_key)
+
+            ext = os.path.splitext(file_path_or_key)[1] or ".pdf"
+            with tempfile.NamedTemporaryFile(delete=False, suffix=ext) as tmp:
+                tmp.write(response['Body'].read())
+                temp_local_path = tmp.name
+                current_file_path = temp_local_path
+
+        publish_progress(task_id, 20, "Converting Document")
+
+        if is_cancelled(task_id):
+            publish_progress(task_id, 0, "Cancelled")
+            return {"status": "cancelled"}
+
+        publish_progress(task_id, 50, "Extracting Content")
+
+        # ── AI extraction path ──────────────────────────────────────────────
+        # Convert PDF to HTML locally, then derive markdown + XML
+        html_doc = process_pdf_to_html(current_file_path)
+        markdown_doc = _html_to_markdown(html_doc)
+        xml_doc = _html_to_xml(html_doc)
+        result_data = {"result": {"html": html_doc,
+                                  "markdown": markdown_doc, "xml": xml_doc}}
+
+        if is_cancelled(task_id):
+            publish_progress(task_id, 0, "Cancelled")
+            return {"status": "cancelled"}
+
+        publish_progress(task_id, 90, "Finalizing")
+
+        with get_db_connection() as conn:
+            from app.modules.documents.models.document_model import Project, ProjectVersion
+            from app.modules.drive.services.activity_service import ActivityService
+            import json
+
+            project = conn.query(Project).filter(
+                Project.session_id == session_id).first()
+            if project:
+                project.status = "completed"
+                project.progress = 100
+
+                xml_content = None
+                markdown_content = None
+                html_content = None
+
+                if isinstance(result_data, dict) and "result" in result_data:
+                    res = result_data["result"]
+                    if isinstance(res, dict):
+                        xml_content = res.get("xml")
+                        markdown_content = res.get("markdown")
+                        html_content = res.get("html")
+
+                        MAX_CONTENT_BYTES = 10 * 1024 * 1024
+                        for field, value in [("xml", xml_content), ("markdown", markdown_content), ("html", html_content)]:
+                            if value and len(value.encode("utf-8")) > MAX_CONTENT_BYTES:
+                                logger.warning(
+                                    "Document session=%s: %s_content exceeds 10 MB (%d bytes) — truncating. "
+                                    "Consider offloading large content to S3.",
+                                    session_id, field, len(
+                                        value.encode("utf-8"))
+                                )
+                        MAX_CONTENT_CHARS = MAX_CONTENT_BYTES
+                        xml_content = xml_content[:MAX_CONTENT_CHARS] if xml_content else None
+                        markdown_content = markdown_content[:
+                                                            MAX_CONTENT_CHARS] if markdown_content else None
+                        html_content = html_content[:MAX_CONTENT_CHARS] if html_content else None
+
+                        project.xml_content = xml_content
+                        project.markdown_content = markdown_content
+                        project.html_content = html_content
+
+                from app.modules.documents.repositories.version_repository import VersionRepository
+                version_repo = VersionRepository(conn)
+                max_v = version_repo.get_max_version_number(project.id)
+
+                version = ProjectVersion(
+                    project_id=project.id,
+                    version_number=max_v + 1,
+                    version_label=f"v{max_v + 1}.0",
+                    changes_description="Initial conversion" if max_v == 0 else f"Conversion update {max_v + 1}",
+                    html_content=html_content,
+                    markdown_content=markdown_content,
+                    xml_content=xml_content,
+                    drive_file_version_id=original_version_id,
+                    is_current=True,
+                    created_by_id=project.user_id
+                )
+                version_repo.set_all_not_current(project.id)
+                conn.add(version)
+
+                activity_service = ActivityService(conn)
+                activity_service.log(
+                    resource_type="document",
+                    resource_id=project.id,
+                    actor_id=project.user_id,
+                    activity_type="conversion_completed",
+                    metadata=json.dumps({
+                        "description": f"Document conversion completed. Version v{max_v + 1}.0 created.",
+                        "version": f"v{max_v + 1}.0"
+                    })
+                )
+
+                if project.drive_file_id:
+                    activity_service.log(
+                        resource_type="file",
+                        resource_id=project.drive_file_id,
+                        actor_id=project.user_id,
+                        activity_type="conversion_completed",
+                        metadata=json.dumps({
+                            "description": f"Conversion completed: Version v{max_v + 1}.0 created for Project '{project.name}'",
+                            "version": f"v{max_v + 1}.0",
+                            "project_id": project.id
+                        })
+                    )
+
+                owner = project.owner
+                log_event(
+                    tenant_id=project.tenant_id,
+                    user_id=owner.id if owner else project.user_id,
+                    user_name=owner.name if owner else None,
+                    user_email=owner.email if owner else None,
+                    module=ActivityLogModule.CONVERSION,
+                    action=ActivityLogAction.CONVERSION_COMPLETED,
+                    target_id=project.drive_file_id or project.id,
+                    target_type=ActivityLogTargetType.DOCUMENT,
+                    target_name=project.name,
+                    metadata={
+                        "from": (project.file_type or "unknown").lower(),
+                        "to": (project.output_format or "unknown").lower(),
+                    },
+                    status=ActivityLogStatus.SUCCESS,
+                )
+                logger.info(
+                    f"Project {project.id} updated successfully with new version v{max_v + 1}.0")
+
+        return {
+            "status": "completed",
+            "session_id": session_id,
+            "result": result_data.get("result"),
+            "version": f"v{max_v + 1}.0" if 'max_v' in locals() else None
+        }
+
+    except Exception as e:
+        if str(e) == "Cancelled":
+            publish_progress(task_id, 0, "Cancelled")
+            return {"status": "cancelled"}
+
+        # Fail fast for non-recoverable connectivity errors (e.g. storage backend).
+        if isinstance(e, requests.exceptions.ConnectionError):
+            logger.error(
+                f"Upstream service unreachable for session {session_id}: {e}")
+            return _mark_failed("A required service is unreachable. Please retry shortly.")
+
+        if isinstance(e, requests.exceptions.Timeout):
+            logger.error(f"Upstream service timeout for session {session_id}: {e}")
+            return _mark_failed("Conversion service timed out. Please retry.")
+
+        if isinstance(e, RuntimeError):
+            logger.error(
+                f"PDF conversion error for session {session_id}: {e}")
+            return _mark_failed("PDF conversion failed. Please check the file and retry.")
+
+        retry_delay = 60 * (2 ** self.request.retries)
+        logger.warning(
+            f"Processing failed (Attempt {self.request.retries + 1}/4). Retrying in {retry_delay}s. Error: {e}")
+
+        try:
+            raise self.retry(exc=e, countdown=retry_delay, max_retries=3)
+        except self.MaxRetriesExceededError:
+            logger.error(
+                f"Max retries exceeded for session {session_id}. Marking as failed.")
+            try:
+                with get_db_connection() as conn:
+                    from app.modules.documents.models.document_model import Project
+                    project = conn.query(Project).filter(
+                        Project.session_id == session_id).first()
+                    if project:
+                        project.status = "failed"
+                        project.progress = 0
+                        conn.commit()
+            except Exception as db_err:
+                logger.error(f"Failed to mark project as failed: {db_err}")
+            raise e
+    finally:
+        if temp_local_path and os.path.exists(temp_local_path):
+            try:
+                os.remove(temp_local_path)
+                logger.info(
+                    f"🗑️ Cleaned up temporary processing file: {temp_local_path}")
+            except Exception as cleanup_err:
+                logger.error(
+                    f"Failed to cleanup temp file {temp_local_path}: {cleanup_err}")
+
+
+@shared_task
+def cleanup_old_trash_task():
+    """
+    Find files/folders in trash older than 30 days and permanently delete.
+    Now optimized to handle multi-tenant storage by grouping deletions by tenant.
+    """
+    from collections import defaultdict
+
+    logger.info("Starting 30-day trash cleanup task (Multi-tenant aware)")
+    cutoff = datetime.utcnow() - timedelta(days=30)
+
+    with get_db_connection() as conn:
+        files_to_delete = conn.query(DriveFile).filter(
+            DriveFile.is_trashed == True,
+            DriveFile.trashed_at < cutoff
+        ).all()
+
+        files_by_tenant = defaultdict(list)
+        for f in files_to_delete:
+            files_by_tenant[f.tenant_id].append(f)
+
+        for tenant_id, tenant_files in files_by_tenant.items():
+            try:
+                s3, quarantine_bucket, clean_bucket = get_storage_client(db=conn, tenant_id=tenant_id)
+                for f in tenant_files:
+                    logger.info(f"Deleting file {f.name} (ID: {f.id}, Tenant: {tenant_id})")
+                    versions = conn.query(DriveFileVersion).filter(DriveFileVersion.file_id == f.id).all()
+                    
+                    for v in versions:
+                        try:
+                            for bucket in [clean_bucket, quarantine_bucket]:
+                                try:
+                                    s3.delete_object(Bucket=bucket, Key=v.s3_key)
+                                except Exception:
+                                    continue
+                        except Exception as e:
+                            logger.error(f"Failed to delete {v.s3_key} from B2 for tenant {tenant_id}: {e}")
+                    
+                    conn.delete(f)
+            except Exception as e:
+                logger.error(f"Failed to initialize storage for tenant {tenant_id} cleanup: {e}")
+
+        folders_to_delete = conn.query(DriveFolder).filter(
+            DriveFolder.is_trashed == True,
+            DriveFolder.trashed_at < cutoff
+        ).all()
+
+        for f in folders_to_delete:
+            logger.info(f"Deleting folder {f.name} (ID: {f.id})")
+            conn.delete(f)
+
+        user_files_to_purge = conn.query(UserFile).filter(
+            UserFile.is_deleted == True,
+            UserFile.deleted_at < cutoff
+        ).all()
+
+        ufiles_by_tenant = defaultdict(list)
+        for uf in user_files_to_purge:
+            ufiles_by_tenant[uf.tenant_id].append(uf)
+
+        for tenant_id, tenant_ufiles in ufiles_by_tenant.items():
+            try:
+                s3, quarantine_bucket, clean_bucket = get_storage_client(db=conn, tenant_id=tenant_id)
+                for uf in tenant_ufiles:
+                    logger.info(f"Purging UserFile {uf.filename} (ID: {uf.id}, Tenant: {tenant_id}, Key: {uf.s3_key})")
+                    try:
+                        for bucket in [clean_bucket, quarantine_bucket]:
+                            try:
+                                s3.delete_object(Bucket=bucket, Key=uf.s3_key)
+                            except Exception:
+                                continue
+                    except Exception as e:
+                        logger.debug(f"Cleanup check for {uf.s3_key} failed for tenant {tenant_id}: {e}")
+                    
+                    conn.delete(uf)
+            except Exception as e:
+                 logger.error(f"Failed to initialize storage for tenant {tenant_id} user-file cleanup: {e}")
+
+        conn.commit()
+
+        # Invalidate Redis trash cache for affected users
+        affected_user_ids = {f.owner_id for f in files_to_delete if hasattr(f, 'owner_id')}.union(
+            {f.owner_id for f in folders_to_delete if hasattr(f, 'owner_id')}
+        )
+        for uid in affected_user_ids:
+            try:
+                from app.db.redis import redis_cache
+                from app.modules.drive.constants import CACHE_USER_TRASH
+                redis_cache.delete(CACHE_USER_TRASH.format(user_id=uid))
+            except Exception as e:
+                logger.debug(f"Redis cache invalidation failed for user {uid}: {e}")
+
+    logger.info("Trash cleanup complete.")