Changes committed

docengine/app/__init__.py

@@ -0,0 +1 @@
+# DocEngine - Document Template Recognition and Reconstruction System

docengine/app/api/router.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from fastapi import APIRouter
+
+from app.api.v1.auth import router as auth_router
+from app.api.v1.documents import router as documents_router
+from app.api.v1.health import router as health_router
+from app.api.v1.templates import router as templates_router
+
+api_v1_router = APIRouter(prefix="/api/v1")
+
+api_v1_router.include_router(health_router)
+api_v1_router.include_router(auth_router)
+api_v1_router.include_router(documents_router)
+api_v1_router.include_router(templates_router)

docengine/app/api/v1/auth.py

@@ -0,0 +1,246 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+
+from fastapi import APIRouter, Depends, HTTPException, status
+from sqlalchemy.orm import Session
+
+from app.core.config import settings
+from app.core.database import get_db
+from app.core.dependencies import CurrentUser
+from app.core.security import (
+    create_access_token,
+    create_refresh_token,
+    decode_token,
+    hash_password,
+    verify_password,
+    InvalidTokenError,
+)
+from app.repositories.user_repository import RefreshTokenRepository, UserRepository
+from app.schemas.auth import (
+    ChangePasswordRequest,
+    LoginRequest,
+    RefreshTokenRequest,
+    RegisterRequest,
+    TokenResponse,
+)
+from app.schemas.common import SuccessResponse
+from app.schemas.user import UserResponse
+
+router = APIRouter(prefix="/auth", tags=["Authentication"])
+
+
+@router.post(
+    "/register",
+    response_model=UserResponse,
+    status_code=status.HTTP_201_CREATED,
+    summary="Register User",
+    description="Register a new user account.",
+)
+def register(
+    payload: RegisterRequest,
+    db: Session = Depends(get_db),
+) -> UserResponse:
+    """Register a new user."""
+    user_repo = UserRepository(db)
+
+    # Check for existing user
+    if user_repo.get_by_username(payload.username):
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT,
+            detail=f"Username '{payload.username}' is already taken",
+        )
+    if user_repo.get_by_email(payload.email):
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT,
+            detail=f"Email '{payload.email}' is already registered",
+        )
+
+    hashed = hash_password(payload.password)
+    user = user_repo.create_user(
+        username=payload.username,
+        email=payload.email,
+        hashed_password=hashed,
+        full_name=payload.full_name,
+        role_names=["user"],
+    )
+    db.commit()
+    db.refresh(user)
+
+    return UserResponse.model_validate(user)
+
+
+@router.post(
+    "/login",
+    response_model=TokenResponse,
+    summary="Login",
+    description="Authenticate with username and password to obtain JWT tokens.",
+)
+def login(
+    payload: LoginRequest,
+    db: Session = Depends(get_db),
+) -> TokenResponse:
+    """Authenticate user and return JWT tokens."""
+    user_repo = UserRepository(db)
+    refresh_repo = RefreshTokenRepository(db)
+
+    user = user_repo.get_by_username(payload.username)
+    if not user or not verify_password(payload.password, user.hashed_password):
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid username or password",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    if not user.is_active:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="User account is deactivated",
+        )
+
+    # Generate tokens
+    access_token = create_access_token(data={"sub": str(user.id), "username": user.username})
+    refresh_token_str = create_refresh_token(data={"sub": str(user.id)})
+
+    # Store refresh token
+    expires_at = datetime.now(UTC) + timedelta(days=settings.jwt_refresh_token_expire_days)
+    refresh_repo.create_token(
+        user_id=user.id,
+        token=refresh_token_str,
+        expires_at=expires_at,
+    )
+
+    # Update last login
+    user_repo.update_last_login(user)
+    db.commit()
+
+    return TokenResponse(
+        access_token=access_token,
+        refresh_token=refresh_token_str,
+        token_type="bearer",
+        expires_in=settings.jwt_access_token_expire_minutes * 60,
+    )
+
+
+@router.post(
+    "/refresh",
+    response_model=TokenResponse,
+    summary="Refresh Token",
+    description="Obtain a new access token using a valid refresh token.",
+)
+def refresh_token(
+    payload: RefreshTokenRequest,
+    db: Session = Depends(get_db),
+) -> TokenResponse:
+    """Refresh access token using a refresh token."""
+    refresh_repo = RefreshTokenRepository(db)
+    user_repo = UserRepository(db)
+
+    # Validate the refresh token
+    try:
+        token_payload = decode_token(payload.refresh_token)
+    except InvalidTokenError:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid or expired refresh token",
+        )
+
+    if token_payload.get("type") != "refresh":
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid token type",
+        )
+
+    # Check if token exists in database and is not revoked
+    stored_token = refresh_repo.get_by_token(payload.refresh_token)
+    if not stored_token:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Refresh token not found or revoked",
+        )
+
+    user = user_repo.get_by_id(token_payload["sub"])
+    if not user or not user.is_active:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="User not found or deactivated",
+        )
+
+    # Revoke old refresh token
+    refresh_repo.revoke_token(payload.refresh_token)
+
+    # Generate new tokens
+    new_access_token = create_access_token(data={"sub": str(user.id), "username": user.username})
+    new_refresh_token = create_refresh_token(data={"sub": str(user.id)})
+
+    expires_at = datetime.now(UTC) + timedelta(days=settings.jwt_refresh_token_expire_days)
+    refresh_repo.create_token(
+        user_id=user.id,
+        token=new_refresh_token,
+        expires_at=expires_at,
+    )
+    db.commit()
+
+    return TokenResponse(
+        access_token=new_access_token,
+        refresh_token=new_refresh_token,
+        token_type="bearer",
+        expires_in=settings.jwt_access_token_expire_minutes * 60,
+    )
+
+
+@router.post(
+    "/logout",
+    response_model=SuccessResponse,
+    summary="Logout",
+    description="Revoke the current refresh token.",
+)
+def logout(
+    payload: RefreshTokenRequest,
+    current_user: CurrentUser,
+    db: Session = Depends(get_db),
+) -> SuccessResponse:
+    """Logout by revoking the refresh token."""
+    refresh_repo = RefreshTokenRepository(db)
+    refresh_repo.revoke_token(payload.refresh_token)
+    db.commit()
+    return SuccessResponse(message="Successfully logged out")
+
+
+@router.post(
+    "/change-password",
+    response_model=SuccessResponse,
+    summary="Change Password",
+    description="Change the current user's password.",
+)
+def change_password(
+    payload: ChangePasswordRequest,
+    current_user: CurrentUser,
+    db: Session = Depends(get_db),
+) -> SuccessResponse:
+    """Change user password."""
+    if not verify_password(payload.current_password, current_user.hashed_password):
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Current password is incorrect",
+        )
+
+    current_user.hashed_password = hash_password(payload.new_password)
+
+    # Revoke all refresh tokens for security
+    refresh_repo = RefreshTokenRepository(db)
+    refresh_repo.revoke_all_user_tokens(current_user.id)
+    db.commit()
+
+    return SuccessResponse(message="Password changed successfully")
+
+
+@router.get(
+    "/me",
+    response_model=UserResponse,
+    summary="Get Current User",
+    description="Get the currently authenticated user's profile.",
+)
+def get_me(current_user: CurrentUser) -> UserResponse:
+    """Get current authenticated user profile."""
+    return UserResponse.model_validate(current_user)

docengine/app/api/v1/documents.py

@@ -0,0 +1,268 @@
+from __future__ import annotations
+
+import uuid
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, File, HTTPException, Query, UploadFile, status
+from sqlalchemy.orm import Session
+
+from app.core.config import settings
+from app.core.database import get_db
+from app.core.dependencies import CurrentUser
+from app.core.exceptions import FileSizeError, UnsupportedFileTypeError
+from app.core.logging_config import get_logger
+from app.models.document import Document
+from app.repositories.document_repository import DocumentRepository
+from app.schemas.common import PaginatedResponse, SuccessResponse
+from app.schemas.document import (
+    DocumentListResponse,
+    DocumentResponse,
+    DocumentUploadResponse,
+    TemplateMatchRequest,
+    TemplateMatchResponse,
+)
+from app.storage.provider import LocalStorageProvider, get_storage_provider
+
+logger = get_logger(__name__)
+
+router = APIRouter(prefix="/documents", tags=["Documents"])
+
+ALLOWED_CONTENT_TYPES = {
+    "image/jpeg": "jpg",
+    "image/png": "png",
+    "image/tiff": "tiff",
+    "application/pdf": "pdf",
+}
+
+ALLOWED_EXTENSIONS = {".jpg", ".jpeg", ".png", ".tiff", ".tif", ".pdf"}
+
+
+def _validate_file(file: UploadFile) -> str:
+    """Validate uploaded file type and size. Returns the content type."""
+    if not file.filename:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Filename is required",
+        )
+
+    # Check extension
+    from pathlib import Path
+    ext = Path(file.filename).suffix.lower()
+    if ext not in ALLOWED_EXTENSIONS:
+        raise UnsupportedFileTypeError(ext)
+
+    # Determine content type
+    content_type = file.content_type or ""
+    if content_type not in ALLOWED_CONTENT_TYPES:
+        # Try to infer from extension
+        ext_to_ct = {
+            ".jpg": "image/jpeg",
+            ".jpeg": "image/jpeg",
+            ".png": "image/png",
+            ".tiff": "image/tiff",
+            ".tif": "image/tiff",
+            ".pdf": "application/pdf",
+        }
+        content_type = ext_to_ct.get(ext, "")
+        if not content_type:
+            raise UnsupportedFileTypeError(file.content_type or "unknown")
+
+    return content_type
+
+
+@router.post(
+    "/upload",
+    response_model=DocumentUploadResponse,
+    status_code=status.HTTP_201_CREATED,
+    summary="Upload Document",
+    description="Upload a document (JPG, JPEG, PNG, TIFF, or PDF) for processing.",
+)
+async def upload_document(
+    file: UploadFile = File(..., description="Document file to upload"),
+    current_user: CurrentUser = None,
+    db: Session = Depends(get_db),
+) -> DocumentUploadResponse:
+    """Upload a document for processing."""
+    content_type = _validate_file(file)
+
+    # Read file data
+    file_data = await file.read()
+
+    # Check file size
+    if len(file_data) > settings.storage_max_file_size_bytes:
+        raise FileSizeError(settings.storage_max_file_size_mb)
+
+    # Store file
+    storage = get_storage_provider()
+    checksum = storage.compute_checksum(file_data)
+    safe_filename = file.filename or "unknown"
+    stored_filename = storage.generate_filename(safe_filename)
+    storage_path = storage.save_file(file_data, "documents", stored_filename)
+
+    # Create document record
+    doc_repo = DocumentRepository(db)
+    document = Document(
+        filename=stored_filename,
+        original_filename=safe_filename,
+        content_type=content_type,
+        file_size=len(file_data),
+        checksum=checksum,
+        storage_path=storage_path,
+        status="pending",
+        uploaded_by=current_user.id if current_user else None,
+    )
+    doc_repo.create(document)
+    db.commit()
+    db.refresh(document)
+
+    logger.info(
+        "document_uploaded",
+        document_id=str(document.id),
+        filename=safe_filename,
+        size=len(file_data),
+        content_type=content_type,
+    )
+
+    # Trigger async processing via Celery
+    try:
+        from app.tasks.document_tasks import process_document_task
+        process_document_task.delay(str(document.id))
+    except Exception as e:
+        logger.warning("celery_dispatch_failed", error=str(e), document_id=str(document.id))
+
+    return DocumentUploadResponse.model_validate(document)
+
+
+@router.get(
+    "/{document_id}",
+    response_model=DocumentResponse,
+    summary="Get Document",
+    description="Retrieve a document by its ID with all extracted content.",
+)
+def get_document(
+    document_id: uuid.UUID,
+    current_user: CurrentUser = None,
+    db: Session = Depends(get_db),
+) -> DocumentResponse:
+    """Get a document by ID."""
+    doc_repo = DocumentRepository(db)
+    document = doc_repo.get_with_pages(document_id)
+    if not document:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Document '{document_id}' not found",
+        )
+    return DocumentResponse.model_validate(document)
+
+
+@router.get(
+    "",
+    response_model=PaginatedResponse[DocumentListResponse],
+    summary="List Documents",
+    description="List all documents with pagination.",
+)
+def list_documents(
+    page: int = Query(default=1, ge=1),
+    page_size: int = Query(default=20, ge=1, le=100),
+    status_filter: str | None = Query(default=None, alias="status"),
+    current_user: CurrentUser = None,
+    db: Session = Depends(get_db),
+) -> PaginatedResponse[DocumentListResponse]:
+    """List documents with pagination and optional status filter."""
+    doc_repo = DocumentRepository(db)
+    offset = (page - 1) * page_size
+    filters = {}
+    if status_filter:
+        filters["status"] = status_filter
+
+    documents = doc_repo.get_all(
+        offset=offset,
+        limit=page_size,
+        filters=filters,
+        order_by="created_at",
+        order_desc=True,
+    )
+    total = doc_repo.count(filters=filters)
+
+    items = [DocumentListResponse.model_validate(doc) for doc in documents]
+    return PaginatedResponse.create(
+        items=items,
+        total=total,
+        page=page,
+        page_size=page_size,
+    )
+
+
+@router.get(
+    "/{document_id}/template",
+    response_model=list[TemplateMatchResponse],
+    summary="Get Document Template Matches",
+    description="Get template matching results for a document.",
+)
+def get_document_template_matches(
+    document_id: uuid.UUID,
+    current_user: CurrentUser = None,
+    db: Session = Depends(get_db),
+) -> list[TemplateMatchResponse]:
+    """Get template matches for a document."""
+    from app.repositories.document_repository import TemplateMatchRepository
+
+    doc_repo = DocumentRepository(db)
+    document = doc_repo.get_by_id(document_id)
+    if not document:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Document '{document_id}' not found",
+        )
+
+    match_repo = TemplateMatchRepository(db)
+    matches = match_repo.get_document_matches(document_id)
+
+    results = []
+    for match in matches:
+        resp = TemplateMatchResponse(
+            id=match.id,
+            document_id=match.document_id,
+            format_id=match.format_id,
+            confidence_score=match.confidence_score,
+            match_details=match.match_details,
+            selected=match.selected,
+            template_name=match.template.name if match.template else None,
+            created_at=match.created_at,
+        )
+        results.append(resp)
+
+    return results
+
+
+@router.delete(
+    "/{document_id}",
+    response_model=SuccessResponse,
+    summary="Delete Document",
+    description="Delete a document and its associated data.",
+)
+def delete_document(
+    document_id: uuid.UUID,
+    current_user: CurrentUser = None,
+    db: Session = Depends(get_db),
+) -> SuccessResponse:
+    """Delete a document."""
+    doc_repo = DocumentRepository(db)
+    document = doc_repo.get_by_id(document_id)
+    if not document:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Document '{document_id}' not found",
+        )
+
+    # Delete stored file
+    try:
+        storage = get_storage_provider()
+        storage.delete_file(document.storage_path)
+    except Exception as e:
+        logger.warning("file_delete_failed", error=str(e), path=document.storage_path)
+
+    doc_repo.delete(document)
+    db.commit()
+
+    return SuccessResponse(message=f"Document '{document_id}' deleted successfully")

docengine/app/api/v1/health.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from datetime import datetime
+
+import redis
+from fastapi import APIRouter, status
+
+from app.core.config import settings
+from app.core.database import check_database_connection
+from app.schemas.common import HealthResponse
+
+router = APIRouter(tags=["Health"])
+
+
+@router.get(
+    "/health",
+    response_model=HealthResponse,
+    status_code=status.HTTP_200_OK,
+    summary="Health Check",
+    description="Check the health status of the application and its dependencies.",
+)
+async def health_check() -> HealthResponse:
+    """Perform health check on all system components."""
+    # Check database
+    db_status = "healthy" if check_database_connection() else "unhealthy"
+
+    # Check Redis
+    redis_status = "healthy"
+    try:
+        r = redis.Redis(
+            host=settings.redis_host,
+            port=settings.redis_port,
+            db=settings.redis_db,
+            password=settings.redis_password or None,
+            socket_timeout=3,
+        )
+        r.ping()
+        r.close()
+    except Exception:
+        redis_status = "unhealthy"
+
+    overall_status = "healthy" if db_status == "healthy" and redis_status == "healthy" else "degraded"
+
+    return HealthResponse(
+        status=overall_status,
+        version=settings.app_version,
+        environment=settings.app_env,
+        database=db_status,
+        redis=redis_status,
+        timestamp=datetime.utcnow(),
+    )

docengine/app/api/v1/templates.py

@@ -0,0 +1,223 @@
+from __future__ import annotations
+
+import uuid
+
+from fastapi import APIRouter, Depends, HTTPException, Query, status
+from fastapi.responses import FileResponse
+from sqlalchemy.orm import Session
+
+from app.core.database import get_db
+from app.core.dependencies import CurrentUser
+from app.core.logging_config import get_logger
+from app.repositories.document_repository import DocumentRepository, TemplateMatchRepository
+from app.repositories.template_repository import TemplateRepository
+from app.schemas.common import PaginatedResponse, SuccessResponse
+from app.schemas.document import TemplateMatchRequest, TemplateMatchResponse
+from app.schemas.template import (
+    TemplateListResponse,
+    TemplateRenderRequest,
+    TemplateRenderResponse,
+    TemplateResponse,
+)
+
+logger = get_logger(__name__)
+
+router = APIRouter(prefix="/templates", tags=["Templates"])
+
+
+@router.get(
+    "",
+    response_model=PaginatedResponse[TemplateListResponse],
+    summary="List Templates",
+    description="List all active templates with pagination.",
+)
+def list_templates(
+    page: int = Query(default=1, ge=1),
+    page_size: int = Query(default=20, ge=1, le=100),
+    current_user: CurrentUser = None,
+    db: Session = Depends(get_db),
+) -> PaginatedResponse[TemplateListResponse]:
+    """List all active templates."""
+    template_repo = TemplateRepository(db)
+    offset = (page - 1) * page_size
+
+    templates = template_repo.get_active_templates(offset=offset, limit=page_size)
+    total = template_repo.count_active()
+
+    items = [TemplateListResponse.model_validate(t) for t in templates]
+    return PaginatedResponse.create(
+        items=items,
+        total=total,
+        page=page,
+        page_size=page_size,
+    )
+
+
+@router.get(
+    "/{template_id}",
+    response_model=TemplateResponse,
+    summary="Get Template",
+    description="Retrieve a template by ID with all its components.",
+)
+def get_template(
+    template_id: uuid.UUID,
+    current_user: CurrentUser = None,
+    db: Session = Depends(get_db),
+) -> TemplateResponse:
+    """Get a template by ID."""
+    template_repo = TemplateRepository(db)
+    template = template_repo.get_by_id(template_id)
+    if not template:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Template '{template_id}' not found",
+        )
+    return TemplateResponse.model_validate(template)
+
+
+@router.delete(
+    "/{template_id}",
+    response_model=SuccessResponse,
+    summary="Delete Template",
+    description="Soft-delete a template by deactivating it.",
+)
+def delete_template(
+    template_id: uuid.UUID,
+    current_user: CurrentUser = None,
+    db: Session = Depends(get_db),
+) -> SuccessResponse:
+    """Soft-delete a template."""
+    template_repo = TemplateRepository(db)
+    template = template_repo.get_by_id(template_id)
+    if not template:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Template '{template_id}' not found",
+        )
+
+    template_repo.deactivate_template(template_id)
+    db.commit()
+
+    return SuccessResponse(message=f"Template '{template_id}' deactivated successfully")
+
+
+@router.post(
+    "/match",
+    response_model=list[TemplateMatchResponse],
+    summary="Match Document to Templates",
+    description="Match a document against existing templates and return ranked results.",
+)
+def match_template(
+    payload: TemplateMatchRequest,
+    current_user: CurrentUser = None,
+    db: Session = Depends(get_db),
+) -> list[TemplateMatchResponse]:
+    """Match a document against existing templates."""
+    doc_repo = DocumentRepository(db)
+    document = doc_repo.get_by_id(payload.document_id)
+    if not document:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Document '{payload.document_id}' not found",
+        )
+
+    if document.status != "completed":
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"Document must be in 'completed' status. Current status: '{document.status}'",
+        )
+
+    # Perform template matching
+    from app.services.matching_service import MatchingService
+    matching_service = MatchingService(db)
+    matches = matching_service.match_document(
+        document_id=payload.document_id,
+        min_confidence=payload.min_confidence,
+        max_results=payload.max_results,
+    )
+    db.commit()
+
+    results = []
+    for match in matches:
+        resp = TemplateMatchResponse(
+            id=match.id,
+            document_id=match.document_id,
+            format_id=match.format_id,
+            confidence_score=match.confidence_score,
+            match_details=match.match_details,
+            selected=match.selected,
+            template_name=match.template.name if match.template else None,
+            created_at=match.created_at,
+        )
+        results.append(resp)
+
+    return results
+
+
+@router.post(
+    "/render",
+    response_model=TemplateRenderResponse,
+    summary="Render Template to PDF",
+    description="Generate a PDF from a stored template with supplied data.",
+)
+def render_template(
+    payload: TemplateRenderRequest,
+    current_user: CurrentUser = None,
+    db: Session = Depends(get_db),
+) -> TemplateRenderResponse:
+    """Render a template to PDF."""
+    template_repo = TemplateRepository(db)
+    template = template_repo.get_by_id(payload.template_id)
+    if not template:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Template '{payload.template_id}' not found",
+        )
+
+    if not template.is_active:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Template is deactivated",
+        )
+
+    from app.services.reconstruction_service import ReconstructionService
+    reconstruction_service = ReconstructionService(db)
+    result = reconstruction_service.render_template(
+        template=template,
+        data=payload.data,
+        output_filename=payload.output_filename,
+        images=payload.images,
+    )
+
+    return result
+
+
+@router.get(
+    "/{template_id}/download",
+    summary="Download Rendered PDF",
+    description="Download a previously rendered PDF.",
+)
+def download_rendered_pdf(
+    template_id: uuid.UUID,
+    filename: str = Query(..., description="Filename of the rendered PDF"),
+    current_user: CurrentUser = None,
+    db: Session = Depends(get_db),
+) -> FileResponse:
+    """Download a rendered PDF."""
+    from app.storage.provider import get_storage_provider
+
+    storage = get_storage_provider()
+    storage_path = f"rendered/{filename}"
+
+    if not storage.file_exists(storage_path):
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Rendered PDF '{filename}' not found",
+        )
+
+    absolute_path = storage.get_absolute_path(storage_path)
+    return FileResponse(
+        path=absolute_path,
+        media_type="application/pdf",
+        filename=filename,
+    )

docengine/app/core/config.py

@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from pydantic import field_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    """Application configuration loaded from environment variables."""
+
+    model_config = SettingsConfigDict(
+        env_file=".env",
+        env_file_encoding="utf-8",
+        case_sensitive=False,
+        extra="ignore",
+    )
+
+    # Application
+    app_name: str = "DocEngine"
+    app_version: str = "1.0.0"
+    app_env: str = "development"
+    app_debug: bool = True
+    app_host: str = "0.0.0.0"
+    app_port: int = 7989
+    app_workers: int = 4
+
+    # Database
+    db_host: str = "192.168.0.111"
+    db_port: int = 7925
+    db_name: str = "document_engine"
+    db_user: str = "postgres"
+    db_password: str = "M@tr!x#149@dm!N"
+    db_schema: str = "admin"
+    db_pool_size: int = 20
+    db_max_overflow: int = 10
+    db_echo: bool = False
+
+    # Redis
+    redis_host: str = "localhost"
+    redis_port: int = 6379
+    redis_db: int = 0
+    redis_password: str = ""
+
+    # Celery
+    celery_broker_url: str = "redis://localhost:6379/0"
+    celery_result_backend: str = "redis://localhost:6379/1"
+
+    # JWT
+    jwt_secret_key: str = "a7f3c9e1d4b8f2a6c0e5d7b3a9f1c4e8d2b6a0f5c3e7d1b9a4f8c2e6d0b5a3"
+    jwt_algorithm: str = "HS256"
+    jwt_access_token_expire_minutes: int = 30
+    jwt_refresh_token_expire_days: int = 7
+
+    # Storage
+    storage_provider: str = "local"
+    storage_local_path: str = "./storage"
+    storage_max_file_size_mb: int = 100
+
+    # OCR
+    ocr_language: str = "en"
+    ocr_use_gpu: bool = False
+
+    # Logging
+    log_level: str = "INFO"
+    log_format: str = "json"
+
+    # CORS
+    cors_origins: list[str] = ["http://localhost:3000", "http://localhost:8080"]
+    cors_allow_credentials: bool = True
+
+    # Rate Limiting
+    rate_limit_requests: int = 100
+    rate_limit_window_seconds: int = 60
+
+    # Prometheus
+    prometheus_enabled: bool = True
+
+    @field_validator("cors_origins", mode="before")
+    @classmethod
+    def parse_cors_origins(cls, v: Any) -> list[str]:
+        if isinstance(v, str):
+            try:
+                parsed = json.loads(v)
+                if isinstance(parsed, list):
+                    return parsed
+            except (json.JSONDecodeError, TypeError):
+                return [origin.strip() for origin in v.split(",") if origin.strip()]
+        return v
+
+    @property
+    def database_url(self) -> str:
+        from urllib.parse import quote_plus
+        password = quote_plus(self.db_password)
+        return f"postgresql+psycopg2://{self.db_user}:{password}@{self.db_host}:{self.db_port}/{self.db_name}"
+
+    @property
+    def async_database_url(self) -> str:
+        from urllib.parse import quote_plus
+        password = quote_plus(self.db_password)
+        return f"postgresql+asyncpg://{self.db_user}:{password}@{self.db_host}:{self.db_port}/{self.db_name}"
+
+    @property
+    def redis_url(self) -> str:
+        if self.redis_password:
+            return f"redis://:{self.redis_password}@{self.redis_host}:{self.redis_port}/{self.redis_db}"
+        return f"redis://{self.redis_host}:{self.redis_port}/{self.redis_db}"
+
+    @property
+    def is_production(self) -> bool:
+        return self.app_env == "production"
+
+    @property
+    def storage_max_file_size_bytes(self) -> int:
+        return self.storage_max_file_size_mb * 1024 * 1024
+
+
+settings = Settings()

docengine/app/core/database.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+from collections.abc import Generator
+from contextlib import contextmanager
+
+from sqlalchemy import MetaData, create_engine, event, text
+from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
+
+from app.core.config import settings
+
+NAMING_CONVENTION = {
+    "ix": "ix_%(column_0_label)s",
+    "uq": "uq_%(table_name)s_%(column_0_name)s",
+    "ck": "ck_%(table_name)s_%(constraint_name)s",
+    "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
+    "pk": "pk_%(table_name)s",
+}
+
+metadata = MetaData(
+    naming_convention=NAMING_CONVENTION,
+    schema=settings.db_schema,
+)
+
+engine = create_engine(
+    settings.database_url,
+    pool_size=settings.db_pool_size,
+    max_overflow=settings.db_max_overflow,
+    echo=settings.db_echo,
+    pool_pre_ping=True,
+    pool_recycle=3600,
+    connect_args={
+        "options": f"-c search_path={settings.db_schema},public"
+    },
+)
+
+
+@event.listens_for(engine, "connect")
+def set_search_path(dbapi_connection: object, connection_record: object) -> None:
+    cursor = dbapi_connection.cursor()  # type: ignore[union-attr]
+    cursor.execute(f"SET search_path TO {settings.db_schema}, public")
+    cursor.close()
+    dbapi_connection.commit()  # type: ignore[union-attr]
+
+
+SessionLocal = sessionmaker(
+    autocommit=False,
+    autoflush=False,
+    bind=engine,
+)
+
+
+class Base(DeclarativeBase):
+    """Base class for all SQLAlchemy models."""
+
+    metadata = metadata
+
+
+def get_db() -> Generator[Session, None, None]:
+    """Dependency to get database session."""
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()
+
+
+@contextmanager
+def get_db_context() -> Generator[Session, None, None]:
+    """Context manager for database session (used outside request scope)."""
+    db = SessionLocal()
+    try:
+        yield db
+        db.commit()
+    except Exception:
+        db.rollback()
+        raise
+    finally:
+        db.close()
+
+
+def check_database_connection() -> bool:
+    """Verify database connectivity."""
+    try:
+        with engine.connect() as conn:
+            conn.execute(text("SELECT 1"))
+        return True
+    except Exception:
+        return False

docengine/app/core/dependencies.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+from typing import Annotated
+
+from fastapi import Depends, HTTPException, status
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+from sqlalchemy.orm import Session
+
+from app.core.database import get_db
+from app.core.security import InvalidTokenError, decode_token
+from app.models.user import User
+from app.repositories.user_repository import UserRepository
+
+security_scheme = HTTPBearer(auto_error=True)
+
+
+def get_current_user(
+    credentials: Annotated[HTTPAuthorizationCredentials, Depends(security_scheme)],
+    db: Annotated[Session, Depends(get_db)],
+) -> User:
+    """Extract and validate the current user from the JWT token."""
+    try:
+        payload = decode_token(credentials.credentials)
+    except InvalidTokenError:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid or expired token",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    token_type = payload.get("type")
+    if token_type != "access":
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid token type. Access token required.",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    user_id: str | None = payload.get("sub")
+    if user_id is None:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Token payload missing subject",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    user_repo = UserRepository(db)
+    user = user_repo.get_by_id(user_id)
+    if user is None:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="User not found",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    if not user.is_active:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="User account is deactivated",
+        )
+
+    return user
+
+
+def get_current_active_user(
+    current_user: Annotated[User, Depends(get_current_user)],
+) -> User:
+    """Ensure the current user is active."""
+    if not current_user.is_active:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="User account is deactivated",
+        )
+    return current_user
+
+
+def require_role(required_roles: list[str]):  # noqa: ANN201
+    """Dependency factory to require specific roles."""
+
+    def role_checker(
+        current_user: Annotated[User, Depends(get_current_user)],
+    ) -> User:
+        user_roles = {role.name for role in current_user.roles}
+        if not user_roles.intersection(required_roles):
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN,
+                detail=f"One of the following roles required: {', '.join(required_roles)}",
+            )
+        return current_user
+
+    return role_checker
+
+
+CurrentUser = Annotated[User, Depends(get_current_user)]
+ActiveUser = Annotated[User, Depends(get_current_active_user)]
+AdminUser = Annotated[User, Depends(require_role(["admin"]))]
+DBSession = Annotated[Session, Depends(get_db)]

docengine/app/core/exceptions.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+from typing import Any
+
+
+class DocEngineException(Exception):
+    """Base exception for DocEngine application."""
+
+    def __init__(self, detail: str, status_code: int = 500, extra: dict[str, Any] | None = None) -> None:
+        self.detail = detail
+        self.status_code = status_code
+        self.extra = extra or {}
+        super().__init__(self.detail)
+
+
+class NotFoundError(DocEngineException):
+    """Resource not found."""
+
+    def __init__(self, resource: str, identifier: str) -> None:
+        super().__init__(
+            detail=f"{resource} with identifier '{identifier}' not found",
+            status_code=404,
+        )
+        self.resource = resource
+        self.identifier = identifier
+
+
+class DuplicateError(DocEngineException):
+    """Resource already exists."""
+
+    def __init__(self, resource: str, field: str, value: str) -> None:
+        super().__init__(
+            detail=f"{resource} with {field} '{value}' already exists",
+            status_code=409,
+        )
+
+
+class ValidationError(DocEngineException):
+    """Input validation error."""
+
+    def __init__(self, detail: str, errors: list[dict[str, Any]] | None = None) -> None:
+        super().__init__(detail=detail, status_code=422)
+        self.errors = errors or []
+
+
+class AuthenticationError(DocEngineException):
+    """Authentication failed."""
+
+    def __init__(self, detail: str = "Authentication failed") -> None:
+        super().__init__(detail=detail, status_code=401)
+
+
+class AuthorizationError(DocEngineException):
+    """Authorization failed."""
+
+    def __init__(self, detail: str = "Insufficient permissions") -> None:
+        super().__init__(detail=detail, status_code=403)
+
+
+class StorageError(DocEngineException):
+    """Storage operation failed."""
+
+    def __init__(self, detail: str) -> None:
+        super().__init__(detail=detail, status_code=500)
+
+
+class ProcessingError(DocEngineException):
+    """Document processing failed."""
+
+    def __init__(self, detail: str, document_id: str | None = None) -> None:
+        super().__init__(detail=detail, status_code=500)
+        self.document_id = document_id
+
+
+class TemplateError(DocEngineException):
+    """Template operation failed."""
+
+    def __init__(self, detail: str) -> None:
+        super().__init__(detail=detail, status_code=500)
+
+
+class RateLimitError(DocEngineException):
+    """Rate limit exceeded."""
+
+    def __init__(self, detail: str = "Rate limit exceeded. Please try again later.") -> None:
+        super().__init__(detail=detail, status_code=429)
+
+
+class FileSizeError(DocEngineException):
+    """File exceeds maximum allowed size."""
+
+    def __init__(self, max_size_mb: int) -> None:
+        super().__init__(
+            detail=f"File size exceeds maximum allowed size of {max_size_mb}MB",
+            status_code=413,
+        )
+
+
+class UnsupportedFileTypeError(DocEngineException):
+    """File type not supported."""
+
+    def __init__(self, file_type: str) -> None:
+        super().__init__(
+            detail=f"File type '{file_type}' is not supported. Supported types: jpg, jpeg, png, tiff, pdf",
+            status_code=415,
+        )

docengine/app/core/logging_config.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+import logging
+import sys
+
+import structlog
+
+from app.core.config import settings
+
+
+def setup_logging() -> None:
+    """Configure structlog for structured JSON logging."""
+    shared_processors: list[structlog.types.Processor] = [
+        structlog.contextvars.merge_contextvars,
+        structlog.stdlib.add_logger_name,
+        structlog.stdlib.add_log_level,
+        structlog.stdlib.PositionalArgumentsFormatter(),
+        structlog.processors.TimeStamper(fmt="iso"),
+        structlog.processors.StackInfoRenderer(),
+        structlog.processors.UnicodeDecoder(),
+    ]
+
+    if settings.log_format == "json":
+        renderer: structlog.types.Processor = structlog.processors.JSONRenderer()
+    else:
+        renderer = structlog.dev.ConsoleRenderer(colors=True)
+
+    structlog.configure(
+        processors=[
+            *shared_processors,
+            structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
+        ],
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        wrapper_class=structlog.stdlib.BoundLogger,
+        cache_logger_on_first_use=True,
+    )
+
+    formatter = structlog.stdlib.ProcessorFormatter(
+        processors=[
+            structlog.stdlib.ProcessorFormatter.remove_processors_meta,
+            renderer,
+        ],
+        foreign_pre_chain=shared_processors,
+    )
+
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setFormatter(formatter)
+
+    root_logger = logging.getLogger()
+    root_logger.handlers.clear()
+    root_logger.addHandler(handler)
+    root_logger.setLevel(getattr(logging, settings.log_level.upper(), logging.INFO))
+
+    # Reduce noise from third-party libraries
+    for logger_name in ("uvicorn.access", "sqlalchemy.engine", "celery"):
+        logging.getLogger(logger_name).setLevel(logging.WARNING)
+
+
+def get_logger(name: str | None = None) -> structlog.stdlib.BoundLogger:
+    """Get a structlog logger instance."""
+    return structlog.get_logger(name)

docengine/app/core/security.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import uuid
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from jose import JWTError, jwt
+from passlib.context import CryptContext
+
+from app.core.config import settings
+
+pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
+
+
+def hash_password(password: str) -> str:
+    """Hash a password using bcrypt."""
+    return pwd_context.hash(password)
+
+
+def verify_password(plain_password: str, hashed_password: str) -> bool:
+    """Verify a plain password against a hashed password."""
+    return pwd_context.verify(plain_password, hashed_password)
+
+
+def create_access_token(data: dict[str, Any], expires_delta: timedelta | None = None) -> str:
+    """Create a JWT access token."""
+    to_encode = data.copy()
+    expire = datetime.now(UTC) + (expires_delta or timedelta(minutes=settings.jwt_access_token_expire_minutes))
+    to_encode.update({"exp": expire, "type": "access"})
+    return jwt.encode(to_encode, settings.jwt_secret_key, algorithm=settings.jwt_algorithm)
+
+
+def create_refresh_token(data: dict[str, Any], expires_delta: timedelta | None = None) -> str:
+    """Create a JWT refresh token."""
+    to_encode = data.copy()
+    expire = datetime.now(UTC) + (expires_delta or timedelta(days=settings.jwt_refresh_token_expire_days))
+    to_encode.update({
+        "exp": expire,
+        "type": "refresh",
+        "jti": str(uuid.uuid4()),
+    })
+    return jwt.encode(to_encode, settings.jwt_secret_key, algorithm=settings.jwt_algorithm)
+
+
+def decode_token(token: str) -> dict[str, Any]:
+    """Decode and validate a JWT token."""
+    try:
+        payload = jwt.decode(token, settings.jwt_secret_key, algorithms=[settings.jwt_algorithm])
+        return payload
+    except JWTError as e:
+        raise InvalidTokenError(str(e)) from e
+
+
+class InvalidTokenError(Exception):
+    """Raised when a JWT token is invalid or expired."""
+
+    def __init__(self, detail: str = "Invalid or expired token") -> None:
+        self.detail = detail
+        super().__init__(self.detail)

docengine/app/events/handlers.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from app.core.logging_config import get_logger, setup_logging
+
+logger = get_logger(__name__)
+
+
+def on_startup() -> None:
+    """Application startup event handler."""
+    setup_logging()
+    logger.info("application_starting", event="startup")
+
+    # Ensure storage directories exist
+    from app.storage.provider import get_storage_provider
+    try:
+        get_storage_provider()
+        logger.info("storage_initialized")
+    except Exception as e:
+        logger.error("storage_init_failed", error=str(e))
+
+    # Verify database connection
+    from app.core.database import check_database_connection
+    if check_database_connection():
+        logger.info("database_connected")
+    else:
+        logger.error("database_connection_failed")
+
+    logger.info("application_started", event="startup_complete")
+
+
+def on_shutdown() -> None:
+    """Application shutdown event handler."""
+    logger.info("application_shutting_down", event="shutdown")
+
+    # Cleanup resources
+    from app.core.database import engine
+    engine.dispose()
+
+    logger.info("application_stopped", event="shutdown_complete")

docengine/app/main.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+from contextlib import asynccontextmanager
+from collections.abc import AsyncGenerator
+from typing import Any
+
+from fastapi import FastAPI, Request, status
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+
+from app.api.router import api_v1_router
+from app.core.config import settings
+from app.core.exceptions import DocEngineException
+from app.events.handlers import on_shutdown, on_startup
+from app.middleware.audit import AuditMiddleware
+from app.middleware.cors import setup_cors
+from app.middleware.metrics import setup_metrics
+from app.middleware.rate_limit import RateLimitMiddleware
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
+    """Application lifespan manager."""
+    on_startup()
+    yield
+    on_shutdown()
+
+
+app = FastAPI(
+    title=settings.app_name,
+    description="Document Template Recognition and Reconstruction System",
+    version=settings.app_version,
+    docs_url="/docs",
+    redoc_url="/redoc",
+    openapi_url="/openapi.json",
+    lifespan=lifespan,
+)
+
+# Setup middleware (order matters: last added = first executed)
+setup_cors(app)
+app.add_middleware(AuditMiddleware)
+app.add_middleware(RateLimitMiddleware)
+
+# Setup Prometheus metrics
+setup_metrics(app)
+
+# Include API routes
+app.include_router(api_v1_router)
+
+
+# Exception handlers
+@app.exception_handler(DocEngineException)
+async def docengine_exception_handler(request: Request, exc: DocEngineException) -> JSONResponse:
+    """Handle application-specific exceptions."""
+    return JSONResponse(
+        status_code=exc.status_code,
+        content={
+            "detail": exc.detail,
+            "error_code": type(exc).__name__,
+            "extra": exc.extra if exc.extra else None,
+        },
+    )
+
+
+@app.exception_handler(RequestValidationError)
+async def validation_exception_handler(request: Request, exc: RequestValidationError) -> JSONResponse:
+    """Handle request validation errors."""
+    errors = []
+    for error in exc.errors():
+        errors.append({
+            "field": ".".join(str(loc) for loc in error.get("loc", [])),
+            "message": error.get("msg", ""),
+            "type": error.get("type", ""),
+        })
+
+    return JSONResponse(
+        status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+        content={
+            "detail": "Request validation failed",
+            "error_code": "ValidationError",
+            "errors": errors,
+        },
+    )
+
+
+@app.exception_handler(Exception)
+async def general_exception_handler(request: Request, exc: Exception) -> JSONResponse:
+    """Handle unexpected exceptions."""
+    return JSONResponse(
+        status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+        content={
+            "detail": "An unexpected error occurred" if settings.is_production else str(exc),
+            "error_code": "InternalServerError",
+        },
+    )
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(
+        "app.main:app",
+        host=settings.app_host,
+        port=settings.app_port,
+        reload=not settings.is_production,
+        workers=1 if settings.app_debug else settings.app_workers,
+    )

docengine/app/middleware/audit.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+import time
+import uuid
+from typing import Any
+
+from fastapi import Request, Response
+from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
+
+from app.core.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+class AuditMiddleware(BaseHTTPMiddleware):
+    """Middleware to log all API requests for audit purposes."""
+
+    EXCLUDED_PATHS = {"/api/v1/health", "/metrics", "/docs", "/openapi.json", "/redoc"}
+
+    async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> Response:
+        if request.url.path in self.EXCLUDED_PATHS:
+            return await call_next(request)
+
+        request_id = str(uuid.uuid4())
+        start_time = time.monotonic()
+
+        # Extract client info
+        client_ip = request.client.host if request.client else "unknown"
+        user_agent = request.headers.get("user-agent", "unknown")
+
+        # Add request ID to request state
+        request.state.request_id = request_id
+
+        logger.info(
+            "request_started",
+            request_id=request_id,
+            method=request.method,
+            path=request.url.path,
+            client_ip=client_ip,
+            user_agent=user_agent[:200],
+        )
+
+        try:
+            response = await call_next(request)
+            duration_ms = (time.monotonic() - start_time) * 1000
+
+            logger.info(
+                "request_completed",
+                request_id=request_id,
+                method=request.method,
+                path=request.url.path,
+                status_code=response.status_code,
+                duration_ms=round(duration_ms, 2),
+                client_ip=client_ip,
+            )
+
+            response.headers["X-Request-ID"] = request_id
+            response.headers["X-Process-Time"] = f"{duration_ms:.2f}ms"
+            return response
+
+        except Exception as exc:
+            duration_ms = (time.monotonic() - start_time) * 1000
+            logger.exception(
+                "request_failed",
+                request_id=request_id,
+                method=request.method,
+                path=request.url.path,
+                duration_ms=round(duration_ms, 2),
+                error=str(exc),
+            )
+            raise

docengine/app/middleware/cors.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+
+from app.core.config import settings
+
+
+def setup_cors(app: FastAPI) -> None:
+    """Configure CORS middleware."""
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=settings.cors_origins,
+        allow_credentials=settings.cors_allow_credentials,
+        allow_methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"],
+        allow_headers=[
+            "Authorization",
+            "Content-Type",
+            "Accept",
+            "X-Request-ID",
+            "X-Requested-With",
+        ],
+        expose_headers=[
+            "X-Request-ID",
+            "X-Process-Time",
+            "X-RateLimit-Limit",
+            "X-RateLimit-Remaining",
+            "X-RateLimit-Reset",
+        ],
+        max_age=600,
+    )

docengine/app/middleware/metrics.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from fastapi import FastAPI
+from prometheus_fastapi_instrumentator import Instrumentator
+
+from app.core.config import settings
+
+
+def setup_metrics(app: FastAPI) -> None:
+    """Configure Prometheus metrics instrumentation."""
+    if not settings.prometheus_enabled:
+        return
+
+    instrumentator = Instrumentator(
+        should_group_status_codes=True,
+        should_ignore_untemplated=True,
+        should_respect_env_var=False,
+        excluded_handlers=["/metrics", "/api/v1/health", "/docs", "/openapi.json"],
+        env_var_name="PROMETHEUS_ENABLED",
+        inprogress_name="docengine_inprogress_requests",
+        inprogress_labels=True,
+    )
+
+    instrumentator.instrument(app).expose(
+        app,
+        endpoint="/metrics",
+        include_in_schema=False,
+        should_gzip=True,
+    )

docengine/app/middleware/rate_limit.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import time
+from collections import defaultdict
+
+from fastapi import Request, Response, status
+from fastapi.responses import JSONResponse
+from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
+
+from app.core.config import settings
+from app.core.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+class RateLimitMiddleware(BaseHTTPMiddleware):
+    """Token bucket rate limiter per client IP."""
+
+    EXCLUDED_PATHS = {"/api/v1/health", "/metrics", "/docs", "/openapi.json", "/redoc"}
+
+    def __init__(self, app, max_requests: int | None = None, window_seconds: int | None = None) -> None:  # noqa: ANN001
+        super().__init__(app)
+        self.max_requests = max_requests or settings.rate_limit_requests
+        self.window_seconds = window_seconds or settings.rate_limit_window_seconds
+        self._requests: dict[str, list[float]] = defaultdict(list)
+
+    def _clean_old_requests(self, client_ip: str, now: float) -> None:
+        """Remove requests outside the current window."""
+        cutoff = now - self.window_seconds
+        self._requests[client_ip] = [
+            ts for ts in self._requests[client_ip] if ts > cutoff
+        ]
+
+    async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> Response:
+        if request.url.path in self.EXCLUDED_PATHS:
+            return await call_next(request)
+
+        client_ip = request.client.host if request.client else "unknown"
+        now = time.monotonic()
+
+        self._clean_old_requests(client_ip, now)
+
+        if len(self._requests[client_ip]) >= self.max_requests:
+            logger.warning(
+                "rate_limit_exceeded",
+                client_ip=client_ip,
+                path=request.url.path,
+                request_count=len(self._requests[client_ip]),
+            )
+            return JSONResponse(
+                status_code=status.HTTP_429_TOO_MANY_REQUESTS,
+                content={
+                    "detail": "Rate limit exceeded. Please try again later.",
+                    "retry_after_seconds": self.window_seconds,
+                },
+                headers={
+                    "Retry-After": str(self.window_seconds),
+                    "X-RateLimit-Limit": str(self.max_requests),
+                    "X-RateLimit-Remaining": "0",
+                    "X-RateLimit-Reset": str(int(now + self.window_seconds)),
+                },
+            )
+
+        self._requests[client_ip].append(now)
+        remaining = self.max_requests - len(self._requests[client_ip])
+
+        response = await call_next(request)
+        response.headers["X-RateLimit-Limit"] = str(self.max_requests)
+        response.headers["X-RateLimit-Remaining"] = str(remaining)
+        response.headers["X-RateLimit-Reset"] = str(int(now + self.window_seconds))
+
+        return response

docengine/app/models/__init__.py

@@ -0,0 +1,43 @@
+from app.models.user import AuditLog, RefreshToken, Role, User, user_roles_table
+from app.models.document import (
+    Document,
+    DocumentImage,
+    DocumentPage,
+    DocumentTable,
+    DocumentTextBlock,
+    TemplateMatch,
+)
+from app.models.template import (
+    DocumentCell,
+    DocumentFormat,
+    DocumentRegion,
+    ImageRegion,
+    TableColumn,
+    TableFormat,
+    TableRow,
+    TemplateFingerprint,
+    Watermark,
+)
+
+__all__ = [
+    "User",
+    "Role",
+    "RefreshToken",
+    "AuditLog",
+    "user_roles_table",
+    "Document",
+    "DocumentPage",
+    "DocumentTextBlock",
+    "DocumentImage",
+    "DocumentTable",
+    "TemplateMatch",
+    "DocumentFormat",
+    "DocumentCell",
+    "DocumentRegion",
+    "TableFormat",
+    "TableColumn",
+    "TableRow",
+    "Watermark",
+    "ImageRegion",
+    "TemplateFingerprint",
+]

docengine/app/models/base.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import uuid
+from datetime import UTC, datetime
+
+from sqlalchemy import DateTime, func
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.orm import Mapped, mapped_column
+
+from app.core.database import Base
+
+
+class TimestampMixin:
+    """Mixin providing created_at and updated_at timestamps."""
+
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=lambda: datetime.now(UTC),
+        server_default=func.now(),
+        nullable=False,
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=lambda: datetime.now(UTC),
+        server_default=func.now(),
+        onupdate=lambda: datetime.now(UTC),
+        nullable=False,
+    )
+
+
+class UUIDPrimaryKeyMixin:
+    """Mixin providing a UUID primary key."""
+
+    id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        primary_key=True,
+        default=uuid.uuid4,
+        nullable=False,
+    )

docengine/app/models/document.py

@@ -0,0 +1,244 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+
+from sqlalchemy import BigInteger, Boolean, DateTime, Float, ForeignKey, Integer, String, Text, func
+from sqlalchemy.dialects.postgresql import JSONB, UUID
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from app.core.database import Base
+from app.models.base import TimestampMixin, UUIDPrimaryKeyMixin
+
+
+class Document(Base, UUIDPrimaryKeyMixin, TimestampMixin):
+    """Uploaded document record."""
+
+    __tablename__ = "documents"
+
+    filename: Mapped[str] = mapped_column(String(500), nullable=False)
+    original_filename: Mapped[str] = mapped_column(String(500), nullable=False)
+    content_type: Mapped[str] = mapped_column(String(100), nullable=False)
+    file_size: Mapped[int] = mapped_column(BigInteger, nullable=False)
+    checksum: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
+    storage_path: Mapped[str] = mapped_column(String(1024), nullable=False)
+    status: Mapped[str] = mapped_column(
+        String(50),
+        default="pending",
+        nullable=False,
+        index=True,
+    )
+    page_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    is_scanned: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
+    document_metadata: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
+    uploaded_by: Mapped[uuid.UUID | None] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("users.id", ondelete="SET NULL"),
+        nullable=True,
+        index=True,
+    )
+
+    pages: Mapped[list[DocumentPage]] = relationship(
+        "DocumentPage",
+        back_populates="document",
+        cascade="all, delete-orphan",
+        order_by="DocumentPage.page_number",
+        lazy="selectin",
+    )
+    template_matches: Mapped[list[TemplateMatch]] = relationship(
+        "TemplateMatch",
+        back_populates="document",
+        cascade="all, delete-orphan",
+        lazy="dynamic",
+    )
+
+    def __repr__(self) -> str:
+        return f"<Document(id={self.id}, filename={self.original_filename}, status={self.status})>"
+
+
+class DocumentPage(Base, UUIDPrimaryKeyMixin):
+    """Individual page within a document."""
+
+    __tablename__ = "document_pages"
+
+    document_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("documents.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    page_number: Mapped[int] = mapped_column(Integer, nullable=False)
+    width: Mapped[float] = mapped_column(Float, nullable=False)
+    height: Mapped[float] = mapped_column(Float, nullable=False)
+    image_path: Mapped[str | None] = mapped_column(String(1024), nullable=True)
+    text_content: Mapped[str | None] = mapped_column(Text, nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    document: Mapped[Document] = relationship("Document", back_populates="pages")
+    text_blocks: Mapped[list[DocumentTextBlock]] = relationship(
+        "DocumentTextBlock",
+        back_populates="page",
+        cascade="all, delete-orphan",
+        order_by="DocumentTextBlock.sequence",
+        lazy="selectin",
+    )
+    images: Mapped[list[DocumentImage]] = relationship(
+        "DocumentImage",
+        back_populates="page",
+        cascade="all, delete-orphan",
+        lazy="selectin",
+    )
+    tables: Mapped[list[DocumentTable]] = relationship(
+        "DocumentTable",
+        back_populates="page",
+        cascade="all, delete-orphan",
+        lazy="selectin",
+    )
+
+    def __repr__(self) -> str:
+        return f"<DocumentPage(id={self.id}, document_id={self.document_id}, page={self.page_number})>"
+
+
+class DocumentTextBlock(Base, UUIDPrimaryKeyMixin):
+    """Extracted text block from a document page."""
+
+    __tablename__ = "document_text_blocks"
+
+    page_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("document_pages.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    text: Mapped[str] = mapped_column(Text, nullable=False)
+    x: Mapped[float] = mapped_column(Float, nullable=False)
+    y: Mapped[float] = mapped_column(Float, nullable=False)
+    width: Mapped[float] = mapped_column(Float, nullable=False)
+    height: Mapped[float] = mapped_column(Float, nullable=False)
+    confidence: Mapped[float | None] = mapped_column(Float, nullable=True)
+    font_family: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    font_size: Mapped[float | None] = mapped_column(Float, nullable=True)
+    font_color: Mapped[str | None] = mapped_column(String(50), nullable=True)
+    font_style: Mapped[str | None] = mapped_column(String(50), nullable=True)
+    block_type: Mapped[str] = mapped_column(
+        String(50),
+        default="text",
+        nullable=False,
+    )
+    sequence: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    page: Mapped[DocumentPage] = relationship("DocumentPage", back_populates="text_blocks")
+
+    def __repr__(self) -> str:
+        return f"<DocumentTextBlock(id={self.id}, type={self.block_type}, text={self.text[:50]})>"
+
+
+class DocumentImage(Base, UUIDPrimaryKeyMixin):
+    """Extracted image from a document page."""
+
+    __tablename__ = "document_images"
+
+    page_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("document_pages.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    x: Mapped[float] = mapped_column(Float, nullable=False)
+    y: Mapped[float] = mapped_column(Float, nullable=False)
+    width: Mapped[float] = mapped_column(Float, nullable=False)
+    height: Mapped[float] = mapped_column(Float, nullable=False)
+    image_path: Mapped[str] = mapped_column(String(1024), nullable=False)
+    image_type: Mapped[str] = mapped_column(
+        String(50),
+        default="figure",
+        nullable=False,
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    page: Mapped[DocumentPage] = relationship("DocumentPage", back_populates="images")
+
+    def __repr__(self) -> str:
+        return f"<DocumentImage(id={self.id}, type={self.image_type})>"
+
+
+class DocumentTable(Base, UUIDPrimaryKeyMixin):
+    """Extracted table from a document page."""
+
+    __tablename__ = "document_tables"
+
+    page_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("document_pages.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    x: Mapped[float] = mapped_column(Float, nullable=False)
+    y: Mapped[float] = mapped_column(Float, nullable=False)
+    width: Mapped[float] = mapped_column(Float, nullable=False)
+    height: Mapped[float] = mapped_column(Float, nullable=False)
+    rows: Mapped[int] = mapped_column(Integer, nullable=False)
+    columns: Mapped[int] = mapped_column(Integer, nullable=False)
+    data: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    page: Mapped[DocumentPage] = relationship("DocumentPage", back_populates="tables")
+
+    def __repr__(self) -> str:
+        return f"<DocumentTable(id={self.id}, rows={self.rows}, cols={self.columns})>"
+
+
+class TemplateMatch(Base, UUIDPrimaryKeyMixin):
+    """Template matching result for a document."""
+
+    __tablename__ = "template_matches"
+
+    document_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("documents.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    format_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("document_formats.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    confidence_score: Mapped[float] = mapped_column(Float, nullable=False)
+    match_details: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    selected: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    document: Mapped[Document] = relationship("Document", back_populates="template_matches")
+    template: Mapped[DocumentFormat] = relationship("DocumentFormat")
+
+    def __repr__(self) -> str:
+        return f"<TemplateMatch(id={self.id}, doc={self.document_id}, score={self.confidence_score})>"

docengine/app/models/template.py

@@ -0,0 +1,378 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+
+from sqlalchemy import Boolean, DateTime, Float, ForeignKey, Integer, String, Text, func
+from sqlalchemy.dialects.postgresql import JSONB, UUID
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from app.core.database import Base
+from app.models.base import TimestampMixin, UUIDPrimaryKeyMixin
+
+
+class DocumentFormat(Base, UUIDPrimaryKeyMixin, TimestampMixin):
+    """Reusable document template format."""
+
+    __tablename__ = "document_formats"
+
+    name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
+    description: Mapped[str | None] = mapped_column(Text, nullable=True)
+    page_width: Mapped[float] = mapped_column(Float, nullable=False)
+    page_height: Mapped[float] = mapped_column(Float, nullable=False)
+    page_count: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
+    margin_top: Mapped[float] = mapped_column(Float, default=72.0, nullable=False)
+    margin_right: Mapped[float] = mapped_column(Float, default=72.0, nullable=False)
+    margin_bottom: Mapped[float] = mapped_column(Float, default=72.0, nullable=False)
+    margin_left: Mapped[float] = mapped_column(Float, default=72.0, nullable=False)
+    fingerprint: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    source_document_id: Mapped[uuid.UUID | None] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("documents.id", ondelete="SET NULL"),
+        nullable=True,
+        index=True,
+    )
+    version: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
+    is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False, index=True)
+    created_by: Mapped[uuid.UUID | None] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("users.id", ondelete="SET NULL"),
+        nullable=True,
+    )
+
+    cells: Mapped[list[DocumentCell]] = relationship(
+        "DocumentCell",
+        back_populates="format",
+        cascade="all, delete-orphan",
+        order_by="DocumentCell.sequence",
+        lazy="selectin",
+    )
+    regions: Mapped[list[DocumentRegion]] = relationship(
+        "DocumentRegion",
+        back_populates="format",
+        cascade="all, delete-orphan",
+        order_by="DocumentRegion.sequence",
+        lazy="selectin",
+    )
+    table_formats: Mapped[list[TableFormat]] = relationship(
+        "TableFormat",
+        back_populates="format",
+        cascade="all, delete-orphan",
+        lazy="selectin",
+    )
+    watermarks: Mapped[list[Watermark]] = relationship(
+        "Watermark",
+        back_populates="format",
+        cascade="all, delete-orphan",
+        lazy="selectin",
+    )
+    image_regions: Mapped[list[ImageRegion]] = relationship(
+        "ImageRegion",
+        back_populates="format",
+        cascade="all, delete-orphan",
+        lazy="selectin",
+    )
+    fingerprint_record: Mapped[TemplateFingerprint | None] = relationship(
+        "TemplateFingerprint",
+        back_populates="format",
+        uselist=False,
+        cascade="all, delete-orphan",
+        lazy="selectin",
+    )
+
+    def __repr__(self) -> str:
+        return f"<DocumentFormat(id={self.id}, name={self.name}, v{self.version})>"
+
+
+class DocumentCell(Base, UUIDPrimaryKeyMixin):
+    """Cell definition within a document template."""
+
+    __tablename__ = "document_cells"
+
+    format_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("document_formats.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    page_number: Mapped[int] = mapped_column(Integer, nullable=False)
+    x: Mapped[float] = mapped_column(Float, nullable=False)
+    y: Mapped[float] = mapped_column(Float, nullable=False)
+    width: Mapped[float] = mapped_column(Float, nullable=False)
+    height: Mapped[float] = mapped_column(Float, nullable=False)
+    row_no: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+    column_no: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+    data_type: Mapped[str] = mapped_column(String(50), default="text", nullable=False)
+    font_family: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    font_size: Mapped[float | None] = mapped_column(Float, nullable=True)
+    font_style: Mapped[str | None] = mapped_column(String(50), nullable=True)
+    font_color: Mapped[str | None] = mapped_column(String(50), nullable=True)
+    background_color: Mapped[str | None] = mapped_column(String(50), nullable=True)
+    border_top: Mapped[str | None] = mapped_column(String(100), nullable=True)
+    border_right: Mapped[str | None] = mapped_column(String(100), nullable=True)
+    border_bottom: Mapped[str | None] = mapped_column(String(100), nullable=True)
+    border_left: Mapped[str | None] = mapped_column(String(100), nullable=True)
+    padding_top: Mapped[float] = mapped_column(Float, default=0.0, nullable=False)
+    padding_right: Mapped[float] = mapped_column(Float, default=0.0, nullable=False)
+    padding_bottom: Mapped[float] = mapped_column(Float, default=0.0, nullable=False)
+    padding_left: Mapped[float] = mapped_column(Float, default=0.0, nullable=False)
+    alignment: Mapped[str] = mapped_column(String(20), default="left", nullable=False)
+    vertical_alignment: Mapped[str] = mapped_column(String(20), default="top", nullable=False)
+    rowspan: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
+    colspan: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
+    static_text: Mapped[str | None] = mapped_column(Text, nullable=True)
+    field_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    sequence: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+    is_dynamic: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    format: Mapped[DocumentFormat] = relationship("DocumentFormat", back_populates="cells")
+
+    def __repr__(self) -> str:
+        return f"<DocumentCell(id={self.id}, page={self.page_number}, row={self.row_no}, col={self.column_no})>"
+
+
+class DocumentRegion(Base, UUIDPrimaryKeyMixin):
+    """Region definition within a document template."""
+
+    __tablename__ = "document_regions"
+
+    format_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("document_formats.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    page_number: Mapped[int] = mapped_column(Integer, nullable=False)
+    region_type: Mapped[str] = mapped_column(String(50), nullable=False)
+    x: Mapped[float] = mapped_column(Float, nullable=False)
+    y: Mapped[float] = mapped_column(Float, nullable=False)
+    width: Mapped[float] = mapped_column(Float, nullable=False)
+    height: Mapped[float] = mapped_column(Float, nullable=False)
+    content: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    sequence: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    format: Mapped[DocumentFormat] = relationship("DocumentFormat", back_populates="regions")
+
+    def __repr__(self) -> str:
+        return f"<DocumentRegion(id={self.id}, type={self.region_type}, page={self.page_number})>"
+
+
+class TableFormat(Base, UUIDPrimaryKeyMixin):
+    """Table definition within a document template."""
+
+    __tablename__ = "table_formats"
+
+    format_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("document_formats.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    page_number: Mapped[int] = mapped_column(Integer, nullable=False)
+    x: Mapped[float] = mapped_column(Float, nullable=False)
+    y: Mapped[float] = mapped_column(Float, nullable=False)
+    width: Mapped[float] = mapped_column(Float, nullable=False)
+    height: Mapped[float] = mapped_column(Float, nullable=False)
+    rows: Mapped[int] = mapped_column(Integer, nullable=False)
+    columns: Mapped[int] = mapped_column(Integer, nullable=False)
+    border_style: Mapped[str] = mapped_column(String(50), default="solid", nullable=False)
+    border_width: Mapped[float] = mapped_column(Float, default=1.0, nullable=False)
+    border_color: Mapped[str] = mapped_column(String(50), default="#000000", nullable=False)
+    header_rows: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    format: Mapped[DocumentFormat] = relationship("DocumentFormat", back_populates="table_formats")
+    table_columns: Mapped[list[TableColumn]] = relationship(
+        "TableColumn",
+        back_populates="table_format",
+        cascade="all, delete-orphan",
+        order_by="TableColumn.column_index",
+        lazy="selectin",
+    )
+    table_rows: Mapped[list[TableRow]] = relationship(
+        "TableRow",
+        back_populates="table_format",
+        cascade="all, delete-orphan",
+        order_by="TableRow.row_index",
+        lazy="selectin",
+    )
+
+    def __repr__(self) -> str:
+        return f"<TableFormat(id={self.id}, rows={self.rows}, cols={self.columns})>"
+
+
+class TableColumn(Base, UUIDPrimaryKeyMixin):
+    """Column definition within a table format."""
+
+    __tablename__ = "table_columns"
+
+    table_format_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("table_formats.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    column_index: Mapped[int] = mapped_column(Integer, nullable=False)
+    width: Mapped[float] = mapped_column(Float, nullable=False)
+    header_text: Mapped[str | None] = mapped_column(String(500), nullable=True)
+    data_type: Mapped[str] = mapped_column(String(50), default="text", nullable=False)
+    alignment: Mapped[str] = mapped_column(String(20), default="left", nullable=False)
+    font_family: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    font_size: Mapped[float | None] = mapped_column(Float, nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    table_format: Mapped[TableFormat] = relationship("TableFormat", back_populates="table_columns")
+
+    def __repr__(self) -> str:
+        return f"<TableColumn(id={self.id}, index={self.column_index}, header={self.header_text})>"
+
+
+class TableRow(Base, UUIDPrimaryKeyMixin):
+    """Row definition within a table format."""
+
+    __tablename__ = "table_rows"
+
+    table_format_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("table_formats.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    row_index: Mapped[int] = mapped_column(Integer, nullable=False)
+    height: Mapped[float] = mapped_column(Float, default=20.0, nullable=False)
+    is_header: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
+    background_color: Mapped[str | None] = mapped_column(String(50), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    table_format: Mapped[TableFormat] = relationship("TableFormat", back_populates="table_rows")
+
+    def __repr__(self) -> str:
+        return f"<TableRow(id={self.id}, index={self.row_index}, is_header={self.is_header})>"
+
+
+class Watermark(Base, UUIDPrimaryKeyMixin):
+    """Watermark definition within a document template."""
+
+    __tablename__ = "watermarks"
+
+    format_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("document_formats.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    page_number: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    text: Mapped[str | None] = mapped_column(String(500), nullable=True)
+    image_path: Mapped[str | None] = mapped_column(String(1024), nullable=True)
+    x: Mapped[float] = mapped_column(Float, nullable=False)
+    y: Mapped[float] = mapped_column(Float, nullable=False)
+    width: Mapped[float] = mapped_column(Float, nullable=False)
+    height: Mapped[float] = mapped_column(Float, nullable=False)
+    opacity: Mapped[float] = mapped_column(Float, default=0.3, nullable=False)
+    rotation: Mapped[float] = mapped_column(Float, default=0.0, nullable=False)
+    font_family: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    font_size: Mapped[float | None] = mapped_column(Float, nullable=True)
+    font_color: Mapped[str | None] = mapped_column(String(50), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    format: Mapped[DocumentFormat] = relationship("DocumentFormat", back_populates="watermarks")
+
+    def __repr__(self) -> str:
+        return f"<Watermark(id={self.id}, text={self.text})>"
+
+
+class ImageRegion(Base, UUIDPrimaryKeyMixin):
+    """Image region within a document template."""
+
+    __tablename__ = "image_regions"
+
+    format_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("document_formats.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    page_number: Mapped[int] = mapped_column(Integer, nullable=False)
+    x: Mapped[float] = mapped_column(Float, nullable=False)
+    y: Mapped[float] = mapped_column(Float, nullable=False)
+    width: Mapped[float] = mapped_column(Float, nullable=False)
+    height: Mapped[float] = mapped_column(Float, nullable=False)
+    image_path: Mapped[str | None] = mapped_column(String(1024), nullable=True)
+    image_type: Mapped[str] = mapped_column(String(50), default="figure", nullable=False)
+    is_static: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
+    field_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    format: Mapped[DocumentFormat] = relationship("DocumentFormat", back_populates="image_regions")
+
+    def __repr__(self) -> str:
+        return f"<ImageRegion(id={self.id}, type={self.image_type}, page={self.page_number})>"
+
+
+class TemplateFingerprint(Base, UUIDPrimaryKeyMixin):
+    """Layout fingerprint for template matching."""
+
+    __tablename__ = "template_fingerprints"
+
+    format_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("document_formats.id", ondelete="CASCADE"),
+        nullable=False,
+        unique=True,
+        index=True,
+    )
+    page_dimensions: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    logo_coordinates: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    header_coordinates: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    footer_coordinates: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    table_coordinates: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    cell_coordinates: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    fingerprint_hash: Mapped[str] = mapped_column(String(256), nullable=False, index=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    format: Mapped[DocumentFormat] = relationship("DocumentFormat", back_populates="fingerprint_record")
+
+    def __repr__(self) -> str:
+        return f"<TemplateFingerprint(id={self.id}, format_id={self.format_id}, hash={self.fingerprint_hash[:16]})>"

docengine/app/models/user.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+
+from sqlalchemy import Boolean, DateTime, ForeignKey, String, Table, Text, func
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from app.core.database import Base
+from app.models.base import TimestampMixin, UUIDPrimaryKeyMixin
+
+user_roles_table = Table(
+    "user_roles",
+    Base.metadata,
+    mapped_column("user_id", UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True),
+    mapped_column("role_id", UUID(as_uuid=True), ForeignKey("roles.id", ondelete="CASCADE"), primary_key=True),
+)
+
+
+class User(Base, UUIDPrimaryKeyMixin, TimestampMixin):
+    """User account model."""
+
+    __tablename__ = "users"
+
+    username: Mapped[str] = mapped_column(String(150), unique=True, nullable=False, index=True)
+    email: Mapped[str] = mapped_column(String(255), unique=True, nullable=False, index=True)
+    hashed_password: Mapped[str] = mapped_column(String(255), nullable=False)
+    full_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
+    is_superuser: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
+    last_login: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+
+    roles: Mapped[list[Role]] = relationship(
+        "Role",
+        secondary=user_roles_table,
+        back_populates="users",
+        lazy="joined",
+    )
+    refresh_tokens: Mapped[list[RefreshToken]] = relationship(
+        "RefreshToken",
+        back_populates="user",
+        cascade="all, delete-orphan",
+        lazy="dynamic",
+    )
+    audit_logs: Mapped[list[AuditLog]] = relationship(
+        "AuditLog",
+        back_populates="user",
+        lazy="dynamic",
+    )
+
+    def __repr__(self) -> str:
+        return f"<User(id={self.id}, username={self.username})>"
+
+
+class Role(Base, UUIDPrimaryKeyMixin):
+    """User role model."""
+
+    __tablename__ = "roles"
+
+    name: Mapped[str] = mapped_column(String(50), unique=True, nullable=False, index=True)
+    description: Mapped[str | None] = mapped_column(Text, nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    users: Mapped[list[User]] = relationship(
+        "User",
+        secondary=user_roles_table,
+        back_populates="roles",
+        lazy="dynamic",
+    )
+
+    def __repr__(self) -> str:
+        return f"<Role(id={self.id}, name={self.name})>"
+
+
+class RefreshToken(Base, UUIDPrimaryKeyMixin):
+    """JWT refresh token storage."""
+
+    __tablename__ = "refresh_tokens"
+
+    user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("users.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    token: Mapped[str] = mapped_column(String(512), unique=True, nullable=False, index=True)
+    expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    revoked: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    user: Mapped[User] = relationship("User", back_populates="refresh_tokens")
+
+    def __repr__(self) -> str:
+        return f"<RefreshToken(id={self.id}, user_id={self.user_id}, revoked={self.revoked})>"
+
+
+class AuditLog(Base, UUIDPrimaryKeyMixin):
+    """Audit trail for user actions."""
+
+    __tablename__ = "audit_logs"
+
+    user_id: Mapped[uuid.UUID | None] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("users.id", ondelete="SET NULL"),
+        nullable=True,
+        index=True,
+    )
+    action: Mapped[str] = mapped_column(String(100), nullable=False, index=True)
+    resource_type: Mapped[str] = mapped_column(String(100), nullable=False, index=True)
+    resource_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    details: Mapped[dict | None] = mapped_column(type_=Text, nullable=True)
+    ip_address: Mapped[str | None] = mapped_column(String(45), nullable=True)
+    user_agent: Mapped[str | None] = mapped_column(String(512), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+        index=True,
+    )
+
+    user: Mapped[User | None] = relationship("User", back_populates="audit_logs")
+
+    def __repr__(self) -> str:
+        return f"<AuditLog(id={self.id}, action={self.action}, resource={self.resource_type})>"

docengine/app/repositories/base.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+import uuid
+from typing import Any, Generic, TypeVar
+
+from sqlalchemy import func, select
+from sqlalchemy.orm import Session
+
+from app.core.database import Base
+
+ModelType = TypeVar("ModelType", bound=Base)
+
+
+class BaseRepository(Generic[ModelType]):
+    """Base repository with common CRUD operations."""
+
+    def __init__(self, db: Session, model: type[ModelType]) -> None:
+        self.db = db
+        self.model = model
+
+    def get_by_id(self, entity_id: str | uuid.UUID) -> ModelType | None:
+        """Get an entity by its primary key."""
+        if isinstance(entity_id, str):
+            entity_id = uuid.UUID(entity_id)
+        return self.db.get(self.model, entity_id)
+
+    def get_all(
+        self,
+        offset: int = 0,
+        limit: int = 100,
+        filters: dict[str, Any] | None = None,
+        order_by: str | None = None,
+        order_desc: bool = False,
+    ) -> list[ModelType]:
+        """Get all entities with optional filtering, pagination, and ordering."""
+        query = select(self.model)
+
+        if filters:
+            for key, value in filters.items():
+                if hasattr(self.model, key) and value is not None:
+                    query = query.where(getattr(self.model, key) == value)
+
+        if order_by and hasattr(self.model, order_by):
+            col = getattr(self.model, order_by)
+            query = query.order_by(col.desc() if order_desc else col.asc())
+
+        query = query.offset(offset).limit(limit)
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def count(self, filters: dict[str, Any] | None = None) -> int:
+        """Count entities with optional filtering."""
+        query = select(func.count()).select_from(self.model)
+
+        if filters:
+            for key, value in filters.items():
+                if hasattr(self.model, key) and value is not None:
+                    query = query.where(getattr(self.model, key) == value)
+
+        result = self.db.execute(query)
+        return result.scalar_one()
+
+    def create(self, entity: ModelType) -> ModelType:
+        """Create a new entity."""
+        self.db.add(entity)
+        self.db.flush()
+        self.db.refresh(entity)
+        return entity
+
+    def create_many(self, entities: list[ModelType]) -> list[ModelType]:
+        """Create multiple entities."""
+        self.db.add_all(entities)
+        self.db.flush()
+        for entity in entities:
+            self.db.refresh(entity)
+        return entities
+
+    def update(self, entity: ModelType, update_data: dict[str, Any]) -> ModelType:
+        """Update an entity with given data."""
+        for key, value in update_data.items():
+            if hasattr(entity, key) and value is not None:
+                setattr(entity, key, value)
+        self.db.flush()
+        self.db.refresh(entity)
+        return entity
+
+    def delete(self, entity: ModelType) -> None:
+        """Delete an entity."""
+        self.db.delete(entity)
+        self.db.flush()
+
+    def delete_by_id(self, entity_id: str | uuid.UUID) -> bool:
+        """Delete an entity by its ID. Returns True if deleted."""
+        entity = self.get_by_id(entity_id)
+        if entity:
+            self.delete(entity)
+            return True
+        return False
+
+    def exists(self, entity_id: str | uuid.UUID) -> bool:
+        """Check if an entity exists by ID."""
+        if isinstance(entity_id, str):
+            entity_id = uuid.UUID(entity_id)
+        query = select(func.count()).select_from(self.model).where(self.model.id == entity_id)
+        result = self.db.execute(query)
+        return result.scalar_one() > 0
+
+    def commit(self) -> None:
+        """Commit the current transaction."""
+        self.db.commit()
+
+    def rollback(self) -> None:
+        """Rollback the current transaction."""
+        self.db.rollback()

docengine/app/repositories/document_repository.py

@@ -0,0 +1,335 @@
+from __future__ import annotations
+
+import uuid
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from app.models.document import (
+    Document,
+    DocumentImage,
+    DocumentPage,
+    DocumentTable,
+    DocumentTextBlock,
+    TemplateMatch,
+)
+from app.repositories.base import BaseRepository
+
+
+class DocumentRepository(BaseRepository[Document]):
+    """Repository for Document operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, Document)
+
+    def get_by_checksum(self, checksum: str) -> Document | None:
+        """Get document by file checksum."""
+        query = select(Document).where(Document.checksum == checksum)
+        result = self.db.execute(query)
+        return result.scalars().first()
+
+    def get_by_status(self, status: str, offset: int = 0, limit: int = 100) -> list[Document]:
+        """Get documents by processing status."""
+        query = (
+            select(Document)
+            .where(Document.status == status)
+            .order_by(Document.created_at.desc())
+            .offset(offset)
+            .limit(limit)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def get_user_documents(
+        self,
+        user_id: uuid.UUID,
+        offset: int = 0,
+        limit: int = 100,
+    ) -> list[Document]:
+        """Get documents uploaded by a specific user."""
+        query = (
+            select(Document)
+            .where(Document.uploaded_by == user_id)
+            .order_by(Document.created_at.desc())
+            .offset(offset)
+            .limit(limit)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def update_status(
+        self,
+        document_id: uuid.UUID,
+        status: str,
+        error_message: str | None = None,
+    ) -> Document | None:
+        """Update document processing status."""
+        document = self.get_by_id(document_id)
+        if document:
+            document.status = status
+            if error_message:
+                document.error_message = error_message
+            self.db.flush()
+            self.db.refresh(document)
+        return document
+
+    def get_with_pages(self, document_id: uuid.UUID) -> Document | None:
+        """Get document with all pages eagerly loaded."""
+        return self.get_by_id(document_id)
+
+    def get_pending_documents(self, limit: int = 10) -> list[Document]:
+        """Get pending documents for processing."""
+        return self.get_by_status("pending", limit=limit)
+
+
+class DocumentPageRepository(BaseRepository[DocumentPage]):
+    """Repository for DocumentPage operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, DocumentPage)
+
+    def get_document_pages(self, document_id: uuid.UUID) -> list[DocumentPage]:
+        """Get all pages for a document ordered by page number."""
+        query = (
+            select(DocumentPage)
+            .where(DocumentPage.document_id == document_id)
+            .order_by(DocumentPage.page_number)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def get_page_by_number(self, document_id: uuid.UUID, page_number: int) -> DocumentPage | None:
+        """Get a specific page by document ID and page number."""
+        query = select(DocumentPage).where(
+            DocumentPage.document_id == document_id,
+            DocumentPage.page_number == page_number,
+        )
+        result = self.db.execute(query)
+        return result.scalars().first()
+
+    def create_page(
+        self,
+        document_id: uuid.UUID,
+        page_number: int,
+        width: float,
+        height: float,
+        image_path: str | None = None,
+        text_content: str | None = None,
+    ) -> DocumentPage:
+        """Create a new document page."""
+        page = DocumentPage(
+            document_id=document_id,
+            page_number=page_number,
+            width=width,
+            height=height,
+            image_path=image_path,
+            text_content=text_content,
+        )
+        return self.create(page)
+
+
+class DocumentTextBlockRepository(BaseRepository[DocumentTextBlock]):
+    """Repository for DocumentTextBlock operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, DocumentTextBlock)
+
+    def get_page_text_blocks(self, page_id: uuid.UUID) -> list[DocumentTextBlock]:
+        """Get all text blocks for a page."""
+        query = (
+            select(DocumentTextBlock)
+            .where(DocumentTextBlock.page_id == page_id)
+            .order_by(DocumentTextBlock.sequence)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def get_by_block_type(self, page_id: uuid.UUID, block_type: str) -> list[DocumentTextBlock]:
+        """Get text blocks by type (header, footer, watermark, text)."""
+        query = (
+            select(DocumentTextBlock)
+            .where(
+                DocumentTextBlock.page_id == page_id,
+                DocumentTextBlock.block_type == block_type,
+            )
+            .order_by(DocumentTextBlock.sequence)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def create_text_block(
+        self,
+        page_id: uuid.UUID,
+        text: str,
+        x: float,
+        y: float,
+        width: float,
+        height: float,
+        confidence: float | None = None,
+        font_family: str | None = None,
+        font_size: float | None = None,
+        font_color: str | None = None,
+        font_style: str | None = None,
+        block_type: str = "text",
+        sequence: int = 0,
+    ) -> DocumentTextBlock:
+        """Create a new text block."""
+        text_block = DocumentTextBlock(
+            page_id=page_id,
+            text=text,
+            x=x,
+            y=y,
+            width=width,
+            height=height,
+            confidence=confidence,
+            font_family=font_family,
+            font_size=font_size,
+            font_color=font_color,
+            font_style=font_style,
+            block_type=block_type,
+            sequence=sequence,
+        )
+        return self.create(text_block)
+
+
+class DocumentImageRepository(BaseRepository[DocumentImage]):
+    """Repository for DocumentImage operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, DocumentImage)
+
+    def get_page_images(self, page_id: uuid.UUID) -> list[DocumentImage]:
+        """Get all images for a page."""
+        query = select(DocumentImage).where(DocumentImage.page_id == page_id)
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def create_image(
+        self,
+        page_id: uuid.UUID,
+        x: float,
+        y: float,
+        width: float,
+        height: float,
+        image_path: str,
+        image_type: str = "figure",
+    ) -> DocumentImage:
+        """Create a new document image record."""
+        image = DocumentImage(
+            page_id=page_id,
+            x=x,
+            y=y,
+            width=width,
+            height=height,
+            image_path=image_path,
+            image_type=image_type,
+        )
+        return self.create(image)
+
+
+class DocumentTableRepository(BaseRepository[DocumentTable]):
+    """Repository for DocumentTable operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, DocumentTable)
+
+    def get_page_tables(self, page_id: uuid.UUID) -> list[DocumentTable]:
+        """Get all tables for a page."""
+        query = select(DocumentTable).where(DocumentTable.page_id == page_id)
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def create_table(
+        self,
+        page_id: uuid.UUID,
+        x: float,
+        y: float,
+        width: float,
+        height: float,
+        rows: int,
+        columns: int,
+        data: dict | None = None,
+    ) -> DocumentTable:
+        """Create a new document table record."""
+        table = DocumentTable(
+            page_id=page_id,
+            x=x,
+            y=y,
+            width=width,
+            height=height,
+            rows=rows,
+            columns=columns,
+            data=data,
+        )
+        return self.create(table)
+
+
+class TemplateMatchRepository(BaseRepository[TemplateMatch]):
+    """Repository for TemplateMatch operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, TemplateMatch)
+
+    def get_document_matches(
+        self,
+        document_id: uuid.UUID,
+        min_confidence: float = 0.0,
+    ) -> list[TemplateMatch]:
+        """Get all template matches for a document."""
+        query = (
+            select(TemplateMatch)
+            .where(
+                TemplateMatch.document_id == document_id,
+                TemplateMatch.confidence_score >= min_confidence,
+            )
+            .order_by(TemplateMatch.confidence_score.desc())
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def get_selected_match(self, document_id: uuid.UUID) -> TemplateMatch | None:
+        """Get the selected template match for a document."""
+        query = select(TemplateMatch).where(
+            TemplateMatch.document_id == document_id,
+            TemplateMatch.selected.is_(True),
+        )
+        result = self.db.execute(query)
+        return result.scalars().first()
+
+    def select_match(self, match_id: uuid.UUID) -> TemplateMatch | None:
+        """Select a template match (deselecting all others for the same document)."""
+        match = self.get_by_id(match_id)
+        if not match:
+            return None
+
+        # Deselect all other matches for this document
+        query = select(TemplateMatch).where(
+            TemplateMatch.document_id == match.document_id,
+            TemplateMatch.selected.is_(True),
+        )
+        result = self.db.execute(query)
+        for existing_match in result.scalars().all():
+            existing_match.selected = False
+
+        match.selected = True
+        self.db.flush()
+        self.db.refresh(match)
+        return match
+
+    def create_match(
+        self,
+        document_id: uuid.UUID,
+        format_id: uuid.UUID,
+        confidence_score: float,
+        match_details: dict | None = None,
+        selected: bool = False,
+    ) -> TemplateMatch:
+        """Create a new template match."""
+        template_match = TemplateMatch(
+            document_id=document_id,
+            format_id=format_id,
+            confidence_score=confidence_score,
+            match_details=match_details,
+            selected=selected,
+        )
+        return self.create(template_match)

docengine/app/repositories/template_repository.py

@@ -0,0 +1,353 @@
+from __future__ import annotations
+
+import uuid
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from app.models.template import (
+    DocumentCell,
+    DocumentFormat,
+    DocumentRegion,
+    ImageRegion,
+    TableColumn,
+    TableFormat,
+    TableRow,
+    TemplateFingerprint,
+    Watermark,
+)
+from app.repositories.base import BaseRepository
+
+
+class TemplateRepository(BaseRepository[DocumentFormat]):
+    """Repository for DocumentFormat (template) operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, DocumentFormat)
+
+    def get_active_templates(self, offset: int = 0, limit: int = 100) -> list[DocumentFormat]:
+        """Get all active templates."""
+        query = (
+            select(DocumentFormat)
+            .where(DocumentFormat.is_active.is_(True))
+            .order_by(DocumentFormat.created_at.desc())
+            .offset(offset)
+            .limit(limit)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def count_active(self) -> int:
+        """Count active templates."""
+        return self.count(filters={"is_active": True})
+
+    def get_by_name(self, name: str) -> DocumentFormat | None:
+        """Get template by name."""
+        query = select(DocumentFormat).where(DocumentFormat.name == name)
+        result = self.db.execute(query)
+        return result.scalars().first()
+
+    def get_by_source_document(self, document_id: uuid.UUID) -> DocumentFormat | None:
+        """Get template generated from a specific source document."""
+        query = select(DocumentFormat).where(
+            DocumentFormat.source_document_id == document_id,
+            DocumentFormat.is_active.is_(True),
+        )
+        result = self.db.execute(query)
+        return result.scalars().first()
+
+    def create_template(
+        self,
+        name: str,
+        page_width: float,
+        page_height: float,
+        page_count: int = 1,
+        description: str | None = None,
+        margin_top: float = 72.0,
+        margin_right: float = 72.0,
+        margin_bottom: float = 72.0,
+        margin_left: float = 72.0,
+        fingerprint: dict | None = None,
+        source_document_id: uuid.UUID | None = None,
+        created_by: uuid.UUID | None = None,
+    ) -> DocumentFormat:
+        """Create a new template."""
+        template = DocumentFormat(
+            name=name,
+            page_width=page_width,
+            page_height=page_height,
+            page_count=page_count,
+            description=description,
+            margin_top=margin_top,
+            margin_right=margin_right,
+            margin_bottom=margin_bottom,
+            margin_left=margin_left,
+            fingerprint=fingerprint,
+            source_document_id=source_document_id,
+            created_by=created_by,
+        )
+        return self.create(template)
+
+    def deactivate_template(self, template_id: uuid.UUID) -> DocumentFormat | None:
+        """Soft-delete a template by deactivating it."""
+        template = self.get_by_id(template_id)
+        if template:
+            template.is_active = False
+            self.db.flush()
+            self.db.refresh(template)
+        return template
+
+    def get_all_with_fingerprints(self) -> list[DocumentFormat]:
+        """Get all active templates with their fingerprints."""
+        query = (
+            select(DocumentFormat)
+            .where(DocumentFormat.is_active.is_(True))
+            .order_by(DocumentFormat.created_at.desc())
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+
+class DocumentCellRepository(BaseRepository[DocumentCell]):
+    """Repository for DocumentCell operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, DocumentCell)
+
+    def get_template_cells(self, format_id: uuid.UUID) -> list[DocumentCell]:
+        """Get all cells for a template."""
+        query = (
+            select(DocumentCell)
+            .where(DocumentCell.format_id == format_id)
+            .order_by(DocumentCell.page_number, DocumentCell.sequence)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def get_page_cells(self, format_id: uuid.UUID, page_number: int) -> list[DocumentCell]:
+        """Get cells for a specific page of a template."""
+        query = (
+            select(DocumentCell)
+            .where(
+                DocumentCell.format_id == format_id,
+                DocumentCell.page_number == page_number,
+            )
+            .order_by(DocumentCell.sequence)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def get_dynamic_cells(self, format_id: uuid.UUID) -> list[DocumentCell]:
+        """Get all dynamic cells for a template."""
+        query = (
+            select(DocumentCell)
+            .where(
+                DocumentCell.format_id == format_id,
+                DocumentCell.is_dynamic.is_(True),
+            )
+            .order_by(DocumentCell.page_number, DocumentCell.sequence)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def create_cell(self, format_id: uuid.UUID, **kwargs) -> DocumentCell:  # noqa: ANN003
+        """Create a new cell for a template."""
+        cell = DocumentCell(format_id=format_id, **kwargs)
+        return self.create(cell)
+
+
+class DocumentRegionRepository(BaseRepository[DocumentRegion]):
+    """Repository for DocumentRegion operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, DocumentRegion)
+
+    def get_template_regions(self, format_id: uuid.UUID) -> list[DocumentRegion]:
+        """Get all regions for a template."""
+        query = (
+            select(DocumentRegion)
+            .where(DocumentRegion.format_id == format_id)
+            .order_by(DocumentRegion.page_number, DocumentRegion.sequence)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def get_regions_by_type(self, format_id: uuid.UUID, region_type: str) -> list[DocumentRegion]:
+        """Get regions of a specific type."""
+        query = (
+            select(DocumentRegion)
+            .where(
+                DocumentRegion.format_id == format_id,
+                DocumentRegion.region_type == region_type,
+            )
+            .order_by(DocumentRegion.page_number, DocumentRegion.sequence)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def create_region(self, format_id: uuid.UUID, **kwargs) -> DocumentRegion:  # noqa: ANN003
+        """Create a new region for a template."""
+        region = DocumentRegion(format_id=format_id, **kwargs)
+        return self.create(region)
+
+
+class TableFormatRepository(BaseRepository[TableFormat]):
+    """Repository for TableFormat operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, TableFormat)
+
+    def get_template_tables(self, format_id: uuid.UUID) -> list[TableFormat]:
+        """Get all table formats for a template."""
+        query = (
+            select(TableFormat)
+            .where(TableFormat.format_id == format_id)
+            .order_by(TableFormat.page_number)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def create_table_format(self, format_id: uuid.UUID, **kwargs) -> TableFormat:  # noqa: ANN003
+        """Create a new table format."""
+        table_format = TableFormat(format_id=format_id, **kwargs)
+        return self.create(table_format)
+
+
+class TableColumnRepository(BaseRepository[TableColumn]):
+    """Repository for TableColumn operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, TableColumn)
+
+    def get_table_columns(self, table_format_id: uuid.UUID) -> list[TableColumn]:
+        """Get all columns for a table format."""
+        query = (
+            select(TableColumn)
+            .where(TableColumn.table_format_id == table_format_id)
+            .order_by(TableColumn.column_index)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def create_column(self, table_format_id: uuid.UUID, **kwargs) -> TableColumn:  # noqa: ANN003
+        """Create a new table column."""
+        column = TableColumn(table_format_id=table_format_id, **kwargs)
+        return self.create(column)
+
+
+class TableRowRepository(BaseRepository[TableRow]):
+    """Repository for TableRow operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, TableRow)
+
+    def get_table_rows(self, table_format_id: uuid.UUID) -> list[TableRow]:
+        """Get all rows for a table format."""
+        query = (
+            select(TableRow)
+            .where(TableRow.table_format_id == table_format_id)
+            .order_by(TableRow.row_index)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def create_row(self, table_format_id: uuid.UUID, **kwargs) -> TableRow:  # noqa: ANN003
+        """Create a new table row."""
+        row = TableRow(table_format_id=table_format_id, **kwargs)
+        return self.create(row)
+
+
+class WatermarkRepository(BaseRepository[Watermark]):
+    """Repository for Watermark operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, Watermark)
+
+    def get_template_watermarks(self, format_id: uuid.UUID) -> list[Watermark]:
+        """Get all watermarks for a template."""
+        query = select(Watermark).where(Watermark.format_id == format_id)
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def create_watermark(self, format_id: uuid.UUID, **kwargs) -> Watermark:  # noqa: ANN003
+        """Create a new watermark."""
+        watermark = Watermark(format_id=format_id, **kwargs)
+        return self.create(watermark)
+
+
+class ImageRegionRepository(BaseRepository[ImageRegion]):
+    """Repository for ImageRegion operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, ImageRegion)
+
+    def get_template_images(self, format_id: uuid.UUID) -> list[ImageRegion]:
+        """Get all image regions for a template."""
+        query = select(ImageRegion).where(ImageRegion.format_id == format_id)
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def get_static_images(self, format_id: uuid.UUID) -> list[ImageRegion]:
+        """Get static image regions."""
+        query = select(ImageRegion).where(
+            ImageRegion.format_id == format_id,
+            ImageRegion.is_static.is_(True),
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def create_image_region(self, format_id: uuid.UUID, **kwargs) -> ImageRegion:  # noqa: ANN003
+        """Create a new image region."""
+        image_region = ImageRegion(format_id=format_id, **kwargs)
+        return self.create(image_region)
+
+
+class TemplateFingerprintRepository(BaseRepository[TemplateFingerprint]):
+    """Repository for TemplateFingerprint operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, TemplateFingerprint)
+
+    def get_by_format_id(self, format_id: uuid.UUID) -> TemplateFingerprint | None:
+        """Get fingerprint by template format ID."""
+        query = select(TemplateFingerprint).where(TemplateFingerprint.format_id == format_id)
+        result = self.db.execute(query)
+        return result.scalars().first()
+
+    def get_by_hash(self, fingerprint_hash: str) -> TemplateFingerprint | None:
+        """Get fingerprint by hash."""
+        query = select(TemplateFingerprint).where(
+            TemplateFingerprint.fingerprint_hash == fingerprint_hash
+        )
+        result = self.db.execute(query)
+        return result.scalars().first()
+
+    def get_all_fingerprints(self) -> list[TemplateFingerprint]:
+        """Get all fingerprints."""
+        query = select(TemplateFingerprint)
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def create_fingerprint(
+        self,
+        format_id: uuid.UUID,
+        fingerprint_hash: str,
+        page_dimensions: dict | None = None,
+        logo_coordinates: dict | None = None,
+        header_coordinates: dict | None = None,
+        footer_coordinates: dict | None = None,
+        table_coordinates: dict | None = None,
+        cell_coordinates: dict | None = None,
+    ) -> TemplateFingerprint:
+        """Create a new template fingerprint."""
+        fp = TemplateFingerprint(
+            format_id=format_id,
+            fingerprint_hash=fingerprint_hash,
+            page_dimensions=page_dimensions,
+            logo_coordinates=logo_coordinates,
+            header_coordinates=header_coordinates,
+            footer_coordinates=footer_coordinates,
+            table_coordinates=table_coordinates,
+            cell_coordinates=cell_coordinates,
+        )
+        return self.create(fp)

docengine/app/repositories/user_repository.py

@@ -0,0 +1,236 @@
+from __future__ import annotations
+
+import uuid
+from datetime import UTC, datetime
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from app.models.user import AuditLog, RefreshToken, Role, User, user_roles_table
+from app.repositories.base import BaseRepository
+
+
+class UserRepository(BaseRepository[User]):
+    """Repository for User operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, User)
+
+    def get_by_username(self, username: str) -> User | None:
+        """Get user by username."""
+        query = select(User).where(User.username == username)
+        result = self.db.execute(query)
+        return result.scalars().first()
+
+    def get_by_email(self, email: str) -> User | None:
+        """Get user by email."""
+        query = select(User).where(User.email == email)
+        result = self.db.execute(query)
+        return result.scalars().first()
+
+    def create_user(
+        self,
+        username: str,
+        email: str,
+        hashed_password: str,
+        full_name: str | None = None,
+        is_active: bool = True,
+        is_superuser: bool = False,
+        role_names: list[str] | None = None,
+    ) -> User:
+        """Create a new user with optional roles."""
+        user = User(
+            username=username,
+            email=email,
+            hashed_password=hashed_password,
+            full_name=full_name,
+            is_active=is_active,
+            is_superuser=is_superuser,
+        )
+
+        if role_names:
+            roles = self.get_roles_by_names(role_names)
+            user.roles = roles
+
+        return self.create(user)
+
+    def update_last_login(self, user: User) -> User:
+        """Update user's last login timestamp."""
+        user.last_login = datetime.now(UTC)
+        self.db.flush()
+        self.db.refresh(user)
+        return user
+
+    def get_roles_by_names(self, role_names: list[str]) -> list[Role]:
+        """Get roles by their names."""
+        query = select(Role).where(Role.name.in_(role_names))
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def assign_roles(self, user: User, role_names: list[str]) -> User:
+        """Assign roles to a user, replacing existing roles."""
+        roles = self.get_roles_by_names(role_names)
+        user.roles = roles
+        self.db.flush()
+        self.db.refresh(user)
+        return user
+
+    def get_active_users(self, offset: int = 0, limit: int = 100) -> list[User]:
+        """Get all active users."""
+        query = select(User).where(User.is_active.is_(True)).offset(offset).limit(limit)
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+
+class RoleRepository(BaseRepository[Role]):
+    """Repository for Role operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, Role)
+
+    def get_by_name(self, name: str) -> Role | None:
+        """Get role by name."""
+        query = select(Role).where(Role.name == name)
+        result = self.db.execute(query)
+        return result.scalars().first()
+
+    def create_role(self, name: str, description: str | None = None) -> Role:
+        """Create a new role."""
+        role = Role(name=name, description=description)
+        return self.create(role)
+
+    def get_all_roles(self) -> list[Role]:
+        """Get all roles."""
+        query = select(Role).order_by(Role.name)
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+
+class RefreshTokenRepository(BaseRepository[RefreshToken]):
+    """Repository for RefreshToken operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, RefreshToken)
+
+    def get_by_token(self, token: str) -> RefreshToken | None:
+        """Get refresh token by token string."""
+        query = select(RefreshToken).where(
+            RefreshToken.token == token,
+            RefreshToken.revoked.is_(False),
+            RefreshToken.expires_at > datetime.now(UTC),
+        )
+        result = self.db.execute(query)
+        return result.scalars().first()
+
+    def create_token(self, user_id: uuid.UUID, token: str, expires_at: datetime) -> RefreshToken:
+        """Create a new refresh token."""
+        refresh_token = RefreshToken(
+            user_id=user_id,
+            token=token,
+            expires_at=expires_at,
+        )
+        return self.create(refresh_token)
+
+    def revoke_token(self, token: str) -> bool:
+        """Revoke a refresh token."""
+        refresh_token = self.get_by_token(token)
+        if refresh_token:
+            refresh_token.revoked = True
+            self.db.flush()
+            return True
+        return False
+
+    def revoke_all_user_tokens(self, user_id: uuid.UUID) -> int:
+        """Revoke all refresh tokens for a user."""
+        query = select(RefreshToken).where(
+            RefreshToken.user_id == user_id,
+            RefreshToken.revoked.is_(False),
+        )
+        result = self.db.execute(query)
+        tokens = result.scalars().all()
+        count = 0
+        for token in tokens:
+            token.revoked = True
+            count += 1
+        self.db.flush()
+        return count
+
+    def cleanup_expired_tokens(self) -> int:
+        """Remove expired or revoked tokens."""
+        query = select(RefreshToken).where(
+            (RefreshToken.expires_at <= datetime.now(UTC)) | (RefreshToken.revoked.is_(True))
+        )
+        result = self.db.execute(query)
+        tokens = result.scalars().all()
+        count = len(tokens)
+        for token in tokens:
+            self.db.delete(token)
+        self.db.flush()
+        return count
+
+
+class AuditLogRepository(BaseRepository[AuditLog]):
+    """Repository for AuditLog operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, AuditLog)
+
+    def log_action(
+        self,
+        action: str,
+        resource_type: str,
+        resource_id: str | None = None,
+        user_id: uuid.UUID | None = None,
+        details: str | None = None,
+        ip_address: str | None = None,
+        user_agent: str | None = None,
+    ) -> AuditLog:
+        """Create an audit log entry."""
+        audit_log = AuditLog(
+            user_id=user_id,
+            action=action,
+            resource_type=resource_type,
+            resource_id=resource_id,
+            details=details,
+            ip_address=ip_address,
+            user_agent=user_agent,
+        )
+        return self.create(audit_log)
+
+    def get_user_logs(
+        self,
+        user_id: uuid.UUID,
+        offset: int = 0,
+        limit: int = 100,
+    ) -> list[AuditLog]:
+        """Get audit logs for a specific user."""
+        query = (
+            select(AuditLog)
+            .where(AuditLog.user_id == user_id)
+            .order_by(AuditLog.created_at.desc())
+            .offset(offset)
+            .limit(limit)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def get_resource_logs(
+        self,
+        resource_type: str,
+        resource_id: str,
+        offset: int = 0,
+        limit: int = 100,
+    ) -> list[AuditLog]:
+        """Get audit logs for a specific resource."""
+        query = (
+            select(AuditLog)
+            .where(
+                AuditLog.resource_type == resource_type,
+                AuditLog.resource_id == resource_id,
+            )
+            .order_by(AuditLog.created_at.desc())
+            .offset(offset)
+            .limit(limit)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())

docengine/app/schemas/auth.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from pydantic import BaseModel, EmailStr, Field
+
+from app.schemas.common import BaseSchema
+
+
+class LoginRequest(BaseModel):
+    """Login credentials."""
+
+    username: str = Field(..., min_length=3, max_length=150)
+    password: str = Field(..., min_length=8, max_length=128)
+
+
+class RegisterRequest(BaseModel):
+    """User registration payload."""
+
+    username: str = Field(..., min_length=3, max_length=150)
+    email: EmailStr
+    password: str = Field(..., min_length=8, max_length=128)
+    full_name: str | None = Field(None, max_length=255)
+
+
+class TokenResponse(BaseSchema):
+    """JWT token pair response."""
+
+    access_token: str
+    refresh_token: str
+    token_type: str = "bearer"
+    expires_in: int
+
+
+class RefreshTokenRequest(BaseModel):
+    """Refresh token request payload."""
+
+    refresh_token: str
+
+
+class ChangePasswordRequest(BaseModel):
+    """Change password payload."""
+
+    current_password: str = Field(..., min_length=8, max_length=128)
+    new_password: str = Field(..., min_length=8, max_length=128)

docengine/app/schemas/common.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from typing import Any, Generic, TypeVar
+
+from pydantic import BaseModel, ConfigDict, Field
+
+T = TypeVar("T")
+
+
+class BaseSchema(BaseModel):
+    """Base schema with common configuration."""
+
+    model_config = ConfigDict(
+        from_attributes=True,
+        populate_by_name=True,
+        str_strip_whitespace=True,
+    )
+
+
+class PaginationParams(BaseModel):
+    """Pagination query parameters."""
+
+    page: int = Field(default=1, ge=1, description="Page number")
+    page_size: int = Field(default=20, ge=1, le=100, description="Items per page")
+
+    @property
+    def offset(self) -> int:
+        return (self.page - 1) * self.page_size
+
+
+class PaginatedResponse(BaseSchema, Generic[T]):
+    """Paginated response wrapper."""
+
+    items: list[T]
+    total: int
+    page: int
+    page_size: int
+    total_pages: int
+
+    @classmethod
+    def create(cls, items: list[T], total: int, page: int, page_size: int) -> PaginatedResponse[T]:
+        total_pages = (total + page_size - 1) // page_size if page_size > 0 else 0
+        return cls(
+            items=items,
+            total=total,
+            page=page,
+            page_size=page_size,
+            total_pages=total_pages,
+        )
+
+
+class ErrorResponse(BaseSchema):
+    """Standard error response."""
+
+    detail: str
+    error_code: str | None = None
+    errors: list[dict[str, Any]] | None = None
+    timestamp: datetime = Field(default_factory=datetime.utcnow)
+
+
+class SuccessResponse(BaseSchema):
+    """Standard success response."""
+
+    message: str
+    data: dict[str, Any] | None = None
+
+
+class HealthResponse(BaseSchema):
+    """Health check response."""
+
+    status: str
+    version: str
+    environment: str
+    database: str
+    redis: str
+    timestamp: datetime = Field(default_factory=datetime.utcnow)
+
+
+class IDResponse(BaseSchema):
+    """Response containing just an ID."""
+
+    id: uuid.UUID

docengine/app/schemas/document.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from typing import Any
+
+from pydantic import Field
+
+from app.schemas.common import BaseSchema
+
+
+class DocumentUploadResponse(BaseSchema):
+    """Response after document upload."""
+
+    id: uuid.UUID
+    filename: str
+    original_filename: str
+    content_type: str
+    file_size: int
+    checksum: str
+    status: str
+    created_at: datetime
+
+
+class TextBlockResponse(BaseSchema):
+    """Extracted text block."""
+
+    id: uuid.UUID
+    text: str
+    x: float
+    y: float
+    width: float
+    height: float
+    confidence: float | None
+    font_family: str | None
+    font_size: float | None
+    font_color: str | None
+    font_style: str | None
+    block_type: str
+    sequence: int
+
+
+class DocumentImageResponse(BaseSchema):
+    """Extracted document image."""
+
+    id: uuid.UUID
+    x: float
+    y: float
+    width: float
+    height: float
+    image_path: str
+    image_type: str
+
+
+class DocumentTableResponse(BaseSchema):
+    """Extracted document table."""
+
+    id: uuid.UUID
+    x: float
+    y: float
+    width: float
+    height: float
+    rows: int
+    columns: int
+    data: dict[str, Any] | None
+
+
+class DocumentPageResponse(BaseSchema):
+    """Document page with extracted content."""
+
+    id: uuid.UUID
+    page_number: int
+    width: float
+    height: float
+    image_path: str | None
+    text_content: str | None
+    text_blocks: list[TextBlockResponse] = Field(default_factory=list)
+    images: list[DocumentImageResponse] = Field(default_factory=list)
+    tables: list[DocumentTableResponse] = Field(default_factory=list)
+
+
+class DocumentResponse(BaseSchema):
+    """Full document response."""
+
+    id: uuid.UUID
+    filename: str
+    original_filename: str
+    content_type: str
+    file_size: int
+    checksum: str
+    storage_path: str
+    status: str
+    page_count: int | None
+    is_scanned: bool | None
+    document_metadata: dict[str, Any] | None
+    error_message: str | None
+    uploaded_by: uuid.UUID | None
+    pages: list[DocumentPageResponse] = Field(default_factory=list)
+    created_at: datetime
+    updated_at: datetime
+
+
+class DocumentListResponse(BaseSchema):
+    """Minimal document response for lists."""
+
+    id: uuid.UUID
+    original_filename: str
+    content_type: str
+    file_size: int
+    status: str
+    page_count: int | None
+    is_scanned: bool | None
+    created_at: datetime
+
+
+class TemplateMatchResponse(BaseSchema):
+    """Template match result."""
+
+    id: uuid.UUID
+    document_id: uuid.UUID
+    format_id: uuid.UUID
+    confidence_score: float
+    match_details: dict[str, Any] | None
+    selected: bool
+    template_name: str | None = None
+    created_at: datetime
+
+
+class TemplateMatchRequest(BaseSchema):
+    """Request to match a document against templates."""
+
+    document_id: uuid.UUID
+    min_confidence: float = Field(default=0.5, ge=0.0, le=1.0)
+    max_results: int = Field(default=5, ge=1, le=20)

docengine/app/schemas/template.py

@@ -0,0 +1,250 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from typing import Any
+
+from pydantic import Field
+
+from app.schemas.common import BaseSchema
+
+
+class DocumentCellResponse(BaseSchema):
+    """Document cell in a template."""
+
+    id: uuid.UUID
+    format_id: uuid.UUID
+    page_number: int
+    x: float
+    y: float
+    width: float
+    height: float
+    row_no: int
+    column_no: int
+    data_type: str
+    font_family: str | None
+    font_size: float | None
+    font_style: str | None
+    font_color: str | None
+    background_color: str | None
+    border_top: str | None
+    border_right: str | None
+    border_bottom: str | None
+    border_left: str | None
+    padding_top: float
+    padding_right: float
+    padding_bottom: float
+    padding_left: float
+    alignment: str
+    vertical_alignment: str
+    rowspan: int
+    colspan: int
+    static_text: str | None
+    field_name: str | None
+    sequence: int
+    is_dynamic: bool
+
+
+class DocumentCellCreate(BaseSchema):
+    """Create a document cell."""
+
+    page_number: int = Field(..., ge=1)
+    x: float
+    y: float
+    width: float = Field(..., gt=0)
+    height: float = Field(..., gt=0)
+    row_no: int = 0
+    column_no: int = 0
+    data_type: str = "text"
+    font_family: str | None = None
+    font_size: float | None = None
+    font_style: str | None = None
+    font_color: str | None = None
+    background_color: str | None = None
+    border_top: str | None = None
+    border_right: str | None = None
+    border_bottom: str | None = None
+    border_left: str | None = None
+    padding_top: float = 0.0
+    padding_right: float = 0.0
+    padding_bottom: float = 0.0
+    padding_left: float = 0.0
+    alignment: str = "left"
+    vertical_alignment: str = "top"
+    rowspan: int = 1
+    colspan: int = 1
+    static_text: str | None = None
+    field_name: str | None = None
+    sequence: int = 0
+    is_dynamic: bool = False
+
+
+class DocumentRegionResponse(BaseSchema):
+    """Region in a template."""
+
+    id: uuid.UUID
+    format_id: uuid.UUID
+    page_number: int
+    region_type: str
+    x: float
+    y: float
+    width: float
+    height: float
+    content: dict[str, Any] | None
+    sequence: int
+
+
+class TableColumnResponse(BaseSchema):
+    """Table column definition."""
+
+    id: uuid.UUID
+    table_format_id: uuid.UUID
+    column_index: int
+    width: float
+    header_text: str | None
+    data_type: str
+    alignment: str
+    font_family: str | None
+    font_size: float | None
+
+
+class TableRowResponse(BaseSchema):
+    """Table row definition."""
+
+    id: uuid.UUID
+    table_format_id: uuid.UUID
+    row_index: int
+    height: float
+    is_header: bool
+    background_color: str | None
+
+
+class TableFormatResponse(BaseSchema):
+    """Table format in a template."""
+
+    id: uuid.UUID
+    format_id: uuid.UUID
+    page_number: int
+    x: float
+    y: float
+    width: float
+    height: float
+    rows: int
+    columns: int
+    border_style: str
+    border_width: float
+    border_color: str
+    header_rows: int
+    table_columns: list[TableColumnResponse] = Field(default_factory=list)
+    table_rows: list[TableRowResponse] = Field(default_factory=list)
+
+
+class WatermarkResponse(BaseSchema):
+    """Watermark in a template."""
+
+    id: uuid.UUID
+    format_id: uuid.UUID
+    page_number: int | None
+    text: str | None
+    image_path: str | None
+    x: float
+    y: float
+    width: float
+    height: float
+    opacity: float
+    rotation: float
+    font_family: str | None
+    font_size: float | None
+    font_color: str | None
+
+
+class ImageRegionResponse(BaseSchema):
+    """Image region in a template."""
+
+    id: uuid.UUID
+    format_id: uuid.UUID
+    page_number: int
+    x: float
+    y: float
+    width: float
+    height: float
+    image_path: str | None
+    image_type: str
+    is_static: bool
+    field_name: str | None
+
+
+class TemplateFingerprintResponse(BaseSchema):
+    """Template fingerprint."""
+
+    id: uuid.UUID
+    format_id: uuid.UUID
+    page_dimensions: dict[str, Any] | None
+    logo_coordinates: dict[str, Any] | None
+    header_coordinates: dict[str, Any] | None
+    footer_coordinates: dict[str, Any] | None
+    table_coordinates: dict[str, Any] | None
+    cell_coordinates: dict[str, Any] | None
+    fingerprint_hash: str
+
+
+class TemplateResponse(BaseSchema):
+    """Full template response."""
+
+    id: uuid.UUID
+    name: str
+    description: str | None
+    page_width: float
+    page_height: float
+    page_count: int
+    margin_top: float
+    margin_right: float
+    margin_bottom: float
+    margin_left: float
+    fingerprint: dict[str, Any] | None
+    source_document_id: uuid.UUID | None
+    version: int
+    is_active: bool
+    created_by: uuid.UUID | None
+    cells: list[DocumentCellResponse] = Field(default_factory=list)
+    regions: list[DocumentRegionResponse] = Field(default_factory=list)
+    table_formats: list[TableFormatResponse] = Field(default_factory=list)
+    watermarks: list[WatermarkResponse] = Field(default_factory=list)
+    image_regions: list[ImageRegionResponse] = Field(default_factory=list)
+    fingerprint_record: TemplateFingerprintResponse | None = None
+    created_at: datetime
+    updated_at: datetime
+
+
+class TemplateListResponse(BaseSchema):
+    """Minimal template response for lists."""
+
+    id: uuid.UUID
+    name: str
+    description: str | None
+    page_width: float
+    page_height: float
+    page_count: int
+    version: int
+    is_active: bool
+    created_at: datetime
+    updated_at: datetime
+
+
+class TemplateRenderRequest(BaseSchema):
+    """Request to render a template to PDF."""
+
+    template_id: uuid.UUID
+    data: dict[str, Any] = Field(default_factory=dict, description="Data to populate dynamic fields")
+    output_filename: str | None = Field(None, max_length=255, description="Output filename for generated PDF")
+    images: dict[str, str] | None = Field(None, description="Mapping of field_name to image path for dynamic images")
+
+
+class TemplateRenderResponse(BaseSchema):
+    """Response after rendering a template."""
+
+    output_path: str
+    filename: str
+    file_size: int
+    page_count: int
+    rendered_at: datetime

docengine/app/schemas/user.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+
+from pydantic import EmailStr, Field
+
+from app.schemas.common import BaseSchema
+
+
+class UserBase(BaseSchema):
+    """Base user fields."""
+
+    username: str = Field(..., min_length=3, max_length=150)
+    email: EmailStr
+    full_name: str | None = Field(None, max_length=255)
+
+
+class UserCreate(UserBase):
+    """User creation payload."""
+
+    password: str = Field(..., min_length=8, max_length=128)
+    is_active: bool = True
+    is_superuser: bool = False
+    role_names: list[str] = Field(default_factory=list)
+
+
+class UserUpdate(BaseSchema):
+    """User update payload."""
+
+    email: EmailStr | None = None
+    full_name: str | None = None
+    is_active: bool | None = None
+    is_superuser: bool | None = None
+    role_names: list[str] | None = None
+
+
+class RoleResponse(BaseSchema):
+    """Role response."""
+
+    id: uuid.UUID
+    name: str
+    description: str | None
+
+
+class UserResponse(BaseSchema):
+    """Full user response."""
+
+    id: uuid.UUID
+    username: str
+    email: str
+    full_name: str | None
+    is_active: bool
+    is_superuser: bool
+    roles: list[RoleResponse] = Field(default_factory=list)
+    created_at: datetime
+    updated_at: datetime
+    last_login: datetime | None
+
+
+class UserListResponse(BaseSchema):
+    """Minimal user response for lists."""
+
+    id: uuid.UUID
+    username: str
+    email: str
+    full_name: str | None
+    is_active: bool
+    created_at: datetime

docengine/app/services/document_service.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import uuid
+
+from sqlalchemy.orm import Session
+
+from app.core.logging_config import get_logger
+from app.models.document import Document
+from app.repositories.document_repository import DocumentRepository
+from app.services.layout_service import LayoutService
+from app.services.ocr_service import OCRService
+from app.services.pdf_service import NativePDFService
+from app.services.template_service import TemplateService
+
+logger = get_logger(__name__)
+
+
+class DocumentProcessingService:
+    """Orchestrates the full document processing pipeline."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+        self.doc_repo = DocumentRepository(db)
+        self.pdf_service = NativePDFService(db)
+        self.ocr_service = OCRService(db)
+        self.layout_service = LayoutService(db)
+        self.template_service = TemplateService(db)
+
+    def process_document(self, document_id: str | uuid.UUID) -> Document:
+        """Process a document through the full pipeline."""
+        if isinstance(document_id, str):
+            document_id = uuid.UUID(document_id)
+
+        document = self.doc_repo.get_by_id(document_id)
+        if not document:
+            raise ValueError(f"Document '{document_id}' not found")
+
+        logger.info(
+            "processing_started",
+            document_id=str(document_id),
+            content_type=document.content_type,
+        )
+
+        # Update status to processing
+        self.doc_repo.update_status(document_id, "processing")
+        self.db.commit()
+
+        try:
+            # Step 1: Extract content based on document type
+            if document.content_type == "application/pdf":
+                document = self._process_pdf(document)
+            else:
+                document = self._process_image(document)
+
+            # Step 2: Analyze layout
+            layout_results = self.layout_service.analyze_document_layout(document)
+            document.document_metadata = document.document_metadata or {}
+            document.document_metadata["layout"] = layout_results
+
+            # Step 3: Generate template
+            template = self.template_service.generate_template(document)
+
+            # Step 4: Update document status
+            self.doc_repo.update_status(document_id, "completed")
+            self.db.commit()
+
+            logger.info(
+                "processing_completed",
+                document_id=str(document_id),
+                pages=document.page_count,
+                template_id=str(template.id),
+            )
+
+            return document
+
+        except Exception as e:
+            logger.exception(
+                "processing_failed",
+                document_id=str(document_id),
+                error=str(e),
+            )
+            self.doc_repo.update_status(document_id, "failed", error_message=str(e))
+            self.db.commit()
+            raise
+
+    def _process_pdf(self, document: Document) -> Document:
+        """Process a PDF document - either native or scanned."""
+        # First, try native PDF extraction
+        document = self.pdf_service.process_pdf(document)
+        self.db.flush()
+
+        # If scanned, also run OCR
+        if document.is_scanned:
+            logger.info(
+                "scanned_pdf_detected",
+                document_id=str(document.id),
+            )
+            document = self.ocr_service.process_scanned_pdf(document)
+            self.db.flush()
+
+        return document
+
+    def _process_image(self, document: Document) -> Document:
+        """Process an image document with OCR."""
+        document = self.ocr_service.process_image(document)
+        self.db.flush()
+        return document

docengine/app/services/fingerprint_service.py

@@ -0,0 +1,343 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import uuid
+from typing import Any
+
+from sqlalchemy.orm import Session
+
+from app.core.logging_config import get_logger
+from app.models.template import DocumentFormat, TemplateFingerprint
+from app.repositories.template_repository import TemplateFingerprintRepository, TemplateRepository
+
+logger = get_logger(__name__)
+
+
+class FingerprintService:
+    """Generate and manage layout fingerprints for template matching."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+        self.fingerprint_repo = TemplateFingerprintRepository(db)
+        self.template_repo = TemplateRepository(db)
+
+    def generate_fingerprint(self, template: DocumentFormat) -> TemplateFingerprint:
+        """Generate a layout fingerprint for a template."""
+        # Collect page dimensions
+        page_dimensions = {
+            "width": template.page_width,
+            "height": template.page_height,
+            "page_count": template.page_count,
+            "margins": {
+                "top": template.margin_top,
+                "right": template.margin_right,
+                "bottom": template.margin_bottom,
+                "left": template.margin_left,
+            },
+        }
+
+        # Collect logo coordinates
+        logo_coordinates = self._extract_logo_coordinates(template)
+
+        # Collect header coordinates
+        header_coordinates = self._extract_region_coordinates(template, "header")
+
+        # Collect footer coordinates
+        footer_coordinates = self._extract_region_coordinates(template, "footer")
+
+        # Collect table coordinates
+        table_coordinates = self._extract_table_coordinates(template)
+
+        # Collect cell coordinates
+        cell_coordinates = self._extract_cell_coordinates(template)
+
+        # Compute fingerprint hash
+        fingerprint_data = {
+            "page_dimensions": page_dimensions,
+            "logo_coordinates": logo_coordinates,
+            "header_coordinates": header_coordinates,
+            "footer_coordinates": footer_coordinates,
+            "table_coordinates": table_coordinates,
+            "cell_coordinates": cell_coordinates,
+        }
+        fingerprint_hash = self._compute_hash(fingerprint_data)
+
+        # Check for existing fingerprint
+        existing = self.fingerprint_repo.get_by_format_id(template.id)
+        if existing:
+            # Update existing
+            existing.page_dimensions = page_dimensions
+            existing.logo_coordinates = logo_coordinates
+            existing.header_coordinates = header_coordinates
+            existing.footer_coordinates = footer_coordinates
+            existing.table_coordinates = table_coordinates
+            existing.cell_coordinates = cell_coordinates
+            existing.fingerprint_hash = fingerprint_hash
+            self.db.flush()
+            self.db.refresh(existing)
+            return existing
+
+        # Create new fingerprint
+        fingerprint = self.fingerprint_repo.create_fingerprint(
+            format_id=template.id,
+            fingerprint_hash=fingerprint_hash,
+            page_dimensions=page_dimensions,
+            logo_coordinates=logo_coordinates,
+            header_coordinates=header_coordinates,
+            footer_coordinates=footer_coordinates,
+            table_coordinates=table_coordinates,
+            cell_coordinates=cell_coordinates,
+        )
+
+        logger.info(
+            "fingerprint_generated",
+            template_id=str(template.id),
+            hash=fingerprint_hash[:16],
+        )
+
+        return fingerprint
+
+    def _extract_logo_coordinates(self, template: DocumentFormat) -> dict[str, Any] | None:
+        """Extract logo image coordinates from template."""
+        logos = [ir for ir in template.image_regions if ir.image_type == "logo"]
+        if not logos:
+            return None
+
+        return {
+            "items": [
+                {
+                    "page": ir.page_number,
+                    "x": ir.x,
+                    "y": ir.y,
+                    "width": ir.width,
+                    "height": ir.height,
+                }
+                for ir in logos
+            ]
+        }
+
+    def _extract_region_coordinates(
+        self,
+        template: DocumentFormat,
+        region_type: str,
+    ) -> dict[str, Any] | None:
+        """Extract coordinates for a specific region type."""
+        regions = [r for r in template.regions if r.region_type == region_type]
+        if not regions:
+            return None
+
+        return {
+            "items": [
+                {
+                    "page": r.page_number,
+                    "x": r.x,
+                    "y": r.y,
+                    "width": r.width,
+                    "height": r.height,
+                }
+                for r in regions
+            ]
+        }
+
+    def _extract_table_coordinates(self, template: DocumentFormat) -> dict[str, Any] | None:
+        """Extract table coordinates from template."""
+        if not template.table_formats:
+            return None
+
+        return {
+            "items": [
+                {
+                    "page": tf.page_number,
+                    "x": tf.x,
+                    "y": tf.y,
+                    "width": tf.width,
+                    "height": tf.height,
+                    "rows": tf.rows,
+                    "columns": tf.columns,
+                }
+                for tf in template.table_formats
+            ]
+        }
+
+    def _extract_cell_coordinates(self, template: DocumentFormat) -> dict[str, Any] | None:
+        """Extract cell coordinates from template."""
+        if not template.cells:
+            return None
+
+        return {
+            "items": [
+                {
+                    "page": c.page_number,
+                    "x": c.x,
+                    "y": c.y,
+                    "width": c.width,
+                    "height": c.height,
+                    "row": c.row_no,
+                    "col": c.column_no,
+                }
+                for c in template.cells
+            ]
+        }
+
+    def _compute_hash(self, data: dict[str, Any]) -> str:
+        """Compute a deterministic hash of the fingerprint data."""
+        # Normalize coordinates to reduce sensitivity to minor variations
+        normalized = self._normalize_coordinates(data)
+        serialized = json.dumps(normalized, sort_keys=True, default=str)
+        return hashlib.sha256(serialized.encode()).hexdigest()
+
+    def _normalize_coordinates(self, data: dict[str, Any]) -> dict[str, Any]:
+        """Normalize coordinates by rounding to reduce sensitivity to small variations."""
+        if isinstance(data, dict):
+            return {k: self._normalize_coordinates(v) for k, v in data.items()}
+        elif isinstance(data, list):
+            return [self._normalize_coordinates(item) for item in data]
+        elif isinstance(data, float):
+            return round(data, 1)
+        return data
+
+    def compute_similarity(
+        self,
+        fingerprint1: TemplateFingerprint,
+        fingerprint2_data: dict[str, Any],
+    ) -> float:
+        """Compute similarity score between a stored fingerprint and new document data."""
+        scores: list[float] = []
+        weights: list[float] = []
+
+        # Page dimensions similarity (high weight)
+        dim_score = self._compare_dimensions(
+            fingerprint1.page_dimensions,
+            fingerprint2_data.get("page_dimensions"),
+        )
+        scores.append(dim_score)
+        weights.append(3.0)
+
+        # Logo coordinates similarity
+        logo_score = self._compare_coordinates(
+            fingerprint1.logo_coordinates,
+            fingerprint2_data.get("logo_coordinates"),
+        )
+        scores.append(logo_score)
+        weights.append(2.0)
+
+        # Header coordinates similarity
+        header_score = self._compare_coordinates(
+            fingerprint1.header_coordinates,
+            fingerprint2_data.get("header_coordinates"),
+        )
+        scores.append(header_score)
+        weights.append(2.0)
+
+        # Footer coordinates similarity
+        footer_score = self._compare_coordinates(
+            fingerprint1.footer_coordinates,
+            fingerprint2_data.get("footer_coordinates"),
+        )
+        scores.append(footer_score)
+        weights.append(1.5)
+
+        # Table coordinates similarity
+        table_score = self._compare_coordinates(
+            fingerprint1.table_coordinates,
+            fingerprint2_data.get("table_coordinates"),
+        )
+        scores.append(table_score)
+        weights.append(2.5)
+
+        # Cell coordinates similarity
+        cell_score = self._compare_coordinates(
+            fingerprint1.cell_coordinates,
+            fingerprint2_data.get("cell_coordinates"),
+        )
+        scores.append(cell_score)
+        weights.append(1.5)
+
+        # Weighted average
+        total_weight = sum(weights)
+        if total_weight == 0:
+            return 0.0
+
+        weighted_sum = sum(s * w for s, w in zip(scores, weights))
+        return weighted_sum / total_weight
+
+    def _compare_dimensions(
+        self,
+        dims1: dict[str, Any] | None,
+        dims2: dict[str, Any] | None,
+    ) -> float:
+        """Compare page dimensions similarity."""
+        if not dims1 or not dims2:
+            return 0.0 if (dims1 or dims2) else 1.0
+
+        width_ratio = min(dims1.get("width", 0), dims2.get("width", 0)) / max(
+            dims1.get("width", 1), dims2.get("width", 1)
+        )
+        height_ratio = min(dims1.get("height", 0), dims2.get("height", 0)) / max(
+            dims1.get("height", 1), dims2.get("height", 1)
+        )
+        page_count_match = 1.0 if dims1.get("page_count") == dims2.get("page_count") else 0.5
+
+        return (width_ratio + height_ratio + page_count_match) / 3.0
+
+    def _compare_coordinates(
+        self,
+        coords1: dict[str, Any] | None,
+        coords2: dict[str, Any] | None,
+    ) -> float:
+        """Compare coordinate sets for similarity."""
+        if not coords1 and not coords2:
+            return 1.0
+        if not coords1 or not coords2:
+            return 0.0
+
+        items1 = coords1.get("items", [])
+        items2 = coords2.get("items", [])
+
+        if not items1 and not items2:
+            return 1.0
+        if not items1 or not items2:
+            return 0.0
+
+        # Compare number of items
+        count_ratio = min(len(items1), len(items2)) / max(len(items1), len(items2))
+
+        # Compare positions of matched items
+        position_scores = []
+        for item1 in items1:
+            best_match = 0.0
+            for item2 in items2:
+                if item1.get("page") != item2.get("page"):
+                    continue
+                score = self._compute_bbox_iou(item1, item2)
+                best_match = max(best_match, score)
+            position_scores.append(best_match)
+
+        avg_position_score = sum(position_scores) / len(position_scores) if position_scores else 0.0
+
+        return (count_ratio + avg_position_score) / 2.0
+
+    def _compute_bbox_iou(self, bbox1: dict[str, Any], bbox2: dict[str, Any]) -> float:
+        """Compute Intersection over Union for two bounding boxes."""
+        x1 = max(bbox1.get("x", 0), bbox2.get("x", 0))
+        y1 = max(bbox1.get("y", 0), bbox2.get("y", 0))
+        x2 = min(
+            bbox1.get("x", 0) + bbox1.get("width", 0),
+            bbox2.get("x", 0) + bbox2.get("width", 0),
+        )
+        y2 = min(
+            bbox1.get("y", 0) + bbox1.get("height", 0),
+            bbox2.get("y", 0) + bbox2.get("height", 0),
+        )
+
+        intersection = max(0, x2 - x1) * max(0, y2 - y1)
+
+        area1 = bbox1.get("width", 0) * bbox1.get("height", 0)
+        area2 = bbox2.get("width", 0) * bbox2.get("height", 0)
+        union = area1 + area2 - intersection
+
+        if union == 0:
+            return 0.0
+
+        return intersection / union

docengine/app/services/layout_service.py

@@ -0,0 +1,317 @@
+from __future__ import annotations
+
+import uuid
+from pathlib import Path
+from typing import Any
+
+import cv2
+import numpy as np
+from sqlalchemy.orm import Session
+
+from app.core.logging_config import get_logger
+from app.models.document import Document, DocumentPage
+from app.repositories.document_repository import (
+    DocumentPageRepository,
+    DocumentTableRepository,
+    DocumentTextBlockRepository,
+)
+from app.storage.provider import get_storage_provider
+
+logger = get_logger(__name__)
+
+
+class LayoutService:
+    """Document layout analysis service using OpenCV-based detection."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+        self.storage = get_storage_provider()
+        self.page_repo = DocumentPageRepository(db)
+        self.text_block_repo = DocumentTextBlockRepository(db)
+        self.table_repo = DocumentTableRepository(db)
+
+    def analyze_document_layout(self, document: Document) -> dict[str, Any]:
+        """Analyze the layout of all pages in a document."""
+        layout_results: dict[str, Any] = {"pages": []}
+
+        for page in document.pages:
+            page_layout = self._analyze_page_layout(page)
+            layout_results["pages"].append(page_layout)
+
+        return layout_results
+
+    def _analyze_page_layout(self, page: DocumentPage) -> dict[str, Any]:
+        """Analyze layout of a single page."""
+        result: dict[str, Any] = {
+            "page_number": page.page_number,
+            "width": page.width,
+            "height": page.height,
+            "tables": [],
+            "lines": [],
+            "rectangles": [],
+            "text_regions": [],
+            "image_regions": [],
+        }
+
+        if not page.image_path:
+            return result
+
+        image_path = self.storage.get_absolute_path(page.image_path)
+        image = cv2.imread(image_path)
+        if image is None:
+            logger.warning("layout_image_read_failed", page_id=str(page.id))
+            return result
+
+        gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+
+        # Detect lines
+        result["lines"] = self._detect_lines(gray)
+
+        # Detect rectangles (potential table cells/borders)
+        result["rectangles"] = self._detect_rectangles(gray)
+
+        # Detect tables
+        tables = self._detect_tables(gray, image.shape)
+        result["tables"] = tables
+
+        # Store detected tables in the database
+        for table_data in tables:
+            self.table_repo.create_table(
+                page_id=page.id,
+                x=table_data["x"],
+                y=table_data["y"],
+                width=table_data["width"],
+                height=table_data["height"],
+                rows=table_data["rows"],
+                columns=table_data["columns"],
+                data=table_data.get("cells"),
+            )
+
+        # Detect watermarks
+        watermark = self._detect_watermark(gray, image.shape)
+        if watermark:
+            result["watermark"] = watermark
+
+        return result
+
+    def _detect_lines(self, gray: np.ndarray) -> list[dict[str, Any]]:
+        """Detect horizontal and vertical lines in the image."""
+        lines_detected: list[dict[str, Any]] = []
+
+        # Apply edge detection
+        edges = cv2.Canny(gray, 50, 150, apertureSize=3)
+
+        # Detect lines using Hough transform
+        lines = cv2.HoughLinesP(edges, 1, np.pi / 180, threshold=100, minLineLength=50, maxLineGap=10)
+
+        if lines is not None:
+            for line in lines:
+                x1, y1, x2, y2 = line[0]
+                length = np.sqrt((x2 - x1) ** 2 + (y2 - y1) ** 2)
+
+                # Classify as horizontal or vertical
+                angle = np.degrees(np.arctan2(y2 - y1, x2 - x1))
+                if abs(angle) < 5 or abs(angle - 180) < 5:
+                    orientation = "horizontal"
+                elif abs(angle - 90) < 5 or abs(angle + 90) < 5:
+                    orientation = "vertical"
+                else:
+                    orientation = "diagonal"
+
+                lines_detected.append({
+                    "x1": float(x1),
+                    "y1": float(y1),
+                    "x2": float(x2),
+                    "y2": float(y2),
+                    "length": float(length),
+                    "orientation": orientation,
+                })
+
+        return lines_detected
+
+    def _detect_rectangles(self, gray: np.ndarray) -> list[dict[str, Any]]:
+        """Detect rectangular regions in the image."""
+        rectangles: list[dict[str, Any]] = []
+
+        # Binary threshold
+        _, binary = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY_INV)
+
+        # Find contours
+        contours, _ = cv2.findContours(binary, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
+
+        for contour in contours:
+            # Approximate the contour
+            peri = cv2.arcLength(contour, True)
+            approx = cv2.approxPolyDP(contour, 0.02 * peri, True)
+
+            # If approximation has 4 vertices, it's likely a rectangle
+            if len(approx) == 4:
+                x, y, w, h = cv2.boundingRect(approx)
+                # Filter out very small or very large rectangles
+                area = w * h
+                if area > 500 and w > 10 and h > 10:
+                    rectangles.append({
+                        "x": float(x),
+                        "y": float(y),
+                        "width": float(w),
+                        "height": float(h),
+                        "area": float(area),
+                    })
+
+        return rectangles
+
+    def _detect_tables(self, gray: np.ndarray, image_shape: tuple) -> list[dict[str, Any]]:
+        """Detect table structures using morphological operations."""
+        tables: list[dict[str, Any]] = []
+        h, w = image_shape[:2]
+
+        # Binary threshold
+        _, binary = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY_INV)
+
+        # Detect horizontal lines
+        horizontal_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (max(w // 30, 1), 1))
+        horizontal = cv2.morphologyEx(binary, cv2.MORPH_OPEN, horizontal_kernel, iterations=2)
+
+        # Detect vertical lines
+        vertical_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1, max(h // 30, 1)))
+        vertical = cv2.morphologyEx(binary, cv2.MORPH_OPEN, vertical_kernel, iterations=2)
+
+        # Combine horizontal and vertical lines to find intersections
+        table_mask = cv2.add(horizontal, vertical)
+
+        # Find contours of table regions
+        contours, _ = cv2.findContours(table_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+        for contour in contours:
+            x, y, cw, ch = cv2.boundingRect(contour)
+            area = cw * ch
+
+            # Filter: table should be reasonably sized
+            if area < 5000 or cw < 50 or ch < 30:
+                continue
+
+            # Estimate rows and columns
+            rows, columns = self._estimate_table_dimensions(
+                table_mask[y:y+ch, x:x+cw], cw, ch
+            )
+
+            if rows >= 1 and columns >= 1:
+                # Extract cell contents
+                cells = self._extract_table_cells(
+                    gray[y:y+ch, x:x+cw], rows, columns, cw, ch
+                )
+
+                tables.append({
+                    "x": float(x),
+                    "y": float(y),
+                    "width": float(cw),
+                    "height": float(ch),
+                    "rows": rows,
+                    "columns": columns,
+                    "cells": cells,
+                })
+
+        return tables
+
+    def _estimate_table_dimensions(
+        self,
+        table_region: np.ndarray,
+        width: int,
+        height: int,
+    ) -> tuple[int, int]:
+        """Estimate the number of rows and columns in a table region."""
+        # Project horizontal lines
+        h_projection = np.sum(table_region, axis=1)
+        h_peaks = self._count_peaks(h_projection, height)
+
+        # Project vertical lines
+        v_projection = np.sum(table_region, axis=0)
+        v_peaks = self._count_peaks(v_projection, width)
+
+        rows = max(1, h_peaks - 1)
+        columns = max(1, v_peaks - 1)
+
+        return rows, columns
+
+    def _count_peaks(self, projection: np.ndarray, total_length: int) -> int:
+        """Count significant peaks in a projection array."""
+        if len(projection) == 0:
+            return 0
+
+        threshold = np.max(projection) * 0.3
+        above_threshold = projection > threshold
+
+        # Count transitions from below to above threshold
+        peaks = 0
+        in_peak = False
+        for val in above_threshold:
+            if val and not in_peak:
+                peaks += 1
+                in_peak = True
+            elif not val:
+                in_peak = False
+
+        return peaks
+
+    def _extract_table_cells(
+        self,
+        table_gray: np.ndarray,
+        rows: int,
+        columns: int,
+        width: int,
+        height: int,
+    ) -> dict[str, Any]:
+        """Extract cell structure data from a table region."""
+        cell_height = height / max(rows, 1)
+        cell_width = width / max(columns, 1)
+
+        cells: dict[str, Any] = {"rows": rows, "columns": columns, "data": []}
+
+        for r in range(rows):
+            row_data = []
+            for c in range(columns):
+                cell_x = int(c * cell_width)
+                cell_y = int(r * cell_height)
+                cell_w = int(cell_width)
+                cell_h = int(cell_height)
+
+                row_data.append({
+                    "row": r,
+                    "col": c,
+                    "x": cell_x,
+                    "y": cell_y,
+                    "width": cell_w,
+                    "height": cell_h,
+                })
+            cells["data"].append(row_data)
+
+        return cells
+
+    def _detect_watermark(self, gray: np.ndarray, image_shape: tuple) -> dict[str, Any] | None:
+        """Detect potential watermark regions."""
+        h, w = image_shape[:2]
+
+        # Look for semi-transparent or light text in the center region
+        center_region = gray[h // 4 : 3 * h // 4, w // 4 : 3 * w // 4]
+
+        # Apply adaptive threshold to find light text
+        _, binary = cv2.threshold(center_region, 230, 255, cv2.THRESH_BINARY)
+
+        # Find contours in the center region
+        contours, _ = cv2.findContours(binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+        for contour in contours:
+            x, y, cw, ch = cv2.boundingRect(contour)
+            area = cw * ch
+            # Watermark typically covers a significant portion of the center
+            center_area = (w // 2) * (h // 2)
+            if area > center_area * 0.1:
+                return {
+                    "x": float(x + w // 4),
+                    "y": float(y + h // 4),
+                    "width": float(cw),
+                    "height": float(ch),
+                    "detected": True,
+                }
+
+        return None

docengine/app/services/matching_service.py

@@ -0,0 +1,217 @@
+from __future__ import annotations
+
+import uuid
+from typing import Any
+
+from sqlalchemy.orm import Session
+
+from app.core.logging_config import get_logger
+from app.models.document import Document, TemplateMatch
+from app.models.template import DocumentFormat
+from app.repositories.document_repository import DocumentRepository, TemplateMatchRepository
+from app.repositories.template_repository import TemplateFingerprintRepository, TemplateRepository
+from app.services.fingerprint_service import FingerprintService
+
+logger = get_logger(__name__)
+
+
+class MatchingService:
+    """Match documents against existing templates using fingerprint comparison."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+        self.doc_repo = DocumentRepository(db)
+        self.template_repo = TemplateRepository(db)
+        self.match_repo = TemplateMatchRepository(db)
+        self.fingerprint_repo = TemplateFingerprintRepository(db)
+        self.fingerprint_service = FingerprintService(db)
+
+    def match_document(
+        self,
+        document_id: uuid.UUID,
+        min_confidence: float = 0.5,
+        max_results: int = 5,
+    ) -> list[TemplateMatch]:
+        """Match a document against all existing templates."""
+        document = self.doc_repo.get_with_pages(document_id)
+        if not document:
+            raise ValueError(f"Document '{document_id}' not found")
+
+        if not document.pages:
+            raise ValueError(f"Document '{document_id}' has no processed pages")
+
+        # Generate document fingerprint data
+        doc_fingerprint_data = self._build_document_fingerprint(document)
+
+        # Get all templates with fingerprints
+        templates = self.template_repo.get_all_with_fingerprints()
+        fingerprints = self.fingerprint_repo.get_all_fingerprints()
+
+        # Map format_id -> fingerprint
+        fp_map = {fp.format_id: fp for fp in fingerprints}
+
+        matches: list[tuple[DocumentFormat, float, dict[str, Any]]] = []
+
+        for template in templates:
+            fp = fp_map.get(template.id)
+            if not fp:
+                continue
+
+            score = self.fingerprint_service.compute_similarity(fp, doc_fingerprint_data)
+            if score >= min_confidence:
+                match_details = {
+                    "page_dimensions_score": self.fingerprint_service._compare_dimensions(
+                        fp.page_dimensions, doc_fingerprint_data.get("page_dimensions")
+                    ),
+                    "logo_score": self.fingerprint_service._compare_coordinates(
+                        fp.logo_coordinates, doc_fingerprint_data.get("logo_coordinates")
+                    ),
+                    "header_score": self.fingerprint_service._compare_coordinates(
+                        fp.header_coordinates, doc_fingerprint_data.get("header_coordinates")
+                    ),
+                    "footer_score": self.fingerprint_service._compare_coordinates(
+                        fp.footer_coordinates, doc_fingerprint_data.get("footer_coordinates")
+                    ),
+                    "table_score": self.fingerprint_service._compare_coordinates(
+                        fp.table_coordinates, doc_fingerprint_data.get("table_coordinates")
+                    ),
+                    "cell_score": self.fingerprint_service._compare_coordinates(
+                        fp.cell_coordinates, doc_fingerprint_data.get("cell_coordinates")
+                    ),
+                }
+                matches.append((template, score, match_details))
+
+        # Sort by score descending
+        matches.sort(key=lambda x: x[1], reverse=True)
+        matches = matches[:max_results]
+
+        # Store match results
+        result_matches: list[TemplateMatch] = []
+        for idx, (template, score, details) in enumerate(matches):
+            template_match = self.match_repo.create_match(
+                document_id=document_id,
+                format_id=template.id,
+                confidence_score=score,
+                match_details=details,
+                selected=(idx == 0),  # Auto-select best match
+            )
+            result_matches.append(template_match)
+
+        logger.info(
+            "document_matched",
+            document_id=str(document_id),
+            matches_found=len(result_matches),
+            best_score=result_matches[0].confidence_score if result_matches else 0.0,
+        )
+
+        return result_matches
+
+    def _build_document_fingerprint(self, document: Document) -> dict[str, Any]:
+        """Build fingerprint data from a document for comparison."""
+        first_page = document.pages[0] if document.pages else None
+
+        page_dimensions = None
+        if first_page:
+            page_dimensions = {
+                "width": first_page.width,
+                "height": first_page.height,
+                "page_count": document.page_count or len(document.pages),
+            }
+
+        # Extract logo coordinates from images
+        logo_coordinates = None
+        logos = []
+        for page in document.pages:
+            for img in page.images:
+                if img.image_type == "logo":
+                    logos.append({
+                        "page": page.page_number,
+                        "x": img.x,
+                        "y": img.y,
+                        "width": img.width,
+                        "height": img.height,
+                    })
+        if logos:
+            logo_coordinates = {"items": logos}
+
+        # Extract header coordinates
+        header_coordinates = None
+        headers = []
+        for page in document.pages:
+            header_blocks = [b for b in page.text_blocks if b.block_type == "header"]
+            if header_blocks:
+                min_x = min(b.x for b in header_blocks)
+                min_y = min(b.y for b in header_blocks)
+                max_x = max(b.x + b.width for b in header_blocks)
+                max_y = max(b.y + b.height for b in header_blocks)
+                headers.append({
+                    "page": page.page_number,
+                    "x": min_x,
+                    "y": min_y,
+                    "width": max_x - min_x,
+                    "height": max_y - min_y,
+                })
+        if headers:
+            header_coordinates = {"items": headers}
+
+        # Extract footer coordinates
+        footer_coordinates = None
+        footers = []
+        for page in document.pages:
+            footer_blocks = [b for b in page.text_blocks if b.block_type == "footer"]
+            if footer_blocks:
+                min_x = min(b.x for b in footer_blocks)
+                min_y = min(b.y for b in footer_blocks)
+                max_x = max(b.x + b.width for b in footer_blocks)
+                max_y = max(b.y + b.height for b in footer_blocks)
+                footers.append({
+                    "page": page.page_number,
+                    "x": min_x,
+                    "y": min_y,
+                    "width": max_x - min_x,
+                    "height": max_y - min_y,
+                })
+        if footers:
+            footer_coordinates = {"items": footers}
+
+        # Extract table coordinates
+        table_coordinates = None
+        tables = []
+        for page in document.pages:
+            for table in page.tables:
+                tables.append({
+                    "page": page.page_number,
+                    "x": table.x,
+                    "y": table.y,
+                    "width": table.width,
+                    "height": table.height,
+                    "rows": table.rows,
+                    "columns": table.columns,
+                })
+        if tables:
+            table_coordinates = {"items": tables}
+
+        # Extract cell coordinates from text blocks
+        cell_coordinates = None
+        cells = []
+        for page in document.pages:
+            for block in page.text_blocks:
+                if block.block_type == "text":
+                    cells.append({
+                        "page": page.page_number,
+                        "x": block.x,
+                        "y": block.y,
+                        "width": block.width,
+                        "height": block.height,
+                    })
+        if cells:
+            cell_coordinates = {"items": cells}
+
+        return {
+            "page_dimensions": page_dimensions,
+            "logo_coordinates": logo_coordinates,
+            "header_coordinates": header_coordinates,
+            "footer_coordinates": footer_coordinates,
+            "table_coordinates": table_coordinates,
+            "cell_coordinates": cell_coordinates,
+        }

docengine/app/services/ocr_service.py

@@ -0,0 +1,210 @@
+from __future__ import annotations
+
+import uuid
+from pathlib import Path
+
+import cv2
+import numpy as np
+from paddleocr import PaddleOCR
+from sqlalchemy.orm import Session
+
+from app.core.config import settings
+from app.core.logging_config import get_logger
+from app.models.document import Document, DocumentPage
+from app.repositories.document_repository import (
+    DocumentPageRepository,
+    DocumentRepository,
+    DocumentTextBlockRepository,
+)
+from app.storage.provider import get_storage_provider
+
+logger = get_logger(__name__)
+
+_ocr_instance: PaddleOCR | None = None
+
+
+def get_ocr_engine() -> PaddleOCR:
+    """Get or create singleton PaddleOCR instance."""
+    global _ocr_instance
+    if _ocr_instance is None:
+        _ocr_instance = PaddleOCR(
+            use_angle_cls=True,
+            lang=settings.ocr_language,
+            use_gpu=settings.ocr_use_gpu,
+            show_log=False,
+            det_db_thresh=0.3,
+            det_db_box_thresh=0.5,
+            rec_batch_num=6,
+        )
+    return _ocr_instance
+
+
+class OCRService:
+    """OCR processing service using PaddleOCR for scanned documents."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+        self.storage = get_storage_provider()
+        self.doc_repo = DocumentRepository(db)
+        self.page_repo = DocumentPageRepository(db)
+        self.text_block_repo = DocumentTextBlockRepository(db)
+
+    def process_image(self, document: Document) -> Document:
+        """Process a scanned image document with OCR."""
+        file_path = self.storage.get_absolute_path(document.storage_path)
+        image = cv2.imread(file_path)
+        if image is None:
+            raise ValueError(f"Failed to read image: {file_path}")
+
+        height, width = image.shape[:2]
+        document.page_count = 1
+        document.is_scanned = True
+
+        # Save page image
+        image_filename = f"{document.id}_page_1.png"
+        image_bytes = cv2.imencode(".png", image)[1].tobytes()
+        image_path = self.storage.save_file(image_bytes, "images", image_filename)
+
+        # Create page record
+        doc_page = self.page_repo.create_page(
+            document_id=document.id,
+            page_number=1,
+            width=float(width),
+            height=float(height),
+            image_path=image_path,
+        )
+
+        # Run OCR
+        self._run_ocr_on_page(doc_page, file_path)
+
+        return document
+
+    def process_scanned_pdf(self, document: Document) -> Document:
+        """Process a scanned PDF document - convert pages to images and OCR each."""
+        file_path = self.storage.get_absolute_path(document.storage_path)
+
+        try:
+            from pdf2image import convert_from_path
+            images = convert_from_path(file_path, dpi=300)
+        except Exception as e:
+            logger.error("pdf_to_image_failed", document_id=str(document.id), error=str(e))
+            raise
+
+        document.page_count = len(images)
+        document.is_scanned = True
+
+        for page_num, pil_image in enumerate(images, start=1):
+            # Convert PIL to OpenCV format
+            np_image = np.array(pil_image)
+            cv_image = cv2.cvtColor(np_image, cv2.COLOR_RGB2BGR)
+            height, width = cv_image.shape[:2]
+
+            # Save page image
+            image_filename = f"{document.id}_page_{page_num}.png"
+            image_bytes = cv2.imencode(".png", cv_image)[1].tobytes()
+            image_path = self.storage.save_file(image_bytes, "images", image_filename)
+
+            # Create page record
+            doc_page = self.page_repo.create_page(
+                document_id=document.id,
+                page_number=page_num,
+                width=float(width),
+                height=float(height),
+                image_path=image_path,
+            )
+
+            # Run OCR on saved image
+            temp_path = self.storage.get_absolute_path(image_path)
+            self._run_ocr_on_page(doc_page, temp_path)
+
+        return document
+
+    def _run_ocr_on_page(self, doc_page: DocumentPage, image_path: str) -> None:
+        """Run PaddleOCR on a single page image and store results."""
+        ocr = get_ocr_engine()
+
+        try:
+            results = ocr.ocr(image_path, cls=True)
+        except Exception as e:
+            logger.error("ocr_failed", page_id=str(doc_page.id), error=str(e))
+            return
+
+        if not results or not results[0]:
+            logger.info("ocr_no_results", page_id=str(doc_page.id))
+            return
+
+        full_text_parts = []
+        sequence = 0
+
+        for line in results[0]:
+            if not line or len(line) < 2:
+                continue
+
+            bbox_points = line[0]  # List of 4 corner points
+            text_info = line[1]    # (text, confidence)
+
+            text = text_info[0] if isinstance(text_info, (list, tuple)) else str(text_info)
+            confidence = float(text_info[1]) if isinstance(text_info, (list, tuple)) and len(text_info) > 1 else 0.0
+
+            if not text.strip():
+                continue
+
+            # Convert bbox points to x, y, width, height
+            xs = [p[0] for p in bbox_points]
+            ys = [p[1] for p in bbox_points]
+            x = min(xs)
+            y = min(ys)
+            width = max(xs) - x
+            height = max(ys) - y
+
+            # Determine block type based on position
+            block_type = self._classify_text_block(
+                y, height, doc_page.height, text
+            )
+
+            self.text_block_repo.create_text_block(
+                page_id=doc_page.id,
+                text=text,
+                x=x,
+                y=y,
+                width=width,
+                height=height,
+                confidence=confidence,
+                block_type=block_type,
+                sequence=sequence,
+            )
+            full_text_parts.append(text)
+            sequence += 1
+
+        # Update page text content
+        doc_page.text_content = "\n".join(full_text_parts)
+
+    def _classify_text_block(
+        self,
+        y: float,
+        height: float,
+        page_height: float,
+        text: str,
+    ) -> str:
+        """Classify a text block as header, footer, watermark, or regular text."""
+        if page_height <= 0:
+            return "text"
+
+        relative_y = y / page_height
+
+        # Header: top 10%
+        if relative_y < 0.10:
+            return "header"
+
+        # Footer: bottom 10%
+        if relative_y > 0.90:
+            return "footer"
+
+        # Watermark detection heuristic: large text in center
+        if 0.3 < relative_y < 0.7 and height > page_height * 0.05:
+            # Check for common watermark words
+            watermark_keywords = {"confidential", "draft", "copy", "sample", "watermark", "void"}
+            if text.strip().lower() in watermark_keywords:
+                return "watermark"
+
+        return "text"

docengine/app/services/pdf_service.py

@@ -0,0 +1,292 @@
+from __future__ import annotations
+
+import uuid
+from pathlib import Path
+
+import cv2
+import fitz  # PyMuPDF
+import numpy as np
+from sqlalchemy.orm import Session
+
+from app.core.config import settings
+from app.core.logging_config import get_logger
+from app.models.document import Document, DocumentImage, DocumentPage, DocumentTable, DocumentTextBlock
+from app.repositories.document_repository import (
+    DocumentImageRepository,
+    DocumentPageRepository,
+    DocumentRepository,
+    DocumentTableRepository,
+    DocumentTextBlockRepository,
+)
+from app.storage.provider import get_storage_provider
+
+logger = get_logger(__name__)
+
+
+class NativePDFService:
+    """Extract text, fonts, images, and layout from native (non-scanned) PDFs using PyMuPDF."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+        self.storage = get_storage_provider()
+        self.doc_repo = DocumentRepository(db)
+        self.page_repo = DocumentPageRepository(db)
+        self.text_block_repo = DocumentTextBlockRepository(db)
+        self.image_repo = DocumentImageRepository(db)
+        self.table_repo = DocumentTableRepository(db)
+
+    def process_pdf(self, document: Document) -> Document:
+        """Process a native PDF document, extracting all content."""
+        file_path = self.storage.get_absolute_path(document.storage_path)
+
+        try:
+            pdf_doc = fitz.open(file_path)
+        except Exception as e:
+            logger.error("pdf_open_failed", document_id=str(document.id), error=str(e))
+            raise
+
+        document.page_count = len(pdf_doc)
+        document.is_scanned = self._is_scanned_pdf(pdf_doc)
+
+        for page_num in range(len(pdf_doc)):
+            page = pdf_doc[page_num]
+            self._process_page(document, page, page_num + 1)
+
+        pdf_doc.close()
+        return document
+
+    def _is_scanned_pdf(self, pdf_doc: fitz.Document) -> bool:
+        """Determine if a PDF is scanned (image-based) or native."""
+        total_text_chars = 0
+        total_images = 0
+        for page_num in range(min(len(pdf_doc), 3)):
+            page = pdf_doc[page_num]
+            text = page.get_text("text")
+            total_text_chars += len(text.strip())
+            total_images += len(page.get_images(full=True))
+
+        # If very little text but has images, likely scanned
+        if total_text_chars < 50 and total_images > 0:
+            return True
+        return False
+
+    def _process_page(self, document: Document, page: fitz.Page, page_number: int) -> DocumentPage:
+        """Process a single PDF page."""
+        rect = page.rect
+        width = rect.width
+        height = rect.height
+
+        # Save page as image for reference
+        pix = page.get_pixmap(dpi=150)
+        image_filename = f"{document.id}_page_{page_number}.png"
+        image_data = pix.tobytes("png")
+        image_path = self.storage.save_file(image_data, "images", image_filename)
+
+        # Get full text content
+        text_content = page.get_text("text")
+
+        doc_page = self.page_repo.create_page(
+            document_id=document.id,
+            page_number=page_number,
+            width=width,
+            height=height,
+            image_path=image_path,
+            text_content=text_content,
+        )
+
+        # Extract text blocks with font information
+        self._extract_text_blocks(doc_page, page)
+
+        # Extract images
+        self._extract_images(doc_page, page, document)
+
+        # Detect headers and footers
+        self._detect_headers_footers(doc_page, page)
+
+        return doc_page
+
+    def _extract_text_blocks(self, doc_page: DocumentPage, page: fitz.Page) -> None:
+        """Extract text blocks with positioning and font information."""
+        blocks = page.get_text("dict", flags=fitz.TEXT_PRESERVE_WHITESPACE)["blocks"]
+        sequence = 0
+
+        for block in blocks:
+            if block["type"] != 0:  # Skip non-text blocks
+                continue
+
+            block_text_parts = []
+            font_info = {"family": None, "size": None, "color": None, "style": None}
+
+            for line in block.get("lines", []):
+                for span in line.get("spans", []):
+                    text = span.get("text", "").strip()
+                    if text:
+                        block_text_parts.append(text)
+                        # Capture font info from the first non-empty span
+                        if font_info["family"] is None:
+                            font_info["family"] = span.get("font", None)
+                            font_info["size"] = span.get("size", None)
+                            color_int = span.get("color", 0)
+                            font_info["color"] = f"#{color_int:06x}" if isinstance(color_int, int) else None
+                            flags = span.get("flags", 0)
+                            styles = []
+                            if flags & 1:
+                                styles.append("superscript")
+                            if flags & 2:
+                                styles.append("italic")
+                            if flags & 4:
+                                styles.append("serif")
+                            if flags & 8:
+                                styles.append("monospace")
+                            if flags & 16:
+                                styles.append("bold")
+                            font_info["style"] = ",".join(styles) if styles else "regular"
+
+            full_text = " ".join(block_text_parts)
+            if not full_text.strip():
+                continue
+
+            bbox = block["bbox"]
+            self.text_block_repo.create_text_block(
+                page_id=doc_page.id,
+                text=full_text,
+                x=bbox[0],
+                y=bbox[1],
+                width=bbox[2] - bbox[0],
+                height=bbox[3] - bbox[1],
+                font_family=font_info["family"],
+                font_size=font_info["size"],
+                font_color=font_info["color"],
+                font_style=font_info["style"],
+                block_type="text",
+                sequence=sequence,
+            )
+            sequence += 1
+
+    def _extract_images(self, doc_page: DocumentPage, page: fitz.Page, document: Document) -> None:
+        """Extract embedded images from a PDF page."""
+        image_list = page.get_images(full=True)
+
+        for img_index, img_info in enumerate(image_list):
+            xref = img_info[0]
+            try:
+                base_image = page.parent.extract_image(xref)
+                if not base_image:
+                    continue
+
+                image_bytes = base_image["image"]
+                ext = base_image.get("ext", "png")
+                img_filename = f"{document.id}_page_{doc_page.page_number}_img_{img_index}.{ext}"
+                img_storage_path = self.storage.save_file(image_bytes, "images", img_filename)
+
+                # Try to get image position on page
+                img_rects = page.get_image_rects(xref)
+                if img_rects:
+                    rect = img_rects[0]
+                    x, y, x1, y1 = rect.x0, rect.y0, rect.x1, rect.y1
+                else:
+                    x, y, x1, y1 = 0, 0, base_image.get("width", 100), base_image.get("height", 100)
+
+                # Determine image type based on position
+                page_height = doc_page.height
+                page_width = doc_page.width
+                image_type = self._classify_image_type(x, y, x1, y1, page_width, page_height)
+
+                self.image_repo.create_image(
+                    page_id=doc_page.id,
+                    x=x,
+                    y=y,
+                    width=x1 - x,
+                    height=y1 - y,
+                    image_path=img_storage_path,
+                    image_type=image_type,
+                )
+
+            except Exception as e:
+                logger.warning(
+                    "image_extraction_failed",
+                    page_id=str(doc_page.id),
+                    img_index=img_index,
+                    error=str(e),
+                )
+
+    def _classify_image_type(
+        self,
+        x: float,
+        y: float,
+        x1: float,
+        y1: float,
+        page_width: float,
+        page_height: float,
+    ) -> str:
+        """Classify an image as logo, figure, background, or stamp based on position and size."""
+        width = x1 - x
+        height = y1 - y
+        area_ratio = (width * height) / (page_width * page_height) if page_width > 0 and page_height > 0 else 0
+
+        # Background: covers most of the page
+        if area_ratio > 0.8:
+            return "background"
+
+        # Logo: small image in top portion
+        if y < page_height * 0.15 and area_ratio < 0.05:
+            return "logo"
+
+        # Stamp: small image in bottom-right
+        if x > page_width * 0.6 and y > page_height * 0.7 and area_ratio < 0.05:
+            return "stamp"
+
+        return "figure"
+
+    def _detect_headers_footers(self, doc_page: DocumentPage, page: fitz.Page) -> None:
+        """Detect header and footer regions based on vertical position."""
+        page_height = page.rect.height
+        header_threshold = page_height * 0.1
+        footer_threshold = page_height * 0.9
+
+        blocks = page.get_text("dict")["blocks"]
+        header_seq = 0
+        footer_seq = 0
+
+        for block in blocks:
+            if block["type"] != 0:
+                continue
+
+            bbox = block["bbox"]
+            block_y = bbox[1]
+
+            text_parts = []
+            for line in block.get("lines", []):
+                for span in line.get("spans", []):
+                    t = span.get("text", "").strip()
+                    if t:
+                        text_parts.append(t)
+
+            full_text = " ".join(text_parts)
+            if not full_text.strip():
+                continue
+
+            if block_y < header_threshold:
+                self.text_block_repo.create_text_block(
+                    page_id=doc_page.id,
+                    text=full_text,
+                    x=bbox[0],
+                    y=bbox[1],
+                    width=bbox[2] - bbox[0],
+                    height=bbox[3] - bbox[1],
+                    block_type="header",
+                    sequence=header_seq,
+                )
+                header_seq += 1
+            elif block_y > footer_threshold:
+                self.text_block_repo.create_text_block(
+                    page_id=doc_page.id,
+                    text=full_text,
+                    x=bbox[0],
+                    y=bbox[1],
+                    width=bbox[2] - bbox[0],
+                    height=bbox[3] - bbox[1],
+                    block_type="footer",
+                    sequence=footer_seq,
+                )
+                footer_seq += 1

docengine/app/services/reconstruction_service.py

@@ -0,0 +1,455 @@
+from __future__ import annotations
+
+import os
+import uuid
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from reportlab.lib import colors
+from reportlab.lib.pagesizes import letter
+from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
+from reportlab.lib.units import inch, mm
+from reportlab.pdfbase import pdfmetrics
+from reportlab.pdfbase.ttfonts import TTFont
+from reportlab.platypus import (
+    BaseDocTemplate,
+    Frame,
+    Image,
+    NextPageTemplate,
+    PageBreak,
+    PageTemplate,
+    Paragraph,
+    SimpleDocTemplate,
+    Spacer,
+    Table,
+    TableStyle,
+)
+from sqlalchemy.orm import Session
+
+from app.core.logging_config import get_logger
+from app.models.template import DocumentFormat
+from app.schemas.template import TemplateRenderResponse
+from app.storage.provider import get_storage_provider
+
+logger = get_logger(__name__)
+
+
+class ReconstructionService:
+    """Reconstruct documents from templates using ReportLab."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+        self.storage = get_storage_provider()
+        self.styles = getSampleStyleSheet()
+        self._register_fonts()
+
+    def _register_fonts(self) -> None:
+        """Register additional fonts if available."""
+        # ReportLab includes Helvetica, Times-Roman, Courier by default
+        # Custom fonts can be registered here
+        pass
+
+    def render_template(
+        self,
+        template: DocumentFormat,
+        data: dict[str, Any],
+        output_filename: str | None = None,
+        images: dict[str, str] | None = None,
+    ) -> TemplateRenderResponse:
+        """Render a template to PDF with supplied data."""
+        if output_filename is None:
+            output_filename = f"{template.name}_{uuid.uuid4().hex[:8]}.pdf"
+
+        if not output_filename.endswith(".pdf"):
+            output_filename += ".pdf"
+
+        # Determine output path
+        output_storage_path = f"rendered/{output_filename}"
+        absolute_output_path = self.storage.get_absolute_path(output_storage_path)
+
+        # Ensure the rendered directory exists
+        os.makedirs(os.path.dirname(absolute_output_path), exist_ok=True)
+
+        # Build the PDF
+        self._build_pdf(template, data, absolute_output_path, images)
+
+        file_size = os.path.getsize(absolute_output_path)
+
+        logger.info(
+            "template_rendered",
+            template_id=str(template.id),
+            output=output_filename,
+            size=file_size,
+        )
+
+        return TemplateRenderResponse(
+            output_path=output_storage_path,
+            filename=output_filename,
+            file_size=file_size,
+            page_count=template.page_count,
+            rendered_at=datetime.now(UTC),
+        )
+
+    def _build_pdf(
+        self,
+        template: DocumentFormat,
+        data: dict[str, Any],
+        output_path: str,
+        images: dict[str, str] | None = None,
+    ) -> None:
+        """Build a PDF document from template definition."""
+        page_width = template.page_width
+        page_height = template.page_height
+
+        doc = SimpleDocTemplate(
+            output_path,
+            pagesize=(page_width, page_height),
+            topMargin=template.margin_top,
+            rightMargin=template.margin_right,
+            bottomMargin=template.margin_bottom,
+            leftMargin=template.margin_left,
+        )
+
+        # Build story (content elements)
+        story: list[Any] = []
+
+        for page_num in range(1, template.page_count + 1):
+            if page_num > 1:
+                story.append(PageBreak())
+
+            # Add page content
+            page_elements = self._build_page_content(template, page_num, data, images)
+            story.extend(page_elements)
+
+        # Build with watermark/header/footer callbacks
+        def on_page(canvas, doc_obj):  # noqa: ANN001, ANN202
+            self._draw_watermarks(canvas, template, doc_obj.page)
+            self._draw_headers_footers(canvas, template, doc_obj.page, page_width, page_height)
+
+        def on_page_later(canvas, doc_obj):  # noqa: ANN001, ANN202
+            self._draw_watermarks(canvas, template, doc_obj.page)
+            self._draw_headers_footers(canvas, template, doc_obj.page, page_width, page_height)
+
+        doc.build(story, onFirstPage=on_page, onLaterPages=on_page_later)
+
+    def _build_page_content(
+        self,
+        template: DocumentFormat,
+        page_number: int,
+        data: dict[str, Any],
+        images: dict[str, str] | None = None,
+    ) -> list[Any]:
+        """Build content elements for a specific page."""
+        elements: list[Any] = []
+
+        # Get cells for this page, sorted by sequence
+        page_cells = sorted(
+            [c for c in template.cells if c.page_number == page_number],
+            key=lambda c: c.sequence,
+        )
+
+        # Get table formats for this page
+        page_tables = [t for t in template.table_formats if t.page_number == page_number]
+
+        # Get image regions for this page
+        page_images = [i for i in template.image_regions if i.page_number == page_number]
+
+        # Add static and dynamic text cells
+        for cell in page_cells:
+            text = self._resolve_cell_text(cell, data)
+            if text:
+                style = self._create_cell_style(cell)
+                para = Paragraph(text, style)
+                elements.append(para)
+                elements.append(Spacer(1, 2))
+
+        # Add tables
+        for table_format in page_tables:
+            table_element = self._build_table(table_format, data)
+            if table_element:
+                elements.append(table_element)
+                elements.append(Spacer(1, 6))
+
+        # Add images
+        for img_region in page_images:
+            img_element = self._build_image(img_region, images)
+            if img_element:
+                elements.append(img_element)
+                elements.append(Spacer(1, 6))
+
+        if not elements:
+            elements.append(Spacer(1, 12))
+
+        return elements
+
+    def _resolve_cell_text(self, cell: Any, data: dict[str, Any]) -> str:
+        """Resolve cell text from static content or dynamic data."""
+        if cell.is_dynamic and cell.field_name:
+            value = data.get(cell.field_name, "")
+            return str(value) if value else ""
+        return cell.static_text or ""
+
+    def _create_cell_style(self, cell: Any) -> ParagraphStyle:
+        """Create a ReportLab paragraph style from cell properties."""
+        font_name = "Helvetica"
+        if cell.font_family:
+            family = cell.font_family.lower()
+            if "times" in family or "serif" in family:
+                font_name = "Times-Roman"
+            elif "courier" in family or "mono" in family:
+                font_name = "Courier"
+
+        font_size = cell.font_size or 10
+        if cell.font_style and "bold" in (cell.font_style or ""):
+            if font_name == "Helvetica":
+                font_name = "Helvetica-Bold"
+            elif font_name == "Times-Roman":
+                font_name = "Times-Bold"
+            elif font_name == "Courier":
+                font_name = "Courier-Bold"
+
+        text_color = colors.black
+        if cell.font_color:
+            try:
+                text_color = colors.HexColor(cell.font_color)
+            except (ValueError, TypeError):
+                text_color = colors.black
+
+        alignment_map = {"left": 0, "center": 1, "right": 2, "justify": 4}
+        alignment = alignment_map.get(cell.alignment, 0)
+
+        style = ParagraphStyle(
+            name=f"cell_{cell.id}",
+            parent=self.styles["Normal"],
+            fontName=font_name,
+            fontSize=font_size,
+            textColor=text_color,
+            alignment=alignment,
+            leading=font_size * 1.2,
+            spaceBefore=cell.padding_top,
+            spaceAfter=cell.padding_bottom,
+            leftIndent=cell.padding_left,
+            rightIndent=cell.padding_right,
+        )
+
+        return style
+
+    def _build_table(self, table_format: Any, data: dict[str, Any]) -> Table | None:
+        """Build a ReportLab table from a table format definition."""
+        rows = table_format.rows
+        columns = table_format.columns
+
+        if rows <= 0 or columns <= 0:
+            return None
+
+        # Build table data
+        table_data: list[list[str]] = []
+
+        # Header row
+        if table_format.table_columns:
+            header_row = [col.header_text or f"Col {col.column_index + 1}" for col in table_format.table_columns]
+            table_data.append(header_row)
+        else:
+            table_data.append([f"Column {i + 1}" for i in range(columns)])
+
+        # Data rows from supplied data
+        table_field_name = f"table_{table_format.id}"
+        table_rows_data = data.get(table_field_name, data.get("table_data", []))
+
+        if isinstance(table_rows_data, list):
+            for row_data in table_rows_data:
+                if isinstance(row_data, list):
+                    # Pad or trim to match column count
+                    row = row_data[:columns]
+                    while len(row) < columns:
+                        row.append("")
+                    table_data.append([str(v) for v in row])
+                elif isinstance(row_data, dict):
+                    row = []
+                    for col in table_format.table_columns:
+                        key = col.header_text or f"col_{col.column_index}"
+                        row.append(str(row_data.get(key, "")))
+                    table_data.append(row)
+
+        # If no data rows, add empty rows
+        if len(table_data) <= 1:
+            for _ in range(max(rows - 1, 1)):
+                table_data.append([""] * columns)
+
+        # Determine column widths
+        col_widths = []
+        if table_format.table_columns:
+            col_widths = [col.width for col in table_format.table_columns]
+        else:
+            col_width = table_format.width / columns
+            col_widths = [col_width] * columns
+
+        # Build table
+        table = Table(table_data, colWidths=col_widths)
+
+        # Apply table style
+        border_color = colors.black
+        if table_format.border_color:
+            try:
+                border_color = colors.HexColor(table_format.border_color)
+            except (ValueError, TypeError):
+                pass
+
+        style_commands = [
+            ("GRID", (0, 0), (-1, -1), table_format.border_width, border_color),
+            ("FONTNAME", (0, 0), (-1, 0), "Helvetica-Bold"),
+            ("FONTSIZE", (0, 0), (-1, -1), 9),
+            ("ALIGN", (0, 0), (-1, -1), "LEFT"),
+            ("VALIGN", (0, 0), (-1, -1), "MIDDLE"),
+            ("TOPPADDING", (0, 0), (-1, -1), 4),
+            ("BOTTOMPADDING", (0, 0), (-1, -1), 4),
+            ("LEFTPADDING", (0, 0), (-1, -1), 4),
+            ("RIGHTPADDING", (0, 0), (-1, -1), 4),
+        ]
+
+        # Header row background
+        if table_format.table_rows:
+            for row in table_format.table_rows:
+                if row.is_header and row.background_color:
+                    try:
+                        bg_color = colors.HexColor(row.background_color)
+                        style_commands.append(
+                            ("BACKGROUND", (0, row.row_index), (-1, row.row_index), bg_color)
+                        )
+                    except (ValueError, TypeError):
+                        pass
+        else:
+            style_commands.append(("BACKGROUND", (0, 0), (-1, 0), colors.HexColor("#E0E0E0")))
+
+        table.setStyle(TableStyle(style_commands))
+
+        return table
+
+    def _build_image(self, img_region: Any, images: dict[str, str] | None = None) -> Image | None:
+        """Build a ReportLab image from an image region definition."""
+        image_path = None
+
+        # Check dynamic images first
+        if not img_region.is_static and img_region.field_name and images:
+            image_path = images.get(img_region.field_name)
+
+        # Fall back to stored image
+        if not image_path and img_region.image_path:
+            try:
+                image_path = self.storage.get_absolute_path(img_region.image_path)
+            except Exception:
+                image_path = None
+
+        if not image_path or not os.path.exists(image_path):
+            return None
+
+        try:
+            img = Image(image_path, width=img_region.width, height=img_region.height)
+            return img
+        except Exception as e:
+            logger.warning("image_build_failed", error=str(e), path=image_path)
+            return None
+
+    def _draw_watermarks(self, canvas: Any, template: DocumentFormat, current_page: int) -> None:
+        """Draw watermarks on the canvas."""
+        for watermark in template.watermarks:
+            # Apply to all pages if page_number is None, or specific page
+            if watermark.page_number is not None and watermark.page_number != current_page:
+                continue
+
+            canvas.saveState()
+
+            # Set opacity
+            canvas.setFillAlpha(watermark.opacity)
+
+            if watermark.text:
+                # Text watermark
+                font_name = "Helvetica"
+                if watermark.font_family:
+                    family = watermark.font_family.lower()
+                    if "times" in family:
+                        font_name = "Times-Roman"
+                    elif "courier" in family:
+                        font_name = "Courier"
+
+                font_size = watermark.font_size or 48
+
+                if watermark.font_color:
+                    try:
+                        canvas.setFillColor(colors.HexColor(watermark.font_color))
+                    except (ValueError, TypeError):
+                        canvas.setFillColor(colors.grey)
+                else:
+                    canvas.setFillColor(colors.grey)
+
+                canvas.setFont(font_name, font_size)
+
+                # Position and rotate
+                canvas.translate(
+                    watermark.x + watermark.width / 2,
+                    watermark.y + watermark.height / 2,
+                )
+                canvas.rotate(watermark.rotation)
+                canvas.drawCentredString(0, 0, watermark.text)
+
+            elif watermark.image_path:
+                # Image watermark
+                try:
+                    img_path = self.storage.get_absolute_path(watermark.image_path)
+                    if os.path.exists(img_path):
+                        canvas.drawImage(
+                            img_path,
+                            watermark.x,
+                            watermark.y,
+                            width=watermark.width,
+                            height=watermark.height,
+                            mask="auto",
+                        )
+                except Exception as e:
+                    logger.warning("watermark_image_failed", error=str(e))
+
+            canvas.restoreState()
+
+    def _draw_headers_footers(
+        self,
+        canvas: Any,
+        template: DocumentFormat,
+        current_page: int,
+        page_width: float,
+        page_height: float,
+    ) -> None:
+        """Draw header and footer regions on the canvas."""
+        for region in template.regions:
+            if region.page_number != current_page:
+                continue
+
+            content = region.content or {}
+            blocks = content.get("blocks", [])
+
+            canvas.saveState()
+
+            for block in blocks:
+                text = block.get("text", "")
+                if not text:
+                    continue
+
+                x = block.get("x", region.x)
+                y = page_height - block.get("y", region.y) - block.get("height", 12)
+                font_family = block.get("font_family", "Helvetica")
+                font_size = block.get("font_size", 10)
+
+                # Map font family
+                font_name = "Helvetica"
+                if font_family:
+                    fl = font_family.lower()
+                    if "times" in fl or "serif" in fl:
+                        font_name = "Times-Roman"
+                    elif "courier" in fl or "mono" in fl:
+                        font_name = "Courier"
+
+                canvas.setFont(font_name, font_size)
+                canvas.setFillColor(colors.black)
+                canvas.drawString(x, y, text)
+
+            canvas.restoreState()

docengine/app/services/template_service.py

@@ -0,0 +1,401 @@
+from __future__ import annotations
+
+import uuid
+from typing import Any
+
+from sqlalchemy.orm import Session
+
+from app.core.logging_config import get_logger
+from app.models.document import Document
+from app.models.template import (
+    DocumentCell,
+    DocumentFormat,
+    DocumentRegion,
+    ImageRegion,
+    TableColumn,
+    TableFormat,
+    TableRow,
+    Watermark,
+)
+from app.repositories.document_repository import DocumentRepository
+from app.repositories.template_repository import (
+    DocumentCellRepository,
+    DocumentRegionRepository,
+    ImageRegionRepository,
+    TableColumnRepository,
+    TableFormatRepository,
+    TableRowRepository,
+    TemplateRepository,
+    WatermarkRepository,
+)
+from app.services.fingerprint_service import FingerprintService
+
+logger = get_logger(__name__)
+
+
+class TemplateService:
+    """Generate reusable document templates from processed documents."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+        self.doc_repo = DocumentRepository(db)
+        self.template_repo = TemplateRepository(db)
+        self.cell_repo = DocumentCellRepository(db)
+        self.region_repo = DocumentRegionRepository(db)
+        self.table_format_repo = TableFormatRepository(db)
+        self.table_column_repo = TableColumnRepository(db)
+        self.table_row_repo = TableRowRepository(db)
+        self.watermark_repo = WatermarkRepository(db)
+        self.image_region_repo = ImageRegionRepository(db)
+        self.fingerprint_service = FingerprintService(db)
+
+    def generate_template(
+        self,
+        document: Document,
+        user_id: uuid.UUID | None = None,
+    ) -> DocumentFormat:
+        """Generate a reusable template from a processed document."""
+        if not document.pages:
+            raise ValueError(f"Document '{document.id}' has no processed pages")
+
+        # Check if template already exists for this document
+        existing = self.template_repo.get_by_source_document(document.id)
+        if existing:
+            logger.info(
+                "template_already_exists",
+                document_id=str(document.id),
+                template_id=str(existing.id),
+            )
+            return existing
+
+        first_page = document.pages[0]
+        template_name = f"Template_{document.original_filename}_{uuid.uuid4().hex[:8]}"
+
+        # Create template
+        template = self.template_repo.create_template(
+            name=template_name,
+            page_width=first_page.width,
+            page_height=first_page.height,
+            page_count=document.page_count or len(document.pages),
+            description=f"Auto-generated template from {document.original_filename}",
+            source_document_id=document.id,
+            created_by=user_id,
+        )
+
+        # Process each page
+        for page in document.pages:
+            self._process_page_for_template(template, page)
+
+        # Generate fingerprint
+        self.fingerprint_service.generate_fingerprint(template)
+
+        logger.info(
+            "template_generated",
+            template_id=str(template.id),
+            document_id=str(document.id),
+            cells=len(template.cells),
+            regions=len(template.regions),
+        )
+
+        return template
+
+    def _process_page_for_template(
+        self,
+        template: DocumentFormat,
+        page: Any,
+    ) -> None:
+        """Process a document page and create template components."""
+        page_number = page.page_number
+
+        # Create cells from text blocks
+        self._create_cells_from_text_blocks(template, page, page_number)
+
+        # Create regions from headers, footers
+        self._create_regions(template, page, page_number)
+
+        # Create table formats
+        self._create_table_formats(template, page, page_number)
+
+        # Create image regions
+        self._create_image_regions(template, page, page_number)
+
+        # Detect watermarks from text blocks
+        self._create_watermarks(template, page, page_number)
+
+    def _create_cells_from_text_blocks(
+        self,
+        template: DocumentFormat,
+        page: Any,
+        page_number: int,
+    ) -> None:
+        """Create template cells from extracted text blocks."""
+        for seq, block in enumerate(page.text_blocks):
+            if block.block_type in ("header", "footer", "watermark"):
+                continue
+
+            # Determine if this is a dynamic field
+            is_dynamic = self._is_dynamic_field(block.text)
+            field_name = self._generate_field_name(block.text, seq) if is_dynamic else None
+
+            self.cell_repo.create_cell(
+                format_id=template.id,
+                page_number=page_number,
+                x=block.x,
+                y=block.y,
+                width=block.width,
+                height=block.height,
+                data_type=self._infer_data_type(block.text),
+                font_family=block.font_family,
+                font_size=block.font_size,
+                font_style=block.font_style,
+                font_color=block.font_color,
+                alignment=self._infer_alignment(block.x, template.page_width),
+                static_text=block.text if not is_dynamic else None,
+                field_name=field_name,
+                sequence=seq,
+                is_dynamic=is_dynamic,
+            )
+
+    def _create_regions(
+        self,
+        template: DocumentFormat,
+        page: Any,
+        page_number: int,
+    ) -> None:
+        """Create template regions from headers and footers."""
+        header_blocks = [b for b in page.text_blocks if b.block_type == "header"]
+        footer_blocks = [b for b in page.text_blocks if b.block_type == "footer"]
+
+        if header_blocks:
+            # Compute bounding box for all header blocks
+            min_x = min(b.x for b in header_blocks)
+            min_y = min(b.y for b in header_blocks)
+            max_x = max(b.x + b.width for b in header_blocks)
+            max_y = max(b.y + b.height for b in header_blocks)
+
+            content = {
+                "blocks": [
+                    {
+                        "text": b.text,
+                        "x": b.x,
+                        "y": b.y,
+                        "width": b.width,
+                        "height": b.height,
+                        "font_family": b.font_family,
+                        "font_size": b.font_size,
+                    }
+                    for b in header_blocks
+                ]
+            }
+
+            self.region_repo.create_region(
+                format_id=template.id,
+                page_number=page_number,
+                region_type="header",
+                x=min_x,
+                y=min_y,
+                width=max_x - min_x,
+                height=max_y - min_y,
+                content=content,
+                sequence=0,
+            )
+
+        if footer_blocks:
+            min_x = min(b.x for b in footer_blocks)
+            min_y = min(b.y for b in footer_blocks)
+            max_x = max(b.x + b.width for b in footer_blocks)
+            max_y = max(b.y + b.height for b in footer_blocks)
+
+            content = {
+                "blocks": [
+                    {
+                        "text": b.text,
+                        "x": b.x,
+                        "y": b.y,
+                        "width": b.width,
+                        "height": b.height,
+                        "font_family": b.font_family,
+                        "font_size": b.font_size,
+                    }
+                    for b in footer_blocks
+                ]
+            }
+
+            self.region_repo.create_region(
+                format_id=template.id,
+                page_number=page_number,
+                region_type="footer",
+                x=min_x,
+                y=min_y,
+                width=max_x - min_x,
+                height=max_y - min_y,
+                content=content,
+                sequence=1,
+            )
+
+    def _create_table_formats(
+        self,
+        template: DocumentFormat,
+        page: Any,
+        page_number: int,
+    ) -> None:
+        """Create table format definitions from detected tables."""
+        for table in page.tables:
+            table_format = self.table_format_repo.create_table_format(
+                format_id=template.id,
+                page_number=page_number,
+                x=table.x,
+                y=table.y,
+                width=table.width,
+                height=table.height,
+                rows=table.rows,
+                columns=table.columns,
+            )
+
+            # Create columns
+            col_width = table.width / max(table.columns, 1)
+            for col_idx in range(table.columns):
+                self.table_column_repo.create_column(
+                    table_format_id=table_format.id,
+                    column_index=col_idx,
+                    width=col_width,
+                    data_type="text",
+                    alignment="left",
+                )
+
+            # Create rows
+            row_height = table.height / max(table.rows, 1)
+            for row_idx in range(table.rows):
+                self.table_row_repo.create_row(
+                    table_format_id=table_format.id,
+                    row_index=row_idx,
+                    height=row_height,
+                    is_header=(row_idx == 0),
+                )
+
+    def _create_image_regions(
+        self,
+        template: DocumentFormat,
+        page: Any,
+        page_number: int,
+    ) -> None:
+        """Create image region definitions from detected images."""
+        for img in page.images:
+            self.image_region_repo.create_image_region(
+                format_id=template.id,
+                page_number=page_number,
+                x=img.x,
+                y=img.y,
+                width=img.width,
+                height=img.height,
+                image_path=img.image_path,
+                image_type=img.image_type,
+                is_static=True,
+            )
+
+    def _create_watermarks(
+        self,
+        template: DocumentFormat,
+        page: Any,
+        page_number: int,
+    ) -> None:
+        """Create watermark definitions from detected watermark text blocks."""
+        watermark_blocks = [b for b in page.text_blocks if b.block_type == "watermark"]
+        for block in watermark_blocks:
+            self.watermark_repo.create_watermark(
+                format_id=template.id,
+                page_number=page_number,
+                text=block.text,
+                x=block.x,
+                y=block.y,
+                width=block.width,
+                height=block.height,
+                opacity=0.3,
+                rotation=0.0,
+                font_family=block.font_family,
+                font_size=block.font_size,
+                font_color=block.font_color or "#CCCCCC",
+            )
+
+    def _is_dynamic_field(self, text: str) -> bool:
+        """Determine if a text block represents a dynamic (variable) field."""
+        if not text:
+            return False
+
+        # Common patterns indicating dynamic content
+        dynamic_patterns = [
+            "{{", "}}", "${", "##",
+            "__________", "___", "...........",
+        ]
+        for pattern in dynamic_patterns:
+            if pattern in text:
+                return True
+
+        # Short single-word values that might be labels are static
+        # Longer values with numbers/dates tend to be dynamic
+        import re
+        # Date patterns
+        if re.search(r"\d{1,2}[/\-\.]\d{1,2}[/\-\.]\d{2,4}", text):
+            return True
+        # Currency patterns
+        if re.search(r"[$€£¥]\s*[\d,]+\.?\d*", text):
+            return True
+        # Phone patterns
+        if re.search(r"\+?\d[\d\s\-()]{7,}", text):
+            return True
+
+        return False
+
+    def _generate_field_name(self, text: str, sequence: int) -> str:
+        """Generate a field name from text content."""
+        import re
+        # Clean text
+        clean = re.sub(r"[^a-zA-Z0-9\s]", "", text)
+        clean = clean.strip().lower()
+        words = clean.split()[:3]
+        if words:
+            return "_".join(words)
+        return f"field_{sequence}"
+
+    def _infer_data_type(self, text: str) -> str:
+        """Infer the data type from text content."""
+        import re
+
+        if not text:
+            return "text"
+
+        stripped = text.strip()
+
+        # Number
+        if re.match(r"^-?[\d,]+\.?\d*$", stripped.replace(",", "")):
+            return "number"
+
+        # Date
+        if re.search(r"\d{1,2}[/\-\.]\d{1,2}[/\-\.]\d{2,4}", stripped):
+            return "date"
+
+        # Currency
+        if re.search(r"^[$€£¥]\s*[\d,]+\.?\d*$", stripped):
+            return "currency"
+
+        # Email
+        if re.search(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", stripped):
+            return "email"
+
+        return "text"
+
+    def _infer_alignment(self, x: float, page_width: float) -> str:
+        """Infer text alignment based on horizontal position."""
+        if page_width <= 0:
+            return "left"
+
+        relative_x = x / page_width
+
+        if relative_x < 0.15:
+            return "left"
+        elif relative_x > 0.6:
+            return "right"
+        elif 0.35 < relative_x < 0.65:
+            return "center"
+
+        return "left"

docengine/app/tasks/document_tasks.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import uuid
+
+from app.core.logging_config import get_logger
+from app.workers.celery_app import celery_app
+from app.core.database import get_db_context
+from app.services.document_service import DocumentProcessingService
+
+logger = get_logger(__name__)
+
+
+@celery_app.task(
+    name="app.tasks.document_tasks.process_document_task",
+    bind=True,
+    max_retries=3,
+    default_retry_delay=60,
+    acks_late=True,
+)
+def process_document_task(self, document_id: str) -> dict:  # noqa: ANN001
+    """Celery task to process a document asynchronously."""
+    logger.info("task_started", task_id=self.request.id, document_id=document_id)
+
+    try:
+        doc_uuid = uuid.UUID(document_id)
+        with get_db_context() as db:
+            service = DocumentProcessingService(db)
+            document = service.process_document(doc_uuid)
+            logger.info(
+                "task_completed",
+                task_id=self.request.id,
+                document_id=document_id,
+                status=document.status,
+            )
+            return {
+                "document_id": document_id,
+                "status": document.status,
+                "page_count": document.page_count,
+            }
+    except Exception as exc:
+        logger.exception(
+            "task_failed",
+            task_id=self.request.id,
+            document_id=document_id,
+            error=str(exc),
+            retry=self.request.retries,
+        )
+        raise self.retry(exc=exc)
+
+
+@celery_app.task(
+    name="app.tasks.document_tasks.match_document_task",
+    bind=True,
+    max_retries=2,
+    default_retry_delay=30,
+)
+def match_document_task(
+    self,  # noqa: ANN001
+    document_id: str,
+    min_confidence: float = 0.5,
+    max_results: int = 5,
+) -> dict:
+    """Celery task to match a document against templates."""
+    logger.info("match_task_started", task_id=self.request.id, document_id=document_id)
+
+    try:
+        doc_uuid = uuid.UUID(document_id)
+        with get_db_context() as db:
+            from app.services.matching_service import MatchingService
+            service = MatchingService(db)
+            matches = service.match_document(
+                document_id=doc_uuid,
+                min_confidence=min_confidence,
+                max_results=max_results,
+            )
+            logger.info(
+                "match_task_completed",
+                task_id=self.request.id,
+                document_id=document_id,
+                matches=len(matches),
+            )
+            return {
+                "document_id": document_id,
+                "matches": len(matches),
+                "best_score": matches[0].confidence_score if matches else 0.0,
+            }
+    except Exception as exc:
+        logger.exception(
+            "match_task_failed",
+            task_id=self.request.id,
+            document_id=document_id,
+            error=str(exc),
+        )
+        raise self.retry(exc=exc)

docengine/app/tasks/maintenance_tasks.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from app.core.logging_config import get_logger
+from app.workers.celery_app import celery_app
+from app.core.database import get_db_context
+
+logger = get_logger(__name__)
+
+
+@celery_app.task(
+    name="app.tasks.maintenance_tasks.cleanup_expired_tokens",
+    bind=True,
+)
+def cleanup_expired_tokens(self) -> dict:  # noqa: ANN001
+    """Cleanup expired and revoked refresh tokens."""
+    logger.info("cleanup_tokens_started", task_id=self.request.id)
+
+    try:
+        with get_db_context() as db:
+            from app.repositories.user_repository import RefreshTokenRepository
+            repo = RefreshTokenRepository(db)
+            count = repo.cleanup_expired_tokens()
+            logger.info("cleanup_tokens_completed", removed=count)
+            return {"removed_tokens": count}
+    except Exception as exc:
+        logger.exception("cleanup_tokens_failed", error=str(exc))
+        return {"error": str(exc)}
+
+
+@celery_app.task(
+    name="app.tasks.maintenance_tasks.cleanup_temp_storage",
+    bind=True,
+)
+def cleanup_temp_storage(self) -> dict:  # noqa: ANN001
+    """Cleanup temporary storage files."""
+    logger.info("cleanup_temp_started", task_id=self.request.id)
+
+    try:
+        from app.storage.provider import get_storage_provider
+        storage = get_storage_provider()
+        count = storage.cleanup_temp()
+        logger.info("cleanup_temp_completed", removed=count)
+        return {"removed_files": count}
+    except Exception as exc:
+        logger.exception("cleanup_temp_failed", error=str(exc))
+        return {"error": str(exc)}

docengine/app/workers/celery_app.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from celery import Celery
+
+from app.core.config import settings
+
+celery_app = Celery(
+    "docengine",
+    broker=settings.celery_broker_url,
+    backend=settings.celery_result_backend,
+)
+
+celery_app.conf.update(
+    task_serializer="json",
+    accept_content=["json"],
+    result_serializer="json",
+    timezone="UTC",
+    enable_utc=True,
+    task_track_started=True,
+    task_time_limit=3600,
+    task_soft_time_limit=3300,
+    worker_max_tasks_per_child=100,
+    worker_prefetch_multiplier=1,
+    task_acks_late=True,
+    task_reject_on_worker_lost=True,
+    broker_connection_retry_on_startup=True,
+    result_expires=86400,
+    task_routes={
+        "app.tasks.document_tasks.*": {"queue": "document_processing"},
+    },
+    beat_schedule={
+        "cleanup-expired-tokens": {
+            "task": "app.tasks.maintenance_tasks.cleanup_expired_tokens",
+            "schedule": 3600.0,
+        },
+        "cleanup-temp-storage": {
+            "task": "app.tasks.maintenance_tasks.cleanup_temp_storage",
+            "schedule": 7200.0,
+        },
+    },
+)
+
+celery_app.autodiscover_tasks(["app.tasks"])