Changes committed

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "dotrush.roslyn.projectOrSolutionFiles": []
+}

backend/database.py

@@ -1,5 +1,7 @@
 import os
-from sqlalchemy import create_engine, Column, Integer, String, Text, DateTime, Boolean, ForeignKey, LargeBinary
+import urllib.parse
+from sqlalchemy import create_engine, Column, Integer, String, Text, DateTime, Boolean, ForeignKey, LargeBinary, Enum as SqlEnum
+from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker, relationship
 from dotenv import load_dotenv
@@ -7,22 +9,11 @@ from dotenv import load_dotenv
 # Load environment variables
 load_dotenv()
 
-DB_USER = os.getenv("DB_USER")
-DB_PASSWORD = os.getenv("DB_PASSWORD")
-DB_HOST = os.getenv("DB_HOST")
-DB_PORT = os.getenv("DB_PORT")
-import urllib.parse
-
-# ... (imports)
-
-# Load environment variables
-load_dotenv()
-
-DB_USER = os.getenv("DB_USER")
-DB_PASSWORD = os.getenv("DB_PASSWORD")
-DB_HOST = os.getenv("DB_HOST")
-DB_PORT = os.getenv("DB_PORT")
-DB_NAME = os.getenv("DB_NAME")
+DB_USER = os.getenv("DB_USER", "postgres")
+DB_PASSWORD = os.getenv("DB_PASSWORD", "M@tr!x#149@dm!N")
+DB_HOST = os.getenv("DB_HOST", "192.168.0.111")
+DB_PORT = os.getenv("DB_PORT", "7925")
+DB_NAME = os.getenv("DB_NAME", "ocr")
 
 encoded_user = urllib.parse.quote_plus(DB_USER)
 encoded_password = urllib.parse.quote_plus(DB_PASSWORD)
@@ -35,6 +26,26 @@ SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 
 Base = declarative_base()
 
+class Vendor(Base):
+    __tablename__ = "vendors"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    name = Column(String, unique=True, index=True)
+    default_model = Column(String) # 'text' or 'vision'
+    created_at = Column(DateTime)
+
+class Document(Base):
+    __tablename__ = "documents"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    vendor_id = Column(Integer, ForeignKey("vendors.id"), nullable=True)
+    filename = Column(String)
+    upload_date = Column(DateTime)
+    status = Column(String) # 'pending', 'verified'
+    processed_data = Column(JSONB) # Store the verified extraction results
+    
+    vendor = relationship("Vendor")
+
 class Email(Base):
     __tablename__ = "emails"
 
@@ -54,7 +65,7 @@ class Attachment(Base):
     email_id = Column(Integer, ForeignKey("emails.id"))
     filename = Column(String)
     content_type = Column(String)
-    file_content = Column(LargeBinary) # Storing content directly in DB as requested
+    file_content = Column(LargeBinary)
     
     email = relationship("Email", back_populates="attachments")
 

backend/llm_service.py

@@ -0,0 +1,116 @@
+import ollama
+import json
+import base64
+
+INVOICE_SCHEMA = {
+    "document_type": None,
+    "invoice_number": None,
+    "invoice_date": None,
+    "due_date": None,
+    "purchase_order_number": None,
+    "vendor": {
+        "name": None,
+        "address": None,
+        "email": None,
+        "phone": None,
+        "gstin": None,
+        "tax_id": None,
+        "website": None
+    },
+    "customer": {
+        "name": None,
+        "address": None,
+        "gstin": None
+    },
+    "amounts": {
+        "subtotal": None,
+        "tax": None,
+        "discount": None,
+        "shipping": None,
+        "round_off": None,
+        "total": None,
+        "amount_paid": None,
+        "balance_due": None,
+        "currency": None
+    },
+    "tax_breakdown": [
+        {
+            "type": None,
+            "rate": None,
+            "amount": None
+        }
+    ],
+    "line_items": [
+        {
+            "line_no": None,
+            "description": None,
+            "product_code": None,
+            "hsn_sac": None,
+            "quantity": None,
+            "unit": None,
+            "unit_price": None,
+            "discount": None,
+            "tax_rate": None,
+            "tax_amount": None,
+            "total": None
+        }
+    ],
+    "payment_information": {
+        "bank_name": None,
+        "account_number": None,
+        "ifsc": None,
+        "upi_id": None
+    },
+    "metadata": {
+        "pages": None,
+        "ocr_confidence": None,
+        "language": None
+    }
+}
+
+def extract_data(text: str = None, image_path: str = None, model_type: str = "text") -> dict:
+    """
+    Extracts structured data using either Text (Gemma) or Vision (Qwen) models.
+    """
+    
+    prompt = f"""
+    You are an expert data extraction assistant. 
+    Extract every possible detail from the provided document and return it strictly as a SINGLE VALID JSON OBJECT matching the following schema structure:
+    
+    {json.dumps(INVOICE_SCHEMA, indent=4)}
+
+    IMPORTANT: 
+    - Return ONLY the JSON. No markdown formatting, no explanations, no prefix.
+    - If a field is not found or data is not available, use null.
+    """
+    
+    messages = [{'role': 'user', 'content': prompt}]
+    model = 'gemma:2b'
+    
+    if model_type == 'vision':
+        if not image_path:
+            return {"error": "Image path required for vision mode"}
+        
+        # Qwen-VL handles images passed in the message
+        model = 'qwen2.5vl:7b' # Using the installed model ID
+        messages[0]['images'] = [image_path]
+        messages[0]['content'] = "Analyze this image. " + prompt
+    else:
+        # Text Mode
+        if not text:
+            return {"error": "Text required for text mode"}
+        messages[0]['content'] += f"\n\n---\n{text}\n---"
+
+    try:
+        response = ollama.chat(model=model, messages=messages)
+        content = response['message']['content']
+        
+        # Clean up markdown
+        content = content.replace("```json", "").replace("```", "").strip()
+        
+        return json.loads(content)
+
+    except Exception as e:
+        print(f"LLM Extraction Error ({model_type}): {e}")
+        return {"error": str(e), "raw_output": content if 'content' in locals() else ""}
+

backend/main.py

@@ -44,7 +44,7 @@ def extract_text_from_pdf(file_bytes: bytes) -> str:
         return ""
 
 # Internal modules
-from database import get_db, Email
+from database import get_db, Email, Vendor, Document
 from scheduler import start_scheduler, stop_scheduler
 from mail_service import fetch_and_store_emails
 
@@ -87,7 +87,7 @@ class LoginResponse(BaseModel):
 
 class NERResponse(BaseModel):
     text: str
-
+    file_path: str
 
 
 def extract_text_from_image(file_bytes: bytes) -> str:
@@ -104,13 +104,41 @@ async def extract_text(file: UploadFile = File(...)):
     content = await file.read()
     filename = file.filename.lower()
     
+    # Save file for Vision mode
+    file_path = f"uploads/{file.filename}"
+    with open(file_path, "wb") as f:
+        f.write(content)
+    
     extracted_text = ""
     
     if filename.endswith(".pdf"):
         # Try text extraction first
-        extracted_text = extract_text_from_pdf(content)
-        
-        # If text is empty, it might be a scanned PDF. 
+        with pdfplumber.open(io.BytesIO(content)) as pdf:
+            try:
+                text = ""
+                for page in pdf.pages:
+                    page_text = page.extract_text(layout=True)
+                    if page_text:
+                        text += page_text + "\n"
+                if text.strip():
+                    extracted_text = text.strip()
+            except Exception:
+                pass
+
+        if not extracted_text:
+             try:
+                 # Fallback to pypdf
+                 reader = PdfReader(io.BytesIO(content))
+                 text = ""
+                 for page in reader.pages:
+                    page_text = page.extract_text()
+                    if page_text:
+                        text += page_text + "\n"
+                 extracted_text = text.strip()
+             except:
+                 pass
+
+        # If text is still empty, it might be a scanned PDF. 
         if not extracted_text.strip():
              try:
                  images = convert_from_bytes(content)
@@ -125,7 +153,35 @@ async def extract_text(file: UploadFile = File(...)):
     else:
         raise HTTPException(status_code=400, detail="Unsupported file type")
     
-    return NERResponse(text=extracted_text)
+    return NERResponse(text=extracted_text, file_path=file_path)
+
+# 3. AI Extraction Module
+from llm_service import extract_data
+from pdf2image import convert_from_path
+
+class AITextRequest(BaseModel):
+    text: Optional[str] = None
+    file_path: Optional[str] = None
+    model_type: str = "text"
+
+@app.post("/api/extract/ai")
+def extract_ai_data(request: AITextRequest):
+    final_image_path = request.file_path
+    
+    if request.model_type == "vision" and request.file_path and request.file_path.endswith(".pdf"):
+        # Convert PDF first page to image
+        try:
+            images = convert_from_path(request.file_path)
+            if images:
+                # Save temp image
+                temp_img_path = request.file_path + ".jpg"
+                images[0].save(temp_img_path, "JPEG")
+                final_image_path = temp_img_path
+        except Exception as e:
+            print(f"Error converting PDF for vision: {e}")
+            
+    data = extract_data(text=request.text, image_path=final_image_path, model_type=request.model_type)
+    return data
 
 import zipfile
 import mimetypes
@@ -255,3 +311,36 @@ def sync_emails():
 @app.get("/")
 def read_root():
     return {"message": "OCR Backend API is running"}
+
+class DocumentSaveRequest(BaseModel):
+    vendor_name: str
+    file_path: str
+    model_type: str
+    data: dict
+
+@app.post("/api/documents/save")
+def save_document(request: DocumentSaveRequest, db: Session = Depends(get_db)):
+    # 1. Find or Create Vendor
+    vendor = db.query(Vendor).filter(Vendor.name == request.vendor_name).first()
+    if not vendor:
+        vendor = Vendor(name=request.vendor_name, default_model=request.model_type)
+        db.add(vendor)
+        db.commit()
+        db.refresh(vendor)
+    else:
+        # Update preference
+        vendor.default_model = request.model_type
+        db.commit()
+    
+    # 2. Save Document
+    filename = request.file_path.split('/')[-1]
+    doc = Document(
+        vendor_id=vendor.id,
+        filename=filename,
+        status="verified",
+        processed_data=request.data
+    )
+    db.add(doc)
+    db.commit()
+    
+    return {"message": "Document saved and Vendor preference updated", "vendor_id": vendor.id}

backend/requirements.txt

@@ -10,3 +10,4 @@ imap-tools
 apscheduler
 python-dotenv
 pdfplumber
+ollama

docengine/.dockerignore

@@ -0,0 +1,26 @@
+__pycache__
+*.pyc
+*.pyo
+.Python
+.env
+.venv
+env/
+venv/
+*.egg-info
+dist/
+build/
+.git
+.gitignore
+.dockerignore
+*.md
+*.rst
+docs/
+tests/
+htmlcov/
+.coverage
+.pytest_cache
+.mypy_cache
+.ruff_cache
+*.log
+.idea/
+.vscode/

docengine/.env.example

@@ -0,0 +1,59 @@
+# Application
+APP_NAME=DocEngine
+APP_VERSION=1.0.0
+APP_ENV=development
+APP_DEBUG=true
+APP_HOST=0.0.0.0
+APP_PORT=7989
+APP_WORKERS=4
+
+# Database
+DB_HOST=localhost
+DB_PORT=5432
+DB_NAME=document_engine
+DB_USER=postgres
+DB_PASSWORD=changeme
+DB_SCHEMA=admin
+DB_POOL_SIZE=20
+DB_MAX_OVERFLOW=10
+DB_ECHO=false
+
+# Redis
+REDIS_HOST=localhost
+REDIS_PORT=6379
+REDIS_DB=0
+REDIS_PASSWORD=
+
+# Celery
+CELERY_BROKER_URL=redis://localhost:6379/0
+CELERY_RESULT_BACKEND=redis://localhost:6379/1
+
+# JWT
+JWT_SECRET_KEY=change-this-to-a-secure-random-string
+JWT_ALGORITHM=HS256
+JWT_ACCESS_TOKEN_EXPIRE_MINUTES=30
+JWT_REFRESH_TOKEN_EXPIRE_DAYS=7
+
+# Storage
+STORAGE_PROVIDER=local
+STORAGE_LOCAL_PATH=./storage
+STORAGE_MAX_FILE_SIZE_MB=100
+
+# OCR
+OCR_LANGUAGE=en
+OCR_USE_GPU=false
+
+# Logging
+LOG_LEVEL=INFO
+LOG_FORMAT=json
+
+# CORS
+CORS_ORIGINS=["http://localhost:3000","http://localhost:8080"]
+CORS_ALLOW_CREDENTIALS=true
+
+# Rate Limiting
+RATE_LIMIT_REQUESTS=100
+RATE_LIMIT_WINDOW_SECONDS=60
+
+# Prometheus
+PROMETHEUS_ENABLED=true

docengine/.gitignore

@@ -0,0 +1,64 @@
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+*.manifest
+*.spec
+pip-log.txt
+pip-delete-this-directory.txt
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+*.mo
+*.pot
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+instance/
+.webassets-cache
+.scrapy
+docs/_build/
+target/
+.venv
+env/
+venv/
+ENV/
+.env
+!.env.example
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+.DS_Store
+Thumbs.db
+storage/
+*.pid
+celerybeat-schedule
+celerybeat.pid

docengine/Dockerfile

@@ -0,0 +1,39 @@
+FROM python:3.12-slim AS base
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    libpq-dev \
+    libgl1-mesa-glx \
+    libglib2.0-0 \
+    libsm6 \
+    libxext6 \
+    libxrender-dev \
+    libgomp1 \
+    poppler-utils \
+    ghostscript \
+    libmagic1 \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+RUN mkdir -p /app/storage/documents /app/storage/templates /app/storage/images /app/storage/temp
+
+FROM base AS app
+EXPOSE 7989
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7989", "--workers", "4"]
+
+FROM base AS worker
+CMD ["celery", "-A", "app.workers.celery_app", "worker", "--loglevel=info", "--concurrency=4"]
+
+FROM base AS beat
+CMD ["celery", "-A", "app.workers.celery_app", "beat", "--loglevel=info"]

docengine/README.md

@@ -0,0 +1,522 @@
+# DocEngine — Document Template Recognition & Reconstruction System
+
+A production-ready system for scanning documents, detecting layouts, extracting content, generating reusable templates, matching future uploads against stored templates, and reconstructing original layouts as PDF output.
+
+## Architecture
+
+```
+┌───────────────┐      ┌───────────────┐      ┌──────────────┐
+│  FastAPI App   │─────▶│    Celery      │─────▶│   Redis      │
+│  (Port 7989)   │      │  Worker(s)     │      │  (Broker)    │
+└───────┬───────┘      └───────┬───────┘      └──────────────┘
+        │                      │
+        ▼                      ▼
+┌───────────────────────────────────────┐
+│       PostgreSQL (Schema: admin)       │
+│       192.168.0.111:7925              │
+└───────────────────────────────────────┘
+```
+
+**Stack**: Python 3.12, FastAPI, SQLAlchemy 2.x, Pydantic V2, Celery, Redis, PaddleOCR, PyMuPDF, OpenCV, ReportLab, PostgreSQL 16.
+
+## Features
+
+| Capability                       | Implementation                             |
+|----------------------------------|--------------------------------------------|
+| Scanned image OCR                | PaddleOCR (CPU/GPU)                        |
+| Native PDF text extraction       | PyMuPDF (fitz)                             |
+| Layout detection                 | OpenCV + LayoutParser                      |
+| Table extraction                 | Camelot-py + OpenCV contour detection      |
+| Header/footer detection          | Positional heuristics                      |
+| Watermark detection              | Transparency + large-font analysis         |
+| Font info extraction             | PyMuPDF text dict parsing                  |
+| Template generation & storage    | PostgreSQL (admin schema)                  |
+| Template fingerprinting          | SHA-256 structural hashing                 |
+| Template matching                | Multi-signal similarity scoring            |
+| PDF reconstruction               | ReportLab from template definitions        |
+| Async processing                 | Celery + Redis                             |
+| Authentication                   | JWT (access + refresh tokens, bcrypt)      |
+| Monitoring                       | Prometheus + structlog JSON logging        |
+
+## Project Structure
+
+```
+docengine/
+├── app/
+│   ├── main.py                  # FastAPI application entry
+│   ├── api/
+│   │   ├── router.py            # Top-level API router
+│   │   └── v1/
+│   │       ├── auth.py          # Auth endpoints
+│   │       ├── documents.py     # Document endpoints
+│   │       ├── health.py        # Health check
+│   │       └── templates.py     # Template endpoints
+│   ├── core/
+│   │   ├── config.py            # Pydantic Settings
+│   │   ├── database.py          # SQLAlchemy engine & session
+│   │   ├── dependencies.py      # FastAPI DI
+│   │   ├── exceptions.py        # Custom exception hierarchy
+│   │   ├── logging_config.py    # structlog configuration
+│   │   └── security.py          # JWT & bcrypt helpers
+│   ├── models/                  # SQLAlchemy ORM models
+│   ├── schemas/                 # Pydantic request/response schemas
+│   ├── repositories/            # Data access layer
+│   ├── services/                # Business logic
+│   │   ├── document_service.py  # Orchestration pipeline
+│   │   ├── ocr_service.py       # PaddleOCR integration
+│   │   ├── pdf_service.py       # PyMuPDF native PDF parsing
+│   │   ├── layout_service.py    # OpenCV layout detection
+│   │   ├── template_service.py  # Template generation
+│   │   ├── fingerprint_service.py
+│   │   ├── matching_service.py
+│   │   └── reconstruction_service.py
+│   ├── middleware/               # CORS, audit, metrics, rate limit
+│   ├── storage/                  # File storage abstraction
+│   ├── tasks/                    # Celery async tasks
+│   ├── workers/                  # Celery app configuration
+│   └── events/                   # App lifecycle handlers
+├── alembic/                      # Database migrations
+├── sql/                          # Raw SQL scripts
+├── tests/                        # Test suite
+├── docker-compose.yml            # Dev stack
+├── docker-compose.prod.yml       # Production stack
+├── Dockerfile                    # Multi-stage build
+├── requirements.txt
+└── .env
+```
+
+---
+
+## Quick Start
+
+### Prerequisites
+
+- Python 3.12+
+- PostgreSQL 16 (running at `192.168.0.111:7925`)
+- Redis (for Celery)
+- `poppler-utils` and `ghostscript` (for pdf2image/camelot)
+
+### Local Setup
+
+```bash
+# Clone & enter
+cd docengine
+
+# Create virtual environment
+python -m venv .venv
+source .venv/bin/activate
+
+# Install dependencies
+pip install -r requirements.txt
+
+# Create storage directories
+mkdir -p storage/{documents,templates,images,temp,rendered}
+
+# Run database migrations
+alembic upgrade head
+
+# (Optional) Seed default data
+psql -h 192.168.0.111 -p 7925 -U postgres -d document_engine -f sql/003_seed_data.sql
+psql -h 192.168.0.111 -p 7925 -U postgres -d document_engine -f sql/004_indexes.sql
+
+# Start the application
+python -m app.main
+```
+
+The API is now available at `http://localhost:7989`. Interactive docs at `http://localhost:7989/docs`.
+
+### Start Celery Worker (separate terminal)
+
+```bash
+source .venv/bin/activate
+celery -A app.workers.celery_app worker --loglevel=info --concurrency=4
+```
+
+### Docker Setup
+
+```bash
+# Build and start all services (app + worker + db + redis)
+docker compose up --build -d
+
+# Run migrations inside the container
+docker compose exec app alembic upgrade head
+
+# Seed data
+docker compose exec app bash -c "psql -h db -U postgres -d document_engine -f sql/003_seed_data.sql"
+```
+
+---
+
+## API Reference
+
+Base URL: `http://localhost:7989/api/v1`
+
+### Health
+
+```bash
+curl http://localhost:7989/api/v1/health
+```
+
+### Authentication
+
+#### Register
+
+```bash
+curl -X POST http://localhost:7989/api/v1/auth/register \
+  -H "Content-Type: application/json" \
+  -d '{
+    "username": "john",
+    "email": "john@example.com",
+    "password": "SecurePass123!",
+    "full_name": "John Doe"
+  }'
+```
+
+#### Login
+
+```bash
+curl -X POST http://localhost:7989/api/v1/auth/login \
+  -H "Content-Type: application/json" \
+  -d '{
+    "username": "john",
+    "password": "SecurePass123!"
+  }'
+```
+
+Response:
+
+```json
+{
+  "access_token": "eyJhbGciOiJIUzI1NiIs...",
+  "refresh_token": "eyJhbGciOiJIUzI1NiIs...",
+  "token_type": "bearer",
+  "expires_in": 1800
+}
+```
+
+#### Get Current User
+
+```bash
+curl http://localhost:7989/api/v1/auth/me \
+  -H "Authorization: Bearer <access_token>"
+```
+
+#### Refresh Token
+
+```bash
+curl -X POST http://localhost:7989/api/v1/auth/refresh \
+  -H "Content-Type: application/json" \
+  -d '{"refresh_token": "<refresh_token>"}'
+```
+
+#### Change Password
+
+```bash
+curl -X POST http://localhost:7989/api/v1/auth/change-password \
+  -H "Authorization: Bearer <access_token>" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "current_password": "SecurePass123!",
+    "new_password": "NewSecurePass456!"
+  }'
+```
+
+#### Logout
+
+```bash
+curl -X POST http://localhost:7989/api/v1/auth/logout \
+  -H "Authorization: Bearer <access_token>" \
+  -H "Content-Type: application/json" \
+  -d '{"refresh_token": "<refresh_token>"}'
+```
+
+### Documents
+
+#### Upload Document
+
+```bash
+# Upload a PDF
+curl -X POST http://localhost:7989/api/v1/documents/upload \
+  -H "Authorization: Bearer <access_token>" \
+  -F "file=@/path/to/document.pdf"
+
+# Upload a scanned image
+curl -X POST http://localhost:7989/api/v1/documents/upload \
+  -H "Authorization: Bearer <access_token>" \
+  -F "file=@/path/to/scan.jpg"
+```
+
+Response:
+
+```json
+{
+  "id": "550e8400-e29b-41d4-a716-446655440000",
+  "filename": "abc123_document.pdf",
+  "original_filename": "document.pdf",
+  "content_type": "application/pdf",
+  "file_size": 245760,
+  "checksum": "e3b0c44298fc1c149afbf4c8996fb924...",
+  "status": "pending",
+  "created_at": "2026-06-01T12:00:00Z"
+}
+```
+
+#### Get Document
+
+```bash
+curl http://localhost:7989/api/v1/documents/<document_id> \
+  -H "Authorization: Bearer <access_token>"
+```
+
+#### List Documents
+
+```bash
+# With pagination
+curl "http://localhost:7989/api/v1/documents?page=1&page_size=20" \
+  -H "Authorization: Bearer <access_token>"
+
+# Filter by status
+curl "http://localhost:7989/api/v1/documents?status=completed" \
+  -H "Authorization: Bearer <access_token>"
+```
+
+#### Delete Document
+
+```bash
+curl -X DELETE http://localhost:7989/api/v1/documents/<document_id> \
+  -H "Authorization: Bearer <access_token>"
+```
+
+#### Get Template Matches for Document
+
+```bash
+curl http://localhost:7989/api/v1/documents/<document_id>/template \
+  -H "Authorization: Bearer <access_token>"
+```
+
+### Templates
+
+#### List Templates
+
+```bash
+curl "http://localhost:7989/api/v1/templates?page=1&page_size=20" \
+  -H "Authorization: Bearer <access_token>"
+```
+
+#### Get Template
+
+```bash
+curl http://localhost:7989/api/v1/templates/<template_id> \
+  -H "Authorization: Bearer <access_token>"
+```
+
+#### Delete (Deactivate) Template
+
+```bash
+curl -X DELETE http://localhost:7989/api/v1/templates/<template_id> \
+  -H "Authorization: Bearer <access_token>"
+```
+
+#### Match Document to Templates
+
+```bash
+curl -X POST http://localhost:7989/api/v1/templates/match \
+  -H "Authorization: Bearer <access_token>" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "document_id": "<document_id>",
+    "min_confidence": 0.5,
+    "max_results": 5
+  }'
+```
+
+Response:
+
+```json
+[
+  {
+    "id": "...",
+    "document_id": "...",
+    "format_id": "...",
+    "confidence_score": 0.92,
+    "match_details": { "dimension_score": 1.0, "header_score": 0.85 },
+    "selected": true,
+    "template_name": "Invoice Template v1",
+    "created_at": "2026-06-01T12:00:00Z"
+  }
+]
+```
+
+#### Render Template to PDF
+
+```bash
+curl -X POST http://localhost:7989/api/v1/templates/render \
+  -H "Authorization: Bearer <access_token>" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "template_id": "<template_id>",
+    "data": {
+      "company_name": "Acme Corp",
+      "invoice_number": "INV-2026-001",
+      "date": "2026-06-01",
+      "total": "$1,250.00"
+    },
+    "output_filename": "invoice_output.pdf"
+  }'
+```
+
+Response:
+
+```json
+{
+  "output_path": "rendered/invoice_output.pdf",
+  "filename": "invoice_output.pdf",
+  "file_size": 32768,
+  "page_count": 1,
+  "rendered_at": "2026-06-01T12:05:00Z"
+}
+```
+
+#### Download Rendered PDF
+
+```bash
+curl -O http://localhost:7989/api/v1/templates/<template_id>/download?filename=invoice_output.pdf \
+  -H "Authorization: Bearer <access_token>"
+```
+
+---
+
+## Processing Pipeline
+
+When a document is uploaded, the following Celery task pipeline executes asynchronously:
+
+1. **File Type Detection** — Determine if the document is a native PDF or scanned image.
+2. **Page Extraction** — Convert PDF pages to images (for scanned docs) or parse directly (for native PDFs).
+3. **OCR** — Run PaddleOCR on scanned pages to extract text blocks with coordinates, confidence, and bounding boxes.
+4. **Native PDF Parsing** — Use PyMuPDF to extract text, fonts, images, and tables from native PDFs.
+5. **Layout Analysis** — Detect headers, footers, watermarks, tables, and image regions using OpenCV heuristics.
+6. **Template Generation** — Build a reusable template definition from the detected layout, stored in PostgreSQL.
+7. **Fingerprint Generation** — Compute a structural fingerprint (SHA-256) for future matching.
+8. **Status Update** — Mark the document as `completed` (or `failed` with error details).
+
+---
+
+## Database
+
+**Connection**: `postgresql://postgres:***@192.168.0.111:7925/document_engine`
+**Schema**: `admin`
+
+### Migrations
+
+```bash
+# Create a new migration
+alembic revision --autogenerate -m "description"
+
+# Apply migrations
+alembic upgrade head
+
+# Rollback one step
+alembic downgrade -1
+```
+
+### Tables
+
+| Table                    | Purpose                                      |
+|--------------------------|----------------------------------------------|
+| `users`                  | User accounts                                |
+| `roles`                  | Role definitions (admin, user, viewer)        |
+| `user_roles`             | User-role mapping (M2M)                      |
+| `refresh_tokens`         | JWT refresh token storage                    |
+| `audit_logs`             | Action audit trail                           |
+| `documents`              | Uploaded document records                    |
+| `document_pages`         | Per-page data (dimensions, images)           |
+| `document_text_blocks`   | Extracted text with position & font info     |
+| `document_images`        | Extracted images with position               |
+| `document_tables`        | Extracted tables with cell data (JSONB)      |
+| `document_formats`       | Template definitions                         |
+| `document_cells`         | Template cell layout definitions             |
+| `document_regions`       | Template region definitions                  |
+| `table_formats`          | Template table structure definitions         |
+| `table_columns`          | Template table column definitions            |
+| `table_rows`             | Template table row definitions               |
+| `watermarks`             | Template watermark definitions               |
+| `image_regions`          | Template image region definitions            |
+| `template_fingerprints`  | Structural fingerprints for matching         |
+| `template_matches`       | Document-to-template match results           |
+
+---
+
+## Testing
+
+```bash
+# Install dev dependencies
+pip install -r requirements-dev.txt
+
+# Run all tests
+pytest
+
+# Run with coverage
+pytest --cov=app --cov-report=term-missing
+
+# Run specific test categories
+pytest tests/unit/
+pytest tests/api/
+pytest tests/repositories/
+```
+
+---
+
+## Configuration
+
+All configuration is via environment variables (`.env` file). Key settings:
+
+| Variable                           | Default                | Description                     |
+|------------------------------------|------------------------|---------------------------------|
+| `APP_PORT`                         | `7989`                 | Application port                |
+| `DB_HOST`                          | `192.168.0.111`        | PostgreSQL host                 |
+| `DB_PORT`                          | `7925`                 | PostgreSQL port                 |
+| `DB_NAME`                          | `document_engine`      | Database name                   |
+| `DB_SCHEMA`                        | `admin`                | PostgreSQL schema               |
+| `REDIS_HOST`                       | `localhost`            | Redis host                      |
+| `CELERY_BROKER_URL`                | `redis://localhost:6379/0` | Celery broker                |
+| `JWT_SECRET_KEY`                   | *(see .env)*           | JWT signing key                 |
+| `JWT_ACCESS_TOKEN_EXPIRE_MINUTES`  | `30`                   | Access token TTL                |
+| `STORAGE_LOCAL_PATH`               | `./storage`            | Local file storage path         |
+| `STORAGE_MAX_FILE_SIZE_MB`         | `100`                  | Max upload size                 |
+| `OCR_LANGUAGE`                     | `en`                   | PaddleOCR language              |
+| `OCR_USE_GPU`                      | `false`                | Enable GPU for OCR              |
+
+---
+
+## Production Deployment
+
+```bash
+# Using production compose file
+docker compose -f docker-compose.prod.yml up --build -d
+
+# Scale workers
+docker compose -f docker-compose.prod.yml up --scale worker=4 -d
+```
+
+Production compose includes:
+- Resource limits (CPU/memory)
+- Redis authentication
+- App replicas
+- Persistent named volumes
+- Auto-restart policies
+
+---
+
+## Default Credentials
+
+| Username | Password      | Role  |
+|----------|---------------|-------|
+| `admin`  | `Admin@123!`  | admin |
+
+> ⚠️ **Change the default admin password immediately in production.**
+
+---
+
+## License
+
+Proprietary — All rights reserved.

docengine/alembic.ini

@@ -0,0 +1,41 @@
+[alembic]
+script_location = alembic
+prepend_sys_path = .
+version_path_separator = os
+sqlalchemy.url = postgresql+psycopg2://postgres:M%%40tr%%21x%%23149%%40dm%%21N@192.168.0.111:7925/document_engine
+
+[post_write_hooks]
+
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

docengine/alembic/env.py

@@ -0,0 +1,67 @@
+from logging.config import fileConfig
+
+from alembic import context
+from sqlalchemy import engine_from_config, pool, text
+
+from app.core.config import settings
+from app.core.database import Base
+
+# Import all models so Alembic can detect them
+import app.models  # noqa: F401
+
+config = context.config
+
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+target_metadata = Base.metadata
+
+# Override the database URL from settings
+config.set_main_option("sqlalchemy.url", settings.database_url)
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode."""
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+        version_table_schema=settings.db_schema,
+        include_schemas=True,
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode."""
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    with connectable.connect() as connection:
+        # Ensure schema exists
+        connection.execute(text(f"CREATE SCHEMA IF NOT EXISTS {settings.db_schema}"))
+        connection.execute(text(f"SET search_path TO {settings.db_schema}, public"))
+        connection.commit()
+
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            version_table_schema=settings.db_schema,
+            include_schemas=True,
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

docengine/alembic/script.py.mako

@@ -0,0 +1,26 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

docengine/alembic/versions/001_initial.py

@@ -0,0 +1,398 @@
+"""initial schema
+
+Revision ID: 001_initial
+Revises:
+Create Date: 2026-05-31 18:00:00.000000
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+revision: str = "001_initial"
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+SCHEMA = "admin"
+
+
+def upgrade() -> None:
+    # Create schema
+    op.execute(f"CREATE SCHEMA IF NOT EXISTS {SCHEMA}")
+
+    # Users table
+    op.create_table(
+        "users",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("username", sa.String(150), unique=True, nullable=False, index=True),
+        sa.Column("email", sa.String(255), unique=True, nullable=False, index=True),
+        sa.Column("hashed_password", sa.String(255), nullable=False),
+        sa.Column("full_name", sa.String(255), nullable=True),
+        sa.Column("is_active", sa.Boolean, default=True, nullable=False),
+        sa.Column("is_superuser", sa.Boolean, default=False, nullable=False),
+        sa.Column("last_login", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Roles table
+    op.create_table(
+        "roles",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("name", sa.String(50), unique=True, nullable=False, index=True),
+        sa.Column("description", sa.Text, nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # User roles (many-to-many)
+    op.create_table(
+        "user_roles",
+        sa.Column("user_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.users.id", ondelete="CASCADE"), primary_key=True),
+        sa.Column("role_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.roles.id", ondelete="CASCADE"), primary_key=True),
+        schema=SCHEMA,
+    )
+
+    # Refresh tokens
+    op.create_table(
+        "refresh_tokens",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("user_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.users.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("token", sa.String(512), unique=True, nullable=False, index=True),
+        sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("revoked", sa.Boolean, default=False, nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Audit logs
+    op.create_table(
+        "audit_logs",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("user_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.users.id", ondelete="SET NULL"), nullable=True, index=True),
+        sa.Column("action", sa.String(100), nullable=False, index=True),
+        sa.Column("resource_type", sa.String(100), nullable=False, index=True),
+        sa.Column("resource_id", sa.String(255), nullable=True),
+        sa.Column("details", sa.Text, nullable=True),
+        sa.Column("ip_address", sa.String(45), nullable=True),
+        sa.Column("user_agent", sa.String(512), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False, index=True),
+        schema=SCHEMA,
+    )
+
+    # Documents
+    op.create_table(
+        "documents",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("filename", sa.String(500), nullable=False),
+        sa.Column("original_filename", sa.String(500), nullable=False),
+        sa.Column("content_type", sa.String(100), nullable=False),
+        sa.Column("file_size", sa.BigInteger, nullable=False),
+        sa.Column("checksum", sa.String(128), nullable=False, index=True),
+        sa.Column("storage_path", sa.String(1024), nullable=False),
+        sa.Column("status", sa.String(50), default="pending", nullable=False, index=True),
+        sa.Column("page_count", sa.Integer, nullable=True),
+        sa.Column("is_scanned", sa.Boolean, nullable=True),
+        sa.Column("document_metadata", postgresql.JSONB, nullable=True),
+        sa.Column("error_message", sa.Text, nullable=True),
+        sa.Column("uploaded_by", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.users.id", ondelete="SET NULL"), nullable=True, index=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Document pages
+    op.create_table(
+        "document_pages",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("document_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.documents.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("page_number", sa.Integer, nullable=False),
+        sa.Column("width", sa.Float, nullable=False),
+        sa.Column("height", sa.Float, nullable=False),
+        sa.Column("image_path", sa.String(1024), nullable=True),
+        sa.Column("text_content", sa.Text, nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Document text blocks
+    op.create_table(
+        "document_text_blocks",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("page_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_pages.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("text", sa.Text, nullable=False),
+        sa.Column("x", sa.Float, nullable=False),
+        sa.Column("y", sa.Float, nullable=False),
+        sa.Column("width", sa.Float, nullable=False),
+        sa.Column("height", sa.Float, nullable=False),
+        sa.Column("confidence", sa.Float, nullable=True),
+        sa.Column("font_family", sa.String(255), nullable=True),
+        sa.Column("font_size", sa.Float, nullable=True),
+        sa.Column("font_color", sa.String(50), nullable=True),
+        sa.Column("font_style", sa.String(50), nullable=True),
+        sa.Column("block_type", sa.String(50), default="text", nullable=False),
+        sa.Column("sequence", sa.Integer, default=0, nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Document images
+    op.create_table(
+        "document_images",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("page_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_pages.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("x", sa.Float, nullable=False),
+        sa.Column("y", sa.Float, nullable=False),
+        sa.Column("width", sa.Float, nullable=False),
+        sa.Column("height", sa.Float, nullable=False),
+        sa.Column("image_path", sa.String(1024), nullable=False),
+        sa.Column("image_type", sa.String(50), default="figure", nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Document tables
+    op.create_table(
+        "document_tables",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("page_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_pages.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("x", sa.Float, nullable=False),
+        sa.Column("y", sa.Float, nullable=False),
+        sa.Column("width", sa.Float, nullable=False),
+        sa.Column("height", sa.Float, nullable=False),
+        sa.Column("rows", sa.Integer, nullable=False),
+        sa.Column("columns", sa.Integer, nullable=False),
+        sa.Column("data", postgresql.JSONB, nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Document formats (templates)
+    op.create_table(
+        "document_formats",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("name", sa.String(255), nullable=False, index=True),
+        sa.Column("description", sa.Text, nullable=True),
+        sa.Column("page_width", sa.Float, nullable=False),
+        sa.Column("page_height", sa.Float, nullable=False),
+        sa.Column("page_count", sa.Integer, default=1, nullable=False),
+        sa.Column("margin_top", sa.Float, default=72.0, nullable=False),
+        sa.Column("margin_right", sa.Float, default=72.0, nullable=False),
+        sa.Column("margin_bottom", sa.Float, default=72.0, nullable=False),
+        sa.Column("margin_left", sa.Float, default=72.0, nullable=False),
+        sa.Column("fingerprint", postgresql.JSONB, nullable=True),
+        sa.Column("source_document_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.documents.id", ondelete="SET NULL"), nullable=True, index=True),
+        sa.Column("version", sa.Integer, default=1, nullable=False),
+        sa.Column("is_active", sa.Boolean, default=True, nullable=False, index=True),
+        sa.Column("created_by", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.users.id", ondelete="SET NULL"), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Document cells
+    op.create_table(
+        "document_cells",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_formats.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("page_number", sa.Integer, nullable=False),
+        sa.Column("x", sa.Float, nullable=False),
+        sa.Column("y", sa.Float, nullable=False),
+        sa.Column("width", sa.Float, nullable=False),
+        sa.Column("height", sa.Float, nullable=False),
+        sa.Column("row_no", sa.Integer, default=0, nullable=False),
+        sa.Column("column_no", sa.Integer, default=0, nullable=False),
+        sa.Column("data_type", sa.String(50), default="text", nullable=False),
+        sa.Column("font_family", sa.String(255), nullable=True),
+        sa.Column("font_size", sa.Float, nullable=True),
+        sa.Column("font_style", sa.String(50), nullable=True),
+        sa.Column("font_color", sa.String(50), nullable=True),
+        sa.Column("background_color", sa.String(50), nullable=True),
+        sa.Column("border_top", sa.String(100), nullable=True),
+        sa.Column("border_right", sa.String(100), nullable=True),
+        sa.Column("border_bottom", sa.String(100), nullable=True),
+        sa.Column("border_left", sa.String(100), nullable=True),
+        sa.Column("padding_top", sa.Float, default=0.0, nullable=False),
+        sa.Column("padding_right", sa.Float, default=0.0, nullable=False),
+        sa.Column("padding_bottom", sa.Float, default=0.0, nullable=False),
+        sa.Column("padding_left", sa.Float, default=0.0, nullable=False),
+        sa.Column("alignment", sa.String(20), default="left", nullable=False),
+        sa.Column("vertical_alignment", sa.String(20), default="top", nullable=False),
+        sa.Column("rowspan", sa.Integer, default=1, nullable=False),
+        sa.Column("colspan", sa.Integer, default=1, nullable=False),
+        sa.Column("static_text", sa.Text, nullable=True),
+        sa.Column("field_name", sa.String(255), nullable=True),
+        sa.Column("sequence", sa.Integer, default=0, nullable=False),
+        sa.Column("is_dynamic", sa.Boolean, default=False, nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Document regions
+    op.create_table(
+        "document_regions",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_formats.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("page_number", sa.Integer, nullable=False),
+        sa.Column("region_type", sa.String(50), nullable=False),
+        sa.Column("x", sa.Float, nullable=False),
+        sa.Column("y", sa.Float, nullable=False),
+        sa.Column("width", sa.Float, nullable=False),
+        sa.Column("height", sa.Float, nullable=False),
+        sa.Column("content", postgresql.JSONB, nullable=True),
+        sa.Column("sequence", sa.Integer, default=0, nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Table formats
+    op.create_table(
+        "table_formats",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_formats.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("page_number", sa.Integer, nullable=False),
+        sa.Column("x", sa.Float, nullable=False),
+        sa.Column("y", sa.Float, nullable=False),
+        sa.Column("width", sa.Float, nullable=False),
+        sa.Column("height", sa.Float, nullable=False),
+        sa.Column("rows", sa.Integer, nullable=False),
+        sa.Column("columns", sa.Integer, nullable=False),
+        sa.Column("border_style", sa.String(50), default="solid", nullable=False),
+        sa.Column("border_width", sa.Float, default=1.0, nullable=False),
+        sa.Column("border_color", sa.String(50), default="#000000", nullable=False),
+        sa.Column("header_rows", sa.Integer, default=1, nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Table columns
+    op.create_table(
+        "table_columns",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("table_format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.table_formats.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("column_index", sa.Integer, nullable=False),
+        sa.Column("width", sa.Float, nullable=False),
+        sa.Column("header_text", sa.String(500), nullable=True),
+        sa.Column("data_type", sa.String(50), default="text", nullable=False),
+        sa.Column("alignment", sa.String(20), default="left", nullable=False),
+        sa.Column("font_family", sa.String(255), nullable=True),
+        sa.Column("font_size", sa.Float, nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Table rows
+    op.create_table(
+        "table_rows",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("table_format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.table_formats.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("row_index", sa.Integer, nullable=False),
+        sa.Column("height", sa.Float, default=20.0, nullable=False),
+        sa.Column("is_header", sa.Boolean, default=False, nullable=False),
+        sa.Column("background_color", sa.String(50), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Watermarks
+    op.create_table(
+        "watermarks",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_formats.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("page_number", sa.Integer, nullable=True),
+        sa.Column("text", sa.String(500), nullable=True),
+        sa.Column("image_path", sa.String(1024), nullable=True),
+        sa.Column("x", sa.Float, nullable=False),
+        sa.Column("y", sa.Float, nullable=False),
+        sa.Column("width", sa.Float, nullable=False),
+        sa.Column("height", sa.Float, nullable=False),
+        sa.Column("opacity", sa.Float, default=0.3, nullable=False),
+        sa.Column("rotation", sa.Float, default=0.0, nullable=False),
+        sa.Column("font_family", sa.String(255), nullable=True),
+        sa.Column("font_size", sa.Float, nullable=True),
+        sa.Column("font_color", sa.String(50), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Image regions
+    op.create_table(
+        "image_regions",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_formats.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("page_number", sa.Integer, nullable=False),
+        sa.Column("x", sa.Float, nullable=False),
+        sa.Column("y", sa.Float, nullable=False),
+        sa.Column("width", sa.Float, nullable=False),
+        sa.Column("height", sa.Float, nullable=False),
+        sa.Column("image_path", sa.String(1024), nullable=True),
+        sa.Column("image_type", sa.String(50), default="figure", nullable=False),
+        sa.Column("is_static", sa.Boolean, default=True, nullable=False),
+        sa.Column("field_name", sa.String(255), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Template fingerprints
+    op.create_table(
+        "template_fingerprints",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_formats.id", ondelete="CASCADE"), nullable=False, unique=True, index=True),
+        sa.Column("page_dimensions", postgresql.JSONB, nullable=True),
+        sa.Column("logo_coordinates", postgresql.JSONB, nullable=True),
+        sa.Column("header_coordinates", postgresql.JSONB, nullable=True),
+        sa.Column("footer_coordinates", postgresql.JSONB, nullable=True),
+        sa.Column("table_coordinates", postgresql.JSONB, nullable=True),
+        sa.Column("cell_coordinates", postgresql.JSONB, nullable=True),
+        sa.Column("fingerprint_hash", sa.String(256), nullable=False, index=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Template matches
+    op.create_table(
+        "template_matches",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("document_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.documents.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_formats.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("confidence_score", sa.Float, nullable=False),
+        sa.Column("match_details", postgresql.JSONB, nullable=True),
+        sa.Column("selected", sa.Boolean, default=False, nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Additional indexes
+    op.create_index("ix_documents_status_created", "documents", ["status", "created_at"], schema=SCHEMA)
+    op.create_index("ix_document_pages_doc_page", "document_pages", ["document_id", "page_number"], schema=SCHEMA)
+    op.create_index("ix_document_text_blocks_type", "document_text_blocks", ["page_id", "block_type"], schema=SCHEMA)
+    op.create_index("ix_document_cells_format_page", "document_cells", ["format_id", "page_number"], schema=SCHEMA)
+    op.create_index("ix_template_matches_doc_score", "template_matches", ["document_id", "confidence_score"], schema=SCHEMA)
+    op.create_index("ix_audit_logs_resource", "audit_logs", ["resource_type", "resource_id"], schema=SCHEMA)
+
+
+def downgrade() -> None:
+    op.drop_table("template_matches", schema=SCHEMA)
+    op.drop_table("template_fingerprints", schema=SCHEMA)
+    op.drop_table("image_regions", schema=SCHEMA)
+    op.drop_table("watermarks", schema=SCHEMA)
+    op.drop_table("table_rows", schema=SCHEMA)
+    op.drop_table("table_columns", schema=SCHEMA)
+    op.drop_table("table_formats", schema=SCHEMA)
+    op.drop_table("document_regions", schema=SCHEMA)
+    op.drop_table("document_cells", schema=SCHEMA)
+    op.drop_table("document_formats", schema=SCHEMA)
+    op.drop_table("document_tables", schema=SCHEMA)
+    op.drop_table("document_images", schema=SCHEMA)
+    op.drop_table("document_text_blocks", schema=SCHEMA)
+    op.drop_table("document_pages", schema=SCHEMA)
+    op.drop_table("documents", schema=SCHEMA)
+    op.drop_table("audit_logs", schema=SCHEMA)
+    op.drop_table("refresh_tokens", schema=SCHEMA)
+    op.drop_table("user_roles", schema=SCHEMA)
+    op.drop_table("roles", schema=SCHEMA)
+    op.drop_table("users", schema=SCHEMA)

docengine/app.py

@@ -0,0 +1,11 @@
+
+from fastapi import FastAPI
+app = FastAPI()
+
+@app.get("/health")
+def health():
+    return {"status":"UP"}
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=7989)

docengine/app/__init__.py

@@ -0,0 +1 @@
+# DocEngine - Document Template Recognition and Reconstruction System

docengine/app/api/router.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from fastapi import APIRouter
+
+from app.api.v1.auth import router as auth_router
+from app.api.v1.documents import router as documents_router
+from app.api.v1.health import router as health_router
+from app.api.v1.templates import router as templates_router
+
+api_v1_router = APIRouter(prefix="/api/v1")
+
+api_v1_router.include_router(health_router)
+api_v1_router.include_router(auth_router)
+api_v1_router.include_router(documents_router)
+api_v1_router.include_router(templates_router)

docengine/app/api/v1/auth.py

@@ -0,0 +1,246 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+
+from fastapi import APIRouter, Depends, HTTPException, status
+from sqlalchemy.orm import Session
+
+from app.core.config import settings
+from app.core.database import get_db
+from app.core.dependencies import CurrentUser
+from app.core.security import (
+    create_access_token,
+    create_refresh_token,
+    decode_token,
+    hash_password,
+    verify_password,
+    InvalidTokenError,
+)
+from app.repositories.user_repository import RefreshTokenRepository, UserRepository
+from app.schemas.auth import (
+    ChangePasswordRequest,
+    LoginRequest,
+    RefreshTokenRequest,
+    RegisterRequest,
+    TokenResponse,
+)
+from app.schemas.common import SuccessResponse
+from app.schemas.user import UserResponse
+
+router = APIRouter(prefix="/auth", tags=["Authentication"])
+
+
+@router.post(
+    "/register",
+    response_model=UserResponse,
+    status_code=status.HTTP_201_CREATED,
+    summary="Register User",
+    description="Register a new user account.",
+)
+def register(
+    payload: RegisterRequest,
+    db: Session = Depends(get_db),
+) -> UserResponse:
+    """Register a new user."""
+    user_repo = UserRepository(db)
+
+    # Check for existing user
+    if user_repo.get_by_username(payload.username):
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT,
+            detail=f"Username '{payload.username}' is already taken",
+        )
+    if user_repo.get_by_email(payload.email):
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT,
+            detail=f"Email '{payload.email}' is already registered",
+        )
+
+    hashed = hash_password(payload.password)
+    user = user_repo.create_user(
+        username=payload.username,
+        email=payload.email,
+        hashed_password=hashed,
+        full_name=payload.full_name,
+        role_names=["user"],
+    )
+    db.commit()
+    db.refresh(user)
+
+    return UserResponse.model_validate(user)
+
+
+@router.post(
+    "/login",
+    response_model=TokenResponse,
+    summary="Login",
+    description="Authenticate with username and password to obtain JWT tokens.",
+)
+def login(
+    payload: LoginRequest,
+    db: Session = Depends(get_db),
+) -> TokenResponse:
+    """Authenticate user and return JWT tokens."""
+    user_repo = UserRepository(db)
+    refresh_repo = RefreshTokenRepository(db)
+
+    user = user_repo.get_by_username(payload.username)
+    if not user or not verify_password(payload.password, user.hashed_password):
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid username or password",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    if not user.is_active:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="User account is deactivated",
+        )
+
+    # Generate tokens
+    access_token = create_access_token(data={"sub": str(user.id), "username": user.username})
+    refresh_token_str = create_refresh_token(data={"sub": str(user.id)})
+
+    # Store refresh token
+    expires_at = datetime.now(UTC) + timedelta(days=settings.jwt_refresh_token_expire_days)
+    refresh_repo.create_token(
+        user_id=user.id,
+        token=refresh_token_str,
+        expires_at=expires_at,
+    )
+
+    # Update last login
+    user_repo.update_last_login(user)
+    db.commit()
+
+    return TokenResponse(
+        access_token=access_token,
+        refresh_token=refresh_token_str,
+        token_type="bearer",
+        expires_in=settings.jwt_access_token_expire_minutes * 60,
+    )
+
+
+@router.post(
+    "/refresh",
+    response_model=TokenResponse,
+    summary="Refresh Token",
+    description="Obtain a new access token using a valid refresh token.",
+)
+def refresh_token(
+    payload: RefreshTokenRequest,
+    db: Session = Depends(get_db),
+) -> TokenResponse:
+    """Refresh access token using a refresh token."""
+    refresh_repo = RefreshTokenRepository(db)
+    user_repo = UserRepository(db)
+
+    # Validate the refresh token
+    try:
+        token_payload = decode_token(payload.refresh_token)
+    except InvalidTokenError:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid or expired refresh token",
+        )
+
+    if token_payload.get("type") != "refresh":
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid token type",
+        )
+
+    # Check if token exists in database and is not revoked
+    stored_token = refresh_repo.get_by_token(payload.refresh_token)
+    if not stored_token:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Refresh token not found or revoked",
+        )
+
+    user = user_repo.get_by_id(token_payload["sub"])
+    if not user or not user.is_active:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="User not found or deactivated",
+        )
+
+    # Revoke old refresh token
+    refresh_repo.revoke_token(payload.refresh_token)
+
+    # Generate new tokens
+    new_access_token = create_access_token(data={"sub": str(user.id), "username": user.username})
+    new_refresh_token = create_refresh_token(data={"sub": str(user.id)})
+
+    expires_at = datetime.now(UTC) + timedelta(days=settings.jwt_refresh_token_expire_days)
+    refresh_repo.create_token(
+        user_id=user.id,
+        token=new_refresh_token,
+        expires_at=expires_at,
+    )
+    db.commit()
+
+    return TokenResponse(
+        access_token=new_access_token,
+        refresh_token=new_refresh_token,
+        token_type="bearer",
+        expires_in=settings.jwt_access_token_expire_minutes * 60,
+    )
+
+
+@router.post(
+    "/logout",
+    response_model=SuccessResponse,
+    summary="Logout",
+    description="Revoke the current refresh token.",
+)
+def logout(
+    payload: RefreshTokenRequest,
+    current_user: CurrentUser,
+    db: Session = Depends(get_db),
+) -> SuccessResponse:
+    """Logout by revoking the refresh token."""
+    refresh_repo = RefreshTokenRepository(db)
+    refresh_repo.revoke_token(payload.refresh_token)
+    db.commit()
+    return SuccessResponse(message="Successfully logged out")
+
+
+@router.post(
+    "/change-password",
+    response_model=SuccessResponse,
+    summary="Change Password",
+    description="Change the current user's password.",
+)
+def change_password(
+    payload: ChangePasswordRequest,
+    current_user: CurrentUser,
+    db: Session = Depends(get_db),
+) -> SuccessResponse:
+    """Change user password."""
+    if not verify_password(payload.current_password, current_user.hashed_password):
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Current password is incorrect",
+        )
+
+    current_user.hashed_password = hash_password(payload.new_password)
+
+    # Revoke all refresh tokens for security
+    refresh_repo = RefreshTokenRepository(db)
+    refresh_repo.revoke_all_user_tokens(current_user.id)
+    db.commit()
+
+    return SuccessResponse(message="Password changed successfully")
+
+
+@router.get(
+    "/me",
+    response_model=UserResponse,
+    summary="Get Current User",
+    description="Get the currently authenticated user's profile.",
+)
+def get_me(current_user: CurrentUser) -> UserResponse:
+    """Get current authenticated user profile."""
+    return UserResponse.model_validate(current_user)

docengine/app/api/v1/documents.py

@@ -0,0 +1,268 @@
+from __future__ import annotations
+
+import uuid
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, File, HTTPException, Query, UploadFile, status
+from sqlalchemy.orm import Session
+
+from app.core.config import settings
+from app.core.database import get_db
+from app.core.dependencies import CurrentUser
+from app.core.exceptions import FileSizeError, UnsupportedFileTypeError
+from app.core.logging_config import get_logger
+from app.models.document import Document
+from app.repositories.document_repository import DocumentRepository
+from app.schemas.common import PaginatedResponse, SuccessResponse
+from app.schemas.document import (
+    DocumentListResponse,
+    DocumentResponse,
+    DocumentUploadResponse,
+    TemplateMatchRequest,
+    TemplateMatchResponse,
+)
+from app.storage.provider import LocalStorageProvider, get_storage_provider
+
+logger = get_logger(__name__)
+
+router = APIRouter(prefix="/documents", tags=["Documents"])
+
+ALLOWED_CONTENT_TYPES = {
+    "image/jpeg": "jpg",
+    "image/png": "png",
+    "image/tiff": "tiff",
+    "application/pdf": "pdf",
+}
+
+ALLOWED_EXTENSIONS = {".jpg", ".jpeg", ".png", ".tiff", ".tif", ".pdf"}
+
+
+def _validate_file(file: UploadFile) -> str:
+    """Validate uploaded file type and size. Returns the content type."""
+    if not file.filename:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Filename is required",
+        )
+
+    # Check extension
+    from pathlib import Path
+    ext = Path(file.filename).suffix.lower()
+    if ext not in ALLOWED_EXTENSIONS:
+        raise UnsupportedFileTypeError(ext)
+
+    # Determine content type
+    content_type = file.content_type or ""
+    if content_type not in ALLOWED_CONTENT_TYPES:
+        # Try to infer from extension
+        ext_to_ct = {
+            ".jpg": "image/jpeg",
+            ".jpeg": "image/jpeg",
+            ".png": "image/png",
+            ".tiff": "image/tiff",
+            ".tif": "image/tiff",
+            ".pdf": "application/pdf",
+        }
+        content_type = ext_to_ct.get(ext, "")
+        if not content_type:
+            raise UnsupportedFileTypeError(file.content_type or "unknown")
+
+    return content_type
+
+
+@router.post(
+    "/upload",
+    response_model=DocumentUploadResponse,
+    status_code=status.HTTP_201_CREATED,
+    summary="Upload Document",
+    description="Upload a document (JPG, JPEG, PNG, TIFF, or PDF) for processing.",
+)
+async def upload_document(
+    file: UploadFile = File(..., description="Document file to upload"),
+    current_user: CurrentUser = None,
+    db: Session = Depends(get_db),
+) -> DocumentUploadResponse:
+    """Upload a document for processing."""
+    content_type = _validate_file(file)
+
+    # Read file data
+    file_data = await file.read()
+
+    # Check file size
+    if len(file_data) > settings.storage_max_file_size_bytes:
+        raise FileSizeError(settings.storage_max_file_size_mb)
+
+    # Store file
+    storage = get_storage_provider()
+    checksum = storage.compute_checksum(file_data)
+    safe_filename = file.filename or "unknown"
+    stored_filename = storage.generate_filename(safe_filename)
+    storage_path = storage.save_file(file_data, "documents", stored_filename)
+
+    # Create document record
+    doc_repo = DocumentRepository(db)
+    document = Document(
+        filename=stored_filename,
+        original_filename=safe_filename,
+        content_type=content_type,
+        file_size=len(file_data),
+        checksum=checksum,
+        storage_path=storage_path,
+        status="pending",
+        uploaded_by=current_user.id if current_user else None,
+    )
+    doc_repo.create(document)
+    db.commit()
+    db.refresh(document)
+
+    logger.info(
+        "document_uploaded",
+        document_id=str(document.id),
+        filename=safe_filename,
+        size=len(file_data),
+        content_type=content_type,
+    )
+
+    # Trigger async processing via Celery
+    try:
+        from app.tasks.document_tasks import process_document_task
+        process_document_task.delay(str(document.id))
+    except Exception as e:
+        logger.warning("celery_dispatch_failed", error=str(e), document_id=str(document.id))
+
+    return DocumentUploadResponse.model_validate(document)
+
+
+@router.get(
+    "/{document_id}",
+    response_model=DocumentResponse,
+    summary="Get Document",
+    description="Retrieve a document by its ID with all extracted content.",
+)
+def get_document(
+    document_id: uuid.UUID,
+    current_user: CurrentUser = None,
+    db: Session = Depends(get_db),
+) -> DocumentResponse:
+    """Get a document by ID."""
+    doc_repo = DocumentRepository(db)
+    document = doc_repo.get_with_pages(document_id)
+    if not document:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Document '{document_id}' not found",
+        )
+    return DocumentResponse.model_validate(document)
+
+
+@router.get(
+    "",
+    response_model=PaginatedResponse[DocumentListResponse],
+    summary="List Documents",
+    description="List all documents with pagination.",
+)
+def list_documents(
+    page: int = Query(default=1, ge=1),
+    page_size: int = Query(default=20, ge=1, le=100),
+    status_filter: str | None = Query(default=None, alias="status"),
+    current_user: CurrentUser = None,
+    db: Session = Depends(get_db),
+) -> PaginatedResponse[DocumentListResponse]:
+    """List documents with pagination and optional status filter."""
+    doc_repo = DocumentRepository(db)
+    offset = (page - 1) * page_size
+    filters = {}
+    if status_filter:
+        filters["status"] = status_filter
+
+    documents = doc_repo.get_all(
+        offset=offset,
+        limit=page_size,
+        filters=filters,
+        order_by="created_at",
+        order_desc=True,
+    )
+    total = doc_repo.count(filters=filters)
+
+    items = [DocumentListResponse.model_validate(doc) for doc in documents]
+    return PaginatedResponse.create(
+        items=items,
+        total=total,
+        page=page,
+        page_size=page_size,
+    )
+
+
+@router.get(
+    "/{document_id}/template",
+    response_model=list[TemplateMatchResponse],
+    summary="Get Document Template Matches",
+    description="Get template matching results for a document.",
+)
+def get_document_template_matches(
+    document_id: uuid.UUID,
+    current_user: CurrentUser = None,
+    db: Session = Depends(get_db),
+) -> list[TemplateMatchResponse]:
+    """Get template matches for a document."""
+    from app.repositories.document_repository import TemplateMatchRepository
+
+    doc_repo = DocumentRepository(db)
+    document = doc_repo.get_by_id(document_id)
+    if not document:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Document '{document_id}' not found",
+        )
+
+    match_repo = TemplateMatchRepository(db)
+    matches = match_repo.get_document_matches(document_id)
+
+    results = []
+    for match in matches:
+        resp = TemplateMatchResponse(
+            id=match.id,
+            document_id=match.document_id,
+            format_id=match.format_id,
+            confidence_score=match.confidence_score,
+            match_details=match.match_details,
+            selected=match.selected,
+            template_name=match.template.name if match.template else None,
+            created_at=match.created_at,
+        )
+        results.append(resp)
+
+    return results
+
+
+@router.delete(
+    "/{document_id}",
+    response_model=SuccessResponse,
+    summary="Delete Document",
+    description="Delete a document and its associated data.",
+)
+def delete_document(
+    document_id: uuid.UUID,
+    current_user: CurrentUser = None,
+    db: Session = Depends(get_db),
+) -> SuccessResponse:
+    """Delete a document."""
+    doc_repo = DocumentRepository(db)
+    document = doc_repo.get_by_id(document_id)
+    if not document:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Document '{document_id}' not found",
+        )
+
+    # Delete stored file
+    try:
+        storage = get_storage_provider()
+        storage.delete_file(document.storage_path)
+    except Exception as e:
+        logger.warning("file_delete_failed", error=str(e), path=document.storage_path)
+
+    doc_repo.delete(document)
+    db.commit()
+
+    return SuccessResponse(message=f"Document '{document_id}' deleted successfully")

docengine/app/api/v1/health.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from datetime import datetime
+
+import redis
+from fastapi import APIRouter, status
+
+from app.core.config import settings
+from app.core.database import check_database_connection
+from app.schemas.common import HealthResponse
+
+router = APIRouter(tags=["Health"])
+
+
+@router.get(
+    "/health",
+    response_model=HealthResponse,
+    status_code=status.HTTP_200_OK,
+    summary="Health Check",
+    description="Check the health status of the application and its dependencies.",
+)
+async def health_check() -> HealthResponse:
+    """Perform health check on all system components."""
+    # Check database
+    db_status = "healthy" if check_database_connection() else "unhealthy"
+
+    # Check Redis
+    redis_status = "healthy"
+    try:
+        r = redis.Redis(
+            host=settings.redis_host,
+            port=settings.redis_port,
+            db=settings.redis_db,
+            password=settings.redis_password or None,
+            socket_timeout=3,
+        )
+        r.ping()
+        r.close()
+    except Exception:
+        redis_status = "unhealthy"
+
+    overall_status = "healthy" if db_status == "healthy" and redis_status == "healthy" else "degraded"
+
+    return HealthResponse(
+        status=overall_status,
+        version=settings.app_version,
+        environment=settings.app_env,
+        database=db_status,
+        redis=redis_status,
+        timestamp=datetime.utcnow(),
+    )

docengine/app/api/v1/templates.py

@@ -0,0 +1,223 @@
+from __future__ import annotations
+
+import uuid
+
+from fastapi import APIRouter, Depends, HTTPException, Query, status
+from fastapi.responses import FileResponse
+from sqlalchemy.orm import Session
+
+from app.core.database import get_db
+from app.core.dependencies import CurrentUser
+from app.core.logging_config import get_logger
+from app.repositories.document_repository import DocumentRepository, TemplateMatchRepository
+from app.repositories.template_repository import TemplateRepository
+from app.schemas.common import PaginatedResponse, SuccessResponse
+from app.schemas.document import TemplateMatchRequest, TemplateMatchResponse
+from app.schemas.template import (
+    TemplateListResponse,
+    TemplateRenderRequest,
+    TemplateRenderResponse,
+    TemplateResponse,
+)
+
+logger = get_logger(__name__)
+
+router = APIRouter(prefix="/templates", tags=["Templates"])
+
+
+@router.get(
+    "",
+    response_model=PaginatedResponse[TemplateListResponse],
+    summary="List Templates",
+    description="List all active templates with pagination.",
+)
+def list_templates(
+    page: int = Query(default=1, ge=1),
+    page_size: int = Query(default=20, ge=1, le=100),
+    current_user: CurrentUser = None,
+    db: Session = Depends(get_db),
+) -> PaginatedResponse[TemplateListResponse]:
+    """List all active templates."""
+    template_repo = TemplateRepository(db)
+    offset = (page - 1) * page_size
+
+    templates = template_repo.get_active_templates(offset=offset, limit=page_size)
+    total = template_repo.count_active()
+
+    items = [TemplateListResponse.model_validate(t) for t in templates]
+    return PaginatedResponse.create(
+        items=items,
+        total=total,
+        page=page,
+        page_size=page_size,
+    )
+
+
+@router.get(
+    "/{template_id}",
+    response_model=TemplateResponse,
+    summary="Get Template",
+    description="Retrieve a template by ID with all its components.",
+)
+def get_template(
+    template_id: uuid.UUID,
+    current_user: CurrentUser = None,
+    db: Session = Depends(get_db),
+) -> TemplateResponse:
+    """Get a template by ID."""
+    template_repo = TemplateRepository(db)
+    template = template_repo.get_by_id(template_id)
+    if not template:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Template '{template_id}' not found",
+        )
+    return TemplateResponse.model_validate(template)
+
+
+@router.delete(
+    "/{template_id}",
+    response_model=SuccessResponse,
+    summary="Delete Template",
+    description="Soft-delete a template by deactivating it.",
+)
+def delete_template(
+    template_id: uuid.UUID,
+    current_user: CurrentUser = None,
+    db: Session = Depends(get_db),
+) -> SuccessResponse:
+    """Soft-delete a template."""
+    template_repo = TemplateRepository(db)
+    template = template_repo.get_by_id(template_id)
+    if not template:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Template '{template_id}' not found",
+        )
+
+    template_repo.deactivate_template(template_id)
+    db.commit()
+
+    return SuccessResponse(message=f"Template '{template_id}' deactivated successfully")
+
+
+@router.post(
+    "/match",
+    response_model=list[TemplateMatchResponse],
+    summary="Match Document to Templates",
+    description="Match a document against existing templates and return ranked results.",
+)
+def match_template(
+    payload: TemplateMatchRequest,
+    current_user: CurrentUser = None,
+    db: Session = Depends(get_db),
+) -> list[TemplateMatchResponse]:
+    """Match a document against existing templates."""
+    doc_repo = DocumentRepository(db)
+    document = doc_repo.get_by_id(payload.document_id)
+    if not document:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Document '{payload.document_id}' not found",
+        )
+
+    if document.status != "completed":
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"Document must be in 'completed' status. Current status: '{document.status}'",
+        )
+
+    # Perform template matching
+    from app.services.matching_service import MatchingService
+    matching_service = MatchingService(db)
+    matches = matching_service.match_document(
+        document_id=payload.document_id,
+        min_confidence=payload.min_confidence,
+        max_results=payload.max_results,
+    )
+    db.commit()
+
+    results = []
+    for match in matches:
+        resp = TemplateMatchResponse(
+            id=match.id,
+            document_id=match.document_id,
+            format_id=match.format_id,
+            confidence_score=match.confidence_score,
+            match_details=match.match_details,
+            selected=match.selected,
+            template_name=match.template.name if match.template else None,
+            created_at=match.created_at,
+        )
+        results.append(resp)
+
+    return results
+
+
+@router.post(
+    "/render",
+    response_model=TemplateRenderResponse,
+    summary="Render Template to PDF",
+    description="Generate a PDF from a stored template with supplied data.",
+)
+def render_template(
+    payload: TemplateRenderRequest,
+    current_user: CurrentUser = None,
+    db: Session = Depends(get_db),
+) -> TemplateRenderResponse:
+    """Render a template to PDF."""
+    template_repo = TemplateRepository(db)
+    template = template_repo.get_by_id(payload.template_id)
+    if not template:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Template '{payload.template_id}' not found",
+        )
+
+    if not template.is_active:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Template is deactivated",
+        )
+
+    from app.services.reconstruction_service import ReconstructionService
+    reconstruction_service = ReconstructionService(db)
+    result = reconstruction_service.render_template(
+        template=template,
+        data=payload.data,
+        output_filename=payload.output_filename,
+        images=payload.images,
+    )
+
+    return result
+
+
+@router.get(
+    "/{template_id}/download",
+    summary="Download Rendered PDF",
+    description="Download a previously rendered PDF.",
+)
+def download_rendered_pdf(
+    template_id: uuid.UUID,
+    filename: str = Query(..., description="Filename of the rendered PDF"),
+    current_user: CurrentUser = None,
+    db: Session = Depends(get_db),
+) -> FileResponse:
+    """Download a rendered PDF."""
+    from app.storage.provider import get_storage_provider
+
+    storage = get_storage_provider()
+    storage_path = f"rendered/{filename}"
+
+    if not storage.file_exists(storage_path):
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Rendered PDF '{filename}' not found",
+        )
+
+    absolute_path = storage.get_absolute_path(storage_path)
+    return FileResponse(
+        path=absolute_path,
+        media_type="application/pdf",
+        filename=filename,
+    )

docengine/app/core/config.py

@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from pydantic import field_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    """Application configuration loaded from environment variables."""
+
+    model_config = SettingsConfigDict(
+        env_file=".env",
+        env_file_encoding="utf-8",
+        case_sensitive=False,
+        extra="ignore",
+    )
+
+    # Application
+    app_name: str = "DocEngine"
+    app_version: str = "1.0.0"
+    app_env: str = "development"
+    app_debug: bool = True
+    app_host: str = "0.0.0.0"
+    app_port: int = 7989
+    app_workers: int = 4
+
+    # Database
+    db_host: str = "192.168.0.111"
+    db_port: int = 7925
+    db_name: str = "document_engine"
+    db_user: str = "postgres"
+    db_password: str = "M@tr!x#149@dm!N"
+    db_schema: str = "admin"
+    db_pool_size: int = 20
+    db_max_overflow: int = 10
+    db_echo: bool = False
+
+    # Redis
+    redis_host: str = "localhost"
+    redis_port: int = 6379
+    redis_db: int = 0
+    redis_password: str = ""
+
+    # Celery
+    celery_broker_url: str = "redis://localhost:6379/0"
+    celery_result_backend: str = "redis://localhost:6379/1"
+
+    # JWT
+    jwt_secret_key: str = "a7f3c9e1d4b8f2a6c0e5d7b3a9f1c4e8d2b6a0f5c3e7d1b9a4f8c2e6d0b5a3"
+    jwt_algorithm: str = "HS256"
+    jwt_access_token_expire_minutes: int = 30
+    jwt_refresh_token_expire_days: int = 7
+
+    # Storage
+    storage_provider: str = "local"
+    storage_local_path: str = "./storage"
+    storage_max_file_size_mb: int = 100
+
+    # OCR
+    ocr_language: str = "en"
+    ocr_use_gpu: bool = False
+
+    # Logging
+    log_level: str = "INFO"
+    log_format: str = "json"
+
+    # CORS
+    cors_origins: list[str] = ["http://localhost:3000", "http://localhost:8080"]
+    cors_allow_credentials: bool = True
+
+    # Rate Limiting
+    rate_limit_requests: int = 100
+    rate_limit_window_seconds: int = 60
+
+    # Prometheus
+    prometheus_enabled: bool = True
+
+    @field_validator("cors_origins", mode="before")
+    @classmethod
+    def parse_cors_origins(cls, v: Any) -> list[str]:
+        if isinstance(v, str):
+            try:
+                parsed = json.loads(v)
+                if isinstance(parsed, list):
+                    return parsed
+            except (json.JSONDecodeError, TypeError):
+                return [origin.strip() for origin in v.split(",") if origin.strip()]
+        return v
+
+    @property
+    def database_url(self) -> str:
+        from urllib.parse import quote_plus
+        password = quote_plus(self.db_password)
+        return f"postgresql+psycopg2://{self.db_user}:{password}@{self.db_host}:{self.db_port}/{self.db_name}"
+
+    @property
+    def async_database_url(self) -> str:
+        from urllib.parse import quote_plus
+        password = quote_plus(self.db_password)
+        return f"postgresql+asyncpg://{self.db_user}:{password}@{self.db_host}:{self.db_port}/{self.db_name}"
+
+    @property
+    def redis_url(self) -> str:
+        if self.redis_password:
+            return f"redis://:{self.redis_password}@{self.redis_host}:{self.redis_port}/{self.redis_db}"
+        return f"redis://{self.redis_host}:{self.redis_port}/{self.redis_db}"
+
+    @property
+    def is_production(self) -> bool:
+        return self.app_env == "production"
+
+    @property
+    def storage_max_file_size_bytes(self) -> int:
+        return self.storage_max_file_size_mb * 1024 * 1024
+
+
+settings = Settings()

docengine/app/core/database.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+from collections.abc import Generator
+from contextlib import contextmanager
+
+from sqlalchemy import MetaData, create_engine, event, text
+from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
+
+from app.core.config import settings
+
+NAMING_CONVENTION = {
+    "ix": "ix_%(column_0_label)s",
+    "uq": "uq_%(table_name)s_%(column_0_name)s",
+    "ck": "ck_%(table_name)s_%(constraint_name)s",
+    "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
+    "pk": "pk_%(table_name)s",
+}
+
+metadata = MetaData(
+    naming_convention=NAMING_CONVENTION,
+    schema=settings.db_schema,
+)
+
+engine = create_engine(
+    settings.database_url,
+    pool_size=settings.db_pool_size,
+    max_overflow=settings.db_max_overflow,
+    echo=settings.db_echo,
+    pool_pre_ping=True,
+    pool_recycle=3600,
+    connect_args={
+        "options": f"-c search_path={settings.db_schema},public"
+    },
+)
+
+
+@event.listens_for(engine, "connect")
+def set_search_path(dbapi_connection: object, connection_record: object) -> None:
+    cursor = dbapi_connection.cursor()  # type: ignore[union-attr]
+    cursor.execute(f"SET search_path TO {settings.db_schema}, public")
+    cursor.close()
+    dbapi_connection.commit()  # type: ignore[union-attr]
+
+
+SessionLocal = sessionmaker(
+    autocommit=False,
+    autoflush=False,
+    bind=engine,
+)
+
+
+class Base(DeclarativeBase):
+    """Base class for all SQLAlchemy models."""
+
+    metadata = metadata
+
+
+def get_db() -> Generator[Session, None, None]:
+    """Dependency to get database session."""
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()
+
+
+@contextmanager
+def get_db_context() -> Generator[Session, None, None]:
+    """Context manager for database session (used outside request scope)."""
+    db = SessionLocal()
+    try:
+        yield db
+        db.commit()
+    except Exception:
+        db.rollback()
+        raise
+    finally:
+        db.close()
+
+
+def check_database_connection() -> bool:
+    """Verify database connectivity."""
+    try:
+        with engine.connect() as conn:
+            conn.execute(text("SELECT 1"))
+        return True
+    except Exception:
+        return False

docengine/app/core/dependencies.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+from typing import Annotated
+
+from fastapi import Depends, HTTPException, status
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+from sqlalchemy.orm import Session
+
+from app.core.database import get_db
+from app.core.security import InvalidTokenError, decode_token
+from app.models.user import User
+from app.repositories.user_repository import UserRepository
+
+security_scheme = HTTPBearer(auto_error=True)
+
+
+def get_current_user(
+    credentials: Annotated[HTTPAuthorizationCredentials, Depends(security_scheme)],
+    db: Annotated[Session, Depends(get_db)],
+) -> User:
+    """Extract and validate the current user from the JWT token."""
+    try:
+        payload = decode_token(credentials.credentials)
+    except InvalidTokenError:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid or expired token",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    token_type = payload.get("type")
+    if token_type != "access":
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid token type. Access token required.",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    user_id: str | None = payload.get("sub")
+    if user_id is None:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Token payload missing subject",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    user_repo = UserRepository(db)
+    user = user_repo.get_by_id(user_id)
+    if user is None:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="User not found",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    if not user.is_active:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="User account is deactivated",
+        )
+
+    return user
+
+
+def get_current_active_user(
+    current_user: Annotated[User, Depends(get_current_user)],
+) -> User:
+    """Ensure the current user is active."""
+    if not current_user.is_active:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="User account is deactivated",
+        )
+    return current_user
+
+
+def require_role(required_roles: list[str]):  # noqa: ANN201
+    """Dependency factory to require specific roles."""
+
+    def role_checker(
+        current_user: Annotated[User, Depends(get_current_user)],
+    ) -> User:
+        user_roles = {role.name for role in current_user.roles}
+        if not user_roles.intersection(required_roles):
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN,
+                detail=f"One of the following roles required: {', '.join(required_roles)}",
+            )
+        return current_user
+
+    return role_checker
+
+
+CurrentUser = Annotated[User, Depends(get_current_user)]
+ActiveUser = Annotated[User, Depends(get_current_active_user)]
+AdminUser = Annotated[User, Depends(require_role(["admin"]))]
+DBSession = Annotated[Session, Depends(get_db)]

docengine/app/core/exceptions.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+from typing import Any
+
+
+class DocEngineException(Exception):
+    """Base exception for DocEngine application."""
+
+    def __init__(self, detail: str, status_code: int = 500, extra: dict[str, Any] | None = None) -> None:
+        self.detail = detail
+        self.status_code = status_code
+        self.extra = extra or {}
+        super().__init__(self.detail)
+
+
+class NotFoundError(DocEngineException):
+    """Resource not found."""
+
+    def __init__(self, resource: str, identifier: str) -> None:
+        super().__init__(
+            detail=f"{resource} with identifier '{identifier}' not found",
+            status_code=404,
+        )
+        self.resource = resource
+        self.identifier = identifier
+
+
+class DuplicateError(DocEngineException):
+    """Resource already exists."""
+
+    def __init__(self, resource: str, field: str, value: str) -> None:
+        super().__init__(
+            detail=f"{resource} with {field} '{value}' already exists",
+            status_code=409,
+        )
+
+
+class ValidationError(DocEngineException):
+    """Input validation error."""
+
+    def __init__(self, detail: str, errors: list[dict[str, Any]] | None = None) -> None:
+        super().__init__(detail=detail, status_code=422)
+        self.errors = errors or []
+
+
+class AuthenticationError(DocEngineException):
+    """Authentication failed."""
+
+    def __init__(self, detail: str = "Authentication failed") -> None:
+        super().__init__(detail=detail, status_code=401)
+
+
+class AuthorizationError(DocEngineException):
+    """Authorization failed."""
+
+    def __init__(self, detail: str = "Insufficient permissions") -> None:
+        super().__init__(detail=detail, status_code=403)
+
+
+class StorageError(DocEngineException):
+    """Storage operation failed."""
+
+    def __init__(self, detail: str) -> None:
+        super().__init__(detail=detail, status_code=500)
+
+
+class ProcessingError(DocEngineException):
+    """Document processing failed."""
+
+    def __init__(self, detail: str, document_id: str | None = None) -> None:
+        super().__init__(detail=detail, status_code=500)
+        self.document_id = document_id
+
+
+class TemplateError(DocEngineException):
+    """Template operation failed."""
+
+    def __init__(self, detail: str) -> None:
+        super().__init__(detail=detail, status_code=500)
+
+
+class RateLimitError(DocEngineException):
+    """Rate limit exceeded."""
+
+    def __init__(self, detail: str = "Rate limit exceeded. Please try again later.") -> None:
+        super().__init__(detail=detail, status_code=429)
+
+
+class FileSizeError(DocEngineException):
+    """File exceeds maximum allowed size."""
+
+    def __init__(self, max_size_mb: int) -> None:
+        super().__init__(
+            detail=f"File size exceeds maximum allowed size of {max_size_mb}MB",
+            status_code=413,
+        )
+
+
+class UnsupportedFileTypeError(DocEngineException):
+    """File type not supported."""
+
+    def __init__(self, file_type: str) -> None:
+        super().__init__(
+            detail=f"File type '{file_type}' is not supported. Supported types: jpg, jpeg, png, tiff, pdf",
+            status_code=415,
+        )

docengine/app/core/logging_config.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+import logging
+import sys
+
+import structlog
+
+from app.core.config import settings
+
+
+def setup_logging() -> None:
+    """Configure structlog for structured JSON logging."""
+    shared_processors: list[structlog.types.Processor] = [
+        structlog.contextvars.merge_contextvars,
+        structlog.stdlib.add_logger_name,
+        structlog.stdlib.add_log_level,
+        structlog.stdlib.PositionalArgumentsFormatter(),
+        structlog.processors.TimeStamper(fmt="iso"),
+        structlog.processors.StackInfoRenderer(),
+        structlog.processors.UnicodeDecoder(),
+    ]
+
+    if settings.log_format == "json":
+        renderer: structlog.types.Processor = structlog.processors.JSONRenderer()
+    else:
+        renderer = structlog.dev.ConsoleRenderer(colors=True)
+
+    structlog.configure(
+        processors=[
+            *shared_processors,
+            structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
+        ],
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        wrapper_class=structlog.stdlib.BoundLogger,
+        cache_logger_on_first_use=True,
+    )
+
+    formatter = structlog.stdlib.ProcessorFormatter(
+        processors=[
+            structlog.stdlib.ProcessorFormatter.remove_processors_meta,
+            renderer,
+        ],
+        foreign_pre_chain=shared_processors,
+    )
+
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setFormatter(formatter)
+
+    root_logger = logging.getLogger()
+    root_logger.handlers.clear()
+    root_logger.addHandler(handler)
+    root_logger.setLevel(getattr(logging, settings.log_level.upper(), logging.INFO))
+
+    # Reduce noise from third-party libraries
+    for logger_name in ("uvicorn.access", "sqlalchemy.engine", "celery"):
+        logging.getLogger(logger_name).setLevel(logging.WARNING)
+
+
+def get_logger(name: str | None = None) -> structlog.stdlib.BoundLogger:
+    """Get a structlog logger instance."""
+    return structlog.get_logger(name)

docengine/app/core/security.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import uuid
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from jose import JWTError, jwt
+from passlib.context import CryptContext
+
+from app.core.config import settings
+
+pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
+
+
+def hash_password(password: str) -> str:
+    """Hash a password using bcrypt."""
+    return pwd_context.hash(password)
+
+
+def verify_password(plain_password: str, hashed_password: str) -> bool:
+    """Verify a plain password against a hashed password."""
+    return pwd_context.verify(plain_password, hashed_password)
+
+
+def create_access_token(data: dict[str, Any], expires_delta: timedelta | None = None) -> str:
+    """Create a JWT access token."""
+    to_encode = data.copy()
+    expire = datetime.now(UTC) + (expires_delta or timedelta(minutes=settings.jwt_access_token_expire_minutes))
+    to_encode.update({"exp": expire, "type": "access"})
+    return jwt.encode(to_encode, settings.jwt_secret_key, algorithm=settings.jwt_algorithm)
+
+
+def create_refresh_token(data: dict[str, Any], expires_delta: timedelta | None = None) -> str:
+    """Create a JWT refresh token."""
+    to_encode = data.copy()
+    expire = datetime.now(UTC) + (expires_delta or timedelta(days=settings.jwt_refresh_token_expire_days))
+    to_encode.update({
+        "exp": expire,
+        "type": "refresh",
+        "jti": str(uuid.uuid4()),
+    })
+    return jwt.encode(to_encode, settings.jwt_secret_key, algorithm=settings.jwt_algorithm)
+
+
+def decode_token(token: str) -> dict[str, Any]:
+    """Decode and validate a JWT token."""
+    try:
+        payload = jwt.decode(token, settings.jwt_secret_key, algorithms=[settings.jwt_algorithm])
+        return payload
+    except JWTError as e:
+        raise InvalidTokenError(str(e)) from e
+
+
+class InvalidTokenError(Exception):
+    """Raised when a JWT token is invalid or expired."""
+
+    def __init__(self, detail: str = "Invalid or expired token") -> None:
+        self.detail = detail
+        super().__init__(self.detail)

docengine/app/events/handlers.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from app.core.logging_config import get_logger, setup_logging
+
+logger = get_logger(__name__)
+
+
+def on_startup() -> None:
+    """Application startup event handler."""
+    setup_logging()
+    logger.info("application_starting", event="startup")
+
+    # Ensure storage directories exist
+    from app.storage.provider import get_storage_provider
+    try:
+        get_storage_provider()
+        logger.info("storage_initialized")
+    except Exception as e:
+        logger.error("storage_init_failed", error=str(e))
+
+    # Verify database connection
+    from app.core.database import check_database_connection
+    if check_database_connection():
+        logger.info("database_connected")
+    else:
+        logger.error("database_connection_failed")
+
+    logger.info("application_started", event="startup_complete")
+
+
+def on_shutdown() -> None:
+    """Application shutdown event handler."""
+    logger.info("application_shutting_down", event="shutdown")
+
+    # Cleanup resources
+    from app.core.database import engine
+    engine.dispose()
+
+    logger.info("application_stopped", event="shutdown_complete")

docengine/app/main.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+from contextlib import asynccontextmanager
+from collections.abc import AsyncGenerator
+from typing import Any
+
+from fastapi import FastAPI, Request, status
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+
+from app.api.router import api_v1_router
+from app.core.config import settings
+from app.core.exceptions import DocEngineException
+from app.events.handlers import on_shutdown, on_startup
+from app.middleware.audit import AuditMiddleware
+from app.middleware.cors import setup_cors
+from app.middleware.metrics import setup_metrics
+from app.middleware.rate_limit import RateLimitMiddleware
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
+    """Application lifespan manager."""
+    on_startup()
+    yield
+    on_shutdown()
+
+
+app = FastAPI(
+    title=settings.app_name,
+    description="Document Template Recognition and Reconstruction System",
+    version=settings.app_version,
+    docs_url="/docs",
+    redoc_url="/redoc",
+    openapi_url="/openapi.json",
+    lifespan=lifespan,
+)
+
+# Setup middleware (order matters: last added = first executed)
+setup_cors(app)
+app.add_middleware(AuditMiddleware)
+app.add_middleware(RateLimitMiddleware)
+
+# Setup Prometheus metrics
+setup_metrics(app)
+
+# Include API routes
+app.include_router(api_v1_router)
+
+
+# Exception handlers
+@app.exception_handler(DocEngineException)
+async def docengine_exception_handler(request: Request, exc: DocEngineException) -> JSONResponse:
+    """Handle application-specific exceptions."""
+    return JSONResponse(
+        status_code=exc.status_code,
+        content={
+            "detail": exc.detail,
+            "error_code": type(exc).__name__,
+            "extra": exc.extra if exc.extra else None,
+        },
+    )
+
+
+@app.exception_handler(RequestValidationError)
+async def validation_exception_handler(request: Request, exc: RequestValidationError) -> JSONResponse:
+    """Handle request validation errors."""
+    errors = []
+    for error in exc.errors():
+        errors.append({
+            "field": ".".join(str(loc) for loc in error.get("loc", [])),
+            "message": error.get("msg", ""),
+            "type": error.get("type", ""),
+        })
+
+    return JSONResponse(
+        status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+        content={
+            "detail": "Request validation failed",
+            "error_code": "ValidationError",
+            "errors": errors,
+        },
+    )
+
+
+@app.exception_handler(Exception)
+async def general_exception_handler(request: Request, exc: Exception) -> JSONResponse:
+    """Handle unexpected exceptions."""
+    return JSONResponse(
+        status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+        content={
+            "detail": "An unexpected error occurred" if settings.is_production else str(exc),
+            "error_code": "InternalServerError",
+        },
+    )
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(
+        "app.main:app",
+        host=settings.app_host,
+        port=settings.app_port,
+        reload=not settings.is_production,
+        workers=1 if settings.app_debug else settings.app_workers,
+    )

docengine/app/middleware/audit.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+import time
+import uuid
+from typing import Any
+
+from fastapi import Request, Response
+from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
+
+from app.core.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+class AuditMiddleware(BaseHTTPMiddleware):
+    """Middleware to log all API requests for audit purposes."""
+
+    EXCLUDED_PATHS = {"/api/v1/health", "/metrics", "/docs", "/openapi.json", "/redoc"}
+
+    async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> Response:
+        if request.url.path in self.EXCLUDED_PATHS:
+            return await call_next(request)
+
+        request_id = str(uuid.uuid4())
+        start_time = time.monotonic()
+
+        # Extract client info
+        client_ip = request.client.host if request.client else "unknown"
+        user_agent = request.headers.get("user-agent", "unknown")
+
+        # Add request ID to request state
+        request.state.request_id = request_id
+
+        logger.info(
+            "request_started",
+            request_id=request_id,
+            method=request.method,
+            path=request.url.path,
+            client_ip=client_ip,
+            user_agent=user_agent[:200],
+        )
+
+        try:
+            response = await call_next(request)
+            duration_ms = (time.monotonic() - start_time) * 1000
+
+            logger.info(
+                "request_completed",
+                request_id=request_id,
+                method=request.method,
+                path=request.url.path,
+                status_code=response.status_code,
+                duration_ms=round(duration_ms, 2),
+                client_ip=client_ip,
+            )
+
+            response.headers["X-Request-ID"] = request_id
+            response.headers["X-Process-Time"] = f"{duration_ms:.2f}ms"
+            return response
+
+        except Exception as exc:
+            duration_ms = (time.monotonic() - start_time) * 1000
+            logger.exception(
+                "request_failed",
+                request_id=request_id,
+                method=request.method,
+                path=request.url.path,
+                duration_ms=round(duration_ms, 2),
+                error=str(exc),
+            )
+            raise

docengine/app/middleware/cors.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+
+from app.core.config import settings
+
+
+def setup_cors(app: FastAPI) -> None:
+    """Configure CORS middleware."""
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=settings.cors_origins,
+        allow_credentials=settings.cors_allow_credentials,
+        allow_methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"],
+        allow_headers=[
+            "Authorization",
+            "Content-Type",
+            "Accept",
+            "X-Request-ID",
+            "X-Requested-With",
+        ],
+        expose_headers=[
+            "X-Request-ID",
+            "X-Process-Time",
+            "X-RateLimit-Limit",
+            "X-RateLimit-Remaining",
+            "X-RateLimit-Reset",
+        ],
+        max_age=600,
+    )

docengine/app/middleware/metrics.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from fastapi import FastAPI
+from prometheus_fastapi_instrumentator import Instrumentator
+
+from app.core.config import settings
+
+
+def setup_metrics(app: FastAPI) -> None:
+    """Configure Prometheus metrics instrumentation."""
+    if not settings.prometheus_enabled:
+        return
+
+    instrumentator = Instrumentator(
+        should_group_status_codes=True,
+        should_ignore_untemplated=True,
+        should_respect_env_var=False,
+        excluded_handlers=["/metrics", "/api/v1/health", "/docs", "/openapi.json"],
+        env_var_name="PROMETHEUS_ENABLED",
+        inprogress_name="docengine_inprogress_requests",
+        inprogress_labels=True,
+    )
+
+    instrumentator.instrument(app).expose(
+        app,
+        endpoint="/metrics",
+        include_in_schema=False,
+        should_gzip=True,
+    )

docengine/app/middleware/rate_limit.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import time
+from collections import defaultdict
+
+from fastapi import Request, Response, status
+from fastapi.responses import JSONResponse
+from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
+
+from app.core.config import settings
+from app.core.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+class RateLimitMiddleware(BaseHTTPMiddleware):
+    """Token bucket rate limiter per client IP."""
+
+    EXCLUDED_PATHS = {"/api/v1/health", "/metrics", "/docs", "/openapi.json", "/redoc"}
+
+    def __init__(self, app, max_requests: int | None = None, window_seconds: int | None = None) -> None:  # noqa: ANN001
+        super().__init__(app)
+        self.max_requests = max_requests or settings.rate_limit_requests
+        self.window_seconds = window_seconds or settings.rate_limit_window_seconds
+        self._requests: dict[str, list[float]] = defaultdict(list)
+
+    def _clean_old_requests(self, client_ip: str, now: float) -> None:
+        """Remove requests outside the current window."""
+        cutoff = now - self.window_seconds
+        self._requests[client_ip] = [
+            ts for ts in self._requests[client_ip] if ts > cutoff
+        ]
+
+    async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> Response:
+        if request.url.path in self.EXCLUDED_PATHS:
+            return await call_next(request)
+
+        client_ip = request.client.host if request.client else "unknown"
+        now = time.monotonic()
+
+        self._clean_old_requests(client_ip, now)
+
+        if len(self._requests[client_ip]) >= self.max_requests:
+            logger.warning(
+                "rate_limit_exceeded",
+                client_ip=client_ip,
+                path=request.url.path,
+                request_count=len(self._requests[client_ip]),
+            )
+            return JSONResponse(
+                status_code=status.HTTP_429_TOO_MANY_REQUESTS,
+                content={
+                    "detail": "Rate limit exceeded. Please try again later.",
+                    "retry_after_seconds": self.window_seconds,
+                },
+                headers={
+                    "Retry-After": str(self.window_seconds),
+                    "X-RateLimit-Limit": str(self.max_requests),
+                    "X-RateLimit-Remaining": "0",
+                    "X-RateLimit-Reset": str(int(now + self.window_seconds)),
+                },
+            )
+
+        self._requests[client_ip].append(now)
+        remaining = self.max_requests - len(self._requests[client_ip])
+
+        response = await call_next(request)
+        response.headers["X-RateLimit-Limit"] = str(self.max_requests)
+        response.headers["X-RateLimit-Remaining"] = str(remaining)
+        response.headers["X-RateLimit-Reset"] = str(int(now + self.window_seconds))
+
+        return response

docengine/app/models/__init__.py

@@ -0,0 +1,43 @@
+from app.models.user import AuditLog, RefreshToken, Role, User, user_roles_table
+from app.models.document import (
+    Document,
+    DocumentImage,
+    DocumentPage,
+    DocumentTable,
+    DocumentTextBlock,
+    TemplateMatch,
+)
+from app.models.template import (
+    DocumentCell,
+    DocumentFormat,
+    DocumentRegion,
+    ImageRegion,
+    TableColumn,
+    TableFormat,
+    TableRow,
+    TemplateFingerprint,
+    Watermark,
+)
+
+__all__ = [
+    "User",
+    "Role",
+    "RefreshToken",
+    "AuditLog",
+    "user_roles_table",
+    "Document",
+    "DocumentPage",
+    "DocumentTextBlock",
+    "DocumentImage",
+    "DocumentTable",
+    "TemplateMatch",
+    "DocumentFormat",
+    "DocumentCell",
+    "DocumentRegion",
+    "TableFormat",
+    "TableColumn",
+    "TableRow",
+    "Watermark",
+    "ImageRegion",
+    "TemplateFingerprint",
+]

docengine/app/models/base.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import uuid
+from datetime import UTC, datetime
+
+from sqlalchemy import DateTime, func
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.orm import Mapped, mapped_column
+
+from app.core.database import Base
+
+
+class TimestampMixin:
+    """Mixin providing created_at and updated_at timestamps."""
+
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=lambda: datetime.now(UTC),
+        server_default=func.now(),
+        nullable=False,
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=lambda: datetime.now(UTC),
+        server_default=func.now(),
+        onupdate=lambda: datetime.now(UTC),
+        nullable=False,
+    )
+
+
+class UUIDPrimaryKeyMixin:
+    """Mixin providing a UUID primary key."""
+
+    id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        primary_key=True,
+        default=uuid.uuid4,
+        nullable=False,
+    )

docengine/app/models/document.py

@@ -0,0 +1,244 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+
+from sqlalchemy import BigInteger, Boolean, DateTime, Float, ForeignKey, Integer, String, Text, func
+from sqlalchemy.dialects.postgresql import JSONB, UUID
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from app.core.database import Base
+from app.models.base import TimestampMixin, UUIDPrimaryKeyMixin
+
+
+class Document(Base, UUIDPrimaryKeyMixin, TimestampMixin):
+    """Uploaded document record."""
+
+    __tablename__ = "documents"
+
+    filename: Mapped[str] = mapped_column(String(500), nullable=False)
+    original_filename: Mapped[str] = mapped_column(String(500), nullable=False)
+    content_type: Mapped[str] = mapped_column(String(100), nullable=False)
+    file_size: Mapped[int] = mapped_column(BigInteger, nullable=False)
+    checksum: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
+    storage_path: Mapped[str] = mapped_column(String(1024), nullable=False)
+    status: Mapped[str] = mapped_column(
+        String(50),
+        default="pending",
+        nullable=False,
+        index=True,
+    )
+    page_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    is_scanned: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
+    document_metadata: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
+    uploaded_by: Mapped[uuid.UUID | None] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("users.id", ondelete="SET NULL"),
+        nullable=True,
+        index=True,
+    )
+
+    pages: Mapped[list[DocumentPage]] = relationship(
+        "DocumentPage",
+        back_populates="document",
+        cascade="all, delete-orphan",
+        order_by="DocumentPage.page_number",
+        lazy="selectin",
+    )
+    template_matches: Mapped[list[TemplateMatch]] = relationship(
+        "TemplateMatch",
+        back_populates="document",
+        cascade="all, delete-orphan",
+        lazy="dynamic",
+    )
+
+    def __repr__(self) -> str:
+        return f"<Document(id={self.id}, filename={self.original_filename}, status={self.status})>"
+
+
+class DocumentPage(Base, UUIDPrimaryKeyMixin):
+    """Individual page within a document."""
+
+    __tablename__ = "document_pages"
+
+    document_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("documents.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    page_number: Mapped[int] = mapped_column(Integer, nullable=False)
+    width: Mapped[float] = mapped_column(Float, nullable=False)
+    height: Mapped[float] = mapped_column(Float, nullable=False)
+    image_path: Mapped[str | None] = mapped_column(String(1024), nullable=True)
+    text_content: Mapped[str | None] = mapped_column(Text, nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    document: Mapped[Document] = relationship("Document", back_populates="pages")
+    text_blocks: Mapped[list[DocumentTextBlock]] = relationship(
+        "DocumentTextBlock",
+        back_populates="page",
+        cascade="all, delete-orphan",
+        order_by="DocumentTextBlock.sequence",
+        lazy="selectin",
+    )
+    images: Mapped[list[DocumentImage]] = relationship(
+        "DocumentImage",
+        back_populates="page",
+        cascade="all, delete-orphan",
+        lazy="selectin",
+    )
+    tables: Mapped[list[DocumentTable]] = relationship(
+        "DocumentTable",
+        back_populates="page",
+        cascade="all, delete-orphan",
+        lazy="selectin",
+    )
+
+    def __repr__(self) -> str:
+        return f"<DocumentPage(id={self.id}, document_id={self.document_id}, page={self.page_number})>"
+
+
+class DocumentTextBlock(Base, UUIDPrimaryKeyMixin):
+    """Extracted text block from a document page."""
+
+    __tablename__ = "document_text_blocks"
+
+    page_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("document_pages.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    text: Mapped[str] = mapped_column(Text, nullable=False)
+    x: Mapped[float] = mapped_column(Float, nullable=False)
+    y: Mapped[float] = mapped_column(Float, nullable=False)
+    width: Mapped[float] = mapped_column(Float, nullable=False)
+    height: Mapped[float] = mapped_column(Float, nullable=False)
+    confidence: Mapped[float | None] = mapped_column(Float, nullable=True)
+    font_family: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    font_size: Mapped[float | None] = mapped_column(Float, nullable=True)
+    font_color: Mapped[str | None] = mapped_column(String(50), nullable=True)
+    font_style: Mapped[str | None] = mapped_column(String(50), nullable=True)
+    block_type: Mapped[str] = mapped_column(
+        String(50),
+        default="text",
+        nullable=False,
+    )
+    sequence: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    page: Mapped[DocumentPage] = relationship("DocumentPage", back_populates="text_blocks")
+
+    def __repr__(self) -> str:
+        return f"<DocumentTextBlock(id={self.id}, type={self.block_type}, text={self.text[:50]})>"
+
+
+class DocumentImage(Base, UUIDPrimaryKeyMixin):
+    """Extracted image from a document page."""
+
+    __tablename__ = "document_images"
+
+    page_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("document_pages.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    x: Mapped[float] = mapped_column(Float, nullable=False)
+    y: Mapped[float] = mapped_column(Float, nullable=False)
+    width: Mapped[float] = mapped_column(Float, nullable=False)
+    height: Mapped[float] = mapped_column(Float, nullable=False)
+    image_path: Mapped[str] = mapped_column(String(1024), nullable=False)
+    image_type: Mapped[str] = mapped_column(
+        String(50),
+        default="figure",
+        nullable=False,
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    page: Mapped[DocumentPage] = relationship("DocumentPage", back_populates="images")
+
+    def __repr__(self) -> str:
+        return f"<DocumentImage(id={self.id}, type={self.image_type})>"
+
+
+class DocumentTable(Base, UUIDPrimaryKeyMixin):
+    """Extracted table from a document page."""
+
+    __tablename__ = "document_tables"
+
+    page_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("document_pages.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    x: Mapped[float] = mapped_column(Float, nullable=False)
+    y: Mapped[float] = mapped_column(Float, nullable=False)
+    width: Mapped[float] = mapped_column(Float, nullable=False)
+    height: Mapped[float] = mapped_column(Float, nullable=False)
+    rows: Mapped[int] = mapped_column(Integer, nullable=False)
+    columns: Mapped[int] = mapped_column(Integer, nullable=False)
+    data: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    page: Mapped[DocumentPage] = relationship("DocumentPage", back_populates="tables")
+
+    def __repr__(self) -> str:
+        return f"<DocumentTable(id={self.id}, rows={self.rows}, cols={self.columns})>"
+
+
+class TemplateMatch(Base, UUIDPrimaryKeyMixin):
+    """Template matching result for a document."""
+
+    __tablename__ = "template_matches"
+
+    document_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("documents.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    format_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("document_formats.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    confidence_score: Mapped[float] = mapped_column(Float, nullable=False)
+    match_details: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    selected: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    document: Mapped[Document] = relationship("Document", back_populates="template_matches")
+    template: Mapped[DocumentFormat] = relationship("DocumentFormat")
+
+    def __repr__(self) -> str:
+        return f"<TemplateMatch(id={self.id}, doc={self.document_id}, score={self.confidence_score})>"

docengine/app/models/template.py

@@ -0,0 +1,378 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+
+from sqlalchemy import Boolean, DateTime, Float, ForeignKey, Integer, String, Text, func
+from sqlalchemy.dialects.postgresql import JSONB, UUID
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from app.core.database import Base
+from app.models.base import TimestampMixin, UUIDPrimaryKeyMixin
+
+
+class DocumentFormat(Base, UUIDPrimaryKeyMixin, TimestampMixin):
+    """Reusable document template format."""
+
+    __tablename__ = "document_formats"
+
+    name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
+    description: Mapped[str | None] = mapped_column(Text, nullable=True)
+    page_width: Mapped[float] = mapped_column(Float, nullable=False)
+    page_height: Mapped[float] = mapped_column(Float, nullable=False)
+    page_count: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
+    margin_top: Mapped[float] = mapped_column(Float, default=72.0, nullable=False)
+    margin_right: Mapped[float] = mapped_column(Float, default=72.0, nullable=False)
+    margin_bottom: Mapped[float] = mapped_column(Float, default=72.0, nullable=False)
+    margin_left: Mapped[float] = mapped_column(Float, default=72.0, nullable=False)
+    fingerprint: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    source_document_id: Mapped[uuid.UUID | None] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("documents.id", ondelete="SET NULL"),
+        nullable=True,
+        index=True,
+    )
+    version: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
+    is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False, index=True)
+    created_by: Mapped[uuid.UUID | None] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("users.id", ondelete="SET NULL"),
+        nullable=True,
+    )
+
+    cells: Mapped[list[DocumentCell]] = relationship(
+        "DocumentCell",
+        back_populates="format",
+        cascade="all, delete-orphan",
+        order_by="DocumentCell.sequence",
+        lazy="selectin",
+    )
+    regions: Mapped[list[DocumentRegion]] = relationship(
+        "DocumentRegion",
+        back_populates="format",
+        cascade="all, delete-orphan",
+        order_by="DocumentRegion.sequence",
+        lazy="selectin",
+    )
+    table_formats: Mapped[list[TableFormat]] = relationship(
+        "TableFormat",
+        back_populates="format",
+        cascade="all, delete-orphan",
+        lazy="selectin",
+    )
+    watermarks: Mapped[list[Watermark]] = relationship(
+        "Watermark",
+        back_populates="format",
+        cascade="all, delete-orphan",
+        lazy="selectin",
+    )
+    image_regions: Mapped[list[ImageRegion]] = relationship(
+        "ImageRegion",
+        back_populates="format",
+        cascade="all, delete-orphan",
+        lazy="selectin",
+    )
+    fingerprint_record: Mapped[TemplateFingerprint | None] = relationship(
+        "TemplateFingerprint",
+        back_populates="format",
+        uselist=False,
+        cascade="all, delete-orphan",
+        lazy="selectin",
+    )
+
+    def __repr__(self) -> str:
+        return f"<DocumentFormat(id={self.id}, name={self.name}, v{self.version})>"
+
+
+class DocumentCell(Base, UUIDPrimaryKeyMixin):
+    """Cell definition within a document template."""
+
+    __tablename__ = "document_cells"
+
+    format_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("document_formats.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    page_number: Mapped[int] = mapped_column(Integer, nullable=False)
+    x: Mapped[float] = mapped_column(Float, nullable=False)
+    y: Mapped[float] = mapped_column(Float, nullable=False)
+    width: Mapped[float] = mapped_column(Float, nullable=False)
+    height: Mapped[float] = mapped_column(Float, nullable=False)
+    row_no: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+    column_no: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+    data_type: Mapped[str] = mapped_column(String(50), default="text", nullable=False)
+    font_family: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    font_size: Mapped[float | None] = mapped_column(Float, nullable=True)
+    font_style: Mapped[str | None] = mapped_column(String(50), nullable=True)
+    font_color: Mapped[str | None] = mapped_column(String(50), nullable=True)
+    background_color: Mapped[str | None] = mapped_column(String(50), nullable=True)
+    border_top: Mapped[str | None] = mapped_column(String(100), nullable=True)
+    border_right: Mapped[str | None] = mapped_column(String(100), nullable=True)
+    border_bottom: Mapped[str | None] = mapped_column(String(100), nullable=True)
+    border_left: Mapped[str | None] = mapped_column(String(100), nullable=True)
+    padding_top: Mapped[float] = mapped_column(Float, default=0.0, nullable=False)
+    padding_right: Mapped[float] = mapped_column(Float, default=0.0, nullable=False)
+    padding_bottom: Mapped[float] = mapped_column(Float, default=0.0, nullable=False)
+    padding_left: Mapped[float] = mapped_column(Float, default=0.0, nullable=False)
+    alignment: Mapped[str] = mapped_column(String(20), default="left", nullable=False)
+    vertical_alignment: Mapped[str] = mapped_column(String(20), default="top", nullable=False)
+    rowspan: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
+    colspan: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
+    static_text: Mapped[str | None] = mapped_column(Text, nullable=True)
+    field_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    sequence: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+    is_dynamic: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    format: Mapped[DocumentFormat] = relationship("DocumentFormat", back_populates="cells")
+
+    def __repr__(self) -> str:
+        return f"<DocumentCell(id={self.id}, page={self.page_number}, row={self.row_no}, col={self.column_no})>"
+
+
+class DocumentRegion(Base, UUIDPrimaryKeyMixin):
+    """Region definition within a document template."""
+
+    __tablename__ = "document_regions"
+
+    format_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("document_formats.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    page_number: Mapped[int] = mapped_column(Integer, nullable=False)
+    region_type: Mapped[str] = mapped_column(String(50), nullable=False)
+    x: Mapped[float] = mapped_column(Float, nullable=False)
+    y: Mapped[float] = mapped_column(Float, nullable=False)
+    width: Mapped[float] = mapped_column(Float, nullable=False)
+    height: Mapped[float] = mapped_column(Float, nullable=False)
+    content: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    sequence: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    format: Mapped[DocumentFormat] = relationship("DocumentFormat", back_populates="regions")
+
+    def __repr__(self) -> str:
+        return f"<DocumentRegion(id={self.id}, type={self.region_type}, page={self.page_number})>"
+
+
+class TableFormat(Base, UUIDPrimaryKeyMixin):
+    """Table definition within a document template."""
+
+    __tablename__ = "table_formats"
+
+    format_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("document_formats.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    page_number: Mapped[int] = mapped_column(Integer, nullable=False)
+    x: Mapped[float] = mapped_column(Float, nullable=False)
+    y: Mapped[float] = mapped_column(Float, nullable=False)
+    width: Mapped[float] = mapped_column(Float, nullable=False)
+    height: Mapped[float] = mapped_column(Float, nullable=False)
+    rows: Mapped[int] = mapped_column(Integer, nullable=False)
+    columns: Mapped[int] = mapped_column(Integer, nullable=False)
+    border_style: Mapped[str] = mapped_column(String(50), default="solid", nullable=False)
+    border_width: Mapped[float] = mapped_column(Float, default=1.0, nullable=False)
+    border_color: Mapped[str] = mapped_column(String(50), default="#000000", nullable=False)
+    header_rows: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    format: Mapped[DocumentFormat] = relationship("DocumentFormat", back_populates="table_formats")
+    table_columns: Mapped[list[TableColumn]] = relationship(
+        "TableColumn",
+        back_populates="table_format",
+        cascade="all, delete-orphan",
+        order_by="TableColumn.column_index",
+        lazy="selectin",
+    )
+    table_rows: Mapped[list[TableRow]] = relationship(
+        "TableRow",
+        back_populates="table_format",
+        cascade="all, delete-orphan",
+        order_by="TableRow.row_index",
+        lazy="selectin",
+    )
+
+    def __repr__(self) -> str:
+        return f"<TableFormat(id={self.id}, rows={self.rows}, cols={self.columns})>"
+
+
+class TableColumn(Base, UUIDPrimaryKeyMixin):
+    """Column definition within a table format."""
+
+    __tablename__ = "table_columns"
+
+    table_format_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("table_formats.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    column_index: Mapped[int] = mapped_column(Integer, nullable=False)
+    width: Mapped[float] = mapped_column(Float, nullable=False)
+    header_text: Mapped[str | None] = mapped_column(String(500), nullable=True)
+    data_type: Mapped[str] = mapped_column(String(50), default="text", nullable=False)
+    alignment: Mapped[str] = mapped_column(String(20), default="left", nullable=False)
+    font_family: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    font_size: Mapped[float | None] = mapped_column(Float, nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    table_format: Mapped[TableFormat] = relationship("TableFormat", back_populates="table_columns")
+
+    def __repr__(self) -> str:
+        return f"<TableColumn(id={self.id}, index={self.column_index}, header={self.header_text})>"
+
+
+class TableRow(Base, UUIDPrimaryKeyMixin):
+    """Row definition within a table format."""
+
+    __tablename__ = "table_rows"
+
+    table_format_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("table_formats.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    row_index: Mapped[int] = mapped_column(Integer, nullable=False)
+    height: Mapped[float] = mapped_column(Float, default=20.0, nullable=False)
+    is_header: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
+    background_color: Mapped[str | None] = mapped_column(String(50), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    table_format: Mapped[TableFormat] = relationship("TableFormat", back_populates="table_rows")
+
+    def __repr__(self) -> str:
+        return f"<TableRow(id={self.id}, index={self.row_index}, is_header={self.is_header})>"
+
+
+class Watermark(Base, UUIDPrimaryKeyMixin):
+    """Watermark definition within a document template."""
+
+    __tablename__ = "watermarks"
+
+    format_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("document_formats.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    page_number: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    text: Mapped[str | None] = mapped_column(String(500), nullable=True)
+    image_path: Mapped[str | None] = mapped_column(String(1024), nullable=True)
+    x: Mapped[float] = mapped_column(Float, nullable=False)
+    y: Mapped[float] = mapped_column(Float, nullable=False)
+    width: Mapped[float] = mapped_column(Float, nullable=False)
+    height: Mapped[float] = mapped_column(Float, nullable=False)
+    opacity: Mapped[float] = mapped_column(Float, default=0.3, nullable=False)
+    rotation: Mapped[float] = mapped_column(Float, default=0.0, nullable=False)
+    font_family: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    font_size: Mapped[float | None] = mapped_column(Float, nullable=True)
+    font_color: Mapped[str | None] = mapped_column(String(50), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    format: Mapped[DocumentFormat] = relationship("DocumentFormat", back_populates="watermarks")
+
+    def __repr__(self) -> str:
+        return f"<Watermark(id={self.id}, text={self.text})>"
+
+
+class ImageRegion(Base, UUIDPrimaryKeyMixin):
+    """Image region within a document template."""
+
+    __tablename__ = "image_regions"
+
+    format_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("document_formats.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    page_number: Mapped[int] = mapped_column(Integer, nullable=False)
+    x: Mapped[float] = mapped_column(Float, nullable=False)
+    y: Mapped[float] = mapped_column(Float, nullable=False)
+    width: Mapped[float] = mapped_column(Float, nullable=False)
+    height: Mapped[float] = mapped_column(Float, nullable=False)
+    image_path: Mapped[str | None] = mapped_column(String(1024), nullable=True)
+    image_type: Mapped[str] = mapped_column(String(50), default="figure", nullable=False)
+    is_static: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
+    field_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    format: Mapped[DocumentFormat] = relationship("DocumentFormat", back_populates="image_regions")
+
+    def __repr__(self) -> str:
+        return f"<ImageRegion(id={self.id}, type={self.image_type}, page={self.page_number})>"
+
+
+class TemplateFingerprint(Base, UUIDPrimaryKeyMixin):
+    """Layout fingerprint for template matching."""
+
+    __tablename__ = "template_fingerprints"
+
+    format_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("document_formats.id", ondelete="CASCADE"),
+        nullable=False,
+        unique=True,
+        index=True,
+    )
+    page_dimensions: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    logo_coordinates: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    header_coordinates: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    footer_coordinates: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    table_coordinates: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    cell_coordinates: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    fingerprint_hash: Mapped[str] = mapped_column(String(256), nullable=False, index=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    format: Mapped[DocumentFormat] = relationship("DocumentFormat", back_populates="fingerprint_record")
+
+    def __repr__(self) -> str:
+        return f"<TemplateFingerprint(id={self.id}, format_id={self.format_id}, hash={self.fingerprint_hash[:16]})>"

docengine/app/models/user.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+
+from sqlalchemy import Boolean, DateTime, ForeignKey, String, Table, Text, func
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from app.core.database import Base
+from app.models.base import TimestampMixin, UUIDPrimaryKeyMixin
+
+user_roles_table = Table(
+    "user_roles",
+    Base.metadata,
+    mapped_column("user_id", UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True),
+    mapped_column("role_id", UUID(as_uuid=True), ForeignKey("roles.id", ondelete="CASCADE"), primary_key=True),
+)
+
+
+class User(Base, UUIDPrimaryKeyMixin, TimestampMixin):
+    """User account model."""
+
+    __tablename__ = "users"
+
+    username: Mapped[str] = mapped_column(String(150), unique=True, nullable=False, index=True)
+    email: Mapped[str] = mapped_column(String(255), unique=True, nullable=False, index=True)
+    hashed_password: Mapped[str] = mapped_column(String(255), nullable=False)
+    full_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
+    is_superuser: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
+    last_login: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+
+    roles: Mapped[list[Role]] = relationship(
+        "Role",
+        secondary=user_roles_table,
+        back_populates="users",
+        lazy="joined",
+    )
+    refresh_tokens: Mapped[list[RefreshToken]] = relationship(
+        "RefreshToken",
+        back_populates="user",
+        cascade="all, delete-orphan",
+        lazy="dynamic",
+    )
+    audit_logs: Mapped[list[AuditLog]] = relationship(
+        "AuditLog",
+        back_populates="user",
+        lazy="dynamic",
+    )
+
+    def __repr__(self) -> str:
+        return f"<User(id={self.id}, username={self.username})>"
+
+
+class Role(Base, UUIDPrimaryKeyMixin):
+    """User role model."""
+
+    __tablename__ = "roles"
+
+    name: Mapped[str] = mapped_column(String(50), unique=True, nullable=False, index=True)
+    description: Mapped[str | None] = mapped_column(Text, nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    users: Mapped[list[User]] = relationship(
+        "User",
+        secondary=user_roles_table,
+        back_populates="roles",
+        lazy="dynamic",
+    )
+
+    def __repr__(self) -> str:
+        return f"<Role(id={self.id}, name={self.name})>"
+
+
+class RefreshToken(Base, UUIDPrimaryKeyMixin):
+    """JWT refresh token storage."""
+
+    __tablename__ = "refresh_tokens"
+
+    user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("users.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    token: Mapped[str] = mapped_column(String(512), unique=True, nullable=False, index=True)
+    expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    revoked: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+    )
+
+    user: Mapped[User] = relationship("User", back_populates="refresh_tokens")
+
+    def __repr__(self) -> str:
+        return f"<RefreshToken(id={self.id}, user_id={self.user_id}, revoked={self.revoked})>"
+
+
+class AuditLog(Base, UUIDPrimaryKeyMixin):
+    """Audit trail for user actions."""
+
+    __tablename__ = "audit_logs"
+
+    user_id: Mapped[uuid.UUID | None] = mapped_column(
+        UUID(as_uuid=True),
+        ForeignKey("users.id", ondelete="SET NULL"),
+        nullable=True,
+        index=True,
+    )
+    action: Mapped[str] = mapped_column(String(100), nullable=False, index=True)
+    resource_type: Mapped[str] = mapped_column(String(100), nullable=False, index=True)
+    resource_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    details: Mapped[dict | None] = mapped_column(type_=Text, nullable=True)
+    ip_address: Mapped[str | None] = mapped_column(String(45), nullable=True)
+    user_agent: Mapped[str | None] = mapped_column(String(512), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=func.now(),
+        server_default=func.now(),
+        nullable=False,
+        index=True,
+    )
+
+    user: Mapped[User | None] = relationship("User", back_populates="audit_logs")
+
+    def __repr__(self) -> str:
+        return f"<AuditLog(id={self.id}, action={self.action}, resource={self.resource_type})>"

docengine/app/repositories/base.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+import uuid
+from typing import Any, Generic, TypeVar
+
+from sqlalchemy import func, select
+from sqlalchemy.orm import Session
+
+from app.core.database import Base
+
+ModelType = TypeVar("ModelType", bound=Base)
+
+
+class BaseRepository(Generic[ModelType]):
+    """Base repository with common CRUD operations."""
+
+    def __init__(self, db: Session, model: type[ModelType]) -> None:
+        self.db = db
+        self.model = model
+
+    def get_by_id(self, entity_id: str | uuid.UUID) -> ModelType | None:
+        """Get an entity by its primary key."""
+        if isinstance(entity_id, str):
+            entity_id = uuid.UUID(entity_id)
+        return self.db.get(self.model, entity_id)
+
+    def get_all(
+        self,
+        offset: int = 0,
+        limit: int = 100,
+        filters: dict[str, Any] | None = None,
+        order_by: str | None = None,
+        order_desc: bool = False,
+    ) -> list[ModelType]:
+        """Get all entities with optional filtering, pagination, and ordering."""
+        query = select(self.model)
+
+        if filters:
+            for key, value in filters.items():
+                if hasattr(self.model, key) and value is not None:
+                    query = query.where(getattr(self.model, key) == value)
+
+        if order_by and hasattr(self.model, order_by):
+            col = getattr(self.model, order_by)
+            query = query.order_by(col.desc() if order_desc else col.asc())
+
+        query = query.offset(offset).limit(limit)
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def count(self, filters: dict[str, Any] | None = None) -> int:
+        """Count entities with optional filtering."""
+        query = select(func.count()).select_from(self.model)
+
+        if filters:
+            for key, value in filters.items():
+                if hasattr(self.model, key) and value is not None:
+                    query = query.where(getattr(self.model, key) == value)
+
+        result = self.db.execute(query)
+        return result.scalar_one()
+
+    def create(self, entity: ModelType) -> ModelType:
+        """Create a new entity."""
+        self.db.add(entity)
+        self.db.flush()
+        self.db.refresh(entity)
+        return entity
+
+    def create_many(self, entities: list[ModelType]) -> list[ModelType]:
+        """Create multiple entities."""
+        self.db.add_all(entities)
+        self.db.flush()
+        for entity in entities:
+            self.db.refresh(entity)
+        return entities
+
+    def update(self, entity: ModelType, update_data: dict[str, Any]) -> ModelType:
+        """Update an entity with given data."""
+        for key, value in update_data.items():
+            if hasattr(entity, key) and value is not None:
+                setattr(entity, key, value)
+        self.db.flush()
+        self.db.refresh(entity)
+        return entity
+
+    def delete(self, entity: ModelType) -> None:
+        """Delete an entity."""
+        self.db.delete(entity)
+        self.db.flush()
+
+    def delete_by_id(self, entity_id: str | uuid.UUID) -> bool:
+        """Delete an entity by its ID. Returns True if deleted."""
+        entity = self.get_by_id(entity_id)
+        if entity:
+            self.delete(entity)
+            return True
+        return False
+
+    def exists(self, entity_id: str | uuid.UUID) -> bool:
+        """Check if an entity exists by ID."""
+        if isinstance(entity_id, str):
+            entity_id = uuid.UUID(entity_id)
+        query = select(func.count()).select_from(self.model).where(self.model.id == entity_id)
+        result = self.db.execute(query)
+        return result.scalar_one() > 0
+
+    def commit(self) -> None:
+        """Commit the current transaction."""
+        self.db.commit()
+
+    def rollback(self) -> None:
+        """Rollback the current transaction."""
+        self.db.rollback()

docengine/app/repositories/document_repository.py

@@ -0,0 +1,335 @@
+from __future__ import annotations
+
+import uuid
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from app.models.document import (
+    Document,
+    DocumentImage,
+    DocumentPage,
+    DocumentTable,
+    DocumentTextBlock,
+    TemplateMatch,
+)
+from app.repositories.base import BaseRepository
+
+
+class DocumentRepository(BaseRepository[Document]):
+    """Repository for Document operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, Document)
+
+    def get_by_checksum(self, checksum: str) -> Document | None:
+        """Get document by file checksum."""
+        query = select(Document).where(Document.checksum == checksum)
+        result = self.db.execute(query)
+        return result.scalars().first()
+
+    def get_by_status(self, status: str, offset: int = 0, limit: int = 100) -> list[Document]:
+        """Get documents by processing status."""
+        query = (
+            select(Document)
+            .where(Document.status == status)
+            .order_by(Document.created_at.desc())
+            .offset(offset)
+            .limit(limit)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def get_user_documents(
+        self,
+        user_id: uuid.UUID,
+        offset: int = 0,
+        limit: int = 100,
+    ) -> list[Document]:
+        """Get documents uploaded by a specific user."""
+        query = (
+            select(Document)
+            .where(Document.uploaded_by == user_id)
+            .order_by(Document.created_at.desc())
+            .offset(offset)
+            .limit(limit)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def update_status(
+        self,
+        document_id: uuid.UUID,
+        status: str,
+        error_message: str | None = None,
+    ) -> Document | None:
+        """Update document processing status."""
+        document = self.get_by_id(document_id)
+        if document:
+            document.status = status
+            if error_message:
+                document.error_message = error_message
+            self.db.flush()
+            self.db.refresh(document)
+        return document
+
+    def get_with_pages(self, document_id: uuid.UUID) -> Document | None:
+        """Get document with all pages eagerly loaded."""
+        return self.get_by_id(document_id)
+
+    def get_pending_documents(self, limit: int = 10) -> list[Document]:
+        """Get pending documents for processing."""
+        return self.get_by_status("pending", limit=limit)
+
+
+class DocumentPageRepository(BaseRepository[DocumentPage]):
+    """Repository for DocumentPage operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, DocumentPage)
+
+    def get_document_pages(self, document_id: uuid.UUID) -> list[DocumentPage]:
+        """Get all pages for a document ordered by page number."""
+        query = (
+            select(DocumentPage)
+            .where(DocumentPage.document_id == document_id)
+            .order_by(DocumentPage.page_number)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def get_page_by_number(self, document_id: uuid.UUID, page_number: int) -> DocumentPage | None:
+        """Get a specific page by document ID and page number."""
+        query = select(DocumentPage).where(
+            DocumentPage.document_id == document_id,
+            DocumentPage.page_number == page_number,
+        )
+        result = self.db.execute(query)
+        return result.scalars().first()
+
+    def create_page(
+        self,
+        document_id: uuid.UUID,
+        page_number: int,
+        width: float,
+        height: float,
+        image_path: str | None = None,
+        text_content: str | None = None,
+    ) -> DocumentPage:
+        """Create a new document page."""
+        page = DocumentPage(
+            document_id=document_id,
+            page_number=page_number,
+            width=width,
+            height=height,
+            image_path=image_path,
+            text_content=text_content,
+        )
+        return self.create(page)
+
+
+class DocumentTextBlockRepository(BaseRepository[DocumentTextBlock]):
+    """Repository for DocumentTextBlock operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, DocumentTextBlock)
+
+    def get_page_text_blocks(self, page_id: uuid.UUID) -> list[DocumentTextBlock]:
+        """Get all text blocks for a page."""
+        query = (
+            select(DocumentTextBlock)
+            .where(DocumentTextBlock.page_id == page_id)
+            .order_by(DocumentTextBlock.sequence)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def get_by_block_type(self, page_id: uuid.UUID, block_type: str) -> list[DocumentTextBlock]:
+        """Get text blocks by type (header, footer, watermark, text)."""
+        query = (
+            select(DocumentTextBlock)
+            .where(
+                DocumentTextBlock.page_id == page_id,
+                DocumentTextBlock.block_type == block_type,
+            )
+            .order_by(DocumentTextBlock.sequence)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def create_text_block(
+        self,
+        page_id: uuid.UUID,
+        text: str,
+        x: float,
+        y: float,
+        width: float,
+        height: float,
+        confidence: float | None = None,
+        font_family: str | None = None,
+        font_size: float | None = None,
+        font_color: str | None = None,
+        font_style: str | None = None,
+        block_type: str = "text",
+        sequence: int = 0,
+    ) -> DocumentTextBlock:
+        """Create a new text block."""
+        text_block = DocumentTextBlock(
+            page_id=page_id,
+            text=text,
+            x=x,
+            y=y,
+            width=width,
+            height=height,
+            confidence=confidence,
+            font_family=font_family,
+            font_size=font_size,
+            font_color=font_color,
+            font_style=font_style,
+            block_type=block_type,
+            sequence=sequence,
+        )
+        return self.create(text_block)
+
+
+class DocumentImageRepository(BaseRepository[DocumentImage]):
+    """Repository for DocumentImage operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, DocumentImage)
+
+    def get_page_images(self, page_id: uuid.UUID) -> list[DocumentImage]:
+        """Get all images for a page."""
+        query = select(DocumentImage).where(DocumentImage.page_id == page_id)
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def create_image(
+        self,
+        page_id: uuid.UUID,
+        x: float,
+        y: float,
+        width: float,
+        height: float,
+        image_path: str,
+        image_type: str = "figure",
+    ) -> DocumentImage:
+        """Create a new document image record."""
+        image = DocumentImage(
+            page_id=page_id,
+            x=x,
+            y=y,
+            width=width,
+            height=height,
+            image_path=image_path,
+            image_type=image_type,
+        )
+        return self.create(image)
+
+
+class DocumentTableRepository(BaseRepository[DocumentTable]):
+    """Repository for DocumentTable operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, DocumentTable)
+
+    def get_page_tables(self, page_id: uuid.UUID) -> list[DocumentTable]:
+        """Get all tables for a page."""
+        query = select(DocumentTable).where(DocumentTable.page_id == page_id)
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def create_table(
+        self,
+        page_id: uuid.UUID,
+        x: float,
+        y: float,
+        width: float,
+        height: float,
+        rows: int,
+        columns: int,
+        data: dict | None = None,
+    ) -> DocumentTable:
+        """Create a new document table record."""
+        table = DocumentTable(
+            page_id=page_id,
+            x=x,
+            y=y,
+            width=width,
+            height=height,
+            rows=rows,
+            columns=columns,
+            data=data,
+        )
+        return self.create(table)
+
+
+class TemplateMatchRepository(BaseRepository[TemplateMatch]):
+    """Repository for TemplateMatch operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, TemplateMatch)
+
+    def get_document_matches(
+        self,
+        document_id: uuid.UUID,
+        min_confidence: float = 0.0,
+    ) -> list[TemplateMatch]:
+        """Get all template matches for a document."""
+        query = (
+            select(TemplateMatch)
+            .where(
+                TemplateMatch.document_id == document_id,
+                TemplateMatch.confidence_score >= min_confidence,
+            )
+            .order_by(TemplateMatch.confidence_score.desc())
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def get_selected_match(self, document_id: uuid.UUID) -> TemplateMatch | None:
+        """Get the selected template match for a document."""
+        query = select(TemplateMatch).where(
+            TemplateMatch.document_id == document_id,
+            TemplateMatch.selected.is_(True),
+        )
+        result = self.db.execute(query)
+        return result.scalars().first()
+
+    def select_match(self, match_id: uuid.UUID) -> TemplateMatch | None:
+        """Select a template match (deselecting all others for the same document)."""
+        match = self.get_by_id(match_id)
+        if not match:
+            return None
+
+        # Deselect all other matches for this document
+        query = select(TemplateMatch).where(
+            TemplateMatch.document_id == match.document_id,
+            TemplateMatch.selected.is_(True),
+        )
+        result = self.db.execute(query)
+        for existing_match in result.scalars().all():
+            existing_match.selected = False
+
+        match.selected = True
+        self.db.flush()
+        self.db.refresh(match)
+        return match
+
+    def create_match(
+        self,
+        document_id: uuid.UUID,
+        format_id: uuid.UUID,
+        confidence_score: float,
+        match_details: dict | None = None,
+        selected: bool = False,
+    ) -> TemplateMatch:
+        """Create a new template match."""
+        template_match = TemplateMatch(
+            document_id=document_id,
+            format_id=format_id,
+            confidence_score=confidence_score,
+            match_details=match_details,
+            selected=selected,
+        )
+        return self.create(template_match)

docengine/app/repositories/template_repository.py

@@ -0,0 +1,353 @@
+from __future__ import annotations
+
+import uuid
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from app.models.template import (
+    DocumentCell,
+    DocumentFormat,
+    DocumentRegion,
+    ImageRegion,
+    TableColumn,
+    TableFormat,
+    TableRow,
+    TemplateFingerprint,
+    Watermark,
+)
+from app.repositories.base import BaseRepository
+
+
+class TemplateRepository(BaseRepository[DocumentFormat]):
+    """Repository for DocumentFormat (template) operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, DocumentFormat)
+
+    def get_active_templates(self, offset: int = 0, limit: int = 100) -> list[DocumentFormat]:
+        """Get all active templates."""
+        query = (
+            select(DocumentFormat)
+            .where(DocumentFormat.is_active.is_(True))
+            .order_by(DocumentFormat.created_at.desc())
+            .offset(offset)
+            .limit(limit)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def count_active(self) -> int:
+        """Count active templates."""
+        return self.count(filters={"is_active": True})
+
+    def get_by_name(self, name: str) -> DocumentFormat | None:
+        """Get template by name."""
+        query = select(DocumentFormat).where(DocumentFormat.name == name)
+        result = self.db.execute(query)
+        return result.scalars().first()
+
+    def get_by_source_document(self, document_id: uuid.UUID) -> DocumentFormat | None:
+        """Get template generated from a specific source document."""
+        query = select(DocumentFormat).where(
+            DocumentFormat.source_document_id == document_id,
+            DocumentFormat.is_active.is_(True),
+        )
+        result = self.db.execute(query)
+        return result.scalars().first()
+
+    def create_template(
+        self,
+        name: str,
+        page_width: float,
+        page_height: float,
+        page_count: int = 1,
+        description: str | None = None,
+        margin_top: float = 72.0,
+        margin_right: float = 72.0,
+        margin_bottom: float = 72.0,
+        margin_left: float = 72.0,
+        fingerprint: dict | None = None,
+        source_document_id: uuid.UUID | None = None,
+        created_by: uuid.UUID | None = None,
+    ) -> DocumentFormat:
+        """Create a new template."""
+        template = DocumentFormat(
+            name=name,
+            page_width=page_width,
+            page_height=page_height,
+            page_count=page_count,
+            description=description,
+            margin_top=margin_top,
+            margin_right=margin_right,
+            margin_bottom=margin_bottom,
+            margin_left=margin_left,
+            fingerprint=fingerprint,
+            source_document_id=source_document_id,
+            created_by=created_by,
+        )
+        return self.create(template)
+
+    def deactivate_template(self, template_id: uuid.UUID) -> DocumentFormat | None:
+        """Soft-delete a template by deactivating it."""
+        template = self.get_by_id(template_id)
+        if template:
+            template.is_active = False
+            self.db.flush()
+            self.db.refresh(template)
+        return template
+
+    def get_all_with_fingerprints(self) -> list[DocumentFormat]:
+        """Get all active templates with their fingerprints."""
+        query = (
+            select(DocumentFormat)
+            .where(DocumentFormat.is_active.is_(True))
+            .order_by(DocumentFormat.created_at.desc())
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+
+class DocumentCellRepository(BaseRepository[DocumentCell]):
+    """Repository for DocumentCell operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, DocumentCell)
+
+    def get_template_cells(self, format_id: uuid.UUID) -> list[DocumentCell]:
+        """Get all cells for a template."""
+        query = (
+            select(DocumentCell)
+            .where(DocumentCell.format_id == format_id)
+            .order_by(DocumentCell.page_number, DocumentCell.sequence)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def get_page_cells(self, format_id: uuid.UUID, page_number: int) -> list[DocumentCell]:
+        """Get cells for a specific page of a template."""
+        query = (
+            select(DocumentCell)
+            .where(
+                DocumentCell.format_id == format_id,
+                DocumentCell.page_number == page_number,
+            )
+            .order_by(DocumentCell.sequence)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def get_dynamic_cells(self, format_id: uuid.UUID) -> list[DocumentCell]:
+        """Get all dynamic cells for a template."""
+        query = (
+            select(DocumentCell)
+            .where(
+                DocumentCell.format_id == format_id,
+                DocumentCell.is_dynamic.is_(True),
+            )
+            .order_by(DocumentCell.page_number, DocumentCell.sequence)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def create_cell(self, format_id: uuid.UUID, **kwargs) -> DocumentCell:  # noqa: ANN003
+        """Create a new cell for a template."""
+        cell = DocumentCell(format_id=format_id, **kwargs)
+        return self.create(cell)
+
+
+class DocumentRegionRepository(BaseRepository[DocumentRegion]):
+    """Repository for DocumentRegion operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, DocumentRegion)
+
+    def get_template_regions(self, format_id: uuid.UUID) -> list[DocumentRegion]:
+        """Get all regions for a template."""
+        query = (
+            select(DocumentRegion)
+            .where(DocumentRegion.format_id == format_id)
+            .order_by(DocumentRegion.page_number, DocumentRegion.sequence)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def get_regions_by_type(self, format_id: uuid.UUID, region_type: str) -> list[DocumentRegion]:
+        """Get regions of a specific type."""
+        query = (
+            select(DocumentRegion)
+            .where(
+                DocumentRegion.format_id == format_id,
+                DocumentRegion.region_type == region_type,
+            )
+            .order_by(DocumentRegion.page_number, DocumentRegion.sequence)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def create_region(self, format_id: uuid.UUID, **kwargs) -> DocumentRegion:  # noqa: ANN003
+        """Create a new region for a template."""
+        region = DocumentRegion(format_id=format_id, **kwargs)
+        return self.create(region)
+
+
+class TableFormatRepository(BaseRepository[TableFormat]):
+    """Repository for TableFormat operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, TableFormat)
+
+    def get_template_tables(self, format_id: uuid.UUID) -> list[TableFormat]:
+        """Get all table formats for a template."""
+        query = (
+            select(TableFormat)
+            .where(TableFormat.format_id == format_id)
+            .order_by(TableFormat.page_number)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def create_table_format(self, format_id: uuid.UUID, **kwargs) -> TableFormat:  # noqa: ANN003
+        """Create a new table format."""
+        table_format = TableFormat(format_id=format_id, **kwargs)
+        return self.create(table_format)
+
+
+class TableColumnRepository(BaseRepository[TableColumn]):
+    """Repository for TableColumn operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, TableColumn)
+
+    def get_table_columns(self, table_format_id: uuid.UUID) -> list[TableColumn]:
+        """Get all columns for a table format."""
+        query = (
+            select(TableColumn)
+            .where(TableColumn.table_format_id == table_format_id)
+            .order_by(TableColumn.column_index)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def create_column(self, table_format_id: uuid.UUID, **kwargs) -> TableColumn:  # noqa: ANN003
+        """Create a new table column."""
+        column = TableColumn(table_format_id=table_format_id, **kwargs)
+        return self.create(column)
+
+
+class TableRowRepository(BaseRepository[TableRow]):
+    """Repository for TableRow operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, TableRow)
+
+    def get_table_rows(self, table_format_id: uuid.UUID) -> list[TableRow]:
+        """Get all rows for a table format."""
+        query = (
+            select(TableRow)
+            .where(TableRow.table_format_id == table_format_id)
+            .order_by(TableRow.row_index)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def create_row(self, table_format_id: uuid.UUID, **kwargs) -> TableRow:  # noqa: ANN003
+        """Create a new table row."""
+        row = TableRow(table_format_id=table_format_id, **kwargs)
+        return self.create(row)
+
+
+class WatermarkRepository(BaseRepository[Watermark]):
+    """Repository for Watermark operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, Watermark)
+
+    def get_template_watermarks(self, format_id: uuid.UUID) -> list[Watermark]:
+        """Get all watermarks for a template."""
+        query = select(Watermark).where(Watermark.format_id == format_id)
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def create_watermark(self, format_id: uuid.UUID, **kwargs) -> Watermark:  # noqa: ANN003
+        """Create a new watermark."""
+        watermark = Watermark(format_id=format_id, **kwargs)
+        return self.create(watermark)
+
+
+class ImageRegionRepository(BaseRepository[ImageRegion]):
+    """Repository for ImageRegion operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, ImageRegion)
+
+    def get_template_images(self, format_id: uuid.UUID) -> list[ImageRegion]:
+        """Get all image regions for a template."""
+        query = select(ImageRegion).where(ImageRegion.format_id == format_id)
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def get_static_images(self, format_id: uuid.UUID) -> list[ImageRegion]:
+        """Get static image regions."""
+        query = select(ImageRegion).where(
+            ImageRegion.format_id == format_id,
+            ImageRegion.is_static.is_(True),
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def create_image_region(self, format_id: uuid.UUID, **kwargs) -> ImageRegion:  # noqa: ANN003
+        """Create a new image region."""
+        image_region = ImageRegion(format_id=format_id, **kwargs)
+        return self.create(image_region)
+
+
+class TemplateFingerprintRepository(BaseRepository[TemplateFingerprint]):
+    """Repository for TemplateFingerprint operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, TemplateFingerprint)
+
+    def get_by_format_id(self, format_id: uuid.UUID) -> TemplateFingerprint | None:
+        """Get fingerprint by template format ID."""
+        query = select(TemplateFingerprint).where(TemplateFingerprint.format_id == format_id)
+        result = self.db.execute(query)
+        return result.scalars().first()
+
+    def get_by_hash(self, fingerprint_hash: str) -> TemplateFingerprint | None:
+        """Get fingerprint by hash."""
+        query = select(TemplateFingerprint).where(
+            TemplateFingerprint.fingerprint_hash == fingerprint_hash
+        )
+        result = self.db.execute(query)
+        return result.scalars().first()
+
+    def get_all_fingerprints(self) -> list[TemplateFingerprint]:
+        """Get all fingerprints."""
+        query = select(TemplateFingerprint)
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def create_fingerprint(
+        self,
+        format_id: uuid.UUID,
+        fingerprint_hash: str,
+        page_dimensions: dict | None = None,
+        logo_coordinates: dict | None = None,
+        header_coordinates: dict | None = None,
+        footer_coordinates: dict | None = None,
+        table_coordinates: dict | None = None,
+        cell_coordinates: dict | None = None,
+    ) -> TemplateFingerprint:
+        """Create a new template fingerprint."""
+        fp = TemplateFingerprint(
+            format_id=format_id,
+            fingerprint_hash=fingerprint_hash,
+            page_dimensions=page_dimensions,
+            logo_coordinates=logo_coordinates,
+            header_coordinates=header_coordinates,
+            footer_coordinates=footer_coordinates,
+            table_coordinates=table_coordinates,
+            cell_coordinates=cell_coordinates,
+        )
+        return self.create(fp)

docengine/app/repositories/user_repository.py

@@ -0,0 +1,236 @@
+from __future__ import annotations
+
+import uuid
+from datetime import UTC, datetime
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from app.models.user import AuditLog, RefreshToken, Role, User, user_roles_table
+from app.repositories.base import BaseRepository
+
+
+class UserRepository(BaseRepository[User]):
+    """Repository for User operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, User)
+
+    def get_by_username(self, username: str) -> User | None:
+        """Get user by username."""
+        query = select(User).where(User.username == username)
+        result = self.db.execute(query)
+        return result.scalars().first()
+
+    def get_by_email(self, email: str) -> User | None:
+        """Get user by email."""
+        query = select(User).where(User.email == email)
+        result = self.db.execute(query)
+        return result.scalars().first()
+
+    def create_user(
+        self,
+        username: str,
+        email: str,
+        hashed_password: str,
+        full_name: str | None = None,
+        is_active: bool = True,
+        is_superuser: bool = False,
+        role_names: list[str] | None = None,
+    ) -> User:
+        """Create a new user with optional roles."""
+        user = User(
+            username=username,
+            email=email,
+            hashed_password=hashed_password,
+            full_name=full_name,
+            is_active=is_active,
+            is_superuser=is_superuser,
+        )
+
+        if role_names:
+            roles = self.get_roles_by_names(role_names)
+            user.roles = roles
+
+        return self.create(user)
+
+    def update_last_login(self, user: User) -> User:
+        """Update user's last login timestamp."""
+        user.last_login = datetime.now(UTC)
+        self.db.flush()
+        self.db.refresh(user)
+        return user
+
+    def get_roles_by_names(self, role_names: list[str]) -> list[Role]:
+        """Get roles by their names."""
+        query = select(Role).where(Role.name.in_(role_names))
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def assign_roles(self, user: User, role_names: list[str]) -> User:
+        """Assign roles to a user, replacing existing roles."""
+        roles = self.get_roles_by_names(role_names)
+        user.roles = roles
+        self.db.flush()
+        self.db.refresh(user)
+        return user
+
+    def get_active_users(self, offset: int = 0, limit: int = 100) -> list[User]:
+        """Get all active users."""
+        query = select(User).where(User.is_active.is_(True)).offset(offset).limit(limit)
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+
+class RoleRepository(BaseRepository[Role]):
+    """Repository for Role operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, Role)
+
+    def get_by_name(self, name: str) -> Role | None:
+        """Get role by name."""
+        query = select(Role).where(Role.name == name)
+        result = self.db.execute(query)
+        return result.scalars().first()
+
+    def create_role(self, name: str, description: str | None = None) -> Role:
+        """Create a new role."""
+        role = Role(name=name, description=description)
+        return self.create(role)
+
+    def get_all_roles(self) -> list[Role]:
+        """Get all roles."""
+        query = select(Role).order_by(Role.name)
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+
+class RefreshTokenRepository(BaseRepository[RefreshToken]):
+    """Repository for RefreshToken operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, RefreshToken)
+
+    def get_by_token(self, token: str) -> RefreshToken | None:
+        """Get refresh token by token string."""
+        query = select(RefreshToken).where(
+            RefreshToken.token == token,
+            RefreshToken.revoked.is_(False),
+            RefreshToken.expires_at > datetime.now(UTC),
+        )
+        result = self.db.execute(query)
+        return result.scalars().first()
+
+    def create_token(self, user_id: uuid.UUID, token: str, expires_at: datetime) -> RefreshToken:
+        """Create a new refresh token."""
+        refresh_token = RefreshToken(
+            user_id=user_id,
+            token=token,
+            expires_at=expires_at,
+        )
+        return self.create(refresh_token)
+
+    def revoke_token(self, token: str) -> bool:
+        """Revoke a refresh token."""
+        refresh_token = self.get_by_token(token)
+        if refresh_token:
+            refresh_token.revoked = True
+            self.db.flush()
+            return True
+        return False
+
+    def revoke_all_user_tokens(self, user_id: uuid.UUID) -> int:
+        """Revoke all refresh tokens for a user."""
+        query = select(RefreshToken).where(
+            RefreshToken.user_id == user_id,
+            RefreshToken.revoked.is_(False),
+        )
+        result = self.db.execute(query)
+        tokens = result.scalars().all()
+        count = 0
+        for token in tokens:
+            token.revoked = True
+            count += 1
+        self.db.flush()
+        return count
+
+    def cleanup_expired_tokens(self) -> int:
+        """Remove expired or revoked tokens."""
+        query = select(RefreshToken).where(
+            (RefreshToken.expires_at <= datetime.now(UTC)) | (RefreshToken.revoked.is_(True))
+        )
+        result = self.db.execute(query)
+        tokens = result.scalars().all()
+        count = len(tokens)
+        for token in tokens:
+            self.db.delete(token)
+        self.db.flush()
+        return count
+
+
+class AuditLogRepository(BaseRepository[AuditLog]):
+    """Repository for AuditLog operations."""
+
+    def __init__(self, db: Session) -> None:
+        super().__init__(db, AuditLog)
+
+    def log_action(
+        self,
+        action: str,
+        resource_type: str,
+        resource_id: str | None = None,
+        user_id: uuid.UUID | None = None,
+        details: str | None = None,
+        ip_address: str | None = None,
+        user_agent: str | None = None,
+    ) -> AuditLog:
+        """Create an audit log entry."""
+        audit_log = AuditLog(
+            user_id=user_id,
+            action=action,
+            resource_type=resource_type,
+            resource_id=resource_id,
+            details=details,
+            ip_address=ip_address,
+            user_agent=user_agent,
+        )
+        return self.create(audit_log)
+
+    def get_user_logs(
+        self,
+        user_id: uuid.UUID,
+        offset: int = 0,
+        limit: int = 100,
+    ) -> list[AuditLog]:
+        """Get audit logs for a specific user."""
+        query = (
+            select(AuditLog)
+            .where(AuditLog.user_id == user_id)
+            .order_by(AuditLog.created_at.desc())
+            .offset(offset)
+            .limit(limit)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())
+
+    def get_resource_logs(
+        self,
+        resource_type: str,
+        resource_id: str,
+        offset: int = 0,
+        limit: int = 100,
+    ) -> list[AuditLog]:
+        """Get audit logs for a specific resource."""
+        query = (
+            select(AuditLog)
+            .where(
+                AuditLog.resource_type == resource_type,
+                AuditLog.resource_id == resource_id,
+            )
+            .order_by(AuditLog.created_at.desc())
+            .offset(offset)
+            .limit(limit)
+        )
+        result = self.db.execute(query)
+        return list(result.scalars().all())

docengine/app/schemas/auth.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from pydantic import BaseModel, EmailStr, Field
+
+from app.schemas.common import BaseSchema
+
+
+class LoginRequest(BaseModel):
+    """Login credentials."""
+
+    username: str = Field(..., min_length=3, max_length=150)
+    password: str = Field(..., min_length=8, max_length=128)
+
+
+class RegisterRequest(BaseModel):
+    """User registration payload."""
+
+    username: str = Field(..., min_length=3, max_length=150)
+    email: EmailStr
+    password: str = Field(..., min_length=8, max_length=128)
+    full_name: str | None = Field(None, max_length=255)
+
+
+class TokenResponse(BaseSchema):
+    """JWT token pair response."""
+
+    access_token: str
+    refresh_token: str
+    token_type: str = "bearer"
+    expires_in: int
+
+
+class RefreshTokenRequest(BaseModel):
+    """Refresh token request payload."""
+
+    refresh_token: str
+
+
+class ChangePasswordRequest(BaseModel):
+    """Change password payload."""
+
+    current_password: str = Field(..., min_length=8, max_length=128)
+    new_password: str = Field(..., min_length=8, max_length=128)

docengine/app/schemas/common.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from typing import Any, Generic, TypeVar
+
+from pydantic import BaseModel, ConfigDict, Field
+
+T = TypeVar("T")
+
+
+class BaseSchema(BaseModel):
+    """Base schema with common configuration."""
+
+    model_config = ConfigDict(
+        from_attributes=True,
+        populate_by_name=True,
+        str_strip_whitespace=True,
+    )
+
+
+class PaginationParams(BaseModel):
+    """Pagination query parameters."""
+
+    page: int = Field(default=1, ge=1, description="Page number")
+    page_size: int = Field(default=20, ge=1, le=100, description="Items per page")
+
+    @property
+    def offset(self) -> int:
+        return (self.page - 1) * self.page_size
+
+
+class PaginatedResponse(BaseSchema, Generic[T]):
+    """Paginated response wrapper."""
+
+    items: list[T]
+    total: int
+    page: int
+    page_size: int
+    total_pages: int
+
+    @classmethod
+    def create(cls, items: list[T], total: int, page: int, page_size: int) -> PaginatedResponse[T]:
+        total_pages = (total + page_size - 1) // page_size if page_size > 0 else 0
+        return cls(
+            items=items,
+            total=total,
+            page=page,
+            page_size=page_size,
+            total_pages=total_pages,
+        )
+
+
+class ErrorResponse(BaseSchema):
+    """Standard error response."""
+
+    detail: str
+    error_code: str | None = None
+    errors: list[dict[str, Any]] | None = None
+    timestamp: datetime = Field(default_factory=datetime.utcnow)
+
+
+class SuccessResponse(BaseSchema):
+    """Standard success response."""
+
+    message: str
+    data: dict[str, Any] | None = None
+
+
+class HealthResponse(BaseSchema):
+    """Health check response."""
+
+    status: str
+    version: str
+    environment: str
+    database: str
+    redis: str
+    timestamp: datetime = Field(default_factory=datetime.utcnow)
+
+
+class IDResponse(BaseSchema):
+    """Response containing just an ID."""
+
+    id: uuid.UUID

docengine/app/schemas/document.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from typing import Any
+
+from pydantic import Field
+
+from app.schemas.common import BaseSchema
+
+
+class DocumentUploadResponse(BaseSchema):
+    """Response after document upload."""
+
+    id: uuid.UUID
+    filename: str
+    original_filename: str
+    content_type: str
+    file_size: int
+    checksum: str
+    status: str
+    created_at: datetime
+
+
+class TextBlockResponse(BaseSchema):
+    """Extracted text block."""
+
+    id: uuid.UUID
+    text: str
+    x: float
+    y: float
+    width: float
+    height: float
+    confidence: float | None
+    font_family: str | None
+    font_size: float | None
+    font_color: str | None
+    font_style: str | None
+    block_type: str
+    sequence: int
+
+
+class DocumentImageResponse(BaseSchema):
+    """Extracted document image."""
+
+    id: uuid.UUID
+    x: float
+    y: float
+    width: float
+    height: float
+    image_path: str
+    image_type: str
+
+
+class DocumentTableResponse(BaseSchema):
+    """Extracted document table."""
+
+    id: uuid.UUID
+    x: float
+    y: float
+    width: float
+    height: float
+    rows: int
+    columns: int
+    data: dict[str, Any] | None
+
+
+class DocumentPageResponse(BaseSchema):
+    """Document page with extracted content."""
+
+    id: uuid.UUID
+    page_number: int
+    width: float
+    height: float
+    image_path: str | None
+    text_content: str | None
+    text_blocks: list[TextBlockResponse] = Field(default_factory=list)
+    images: list[DocumentImageResponse] = Field(default_factory=list)
+    tables: list[DocumentTableResponse] = Field(default_factory=list)
+
+
+class DocumentResponse(BaseSchema):
+    """Full document response."""
+
+    id: uuid.UUID
+    filename: str
+    original_filename: str
+    content_type: str
+    file_size: int
+    checksum: str
+    storage_path: str
+    status: str
+    page_count: int | None
+    is_scanned: bool | None
+    document_metadata: dict[str, Any] | None
+    error_message: str | None
+    uploaded_by: uuid.UUID | None
+    pages: list[DocumentPageResponse] = Field(default_factory=list)
+    created_at: datetime
+    updated_at: datetime
+
+
+class DocumentListResponse(BaseSchema):
+    """Minimal document response for lists."""
+
+    id: uuid.UUID
+    original_filename: str
+    content_type: str
+    file_size: int
+    status: str
+    page_count: int | None
+    is_scanned: bool | None
+    created_at: datetime
+
+
+class TemplateMatchResponse(BaseSchema):
+    """Template match result."""
+
+    id: uuid.UUID
+    document_id: uuid.UUID
+    format_id: uuid.UUID
+    confidence_score: float
+    match_details: dict[str, Any] | None
+    selected: bool
+    template_name: str | None = None
+    created_at: datetime
+
+
+class TemplateMatchRequest(BaseSchema):
+    """Request to match a document against templates."""
+
+    document_id: uuid.UUID
+    min_confidence: float = Field(default=0.5, ge=0.0, le=1.0)
+    max_results: int = Field(default=5, ge=1, le=20)

docengine/app/schemas/template.py

@@ -0,0 +1,250 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from typing import Any
+
+from pydantic import Field
+
+from app.schemas.common import BaseSchema
+
+
+class DocumentCellResponse(BaseSchema):
+    """Document cell in a template."""
+
+    id: uuid.UUID
+    format_id: uuid.UUID
+    page_number: int
+    x: float
+    y: float
+    width: float
+    height: float
+    row_no: int
+    column_no: int
+    data_type: str
+    font_family: str | None
+    font_size: float | None
+    font_style: str | None
+    font_color: str | None
+    background_color: str | None
+    border_top: str | None
+    border_right: str | None
+    border_bottom: str | None
+    border_left: str | None
+    padding_top: float
+    padding_right: float
+    padding_bottom: float
+    padding_left: float
+    alignment: str
+    vertical_alignment: str
+    rowspan: int
+    colspan: int
+    static_text: str | None
+    field_name: str | None
+    sequence: int
+    is_dynamic: bool
+
+
+class DocumentCellCreate(BaseSchema):
+    """Create a document cell."""
+
+    page_number: int = Field(..., ge=1)
+    x: float
+    y: float
+    width: float = Field(..., gt=0)
+    height: float = Field(..., gt=0)
+    row_no: int = 0
+    column_no: int = 0
+    data_type: str = "text"
+    font_family: str | None = None
+    font_size: float | None = None
+    font_style: str | None = None
+    font_color: str | None = None
+    background_color: str | None = None
+    border_top: str | None = None
+    border_right: str | None = None
+    border_bottom: str | None = None
+    border_left: str | None = None
+    padding_top: float = 0.0
+    padding_right: float = 0.0
+    padding_bottom: float = 0.0
+    padding_left: float = 0.0
+    alignment: str = "left"
+    vertical_alignment: str = "top"
+    rowspan: int = 1
+    colspan: int = 1
+    static_text: str | None = None
+    field_name: str | None = None
+    sequence: int = 0
+    is_dynamic: bool = False
+
+
+class DocumentRegionResponse(BaseSchema):
+    """Region in a template."""
+
+    id: uuid.UUID
+    format_id: uuid.UUID
+    page_number: int
+    region_type: str
+    x: float
+    y: float
+    width: float
+    height: float
+    content: dict[str, Any] | None
+    sequence: int
+
+
+class TableColumnResponse(BaseSchema):
+    """Table column definition."""
+
+    id: uuid.UUID
+    table_format_id: uuid.UUID
+    column_index: int
+    width: float
+    header_text: str | None
+    data_type: str
+    alignment: str
+    font_family: str | None
+    font_size: float | None
+
+
+class TableRowResponse(BaseSchema):
+    """Table row definition."""
+
+    id: uuid.UUID
+    table_format_id: uuid.UUID
+    row_index: int
+    height: float
+    is_header: bool
+    background_color: str | None
+
+
+class TableFormatResponse(BaseSchema):
+    """Table format in a template."""
+
+    id: uuid.UUID
+    format_id: uuid.UUID
+    page_number: int
+    x: float
+    y: float
+    width: float
+    height: float
+    rows: int
+    columns: int
+    border_style: str
+    border_width: float
+    border_color: str
+    header_rows: int
+    table_columns: list[TableColumnResponse] = Field(default_factory=list)
+    table_rows: list[TableRowResponse] = Field(default_factory=list)
+
+
+class WatermarkResponse(BaseSchema):
+    """Watermark in a template."""
+
+    id: uuid.UUID
+    format_id: uuid.UUID
+    page_number: int | None
+    text: str | None
+    image_path: str | None
+    x: float
+    y: float
+    width: float
+    height: float
+    opacity: float
+    rotation: float
+    font_family: str | None
+    font_size: float | None
+    font_color: str | None
+
+
+class ImageRegionResponse(BaseSchema):
+    """Image region in a template."""
+
+    id: uuid.UUID
+    format_id: uuid.UUID
+    page_number: int
+    x: float
+    y: float
+    width: float
+    height: float
+    image_path: str | None
+    image_type: str
+    is_static: bool
+    field_name: str | None
+
+
+class TemplateFingerprintResponse(BaseSchema):
+    """Template fingerprint."""
+
+    id: uuid.UUID
+    format_id: uuid.UUID
+    page_dimensions: dict[str, Any] | None
+    logo_coordinates: dict[str, Any] | None
+    header_coordinates: dict[str, Any] | None
+    footer_coordinates: dict[str, Any] | None
+    table_coordinates: dict[str, Any] | None
+    cell_coordinates: dict[str, Any] | None
+    fingerprint_hash: str
+
+
+class TemplateResponse(BaseSchema):
+    """Full template response."""
+
+    id: uuid.UUID
+    name: str
+    description: str | None
+    page_width: float
+    page_height: float
+    page_count: int
+    margin_top: float
+    margin_right: float
+    margin_bottom: float
+    margin_left: float
+    fingerprint: dict[str, Any] | None
+    source_document_id: uuid.UUID | None
+    version: int
+    is_active: bool
+    created_by: uuid.UUID | None
+    cells: list[DocumentCellResponse] = Field(default_factory=list)
+    regions: list[DocumentRegionResponse] = Field(default_factory=list)
+    table_formats: list[TableFormatResponse] = Field(default_factory=list)
+    watermarks: list[WatermarkResponse] = Field(default_factory=list)
+    image_regions: list[ImageRegionResponse] = Field(default_factory=list)
+    fingerprint_record: TemplateFingerprintResponse | None = None
+    created_at: datetime
+    updated_at: datetime
+
+
+class TemplateListResponse(BaseSchema):
+    """Minimal template response for lists."""
+
+    id: uuid.UUID
+    name: str
+    description: str | None
+    page_width: float
+    page_height: float
+    page_count: int
+    version: int
+    is_active: bool
+    created_at: datetime
+    updated_at: datetime
+
+
+class TemplateRenderRequest(BaseSchema):
+    """Request to render a template to PDF."""
+
+    template_id: uuid.UUID
+    data: dict[str, Any] = Field(default_factory=dict, description="Data to populate dynamic fields")
+    output_filename: str | None = Field(None, max_length=255, description="Output filename for generated PDF")
+    images: dict[str, str] | None = Field(None, description="Mapping of field_name to image path for dynamic images")
+
+
+class TemplateRenderResponse(BaseSchema):
+    """Response after rendering a template."""
+
+    output_path: str
+    filename: str
+    file_size: int
+    page_count: int
+    rendered_at: datetime

docengine/app/schemas/user.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+
+from pydantic import EmailStr, Field
+
+from app.schemas.common import BaseSchema
+
+
+class UserBase(BaseSchema):
+    """Base user fields."""
+
+    username: str = Field(..., min_length=3, max_length=150)
+    email: EmailStr
+    full_name: str | None = Field(None, max_length=255)
+
+
+class UserCreate(UserBase):
+    """User creation payload."""
+
+    password: str = Field(..., min_length=8, max_length=128)
+    is_active: bool = True
+    is_superuser: bool = False
+    role_names: list[str] = Field(default_factory=list)
+
+
+class UserUpdate(BaseSchema):
+    """User update payload."""
+
+    email: EmailStr | None = None
+    full_name: str | None = None
+    is_active: bool | None = None
+    is_superuser: bool | None = None
+    role_names: list[str] | None = None
+
+
+class RoleResponse(BaseSchema):
+    """Role response."""
+
+    id: uuid.UUID
+    name: str
+    description: str | None
+
+
+class UserResponse(BaseSchema):
+    """Full user response."""
+
+    id: uuid.UUID
+    username: str
+    email: str
+    full_name: str | None
+    is_active: bool
+    is_superuser: bool
+    roles: list[RoleResponse] = Field(default_factory=list)
+    created_at: datetime
+    updated_at: datetime
+    last_login: datetime | None
+
+
+class UserListResponse(BaseSchema):
+    """Minimal user response for lists."""
+
+    id: uuid.UUID
+    username: str
+    email: str
+    full_name: str | None
+    is_active: bool
+    created_at: datetime

docengine/app/services/document_service.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import uuid
+
+from sqlalchemy.orm import Session
+
+from app.core.logging_config import get_logger
+from app.models.document import Document
+from app.repositories.document_repository import DocumentRepository
+from app.services.layout_service import LayoutService
+from app.services.ocr_service import OCRService
+from app.services.pdf_service import NativePDFService
+from app.services.template_service import TemplateService
+
+logger = get_logger(__name__)
+
+
+class DocumentProcessingService:
+    """Orchestrates the full document processing pipeline."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+        self.doc_repo = DocumentRepository(db)
+        self.pdf_service = NativePDFService(db)
+        self.ocr_service = OCRService(db)
+        self.layout_service = LayoutService(db)
+        self.template_service = TemplateService(db)
+
+    def process_document(self, document_id: str | uuid.UUID) -> Document:
+        """Process a document through the full pipeline."""
+        if isinstance(document_id, str):
+            document_id = uuid.UUID(document_id)
+
+        document = self.doc_repo.get_by_id(document_id)
+        if not document:
+            raise ValueError(f"Document '{document_id}' not found")
+
+        logger.info(
+            "processing_started",
+            document_id=str(document_id),
+            content_type=document.content_type,
+        )
+
+        # Update status to processing
+        self.doc_repo.update_status(document_id, "processing")
+        self.db.commit()
+
+        try:
+            # Step 1: Extract content based on document type
+            if document.content_type == "application/pdf":
+                document = self._process_pdf(document)
+            else:
+                document = self._process_image(document)
+
+            # Step 2: Analyze layout
+            layout_results = self.layout_service.analyze_document_layout(document)
+            document.document_metadata = document.document_metadata or {}
+            document.document_metadata["layout"] = layout_results
+
+            # Step 3: Generate template
+            template = self.template_service.generate_template(document)
+
+            # Step 4: Update document status
+            self.doc_repo.update_status(document_id, "completed")
+            self.db.commit()
+
+            logger.info(
+                "processing_completed",
+                document_id=str(document_id),
+                pages=document.page_count,
+                template_id=str(template.id),
+            )
+
+            return document
+
+        except Exception as e:
+            logger.exception(
+                "processing_failed",
+                document_id=str(document_id),
+                error=str(e),
+            )
+            self.doc_repo.update_status(document_id, "failed", error_message=str(e))
+            self.db.commit()
+            raise
+
+    def _process_pdf(self, document: Document) -> Document:
+        """Process a PDF document - either native or scanned."""
+        # First, try native PDF extraction
+        document = self.pdf_service.process_pdf(document)
+        self.db.flush()
+
+        # If scanned, also run OCR
+        if document.is_scanned:
+            logger.info(
+                "scanned_pdf_detected",
+                document_id=str(document.id),
+            )
+            document = self.ocr_service.process_scanned_pdf(document)
+            self.db.flush()
+
+        return document
+
+    def _process_image(self, document: Document) -> Document:
+        """Process an image document with OCR."""
+        document = self.ocr_service.process_image(document)
+        self.db.flush()
+        return document

docengine/app/services/fingerprint_service.py

@@ -0,0 +1,343 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import uuid
+from typing import Any
+
+from sqlalchemy.orm import Session
+
+from app.core.logging_config import get_logger
+from app.models.template import DocumentFormat, TemplateFingerprint
+from app.repositories.template_repository import TemplateFingerprintRepository, TemplateRepository
+
+logger = get_logger(__name__)
+
+
+class FingerprintService:
+    """Generate and manage layout fingerprints for template matching."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+        self.fingerprint_repo = TemplateFingerprintRepository(db)
+        self.template_repo = TemplateRepository(db)
+
+    def generate_fingerprint(self, template: DocumentFormat) -> TemplateFingerprint:
+        """Generate a layout fingerprint for a template."""
+        # Collect page dimensions
+        page_dimensions = {
+            "width": template.page_width,
+            "height": template.page_height,
+            "page_count": template.page_count,
+            "margins": {
+                "top": template.margin_top,
+                "right": template.margin_right,
+                "bottom": template.margin_bottom,
+                "left": template.margin_left,
+            },
+        }
+
+        # Collect logo coordinates
+        logo_coordinates = self._extract_logo_coordinates(template)
+
+        # Collect header coordinates
+        header_coordinates = self._extract_region_coordinates(template, "header")
+
+        # Collect footer coordinates
+        footer_coordinates = self._extract_region_coordinates(template, "footer")
+
+        # Collect table coordinates
+        table_coordinates = self._extract_table_coordinates(template)
+
+        # Collect cell coordinates
+        cell_coordinates = self._extract_cell_coordinates(template)
+
+        # Compute fingerprint hash
+        fingerprint_data = {
+            "page_dimensions": page_dimensions,
+            "logo_coordinates": logo_coordinates,
+            "header_coordinates": header_coordinates,
+            "footer_coordinates": footer_coordinates,
+            "table_coordinates": table_coordinates,
+            "cell_coordinates": cell_coordinates,
+        }
+        fingerprint_hash = self._compute_hash(fingerprint_data)
+
+        # Check for existing fingerprint
+        existing = self.fingerprint_repo.get_by_format_id(template.id)
+        if existing:
+            # Update existing
+            existing.page_dimensions = page_dimensions
+            existing.logo_coordinates = logo_coordinates
+            existing.header_coordinates = header_coordinates
+            existing.footer_coordinates = footer_coordinates
+            existing.table_coordinates = table_coordinates
+            existing.cell_coordinates = cell_coordinates
+            existing.fingerprint_hash = fingerprint_hash
+            self.db.flush()
+            self.db.refresh(existing)
+            return existing
+
+        # Create new fingerprint
+        fingerprint = self.fingerprint_repo.create_fingerprint(
+            format_id=template.id,
+            fingerprint_hash=fingerprint_hash,
+            page_dimensions=page_dimensions,
+            logo_coordinates=logo_coordinates,
+            header_coordinates=header_coordinates,
+            footer_coordinates=footer_coordinates,
+            table_coordinates=table_coordinates,
+            cell_coordinates=cell_coordinates,
+        )
+
+        logger.info(
+            "fingerprint_generated",
+            template_id=str(template.id),
+            hash=fingerprint_hash[:16],
+        )
+
+        return fingerprint
+
+    def _extract_logo_coordinates(self, template: DocumentFormat) -> dict[str, Any] | None:
+        """Extract logo image coordinates from template."""
+        logos = [ir for ir in template.image_regions if ir.image_type == "logo"]
+        if not logos:
+            return None
+
+        return {
+            "items": [
+                {
+                    "page": ir.page_number,
+                    "x": ir.x,
+                    "y": ir.y,
+                    "width": ir.width,
+                    "height": ir.height,
+                }
+                for ir in logos
+            ]
+        }
+
+    def _extract_region_coordinates(
+        self,
+        template: DocumentFormat,
+        region_type: str,
+    ) -> dict[str, Any] | None:
+        """Extract coordinates for a specific region type."""
+        regions = [r for r in template.regions if r.region_type == region_type]
+        if not regions:
+            return None
+
+        return {
+            "items": [
+                {
+                    "page": r.page_number,
+                    "x": r.x,
+                    "y": r.y,
+                    "width": r.width,
+                    "height": r.height,
+                }
+                for r in regions
+            ]
+        }
+
+    def _extract_table_coordinates(self, template: DocumentFormat) -> dict[str, Any] | None:
+        """Extract table coordinates from template."""
+        if not template.table_formats:
+            return None
+
+        return {
+            "items": [
+                {
+                    "page": tf.page_number,
+                    "x": tf.x,
+                    "y": tf.y,
+                    "width": tf.width,
+                    "height": tf.height,
+                    "rows": tf.rows,
+                    "columns": tf.columns,
+                }
+                for tf in template.table_formats
+            ]
+        }
+
+    def _extract_cell_coordinates(self, template: DocumentFormat) -> dict[str, Any] | None:
+        """Extract cell coordinates from template."""
+        if not template.cells:
+            return None
+
+        return {
+            "items": [
+                {
+                    "page": c.page_number,
+                    "x": c.x,
+                    "y": c.y,
+                    "width": c.width,
+                    "height": c.height,
+                    "row": c.row_no,
+                    "col": c.column_no,
+                }
+                for c in template.cells
+            ]
+        }
+
+    def _compute_hash(self, data: dict[str, Any]) -> str:
+        """Compute a deterministic hash of the fingerprint data."""
+        # Normalize coordinates to reduce sensitivity to minor variations
+        normalized = self._normalize_coordinates(data)
+        serialized = json.dumps(normalized, sort_keys=True, default=str)
+        return hashlib.sha256(serialized.encode()).hexdigest()
+
+    def _normalize_coordinates(self, data: dict[str, Any]) -> dict[str, Any]:
+        """Normalize coordinates by rounding to reduce sensitivity to small variations."""
+        if isinstance(data, dict):
+            return {k: self._normalize_coordinates(v) for k, v in data.items()}
+        elif isinstance(data, list):
+            return [self._normalize_coordinates(item) for item in data]
+        elif isinstance(data, float):
+            return round(data, 1)
+        return data
+
+    def compute_similarity(
+        self,
+        fingerprint1: TemplateFingerprint,
+        fingerprint2_data: dict[str, Any],
+    ) -> float:
+        """Compute similarity score between a stored fingerprint and new document data."""
+        scores: list[float] = []
+        weights: list[float] = []
+
+        # Page dimensions similarity (high weight)
+        dim_score = self._compare_dimensions(
+            fingerprint1.page_dimensions,
+            fingerprint2_data.get("page_dimensions"),
+        )
+        scores.append(dim_score)
+        weights.append(3.0)
+
+        # Logo coordinates similarity
+        logo_score = self._compare_coordinates(
+            fingerprint1.logo_coordinates,
+            fingerprint2_data.get("logo_coordinates"),
+        )
+        scores.append(logo_score)
+        weights.append(2.0)
+
+        # Header coordinates similarity
+        header_score = self._compare_coordinates(
+            fingerprint1.header_coordinates,
+            fingerprint2_data.get("header_coordinates"),
+        )
+        scores.append(header_score)
+        weights.append(2.0)
+
+        # Footer coordinates similarity
+        footer_score = self._compare_coordinates(
+            fingerprint1.footer_coordinates,
+            fingerprint2_data.get("footer_coordinates"),
+        )
+        scores.append(footer_score)
+        weights.append(1.5)
+
+        # Table coordinates similarity
+        table_score = self._compare_coordinates(
+            fingerprint1.table_coordinates,
+            fingerprint2_data.get("table_coordinates"),
+        )
+        scores.append(table_score)
+        weights.append(2.5)
+
+        # Cell coordinates similarity
+        cell_score = self._compare_coordinates(
+            fingerprint1.cell_coordinates,
+            fingerprint2_data.get("cell_coordinates"),
+        )
+        scores.append(cell_score)
+        weights.append(1.5)
+
+        # Weighted average
+        total_weight = sum(weights)
+        if total_weight == 0:
+            return 0.0
+
+        weighted_sum = sum(s * w for s, w in zip(scores, weights))
+        return weighted_sum / total_weight
+
+    def _compare_dimensions(
+        self,
+        dims1: dict[str, Any] | None,
+        dims2: dict[str, Any] | None,
+    ) -> float:
+        """Compare page dimensions similarity."""
+        if not dims1 or not dims2:
+            return 0.0 if (dims1 or dims2) else 1.0
+
+        width_ratio = min(dims1.get("width", 0), dims2.get("width", 0)) / max(
+            dims1.get("width", 1), dims2.get("width", 1)
+        )
+        height_ratio = min(dims1.get("height", 0), dims2.get("height", 0)) / max(
+            dims1.get("height", 1), dims2.get("height", 1)
+        )
+        page_count_match = 1.0 if dims1.get("page_count") == dims2.get("page_count") else 0.5
+
+        return (width_ratio + height_ratio + page_count_match) / 3.0
+
+    def _compare_coordinates(
+        self,
+        coords1: dict[str, Any] | None,
+        coords2: dict[str, Any] | None,
+    ) -> float:
+        """Compare coordinate sets for similarity."""
+        if not coords1 and not coords2:
+            return 1.0
+        if not coords1 or not coords2:
+            return 0.0
+
+        items1 = coords1.get("items", [])
+        items2 = coords2.get("items", [])
+
+        if not items1 and not items2:
+            return 1.0
+        if not items1 or not items2:
+            return 0.0
+
+        # Compare number of items
+        count_ratio = min(len(items1), len(items2)) / max(len(items1), len(items2))
+
+        # Compare positions of matched items
+        position_scores = []
+        for item1 in items1:
+            best_match = 0.0
+            for item2 in items2:
+                if item1.get("page") != item2.get("page"):
+                    continue
+                score = self._compute_bbox_iou(item1, item2)
+                best_match = max(best_match, score)
+            position_scores.append(best_match)
+
+        avg_position_score = sum(position_scores) / len(position_scores) if position_scores else 0.0
+
+        return (count_ratio + avg_position_score) / 2.0
+
+    def _compute_bbox_iou(self, bbox1: dict[str, Any], bbox2: dict[str, Any]) -> float:
+        """Compute Intersection over Union for two bounding boxes."""
+        x1 = max(bbox1.get("x", 0), bbox2.get("x", 0))
+        y1 = max(bbox1.get("y", 0), bbox2.get("y", 0))
+        x2 = min(
+            bbox1.get("x", 0) + bbox1.get("width", 0),
+            bbox2.get("x", 0) + bbox2.get("width", 0),
+        )
+        y2 = min(
+            bbox1.get("y", 0) + bbox1.get("height", 0),
+            bbox2.get("y", 0) + bbox2.get("height", 0),
+        )
+
+        intersection = max(0, x2 - x1) * max(0, y2 - y1)
+
+        area1 = bbox1.get("width", 0) * bbox1.get("height", 0)
+        area2 = bbox2.get("width", 0) * bbox2.get("height", 0)
+        union = area1 + area2 - intersection
+
+        if union == 0:
+            return 0.0
+
+        return intersection / union

docengine/app/services/layout_service.py

@@ -0,0 +1,317 @@
+from __future__ import annotations
+
+import uuid
+from pathlib import Path
+from typing import Any
+
+import cv2
+import numpy as np
+from sqlalchemy.orm import Session
+
+from app.core.logging_config import get_logger
+from app.models.document import Document, DocumentPage
+from app.repositories.document_repository import (
+    DocumentPageRepository,
+    DocumentTableRepository,
+    DocumentTextBlockRepository,
+)
+from app.storage.provider import get_storage_provider
+
+logger = get_logger(__name__)
+
+
+class LayoutService:
+    """Document layout analysis service using OpenCV-based detection."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+        self.storage = get_storage_provider()
+        self.page_repo = DocumentPageRepository(db)
+        self.text_block_repo = DocumentTextBlockRepository(db)
+        self.table_repo = DocumentTableRepository(db)
+
+    def analyze_document_layout(self, document: Document) -> dict[str, Any]:
+        """Analyze the layout of all pages in a document."""
+        layout_results: dict[str, Any] = {"pages": []}
+
+        for page in document.pages:
+            page_layout = self._analyze_page_layout(page)
+            layout_results["pages"].append(page_layout)
+
+        return layout_results
+
+    def _analyze_page_layout(self, page: DocumentPage) -> dict[str, Any]:
+        """Analyze layout of a single page."""
+        result: dict[str, Any] = {
+            "page_number": page.page_number,
+            "width": page.width,
+            "height": page.height,
+            "tables": [],
+            "lines": [],
+            "rectangles": [],
+            "text_regions": [],
+            "image_regions": [],
+        }
+
+        if not page.image_path:
+            return result
+
+        image_path = self.storage.get_absolute_path(page.image_path)
+        image = cv2.imread(image_path)
+        if image is None:
+            logger.warning("layout_image_read_failed", page_id=str(page.id))
+            return result
+
+        gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+
+        # Detect lines
+        result["lines"] = self._detect_lines(gray)
+
+        # Detect rectangles (potential table cells/borders)
+        result["rectangles"] = self._detect_rectangles(gray)
+
+        # Detect tables
+        tables = self._detect_tables(gray, image.shape)
+        result["tables"] = tables
+
+        # Store detected tables in the database
+        for table_data in tables:
+            self.table_repo.create_table(
+                page_id=page.id,
+                x=table_data["x"],
+                y=table_data["y"],
+                width=table_data["width"],
+                height=table_data["height"],
+                rows=table_data["rows"],
+                columns=table_data["columns"],
+                data=table_data.get("cells"),
+            )
+
+        # Detect watermarks
+        watermark = self._detect_watermark(gray, image.shape)
+        if watermark:
+            result["watermark"] = watermark
+
+        return result
+
+    def _detect_lines(self, gray: np.ndarray) -> list[dict[str, Any]]:
+        """Detect horizontal and vertical lines in the image."""
+        lines_detected: list[dict[str, Any]] = []
+
+        # Apply edge detection
+        edges = cv2.Canny(gray, 50, 150, apertureSize=3)
+
+        # Detect lines using Hough transform
+        lines = cv2.HoughLinesP(edges, 1, np.pi / 180, threshold=100, minLineLength=50, maxLineGap=10)
+
+        if lines is not None:
+            for line in lines:
+                x1, y1, x2, y2 = line[0]
+                length = np.sqrt((x2 - x1) ** 2 + (y2 - y1) ** 2)
+
+                # Classify as horizontal or vertical
+                angle = np.degrees(np.arctan2(y2 - y1, x2 - x1))
+                if abs(angle) < 5 or abs(angle - 180) < 5:
+                    orientation = "horizontal"
+                elif abs(angle - 90) < 5 or abs(angle + 90) < 5:
+                    orientation = "vertical"
+                else:
+                    orientation = "diagonal"
+
+                lines_detected.append({
+                    "x1": float(x1),
+                    "y1": float(y1),
+                    "x2": float(x2),
+                    "y2": float(y2),
+                    "length": float(length),
+                    "orientation": orientation,
+                })
+
+        return lines_detected
+
+    def _detect_rectangles(self, gray: np.ndarray) -> list[dict[str, Any]]:
+        """Detect rectangular regions in the image."""
+        rectangles: list[dict[str, Any]] = []
+
+        # Binary threshold
+        _, binary = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY_INV)
+
+        # Find contours
+        contours, _ = cv2.findContours(binary, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
+
+        for contour in contours:
+            # Approximate the contour
+            peri = cv2.arcLength(contour, True)
+            approx = cv2.approxPolyDP(contour, 0.02 * peri, True)
+
+            # If approximation has 4 vertices, it's likely a rectangle
+            if len(approx) == 4:
+                x, y, w, h = cv2.boundingRect(approx)
+                # Filter out very small or very large rectangles
+                area = w * h
+                if area > 500 and w > 10 and h > 10:
+                    rectangles.append({
+                        "x": float(x),
+                        "y": float(y),
+                        "width": float(w),
+                        "height": float(h),
+                        "area": float(area),
+                    })
+
+        return rectangles
+
+    def _detect_tables(self, gray: np.ndarray, image_shape: tuple) -> list[dict[str, Any]]:
+        """Detect table structures using morphological operations."""
+        tables: list[dict[str, Any]] = []
+        h, w = image_shape[:2]
+
+        # Binary threshold
+        _, binary = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY_INV)
+
+        # Detect horizontal lines
+        horizontal_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (max(w // 30, 1), 1))
+        horizontal = cv2.morphologyEx(binary, cv2.MORPH_OPEN, horizontal_kernel, iterations=2)
+
+        # Detect vertical lines
+        vertical_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1, max(h // 30, 1)))
+        vertical = cv2.morphologyEx(binary, cv2.MORPH_OPEN, vertical_kernel, iterations=2)
+
+        # Combine horizontal and vertical lines to find intersections
+        table_mask = cv2.add(horizontal, vertical)
+
+        # Find contours of table regions
+        contours, _ = cv2.findContours(table_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+        for contour in contours:
+            x, y, cw, ch = cv2.boundingRect(contour)
+            area = cw * ch
+
+            # Filter: table should be reasonably sized
+            if area < 5000 or cw < 50 or ch < 30:
+                continue
+
+            # Estimate rows and columns
+            rows, columns = self._estimate_table_dimensions(
+                table_mask[y:y+ch, x:x+cw], cw, ch
+            )
+
+            if rows >= 1 and columns >= 1:
+                # Extract cell contents
+                cells = self._extract_table_cells(
+                    gray[y:y+ch, x:x+cw], rows, columns, cw, ch
+                )
+
+                tables.append({
+                    "x": float(x),
+                    "y": float(y),
+                    "width": float(cw),
+                    "height": float(ch),
+                    "rows": rows,
+                    "columns": columns,
+                    "cells": cells,
+                })
+
+        return tables
+
+    def _estimate_table_dimensions(
+        self,
+        table_region: np.ndarray,
+        width: int,
+        height: int,
+    ) -> tuple[int, int]:
+        """Estimate the number of rows and columns in a table region."""
+        # Project horizontal lines
+        h_projection = np.sum(table_region, axis=1)
+        h_peaks = self._count_peaks(h_projection, height)
+
+        # Project vertical lines
+        v_projection = np.sum(table_region, axis=0)
+        v_peaks = self._count_peaks(v_projection, width)
+
+        rows = max(1, h_peaks - 1)
+        columns = max(1, v_peaks - 1)
+
+        return rows, columns
+
+    def _count_peaks(self, projection: np.ndarray, total_length: int) -> int:
+        """Count significant peaks in a projection array."""
+        if len(projection) == 0:
+            return 0
+
+        threshold = np.max(projection) * 0.3
+        above_threshold = projection > threshold
+
+        # Count transitions from below to above threshold
+        peaks = 0
+        in_peak = False
+        for val in above_threshold:
+            if val and not in_peak:
+                peaks += 1
+                in_peak = True
+            elif not val:
+                in_peak = False
+
+        return peaks
+
+    def _extract_table_cells(
+        self,
+        table_gray: np.ndarray,
+        rows: int,
+        columns: int,
+        width: int,
+        height: int,
+    ) -> dict[str, Any]:
+        """Extract cell structure data from a table region."""
+        cell_height = height / max(rows, 1)
+        cell_width = width / max(columns, 1)
+
+        cells: dict[str, Any] = {"rows": rows, "columns": columns, "data": []}
+
+        for r in range(rows):
+            row_data = []
+            for c in range(columns):
+                cell_x = int(c * cell_width)
+                cell_y = int(r * cell_height)
+                cell_w = int(cell_width)
+                cell_h = int(cell_height)
+
+                row_data.append({
+                    "row": r,
+                    "col": c,
+                    "x": cell_x,
+                    "y": cell_y,
+                    "width": cell_w,
+                    "height": cell_h,
+                })
+            cells["data"].append(row_data)
+
+        return cells
+
+    def _detect_watermark(self, gray: np.ndarray, image_shape: tuple) -> dict[str, Any] | None:
+        """Detect potential watermark regions."""
+        h, w = image_shape[:2]
+
+        # Look for semi-transparent or light text in the center region
+        center_region = gray[h // 4 : 3 * h // 4, w // 4 : 3 * w // 4]
+
+        # Apply adaptive threshold to find light text
+        _, binary = cv2.threshold(center_region, 230, 255, cv2.THRESH_BINARY)
+
+        # Find contours in the center region
+        contours, _ = cv2.findContours(binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+        for contour in contours:
+            x, y, cw, ch = cv2.boundingRect(contour)
+            area = cw * ch
+            # Watermark typically covers a significant portion of the center
+            center_area = (w // 2) * (h // 2)
+            if area > center_area * 0.1:
+                return {
+                    "x": float(x + w // 4),
+                    "y": float(y + h // 4),
+                    "width": float(cw),
+                    "height": float(ch),
+                    "detected": True,
+                }
+
+        return None

docengine/app/services/matching_service.py

@@ -0,0 +1,217 @@
+from __future__ import annotations
+
+import uuid
+from typing import Any
+
+from sqlalchemy.orm import Session
+
+from app.core.logging_config import get_logger
+from app.models.document import Document, TemplateMatch
+from app.models.template import DocumentFormat
+from app.repositories.document_repository import DocumentRepository, TemplateMatchRepository
+from app.repositories.template_repository import TemplateFingerprintRepository, TemplateRepository
+from app.services.fingerprint_service import FingerprintService
+
+logger = get_logger(__name__)
+
+
+class MatchingService:
+    """Match documents against existing templates using fingerprint comparison."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+        self.doc_repo = DocumentRepository(db)
+        self.template_repo = TemplateRepository(db)
+        self.match_repo = TemplateMatchRepository(db)
+        self.fingerprint_repo = TemplateFingerprintRepository(db)
+        self.fingerprint_service = FingerprintService(db)
+
+    def match_document(
+        self,
+        document_id: uuid.UUID,
+        min_confidence: float = 0.5,
+        max_results: int = 5,
+    ) -> list[TemplateMatch]:
+        """Match a document against all existing templates."""
+        document = self.doc_repo.get_with_pages(document_id)
+        if not document:
+            raise ValueError(f"Document '{document_id}' not found")
+
+        if not document.pages:
+            raise ValueError(f"Document '{document_id}' has no processed pages")
+
+        # Generate document fingerprint data
+        doc_fingerprint_data = self._build_document_fingerprint(document)
+
+        # Get all templates with fingerprints
+        templates = self.template_repo.get_all_with_fingerprints()
+        fingerprints = self.fingerprint_repo.get_all_fingerprints()
+
+        # Map format_id -> fingerprint
+        fp_map = {fp.format_id: fp for fp in fingerprints}
+
+        matches: list[tuple[DocumentFormat, float, dict[str, Any]]] = []
+
+        for template in templates:
+            fp = fp_map.get(template.id)
+            if not fp:
+                continue
+
+            score = self.fingerprint_service.compute_similarity(fp, doc_fingerprint_data)
+            if score >= min_confidence:
+                match_details = {
+                    "page_dimensions_score": self.fingerprint_service._compare_dimensions(
+                        fp.page_dimensions, doc_fingerprint_data.get("page_dimensions")
+                    ),
+                    "logo_score": self.fingerprint_service._compare_coordinates(
+                        fp.logo_coordinates, doc_fingerprint_data.get("logo_coordinates")
+                    ),
+                    "header_score": self.fingerprint_service._compare_coordinates(
+                        fp.header_coordinates, doc_fingerprint_data.get("header_coordinates")
+                    ),
+                    "footer_score": self.fingerprint_service._compare_coordinates(
+                        fp.footer_coordinates, doc_fingerprint_data.get("footer_coordinates")
+                    ),
+                    "table_score": self.fingerprint_service._compare_coordinates(
+                        fp.table_coordinates, doc_fingerprint_data.get("table_coordinates")
+                    ),
+                    "cell_score": self.fingerprint_service._compare_coordinates(
+                        fp.cell_coordinates, doc_fingerprint_data.get("cell_coordinates")
+                    ),
+                }
+                matches.append((template, score, match_details))
+
+        # Sort by score descending
+        matches.sort(key=lambda x: x[1], reverse=True)
+        matches = matches[:max_results]
+
+        # Store match results
+        result_matches: list[TemplateMatch] = []
+        for idx, (template, score, details) in enumerate(matches):
+            template_match = self.match_repo.create_match(
+                document_id=document_id,
+                format_id=template.id,
+                confidence_score=score,
+                match_details=details,
+                selected=(idx == 0),  # Auto-select best match
+            )
+            result_matches.append(template_match)
+
+        logger.info(
+            "document_matched",
+            document_id=str(document_id),
+            matches_found=len(result_matches),
+            best_score=result_matches[0].confidence_score if result_matches else 0.0,
+        )
+
+        return result_matches
+
+    def _build_document_fingerprint(self, document: Document) -> dict[str, Any]:
+        """Build fingerprint data from a document for comparison."""
+        first_page = document.pages[0] if document.pages else None
+
+        page_dimensions = None
+        if first_page:
+            page_dimensions = {
+                "width": first_page.width,
+                "height": first_page.height,
+                "page_count": document.page_count or len(document.pages),
+            }
+
+        # Extract logo coordinates from images
+        logo_coordinates = None
+        logos = []
+        for page in document.pages:
+            for img in page.images:
+                if img.image_type == "logo":
+                    logos.append({
+                        "page": page.page_number,
+                        "x": img.x,
+                        "y": img.y,
+                        "width": img.width,
+                        "height": img.height,
+                    })
+        if logos:
+            logo_coordinates = {"items": logos}
+
+        # Extract header coordinates
+        header_coordinates = None
+        headers = []
+        for page in document.pages:
+            header_blocks = [b for b in page.text_blocks if b.block_type == "header"]
+            if header_blocks:
+                min_x = min(b.x for b in header_blocks)
+                min_y = min(b.y for b in header_blocks)
+                max_x = max(b.x + b.width for b in header_blocks)
+                max_y = max(b.y + b.height for b in header_blocks)
+                headers.append({
+                    "page": page.page_number,
+                    "x": min_x,
+                    "y": min_y,
+                    "width": max_x - min_x,
+                    "height": max_y - min_y,
+                })
+        if headers:
+            header_coordinates = {"items": headers}
+
+        # Extract footer coordinates
+        footer_coordinates = None
+        footers = []
+        for page in document.pages:
+            footer_blocks = [b for b in page.text_blocks if b.block_type == "footer"]
+            if footer_blocks:
+                min_x = min(b.x for b in footer_blocks)
+                min_y = min(b.y for b in footer_blocks)
+                max_x = max(b.x + b.width for b in footer_blocks)
+                max_y = max(b.y + b.height for b in footer_blocks)
+                footers.append({
+                    "page": page.page_number,
+                    "x": min_x,
+                    "y": min_y,
+                    "width": max_x - min_x,
+                    "height": max_y - min_y,
+                })
+        if footers:
+            footer_coordinates = {"items": footers}
+
+        # Extract table coordinates
+        table_coordinates = None
+        tables = []
+        for page in document.pages:
+            for table in page.tables:
+                tables.append({
+                    "page": page.page_number,
+                    "x": table.x,
+                    "y": table.y,
+                    "width": table.width,
+                    "height": table.height,
+                    "rows": table.rows,
+                    "columns": table.columns,
+                })
+        if tables:
+            table_coordinates = {"items": tables}
+
+        # Extract cell coordinates from text blocks
+        cell_coordinates = None
+        cells = []
+        for page in document.pages:
+            for block in page.text_blocks:
+                if block.block_type == "text":
+                    cells.append({
+                        "page": page.page_number,
+                        "x": block.x,
+                        "y": block.y,
+                        "width": block.width,
+                        "height": block.height,
+                    })
+        if cells:
+            cell_coordinates = {"items": cells}
+
+        return {
+            "page_dimensions": page_dimensions,
+            "logo_coordinates": logo_coordinates,
+            "header_coordinates": header_coordinates,
+            "footer_coordinates": footer_coordinates,
+            "table_coordinates": table_coordinates,
+            "cell_coordinates": cell_coordinates,
+        }

docengine/app/services/ocr_service.py

@@ -0,0 +1,210 @@
+from __future__ import annotations
+
+import uuid
+from pathlib import Path
+
+import cv2
+import numpy as np
+from paddleocr import PaddleOCR
+from sqlalchemy.orm import Session
+
+from app.core.config import settings
+from app.core.logging_config import get_logger
+from app.models.document import Document, DocumentPage
+from app.repositories.document_repository import (
+    DocumentPageRepository,
+    DocumentRepository,
+    DocumentTextBlockRepository,
+)
+from app.storage.provider import get_storage_provider
+
+logger = get_logger(__name__)
+
+_ocr_instance: PaddleOCR | None = None
+
+
+def get_ocr_engine() -> PaddleOCR:
+    """Get or create singleton PaddleOCR instance."""
+    global _ocr_instance
+    if _ocr_instance is None:
+        _ocr_instance = PaddleOCR(
+            use_angle_cls=True,
+            lang=settings.ocr_language,
+            use_gpu=settings.ocr_use_gpu,
+            show_log=False,
+            det_db_thresh=0.3,
+            det_db_box_thresh=0.5,
+            rec_batch_num=6,
+        )
+    return _ocr_instance
+
+
+class OCRService:
+    """OCR processing service using PaddleOCR for scanned documents."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+        self.storage = get_storage_provider()
+        self.doc_repo = DocumentRepository(db)
+        self.page_repo = DocumentPageRepository(db)
+        self.text_block_repo = DocumentTextBlockRepository(db)
+
+    def process_image(self, document: Document) -> Document:
+        """Process a scanned image document with OCR."""
+        file_path = self.storage.get_absolute_path(document.storage_path)
+        image = cv2.imread(file_path)
+        if image is None:
+            raise ValueError(f"Failed to read image: {file_path}")
+
+        height, width = image.shape[:2]
+        document.page_count = 1
+        document.is_scanned = True
+
+        # Save page image
+        image_filename = f"{document.id}_page_1.png"
+        image_bytes = cv2.imencode(".png", image)[1].tobytes()
+        image_path = self.storage.save_file(image_bytes, "images", image_filename)
+
+        # Create page record
+        doc_page = self.page_repo.create_page(
+            document_id=document.id,
+            page_number=1,
+            width=float(width),
+            height=float(height),
+            image_path=image_path,
+        )
+
+        # Run OCR
+        self._run_ocr_on_page(doc_page, file_path)
+
+        return document
+
+    def process_scanned_pdf(self, document: Document) -> Document:
+        """Process a scanned PDF document - convert pages to images and OCR each."""
+        file_path = self.storage.get_absolute_path(document.storage_path)
+
+        try:
+            from pdf2image import convert_from_path
+            images = convert_from_path(file_path, dpi=300)
+        except Exception as e:
+            logger.error("pdf_to_image_failed", document_id=str(document.id), error=str(e))
+            raise
+
+        document.page_count = len(images)
+        document.is_scanned = True
+
+        for page_num, pil_image in enumerate(images, start=1):
+            # Convert PIL to OpenCV format
+            np_image = np.array(pil_image)
+            cv_image = cv2.cvtColor(np_image, cv2.COLOR_RGB2BGR)
+            height, width = cv_image.shape[:2]
+
+            # Save page image
+            image_filename = f"{document.id}_page_{page_num}.png"
+            image_bytes = cv2.imencode(".png", cv_image)[1].tobytes()
+            image_path = self.storage.save_file(image_bytes, "images", image_filename)
+
+            # Create page record
+            doc_page = self.page_repo.create_page(
+                document_id=document.id,
+                page_number=page_num,
+                width=float(width),
+                height=float(height),
+                image_path=image_path,
+            )
+
+            # Run OCR on saved image
+            temp_path = self.storage.get_absolute_path(image_path)
+            self._run_ocr_on_page(doc_page, temp_path)
+
+        return document
+
+    def _run_ocr_on_page(self, doc_page: DocumentPage, image_path: str) -> None:
+        """Run PaddleOCR on a single page image and store results."""
+        ocr = get_ocr_engine()
+
+        try:
+            results = ocr.ocr(image_path, cls=True)
+        except Exception as e:
+            logger.error("ocr_failed", page_id=str(doc_page.id), error=str(e))
+            return
+
+        if not results or not results[0]:
+            logger.info("ocr_no_results", page_id=str(doc_page.id))
+            return
+
+        full_text_parts = []
+        sequence = 0
+
+        for line in results[0]:
+            if not line or len(line) < 2:
+                continue
+
+            bbox_points = line[0]  # List of 4 corner points
+            text_info = line[1]    # (text, confidence)
+
+            text = text_info[0] if isinstance(text_info, (list, tuple)) else str(text_info)
+            confidence = float(text_info[1]) if isinstance(text_info, (list, tuple)) and len(text_info) > 1 else 0.0
+
+            if not text.strip():
+                continue
+
+            # Convert bbox points to x, y, width, height
+            xs = [p[0] for p in bbox_points]
+            ys = [p[1] for p in bbox_points]
+            x = min(xs)
+            y = min(ys)
+            width = max(xs) - x
+            height = max(ys) - y
+
+            # Determine block type based on position
+            block_type = self._classify_text_block(
+                y, height, doc_page.height, text
+            )
+
+            self.text_block_repo.create_text_block(
+                page_id=doc_page.id,
+                text=text,
+                x=x,
+                y=y,
+                width=width,
+                height=height,
+                confidence=confidence,
+                block_type=block_type,
+                sequence=sequence,
+            )
+            full_text_parts.append(text)
+            sequence += 1
+
+        # Update page text content
+        doc_page.text_content = "\n".join(full_text_parts)
+
+    def _classify_text_block(
+        self,
+        y: float,
+        height: float,
+        page_height: float,
+        text: str,
+    ) -> str:
+        """Classify a text block as header, footer, watermark, or regular text."""
+        if page_height <= 0:
+            return "text"
+
+        relative_y = y / page_height
+
+        # Header: top 10%
+        if relative_y < 0.10:
+            return "header"
+
+        # Footer: bottom 10%
+        if relative_y > 0.90:
+            return "footer"
+
+        # Watermark detection heuristic: large text in center
+        if 0.3 < relative_y < 0.7 and height > page_height * 0.05:
+            # Check for common watermark words
+            watermark_keywords = {"confidential", "draft", "copy", "sample", "watermark", "void"}
+            if text.strip().lower() in watermark_keywords:
+                return "watermark"
+
+        return "text"

docengine/app/services/pdf_service.py

@@ -0,0 +1,292 @@
+from __future__ import annotations
+
+import uuid
+from pathlib import Path
+
+import cv2
+import fitz  # PyMuPDF
+import numpy as np
+from sqlalchemy.orm import Session
+
+from app.core.config import settings
+from app.core.logging_config import get_logger
+from app.models.document import Document, DocumentImage, DocumentPage, DocumentTable, DocumentTextBlock
+from app.repositories.document_repository import (
+    DocumentImageRepository,
+    DocumentPageRepository,
+    DocumentRepository,
+    DocumentTableRepository,
+    DocumentTextBlockRepository,
+)
+from app.storage.provider import get_storage_provider
+
+logger = get_logger(__name__)
+
+
+class NativePDFService:
+    """Extract text, fonts, images, and layout from native (non-scanned) PDFs using PyMuPDF."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+        self.storage = get_storage_provider()
+        self.doc_repo = DocumentRepository(db)
+        self.page_repo = DocumentPageRepository(db)
+        self.text_block_repo = DocumentTextBlockRepository(db)
+        self.image_repo = DocumentImageRepository(db)
+        self.table_repo = DocumentTableRepository(db)
+
+    def process_pdf(self, document: Document) -> Document:
+        """Process a native PDF document, extracting all content."""
+        file_path = self.storage.get_absolute_path(document.storage_path)
+
+        try:
+            pdf_doc = fitz.open(file_path)
+        except Exception as e:
+            logger.error("pdf_open_failed", document_id=str(document.id), error=str(e))
+            raise
+
+        document.page_count = len(pdf_doc)
+        document.is_scanned = self._is_scanned_pdf(pdf_doc)
+
+        for page_num in range(len(pdf_doc)):
+            page = pdf_doc[page_num]
+            self._process_page(document, page, page_num + 1)
+
+        pdf_doc.close()
+        return document
+
+    def _is_scanned_pdf(self, pdf_doc: fitz.Document) -> bool:
+        """Determine if a PDF is scanned (image-based) or native."""
+        total_text_chars = 0
+        total_images = 0
+        for page_num in range(min(len(pdf_doc), 3)):
+            page = pdf_doc[page_num]
+            text = page.get_text("text")
+            total_text_chars += len(text.strip())
+            total_images += len(page.get_images(full=True))
+
+        # If very little text but has images, likely scanned
+        if total_text_chars < 50 and total_images > 0:
+            return True
+        return False
+
+    def _process_page(self, document: Document, page: fitz.Page, page_number: int) -> DocumentPage:
+        """Process a single PDF page."""
+        rect = page.rect
+        width = rect.width
+        height = rect.height
+
+        # Save page as image for reference
+        pix = page.get_pixmap(dpi=150)
+        image_filename = f"{document.id}_page_{page_number}.png"
+        image_data = pix.tobytes("png")
+        image_path = self.storage.save_file(image_data, "images", image_filename)
+
+        # Get full text content
+        text_content = page.get_text("text")
+
+        doc_page = self.page_repo.create_page(
+            document_id=document.id,
+            page_number=page_number,
+            width=width,
+            height=height,
+            image_path=image_path,
+            text_content=text_content,
+        )
+
+        # Extract text blocks with font information
+        self._extract_text_blocks(doc_page, page)
+
+        # Extract images
+        self._extract_images(doc_page, page, document)
+
+        # Detect headers and footers
+        self._detect_headers_footers(doc_page, page)
+
+        return doc_page
+
+    def _extract_text_blocks(self, doc_page: DocumentPage, page: fitz.Page) -> None:
+        """Extract text blocks with positioning and font information."""
+        blocks = page.get_text("dict", flags=fitz.TEXT_PRESERVE_WHITESPACE)["blocks"]
+        sequence = 0
+
+        for block in blocks:
+            if block["type"] != 0:  # Skip non-text blocks
+                continue
+
+            block_text_parts = []
+            font_info = {"family": None, "size": None, "color": None, "style": None}
+
+            for line in block.get("lines", []):
+                for span in line.get("spans", []):
+                    text = span.get("text", "").strip()
+                    if text:
+                        block_text_parts.append(text)
+                        # Capture font info from the first non-empty span
+                        if font_info["family"] is None:
+                            font_info["family"] = span.get("font", None)
+                            font_info["size"] = span.get("size", None)
+                            color_int = span.get("color", 0)
+                            font_info["color"] = f"#{color_int:06x}" if isinstance(color_int, int) else None
+                            flags = span.get("flags", 0)
+                            styles = []
+                            if flags & 1:
+                                styles.append("superscript")
+                            if flags & 2:
+                                styles.append("italic")
+                            if flags & 4:
+                                styles.append("serif")
+                            if flags & 8:
+                                styles.append("monospace")
+                            if flags & 16:
+                                styles.append("bold")
+                            font_info["style"] = ",".join(styles) if styles else "regular"
+
+            full_text = " ".join(block_text_parts)
+            if not full_text.strip():
+                continue
+
+            bbox = block["bbox"]
+            self.text_block_repo.create_text_block(
+                page_id=doc_page.id,
+                text=full_text,
+                x=bbox[0],
+                y=bbox[1],
+                width=bbox[2] - bbox[0],
+                height=bbox[3] - bbox[1],
+                font_family=font_info["family"],
+                font_size=font_info["size"],
+                font_color=font_info["color"],
+                font_style=font_info["style"],
+                block_type="text",
+                sequence=sequence,
+            )
+            sequence += 1
+
+    def _extract_images(self, doc_page: DocumentPage, page: fitz.Page, document: Document) -> None:
+        """Extract embedded images from a PDF page."""
+        image_list = page.get_images(full=True)
+
+        for img_index, img_info in enumerate(image_list):
+            xref = img_info[0]
+            try:
+                base_image = page.parent.extract_image(xref)
+                if not base_image:
+                    continue
+
+                image_bytes = base_image["image"]
+                ext = base_image.get("ext", "png")
+                img_filename = f"{document.id}_page_{doc_page.page_number}_img_{img_index}.{ext}"
+                img_storage_path = self.storage.save_file(image_bytes, "images", img_filename)
+
+                # Try to get image position on page
+                img_rects = page.get_image_rects(xref)
+                if img_rects:
+                    rect = img_rects[0]
+                    x, y, x1, y1 = rect.x0, rect.y0, rect.x1, rect.y1
+                else:
+                    x, y, x1, y1 = 0, 0, base_image.get("width", 100), base_image.get("height", 100)
+
+                # Determine image type based on position
+                page_height = doc_page.height
+                page_width = doc_page.width
+                image_type = self._classify_image_type(x, y, x1, y1, page_width, page_height)
+
+                self.image_repo.create_image(
+                    page_id=doc_page.id,
+                    x=x,
+                    y=y,
+                    width=x1 - x,
+                    height=y1 - y,
+                    image_path=img_storage_path,
+                    image_type=image_type,
+                )
+
+            except Exception as e:
+                logger.warning(
+                    "image_extraction_failed",
+                    page_id=str(doc_page.id),
+                    img_index=img_index,
+                    error=str(e),
+                )
+
+    def _classify_image_type(
+        self,
+        x: float,
+        y: float,
+        x1: float,
+        y1: float,
+        page_width: float,
+        page_height: float,
+    ) -> str:
+        """Classify an image as logo, figure, background, or stamp based on position and size."""
+        width = x1 - x
+        height = y1 - y
+        area_ratio = (width * height) / (page_width * page_height) if page_width > 0 and page_height > 0 else 0
+
+        # Background: covers most of the page
+        if area_ratio > 0.8:
+            return "background"
+
+        # Logo: small image in top portion
+        if y < page_height * 0.15 and area_ratio < 0.05:
+            return "logo"
+
+        # Stamp: small image in bottom-right
+        if x > page_width * 0.6 and y > page_height * 0.7 and area_ratio < 0.05:
+            return "stamp"
+
+        return "figure"
+
+    def _detect_headers_footers(self, doc_page: DocumentPage, page: fitz.Page) -> None:
+        """Detect header and footer regions based on vertical position."""
+        page_height = page.rect.height
+        header_threshold = page_height * 0.1
+        footer_threshold = page_height * 0.9
+
+        blocks = page.get_text("dict")["blocks"]
+        header_seq = 0
+        footer_seq = 0
+
+        for block in blocks:
+            if block["type"] != 0:
+                continue
+
+            bbox = block["bbox"]
+            block_y = bbox[1]
+
+            text_parts = []
+            for line in block.get("lines", []):
+                for span in line.get("spans", []):
+                    t = span.get("text", "").strip()
+                    if t:
+                        text_parts.append(t)
+
+            full_text = " ".join(text_parts)
+            if not full_text.strip():
+                continue
+
+            if block_y < header_threshold:
+                self.text_block_repo.create_text_block(
+                    page_id=doc_page.id,
+                    text=full_text,
+                    x=bbox[0],
+                    y=bbox[1],
+                    width=bbox[2] - bbox[0],
+                    height=bbox[3] - bbox[1],
+                    block_type="header",
+                    sequence=header_seq,
+                )
+                header_seq += 1
+            elif block_y > footer_threshold:
+                self.text_block_repo.create_text_block(
+                    page_id=doc_page.id,
+                    text=full_text,
+                    x=bbox[0],
+                    y=bbox[1],
+                    width=bbox[2] - bbox[0],
+                    height=bbox[3] - bbox[1],
+                    block_type="footer",
+                    sequence=footer_seq,
+                )
+                footer_seq += 1

docengine/app/services/reconstruction_service.py

@@ -0,0 +1,455 @@
+from __future__ import annotations
+
+import os
+import uuid
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from reportlab.lib import colors
+from reportlab.lib.pagesizes import letter
+from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
+from reportlab.lib.units import inch, mm
+from reportlab.pdfbase import pdfmetrics
+from reportlab.pdfbase.ttfonts import TTFont
+from reportlab.platypus import (
+    BaseDocTemplate,
+    Frame,
+    Image,
+    NextPageTemplate,
+    PageBreak,
+    PageTemplate,
+    Paragraph,
+    SimpleDocTemplate,
+    Spacer,
+    Table,
+    TableStyle,
+)
+from sqlalchemy.orm import Session
+
+from app.core.logging_config import get_logger
+from app.models.template import DocumentFormat
+from app.schemas.template import TemplateRenderResponse
+from app.storage.provider import get_storage_provider
+
+logger = get_logger(__name__)
+
+
+class ReconstructionService:
+    """Reconstruct documents from templates using ReportLab."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+        self.storage = get_storage_provider()
+        self.styles = getSampleStyleSheet()
+        self._register_fonts()
+
+    def _register_fonts(self) -> None:
+        """Register additional fonts if available."""
+        # ReportLab includes Helvetica, Times-Roman, Courier by default
+        # Custom fonts can be registered here
+        pass
+
+    def render_template(
+        self,
+        template: DocumentFormat,
+        data: dict[str, Any],
+        output_filename: str | None = None,
+        images: dict[str, str] | None = None,
+    ) -> TemplateRenderResponse:
+        """Render a template to PDF with supplied data."""
+        if output_filename is None:
+            output_filename = f"{template.name}_{uuid.uuid4().hex[:8]}.pdf"
+
+        if not output_filename.endswith(".pdf"):
+            output_filename += ".pdf"
+
+        # Determine output path
+        output_storage_path = f"rendered/{output_filename}"
+        absolute_output_path = self.storage.get_absolute_path(output_storage_path)
+
+        # Ensure the rendered directory exists
+        os.makedirs(os.path.dirname(absolute_output_path), exist_ok=True)
+
+        # Build the PDF
+        self._build_pdf(template, data, absolute_output_path, images)
+
+        file_size = os.path.getsize(absolute_output_path)
+
+        logger.info(
+            "template_rendered",
+            template_id=str(template.id),
+            output=output_filename,
+            size=file_size,
+        )
+
+        return TemplateRenderResponse(
+            output_path=output_storage_path,
+            filename=output_filename,
+            file_size=file_size,
+            page_count=template.page_count,
+            rendered_at=datetime.now(UTC),
+        )
+
+    def _build_pdf(
+        self,
+        template: DocumentFormat,
+        data: dict[str, Any],
+        output_path: str,
+        images: dict[str, str] | None = None,
+    ) -> None:
+        """Build a PDF document from template definition."""
+        page_width = template.page_width
+        page_height = template.page_height
+
+        doc = SimpleDocTemplate(
+            output_path,
+            pagesize=(page_width, page_height),
+            topMargin=template.margin_top,
+            rightMargin=template.margin_right,
+            bottomMargin=template.margin_bottom,
+            leftMargin=template.margin_left,
+        )
+
+        # Build story (content elements)
+        story: list[Any] = []
+
+        for page_num in range(1, template.page_count + 1):
+            if page_num > 1:
+                story.append(PageBreak())
+
+            # Add page content
+            page_elements = self._build_page_content(template, page_num, data, images)
+            story.extend(page_elements)
+
+        # Build with watermark/header/footer callbacks
+        def on_page(canvas, doc_obj):  # noqa: ANN001, ANN202
+            self._draw_watermarks(canvas, template, doc_obj.page)
+            self._draw_headers_footers(canvas, template, doc_obj.page, page_width, page_height)
+
+        def on_page_later(canvas, doc_obj):  # noqa: ANN001, ANN202
+            self._draw_watermarks(canvas, template, doc_obj.page)
+            self._draw_headers_footers(canvas, template, doc_obj.page, page_width, page_height)
+
+        doc.build(story, onFirstPage=on_page, onLaterPages=on_page_later)
+
+    def _build_page_content(
+        self,
+        template: DocumentFormat,
+        page_number: int,
+        data: dict[str, Any],
+        images: dict[str, str] | None = None,
+    ) -> list[Any]:
+        """Build content elements for a specific page."""
+        elements: list[Any] = []
+
+        # Get cells for this page, sorted by sequence
+        page_cells = sorted(
+            [c for c in template.cells if c.page_number == page_number],
+            key=lambda c: c.sequence,
+        )
+
+        # Get table formats for this page
+        page_tables = [t for t in template.table_formats if t.page_number == page_number]
+
+        # Get image regions for this page
+        page_images = [i for i in template.image_regions if i.page_number == page_number]
+
+        # Add static and dynamic text cells
+        for cell in page_cells:
+            text = self._resolve_cell_text(cell, data)
+            if text:
+                style = self._create_cell_style(cell)
+                para = Paragraph(text, style)
+                elements.append(para)
+                elements.append(Spacer(1, 2))
+
+        # Add tables
+        for table_format in page_tables:
+            table_element = self._build_table(table_format, data)
+            if table_element:
+                elements.append(table_element)
+                elements.append(Spacer(1, 6))
+
+        # Add images
+        for img_region in page_images:
+            img_element = self._build_image(img_region, images)
+            if img_element:
+                elements.append(img_element)
+                elements.append(Spacer(1, 6))
+
+        if not elements:
+            elements.append(Spacer(1, 12))
+
+        return elements
+
+    def _resolve_cell_text(self, cell: Any, data: dict[str, Any]) -> str:
+        """Resolve cell text from static content or dynamic data."""
+        if cell.is_dynamic and cell.field_name:
+            value = data.get(cell.field_name, "")
+            return str(value) if value else ""
+        return cell.static_text or ""
+
+    def _create_cell_style(self, cell: Any) -> ParagraphStyle:
+        """Create a ReportLab paragraph style from cell properties."""
+        font_name = "Helvetica"
+        if cell.font_family:
+            family = cell.font_family.lower()
+            if "times" in family or "serif" in family:
+                font_name = "Times-Roman"
+            elif "courier" in family or "mono" in family:
+                font_name = "Courier"
+
+        font_size = cell.font_size or 10
+        if cell.font_style and "bold" in (cell.font_style or ""):
+            if font_name == "Helvetica":
+                font_name = "Helvetica-Bold"
+            elif font_name == "Times-Roman":
+                font_name = "Times-Bold"
+            elif font_name == "Courier":
+                font_name = "Courier-Bold"
+
+        text_color = colors.black
+        if cell.font_color:
+            try:
+                text_color = colors.HexColor(cell.font_color)
+            except (ValueError, TypeError):
+                text_color = colors.black
+
+        alignment_map = {"left": 0, "center": 1, "right": 2, "justify": 4}
+        alignment = alignment_map.get(cell.alignment, 0)
+
+        style = ParagraphStyle(
+            name=f"cell_{cell.id}",
+            parent=self.styles["Normal"],
+            fontName=font_name,
+            fontSize=font_size,
+            textColor=text_color,
+            alignment=alignment,
+            leading=font_size * 1.2,
+            spaceBefore=cell.padding_top,
+            spaceAfter=cell.padding_bottom,
+            leftIndent=cell.padding_left,
+            rightIndent=cell.padding_right,
+        )
+
+        return style
+
+    def _build_table(self, table_format: Any, data: dict[str, Any]) -> Table | None:
+        """Build a ReportLab table from a table format definition."""
+        rows = table_format.rows
+        columns = table_format.columns
+
+        if rows <= 0 or columns <= 0:
+            return None
+
+        # Build table data
+        table_data: list[list[str]] = []
+
+        # Header row
+        if table_format.table_columns:
+            header_row = [col.header_text or f"Col {col.column_index + 1}" for col in table_format.table_columns]
+            table_data.append(header_row)
+        else:
+            table_data.append([f"Column {i + 1}" for i in range(columns)])
+
+        # Data rows from supplied data
+        table_field_name = f"table_{table_format.id}"
+        table_rows_data = data.get(table_field_name, data.get("table_data", []))
+
+        if isinstance(table_rows_data, list):
+            for row_data in table_rows_data:
+                if isinstance(row_data, list):
+                    # Pad or trim to match column count
+                    row = row_data[:columns]
+                    while len(row) < columns:
+                        row.append("")
+                    table_data.append([str(v) for v in row])
+                elif isinstance(row_data, dict):
+                    row = []
+                    for col in table_format.table_columns:
+                        key = col.header_text or f"col_{col.column_index}"
+                        row.append(str(row_data.get(key, "")))
+                    table_data.append(row)
+
+        # If no data rows, add empty rows
+        if len(table_data) <= 1:
+            for _ in range(max(rows - 1, 1)):
+                table_data.append([""] * columns)
+
+        # Determine column widths
+        col_widths = []
+        if table_format.table_columns:
+            col_widths = [col.width for col in table_format.table_columns]
+        else:
+            col_width = table_format.width / columns
+            col_widths = [col_width] * columns
+
+        # Build table
+        table = Table(table_data, colWidths=col_widths)
+
+        # Apply table style
+        border_color = colors.black
+        if table_format.border_color:
+            try:
+                border_color = colors.HexColor(table_format.border_color)
+            except (ValueError, TypeError):
+                pass
+
+        style_commands = [
+            ("GRID", (0, 0), (-1, -1), table_format.border_width, border_color),
+            ("FONTNAME", (0, 0), (-1, 0), "Helvetica-Bold"),
+            ("FONTSIZE", (0, 0), (-1, -1), 9),
+            ("ALIGN", (0, 0), (-1, -1), "LEFT"),
+            ("VALIGN", (0, 0), (-1, -1), "MIDDLE"),
+            ("TOPPADDING", (0, 0), (-1, -1), 4),
+            ("BOTTOMPADDING", (0, 0), (-1, -1), 4),
+            ("LEFTPADDING", (0, 0), (-1, -1), 4),
+            ("RIGHTPADDING", (0, 0), (-1, -1), 4),
+        ]
+
+        # Header row background
+        if table_format.table_rows:
+            for row in table_format.table_rows:
+                if row.is_header and row.background_color:
+                    try:
+                        bg_color = colors.HexColor(row.background_color)
+                        style_commands.append(
+                            ("BACKGROUND", (0, row.row_index), (-1, row.row_index), bg_color)
+                        )
+                    except (ValueError, TypeError):
+                        pass
+        else:
+            style_commands.append(("BACKGROUND", (0, 0), (-1, 0), colors.HexColor("#E0E0E0")))
+
+        table.setStyle(TableStyle(style_commands))
+
+        return table
+
+    def _build_image(self, img_region: Any, images: dict[str, str] | None = None) -> Image | None:
+        """Build a ReportLab image from an image region definition."""
+        image_path = None
+
+        # Check dynamic images first
+        if not img_region.is_static and img_region.field_name and images:
+            image_path = images.get(img_region.field_name)
+
+        # Fall back to stored image
+        if not image_path and img_region.image_path:
+            try:
+                image_path = self.storage.get_absolute_path(img_region.image_path)
+            except Exception:
+                image_path = None
+
+        if not image_path or not os.path.exists(image_path):
+            return None
+
+        try:
+            img = Image(image_path, width=img_region.width, height=img_region.height)
+            return img
+        except Exception as e:
+            logger.warning("image_build_failed", error=str(e), path=image_path)
+            return None
+
+    def _draw_watermarks(self, canvas: Any, template: DocumentFormat, current_page: int) -> None:
+        """Draw watermarks on the canvas."""
+        for watermark in template.watermarks:
+            # Apply to all pages if page_number is None, or specific page
+            if watermark.page_number is not None and watermark.page_number != current_page:
+                continue
+
+            canvas.saveState()
+
+            # Set opacity
+            canvas.setFillAlpha(watermark.opacity)
+
+            if watermark.text:
+                # Text watermark
+                font_name = "Helvetica"
+                if watermark.font_family:
+                    family = watermark.font_family.lower()
+                    if "times" in family:
+                        font_name = "Times-Roman"
+                    elif "courier" in family:
+                        font_name = "Courier"
+
+                font_size = watermark.font_size or 48
+
+                if watermark.font_color:
+                    try:
+                        canvas.setFillColor(colors.HexColor(watermark.font_color))
+                    except (ValueError, TypeError):
+                        canvas.setFillColor(colors.grey)
+                else:
+                    canvas.setFillColor(colors.grey)
+
+                canvas.setFont(font_name, font_size)
+
+                # Position and rotate
+                canvas.translate(
+                    watermark.x + watermark.width / 2,
+                    watermark.y + watermark.height / 2,
+                )
+                canvas.rotate(watermark.rotation)
+                canvas.drawCentredString(0, 0, watermark.text)
+
+            elif watermark.image_path:
+                # Image watermark
+                try:
+                    img_path = self.storage.get_absolute_path(watermark.image_path)
+                    if os.path.exists(img_path):
+                        canvas.drawImage(
+                            img_path,
+                            watermark.x,
+                            watermark.y,
+                            width=watermark.width,
+                            height=watermark.height,
+                            mask="auto",
+                        )
+                except Exception as e:
+                    logger.warning("watermark_image_failed", error=str(e))
+
+            canvas.restoreState()
+
+    def _draw_headers_footers(
+        self,
+        canvas: Any,
+        template: DocumentFormat,
+        current_page: int,
+        page_width: float,
+        page_height: float,
+    ) -> None:
+        """Draw header and footer regions on the canvas."""
+        for region in template.regions:
+            if region.page_number != current_page:
+                continue
+
+            content = region.content or {}
+            blocks = content.get("blocks", [])
+
+            canvas.saveState()
+
+            for block in blocks:
+                text = block.get("text", "")
+                if not text:
+                    continue
+
+                x = block.get("x", region.x)
+                y = page_height - block.get("y", region.y) - block.get("height", 12)
+                font_family = block.get("font_family", "Helvetica")
+                font_size = block.get("font_size", 10)
+
+                # Map font family
+                font_name = "Helvetica"
+                if font_family:
+                    fl = font_family.lower()
+                    if "times" in fl or "serif" in fl:
+                        font_name = "Times-Roman"
+                    elif "courier" in fl or "mono" in fl:
+                        font_name = "Courier"
+
+                canvas.setFont(font_name, font_size)
+                canvas.setFillColor(colors.black)
+                canvas.drawString(x, y, text)
+
+            canvas.restoreState()

docengine/app/services/template_service.py

@@ -0,0 +1,401 @@
+from __future__ import annotations
+
+import uuid
+from typing import Any
+
+from sqlalchemy.orm import Session
+
+from app.core.logging_config import get_logger
+from app.models.document import Document
+from app.models.template import (
+    DocumentCell,
+    DocumentFormat,
+    DocumentRegion,
+    ImageRegion,
+    TableColumn,
+    TableFormat,
+    TableRow,
+    Watermark,
+)
+from app.repositories.document_repository import DocumentRepository
+from app.repositories.template_repository import (
+    DocumentCellRepository,
+    DocumentRegionRepository,
+    ImageRegionRepository,
+    TableColumnRepository,
+    TableFormatRepository,
+    TableRowRepository,
+    TemplateRepository,
+    WatermarkRepository,
+)
+from app.services.fingerprint_service import FingerprintService
+
+logger = get_logger(__name__)
+
+
+class TemplateService:
+    """Generate reusable document templates from processed documents."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+        self.doc_repo = DocumentRepository(db)
+        self.template_repo = TemplateRepository(db)
+        self.cell_repo = DocumentCellRepository(db)
+        self.region_repo = DocumentRegionRepository(db)
+        self.table_format_repo = TableFormatRepository(db)
+        self.table_column_repo = TableColumnRepository(db)
+        self.table_row_repo = TableRowRepository(db)
+        self.watermark_repo = WatermarkRepository(db)
+        self.image_region_repo = ImageRegionRepository(db)
+        self.fingerprint_service = FingerprintService(db)
+
+    def generate_template(
+        self,
+        document: Document,
+        user_id: uuid.UUID | None = None,
+    ) -> DocumentFormat:
+        """Generate a reusable template from a processed document."""
+        if not document.pages:
+            raise ValueError(f"Document '{document.id}' has no processed pages")
+
+        # Check if template already exists for this document
+        existing = self.template_repo.get_by_source_document(document.id)
+        if existing:
+            logger.info(
+                "template_already_exists",
+                document_id=str(document.id),
+                template_id=str(existing.id),
+            )
+            return existing
+
+        first_page = document.pages[0]
+        template_name = f"Template_{document.original_filename}_{uuid.uuid4().hex[:8]}"
+
+        # Create template
+        template = self.template_repo.create_template(
+            name=template_name,
+            page_width=first_page.width,
+            page_height=first_page.height,
+            page_count=document.page_count or len(document.pages),
+            description=f"Auto-generated template from {document.original_filename}",
+            source_document_id=document.id,
+            created_by=user_id,
+        )
+
+        # Process each page
+        for page in document.pages:
+            self._process_page_for_template(template, page)
+
+        # Generate fingerprint
+        self.fingerprint_service.generate_fingerprint(template)
+
+        logger.info(
+            "template_generated",
+            template_id=str(template.id),
+            document_id=str(document.id),
+            cells=len(template.cells),
+            regions=len(template.regions),
+        )
+
+        return template
+
+    def _process_page_for_template(
+        self,
+        template: DocumentFormat,
+        page: Any,
+    ) -> None:
+        """Process a document page and create template components."""
+        page_number = page.page_number
+
+        # Create cells from text blocks
+        self._create_cells_from_text_blocks(template, page, page_number)
+
+        # Create regions from headers, footers
+        self._create_regions(template, page, page_number)
+
+        # Create table formats
+        self._create_table_formats(template, page, page_number)
+
+        # Create image regions
+        self._create_image_regions(template, page, page_number)
+
+        # Detect watermarks from text blocks
+        self._create_watermarks(template, page, page_number)
+
+    def _create_cells_from_text_blocks(
+        self,
+        template: DocumentFormat,
+        page: Any,
+        page_number: int,
+    ) -> None:
+        """Create template cells from extracted text blocks."""
+        for seq, block in enumerate(page.text_blocks):
+            if block.block_type in ("header", "footer", "watermark"):
+                continue
+
+            # Determine if this is a dynamic field
+            is_dynamic = self._is_dynamic_field(block.text)
+            field_name = self._generate_field_name(block.text, seq) if is_dynamic else None
+
+            self.cell_repo.create_cell(
+                format_id=template.id,
+                page_number=page_number,
+                x=block.x,
+                y=block.y,
+                width=block.width,
+                height=block.height,
+                data_type=self._infer_data_type(block.text),
+                font_family=block.font_family,
+                font_size=block.font_size,
+                font_style=block.font_style,
+                font_color=block.font_color,
+                alignment=self._infer_alignment(block.x, template.page_width),
+                static_text=block.text if not is_dynamic else None,
+                field_name=field_name,
+                sequence=seq,
+                is_dynamic=is_dynamic,
+            )
+
+    def _create_regions(
+        self,
+        template: DocumentFormat,
+        page: Any,
+        page_number: int,
+    ) -> None:
+        """Create template regions from headers and footers."""
+        header_blocks = [b for b in page.text_blocks if b.block_type == "header"]
+        footer_blocks = [b for b in page.text_blocks if b.block_type == "footer"]
+
+        if header_blocks:
+            # Compute bounding box for all header blocks
+            min_x = min(b.x for b in header_blocks)
+            min_y = min(b.y for b in header_blocks)
+            max_x = max(b.x + b.width for b in header_blocks)
+            max_y = max(b.y + b.height for b in header_blocks)
+
+            content = {
+                "blocks": [
+                    {
+                        "text": b.text,
+                        "x": b.x,
+                        "y": b.y,
+                        "width": b.width,
+                        "height": b.height,
+                        "font_family": b.font_family,
+                        "font_size": b.font_size,
+                    }
+                    for b in header_blocks
+                ]
+            }
+
+            self.region_repo.create_region(
+                format_id=template.id,
+                page_number=page_number,
+                region_type="header",
+                x=min_x,
+                y=min_y,
+                width=max_x - min_x,
+                height=max_y - min_y,
+                content=content,
+                sequence=0,
+            )
+
+        if footer_blocks:
+            min_x = min(b.x for b in footer_blocks)
+            min_y = min(b.y for b in footer_blocks)
+            max_x = max(b.x + b.width for b in footer_blocks)
+            max_y = max(b.y + b.height for b in footer_blocks)
+
+            content = {
+                "blocks": [
+                    {
+                        "text": b.text,
+                        "x": b.x,
+                        "y": b.y,
+                        "width": b.width,
+                        "height": b.height,
+                        "font_family": b.font_family,
+                        "font_size": b.font_size,
+                    }
+                    for b in footer_blocks
+                ]
+            }
+
+            self.region_repo.create_region(
+                format_id=template.id,
+                page_number=page_number,
+                region_type="footer",
+                x=min_x,
+                y=min_y,
+                width=max_x - min_x,
+                height=max_y - min_y,
+                content=content,
+                sequence=1,
+            )
+
+    def _create_table_formats(
+        self,
+        template: DocumentFormat,
+        page: Any,
+        page_number: int,
+    ) -> None:
+        """Create table format definitions from detected tables."""
+        for table in page.tables:
+            table_format = self.table_format_repo.create_table_format(
+                format_id=template.id,
+                page_number=page_number,
+                x=table.x,
+                y=table.y,
+                width=table.width,
+                height=table.height,
+                rows=table.rows,
+                columns=table.columns,
+            )
+
+            # Create columns
+            col_width = table.width / max(table.columns, 1)
+            for col_idx in range(table.columns):
+                self.table_column_repo.create_column(
+                    table_format_id=table_format.id,
+                    column_index=col_idx,
+                    width=col_width,
+                    data_type="text",
+                    alignment="left",
+                )
+
+            # Create rows
+            row_height = table.height / max(table.rows, 1)
+            for row_idx in range(table.rows):
+                self.table_row_repo.create_row(
+                    table_format_id=table_format.id,
+                    row_index=row_idx,
+                    height=row_height,
+                    is_header=(row_idx == 0),
+                )
+
+    def _create_image_regions(
+        self,
+        template: DocumentFormat,
+        page: Any,
+        page_number: int,
+    ) -> None:
+        """Create image region definitions from detected images."""
+        for img in page.images:
+            self.image_region_repo.create_image_region(
+                format_id=template.id,
+                page_number=page_number,
+                x=img.x,
+                y=img.y,
+                width=img.width,
+                height=img.height,
+                image_path=img.image_path,
+                image_type=img.image_type,
+                is_static=True,
+            )
+
+    def _create_watermarks(
+        self,
+        template: DocumentFormat,
+        page: Any,
+        page_number: int,
+    ) -> None:
+        """Create watermark definitions from detected watermark text blocks."""
+        watermark_blocks = [b for b in page.text_blocks if b.block_type == "watermark"]
+        for block in watermark_blocks:
+            self.watermark_repo.create_watermark(
+                format_id=template.id,
+                page_number=page_number,
+                text=block.text,
+                x=block.x,
+                y=block.y,
+                width=block.width,
+                height=block.height,
+                opacity=0.3,
+                rotation=0.0,
+                font_family=block.font_family,
+                font_size=block.font_size,
+                font_color=block.font_color or "#CCCCCC",
+            )
+
+    def _is_dynamic_field(self, text: str) -> bool:
+        """Determine if a text block represents a dynamic (variable) field."""
+        if not text:
+            return False
+
+        # Common patterns indicating dynamic content
+        dynamic_patterns = [
+            "{{", "}}", "${", "##",
+            "__________", "___", "...........",
+        ]
+        for pattern in dynamic_patterns:
+            if pattern in text:
+                return True
+
+        # Short single-word values that might be labels are static
+        # Longer values with numbers/dates tend to be dynamic
+        import re
+        # Date patterns
+        if re.search(r"\d{1,2}[/\-\.]\d{1,2}[/\-\.]\d{2,4}", text):
+            return True
+        # Currency patterns
+        if re.search(r"[$€£¥]\s*[\d,]+\.?\d*", text):
+            return True
+        # Phone patterns
+        if re.search(r"\+?\d[\d\s\-()]{7,}", text):
+            return True
+
+        return False
+
+    def _generate_field_name(self, text: str, sequence: int) -> str:
+        """Generate a field name from text content."""
+        import re
+        # Clean text
+        clean = re.sub(r"[^a-zA-Z0-9\s]", "", text)
+        clean = clean.strip().lower()
+        words = clean.split()[:3]
+        if words:
+            return "_".join(words)
+        return f"field_{sequence}"
+
+    def _infer_data_type(self, text: str) -> str:
+        """Infer the data type from text content."""
+        import re
+
+        if not text:
+            return "text"
+
+        stripped = text.strip()
+
+        # Number
+        if re.match(r"^-?[\d,]+\.?\d*$", stripped.replace(",", "")):
+            return "number"
+
+        # Date
+        if re.search(r"\d{1,2}[/\-\.]\d{1,2}[/\-\.]\d{2,4}", stripped):
+            return "date"
+
+        # Currency
+        if re.search(r"^[$€£¥]\s*[\d,]+\.?\d*$", stripped):
+            return "currency"
+
+        # Email
+        if re.search(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", stripped):
+            return "email"
+
+        return "text"
+
+    def _infer_alignment(self, x: float, page_width: float) -> str:
+        """Infer text alignment based on horizontal position."""
+        if page_width <= 0:
+            return "left"
+
+        relative_x = x / page_width
+
+        if relative_x < 0.15:
+            return "left"
+        elif relative_x > 0.6:
+            return "right"
+        elif 0.35 < relative_x < 0.65:
+            return "center"
+
+        return "left"

docengine/app/tasks/document_tasks.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import uuid
+
+from app.core.logging_config import get_logger
+from app.workers.celery_app import celery_app
+from app.core.database import get_db_context
+from app.services.document_service import DocumentProcessingService
+
+logger = get_logger(__name__)
+
+
+@celery_app.task(
+    name="app.tasks.document_tasks.process_document_task",
+    bind=True,
+    max_retries=3,
+    default_retry_delay=60,
+    acks_late=True,
+)
+def process_document_task(self, document_id: str) -> dict:  # noqa: ANN001
+    """Celery task to process a document asynchronously."""
+    logger.info("task_started", task_id=self.request.id, document_id=document_id)
+
+    try:
+        doc_uuid = uuid.UUID(document_id)
+        with get_db_context() as db:
+            service = DocumentProcessingService(db)
+            document = service.process_document(doc_uuid)
+            logger.info(
+                "task_completed",
+                task_id=self.request.id,
+                document_id=document_id,
+                status=document.status,
+            )
+            return {
+                "document_id": document_id,
+                "status": document.status,
+                "page_count": document.page_count,
+            }
+    except Exception as exc:
+        logger.exception(
+            "task_failed",
+            task_id=self.request.id,
+            document_id=document_id,
+            error=str(exc),
+            retry=self.request.retries,
+        )
+        raise self.retry(exc=exc)
+
+
+@celery_app.task(
+    name="app.tasks.document_tasks.match_document_task",
+    bind=True,
+    max_retries=2,
+    default_retry_delay=30,
+)
+def match_document_task(
+    self,  # noqa: ANN001
+    document_id: str,
+    min_confidence: float = 0.5,
+    max_results: int = 5,
+) -> dict:
+    """Celery task to match a document against templates."""
+    logger.info("match_task_started", task_id=self.request.id, document_id=document_id)
+
+    try:
+        doc_uuid = uuid.UUID(document_id)
+        with get_db_context() as db:
+            from app.services.matching_service import MatchingService
+            service = MatchingService(db)
+            matches = service.match_document(
+                document_id=doc_uuid,
+                min_confidence=min_confidence,
+                max_results=max_results,
+            )
+            logger.info(
+                "match_task_completed",
+                task_id=self.request.id,
+                document_id=document_id,
+                matches=len(matches),
+            )
+            return {
+                "document_id": document_id,
+                "matches": len(matches),
+                "best_score": matches[0].confidence_score if matches else 0.0,
+            }
+    except Exception as exc:
+        logger.exception(
+            "match_task_failed",
+            task_id=self.request.id,
+            document_id=document_id,
+            error=str(exc),
+        )
+        raise self.retry(exc=exc)

docengine/app/tasks/maintenance_tasks.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from app.core.logging_config import get_logger
+from app.workers.celery_app import celery_app
+from app.core.database import get_db_context
+
+logger = get_logger(__name__)
+
+
+@celery_app.task(
+    name="app.tasks.maintenance_tasks.cleanup_expired_tokens",
+    bind=True,
+)
+def cleanup_expired_tokens(self) -> dict:  # noqa: ANN001
+    """Cleanup expired and revoked refresh tokens."""
+    logger.info("cleanup_tokens_started", task_id=self.request.id)
+
+    try:
+        with get_db_context() as db:
+            from app.repositories.user_repository import RefreshTokenRepository
+            repo = RefreshTokenRepository(db)
+            count = repo.cleanup_expired_tokens()
+            logger.info("cleanup_tokens_completed", removed=count)
+            return {"removed_tokens": count}
+    except Exception as exc:
+        logger.exception("cleanup_tokens_failed", error=str(exc))
+        return {"error": str(exc)}
+
+
+@celery_app.task(
+    name="app.tasks.maintenance_tasks.cleanup_temp_storage",
+    bind=True,
+)
+def cleanup_temp_storage(self) -> dict:  # noqa: ANN001
+    """Cleanup temporary storage files."""
+    logger.info("cleanup_temp_started", task_id=self.request.id)
+
+    try:
+        from app.storage.provider import get_storage_provider
+        storage = get_storage_provider()
+        count = storage.cleanup_temp()
+        logger.info("cleanup_temp_completed", removed=count)
+        return {"removed_files": count}
+    except Exception as exc:
+        logger.exception("cleanup_temp_failed", error=str(exc))
+        return {"error": str(exc)}

docengine/app/workers/celery_app.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from celery import Celery
+
+from app.core.config import settings
+
+celery_app = Celery(
+    "docengine",
+    broker=settings.celery_broker_url,
+    backend=settings.celery_result_backend,
+)
+
+celery_app.conf.update(
+    task_serializer="json",
+    accept_content=["json"],
+    result_serializer="json",
+    timezone="UTC",
+    enable_utc=True,
+    task_track_started=True,
+    task_time_limit=3600,
+    task_soft_time_limit=3300,
+    worker_max_tasks_per_child=100,
+    worker_prefetch_multiplier=1,
+    task_acks_late=True,
+    task_reject_on_worker_lost=True,
+    broker_connection_retry_on_startup=True,
+    result_expires=86400,
+    task_routes={
+        "app.tasks.document_tasks.*": {"queue": "document_processing"},
+    },
+    beat_schedule={
+        "cleanup-expired-tokens": {
+            "task": "app.tasks.maintenance_tasks.cleanup_expired_tokens",
+            "schedule": 3600.0,
+        },
+        "cleanup-temp-storage": {
+            "task": "app.tasks.maintenance_tasks.cleanup_temp_storage",
+            "schedule": 7200.0,
+        },
+    },
+)
+
+celery_app.autodiscover_tasks(["app.tasks"])

docengine/application.properties

@@ -0,0 +1,8 @@
+
+server.port=7989
+
+db.host=192.168.0.111
+db.port=7925
+db.user=postgres
+db.password=M@tr!x#149@dm!N
+db.schema=admin

docengine/docker-compose.prod.yml

@@ -0,0 +1,154 @@
+version: "3.9"
+
+services:
+  db:
+    image: postgres:16-alpine
+    container_name: docengine_db_prod
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: "${DB_PASSWORD}"
+      POSTGRES_DB: document_engine
+    ports:
+      - "7925:5432"
+    volumes:
+      - docengine_pgdata_prod:/var/lib/postgresql/data
+      - ./sql/001_create_schema.sql:/docker-entrypoint-initdb.d/001_create_schema.sql
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres -d document_engine"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    deploy:
+      resources:
+        limits:
+          memory: 2G
+          cpus: "2.0"
+    restart: always
+    networks:
+      - docengine_net_prod
+
+  redis:
+    image: redis:7-alpine
+    container_name: docengine_redis_prod
+    command: redis-server --requirepass "${REDIS_PASSWORD}" --appendonly yes
+    ports:
+      - "6379:6379"
+    volumes:
+      - docengine_redis_data_prod:/data
+    healthcheck:
+      test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    deploy:
+      resources:
+        limits:
+          memory: 1G
+          cpus: "1.0"
+    restart: always
+    networks:
+      - docengine_net_prod
+
+  app:
+    build:
+      context: .
+      target: app
+    container_name: docengine_app_prod
+    env_file:
+      - .env
+    environment:
+      APP_ENV: production
+      APP_DEBUG: "false"
+      DB_HOST: db
+      DB_PORT: 5432
+      REDIS_HOST: redis
+      CELERY_BROKER_URL: "redis://:${REDIS_PASSWORD}@redis:6379/0"
+      CELERY_RESULT_BACKEND: "redis://:${REDIS_PASSWORD}@redis:6379/1"
+    ports:
+      - "7989:7989"
+    volumes:
+      - docengine_storage_prod:/app/storage
+    depends_on:
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    deploy:
+      resources:
+        limits:
+          memory: 4G
+          cpus: "4.0"
+      replicas: 2
+    restart: always
+    networks:
+      - docengine_net_prod
+
+  worker:
+    build:
+      context: .
+      target: worker
+    container_name: docengine_worker_prod
+    env_file:
+      - .env
+    environment:
+      APP_ENV: production
+      APP_DEBUG: "false"
+      DB_HOST: db
+      DB_PORT: 5432
+      REDIS_HOST: redis
+      CELERY_BROKER_URL: "redis://:${REDIS_PASSWORD}@redis:6379/0"
+      CELERY_RESULT_BACKEND: "redis://:${REDIS_PASSWORD}@redis:6379/1"
+    volumes:
+      - docengine_storage_prod:/app/storage
+    depends_on:
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    deploy:
+      resources:
+        limits:
+          memory: 8G
+          cpus: "4.0"
+      replicas: 2
+    restart: always
+    networks:
+      - docengine_net_prod
+
+  beat:
+    build:
+      context: .
+      target: beat
+    container_name: docengine_beat_prod
+    env_file:
+      - .env
+    environment:
+      APP_ENV: production
+      APP_DEBUG: "false"
+      DB_HOST: db
+      DB_PORT: 5432
+      REDIS_HOST: redis
+      CELERY_BROKER_URL: "redis://:${REDIS_PASSWORD}@redis:6379/0"
+      CELERY_RESULT_BACKEND: "redis://:${REDIS_PASSWORD}@redis:6379/1"
+    depends_on:
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    deploy:
+      resources:
+        limits:
+          memory: 512M
+          cpus: "0.5"
+    restart: always
+    networks:
+      - docengine_net_prod
+
+volumes:
+  docengine_pgdata_prod:
+  docengine_redis_data_prod:
+  docengine_storage_prod:
+
+networks:
+  docengine_net_prod:
+    driver: bridge

docengine/docker-compose.yml

@@ -0,0 +1,93 @@
+version: "3.9"
+
+services:
+  db:
+    image: postgres:16-alpine
+    container_name: docengine_db
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: "M@tr!x#149@dm!N"
+      POSTGRES_DB: document_engine
+    ports:
+      - "7925:5432"
+    volumes:
+      - docengine_pgdata:/var/lib/postgresql/data
+      - ./sql/001_create_schema.sql:/docker-entrypoint-initdb.d/001_create_schema.sql
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres -d document_engine"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    networks:
+      - docengine_net
+
+  redis:
+    image: redis:7-alpine
+    container_name: docengine_redis
+    ports:
+      - "6379:6379"
+    volumes:
+      - docengine_redis_data:/data
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    networks:
+      - docengine_net
+
+  app:
+    build:
+      context: .
+      target: app
+    container_name: docengine_app
+    env_file:
+      - .env
+    environment:
+      DB_HOST: db
+      DB_PORT: 5432
+      REDIS_HOST: redis
+      CELERY_BROKER_URL: redis://redis:6379/0
+      CELERY_RESULT_BACKEND: redis://redis:6379/1
+    ports:
+      - "7989:7989"
+    volumes:
+      - ./storage:/app/storage
+    depends_on:
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    networks:
+      - docengine_net
+
+  worker:
+    build:
+      context: .
+      target: worker
+    container_name: docengine_worker
+    env_file:
+      - .env
+    environment:
+      DB_HOST: db
+      DB_PORT: 5432
+      REDIS_HOST: redis
+      CELERY_BROKER_URL: redis://redis:6379/0
+      CELERY_RESULT_BACKEND: redis://redis:6379/1
+    volumes:
+      - ./storage:/app/storage
+    depends_on:
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    networks:
+      - docengine_net
+
+volumes:
+  docengine_pgdata:
+  docengine_redis_data:
+
+networks:
+  docengine_net:
+    driver: bridge

docengine/pyproject.toml

@@ -0,0 +1,45 @@
+[build-system]
+requires = ["setuptools>=75.0", "wheel"]
+build-backend = "setuptools.backends._legacy:_Backend"
+
+[project]
+name = "docengine"
+version = "1.0.0"
+description = "Document Template Recognition and Reconstruction System"
+readme = "README.md"
+requires-python = ">=3.12"
+license = {text = "Proprietary"}
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+asyncio_mode = "auto"
+python_files = ["test_*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = "-v --tb=short --cov=app --cov-report=term-missing --cov-report=html"
+filterwarnings = [
+    "ignore::DeprecationWarning",
+]
+
+[tool.black]
+line-length = 120
+target-version = ["py312"]
+
+[tool.isort]
+profile = "black"
+line_length = 120
+
+[tool.ruff]
+line-length = 120
+target-version = "py312"
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "I", "N", "UP", "S", "B", "A", "C4", "DTZ", "ISC", "PIE", "T20", "RSE", "RET", "SIM", "TCH", "ERA", "PGH", "PLC", "PLE", "PLR", "PLW", "TRY", "RUF"]
+ignore = ["S101", "S603", "S607", "TRY003", "PLR0913", "B008"]
+
+[tool.mypy]
+python_version = "3.12"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = true
+ignore_missing_imports = true

docengine/requirements-dev.txt

@@ -0,0 +1,21 @@
+-r requirements.txt
+
+# Testing
+pytest==8.3.4
+pytest-cov==6.0.0
+pytest-asyncio==0.25.0
+pytest-mock==3.14.0
+httpx==0.28.1
+factory-boy==3.3.1
+
+# Code Quality
+ruff==0.8.6
+mypy==1.14.1
+black==24.10.0
+isort==5.13.2
+
+# Type Stubs
+types-redis==4.6.0.20241004
+types-python-dateutil==2.9.0.20241003
+types-passlib==1.7.7.20240819
+types-aiofiles==24.1.0.20240626

docengine/requirements.txt

@@ -0,0 +1,49 @@
+# Core Framework
+fastapi==0.115.6
+uvicorn[standard]==0.34.0
+python-multipart==0.0.20
+
+# Database
+sqlalchemy[asyncio]==2.0.36
+psycopg2-binary==2.9.10
+alembic==1.14.1
+
+# Validation & Settings
+pydantic==2.10.4
+pydantic-settings==2.7.1
+email-validator==2.2.0
+
+# Authentication & Security
+python-jose[cryptography]==3.3.0
+passlib[bcrypt]==1.7.4
+bcrypt==4.2.1
+
+# Document Processing
+PyMuPDF==1.25.3
+paddleocr==2.9.1
+paddlepaddle==3.0.0b1
+layoutparser==0.3.4
+opencv-python-headless==4.10.0.84
+camelot-py[cv]==0.11.0
+pdf2image==1.17.0
+Pillow==11.1.0
+
+# PDF Generation
+reportlab==4.2.5
+
+# Background Jobs
+celery[redis]==5.4.0
+redis==5.2.1
+
+# Logging
+structlog==24.4.0
+
+# Monitoring
+prometheus-client==0.21.1
+prometheus-fastapi-instrumentator==7.0.2
+
+# Utilities
+python-dateutil==2.9.0
+aiofiles==24.1.0
+httpx==0.28.1
+numpy==1.26.4

docengine/sql/001_create_schema.sql

@@ -0,0 +1,12 @@
+-- Create the admin schema for DocEngine
+CREATE SCHEMA IF NOT EXISTS admin;
+
+-- Set the default search path
+ALTER DATABASE document_engine SET search_path TO admin, public;
+
+-- Grant privileges
+GRANT ALL ON SCHEMA admin TO postgres;
+GRANT USAGE ON SCHEMA admin TO postgres;
+ALTER DEFAULT PRIVILEGES IN SCHEMA admin GRANT ALL ON TABLES TO postgres;
+ALTER DEFAULT PRIVILEGES IN SCHEMA admin GRANT ALL ON SEQUENCES TO postgres;
+ALTER DEFAULT PRIVILEGES IN SCHEMA admin GRANT ALL ON FUNCTIONS TO postgres;

docengine/sql/001_init.sql

@@ -0,0 +1,16 @@
+
+CREATE SCHEMA IF NOT EXISTS admin;
+
+CREATE TABLE IF NOT EXISTS admin.document_format(
+    id BIGSERIAL PRIMARY KEY,
+    name VARCHAR(255),
+    created_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE TABLE IF NOT EXISTS admin.document_cell(
+    id BIGSERIAL PRIMARY KEY,
+    format_id BIGINT REFERENCES admin.document_format(id),
+    data_type VARCHAR(50),
+    font_family VARCHAR(100),
+    font_size INTEGER
+);

docengine/sql/002_create_tables.sql

@@ -0,0 +1,301 @@
+-- DocEngine: Complete table creation script
+-- Schema: admin
+-- Database: document_engine
+
+SET search_path TO admin, public;
+
+-- ============================================
+-- Users & Authentication
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS admin.users (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    username VARCHAR(150) NOT NULL UNIQUE,
+    email VARCHAR(255) NOT NULL UNIQUE,
+    hashed_password VARCHAR(255) NOT NULL,
+    full_name VARCHAR(255),
+    is_active BOOLEAN NOT NULL DEFAULT TRUE,
+    is_superuser BOOLEAN NOT NULL DEFAULT FALSE,
+    last_login TIMESTAMPTZ,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE TABLE IF NOT EXISTS admin.roles (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    name VARCHAR(50) NOT NULL UNIQUE,
+    description TEXT,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE TABLE IF NOT EXISTS admin.user_roles (
+    user_id UUID NOT NULL REFERENCES admin.users(id) ON DELETE CASCADE,
+    role_id UUID NOT NULL REFERENCES admin.roles(id) ON DELETE CASCADE,
+    PRIMARY KEY (user_id, role_id)
+);
+
+CREATE TABLE IF NOT EXISTS admin.refresh_tokens (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    user_id UUID NOT NULL REFERENCES admin.users(id) ON DELETE CASCADE,
+    token VARCHAR(512) NOT NULL UNIQUE,
+    expires_at TIMESTAMPTZ NOT NULL,
+    revoked BOOLEAN NOT NULL DEFAULT FALSE,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE TABLE IF NOT EXISTS admin.audit_logs (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    user_id UUID REFERENCES admin.users(id) ON DELETE SET NULL,
+    action VARCHAR(100) NOT NULL,
+    resource_type VARCHAR(100) NOT NULL,
+    resource_id VARCHAR(255),
+    details TEXT,
+    ip_address VARCHAR(45),
+    user_agent VARCHAR(512),
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+-- ============================================
+-- Documents
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS admin.documents (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    filename VARCHAR(500) NOT NULL,
+    original_filename VARCHAR(500) NOT NULL,
+    content_type VARCHAR(100) NOT NULL,
+    file_size BIGINT NOT NULL,
+    checksum VARCHAR(128) NOT NULL,
+    storage_path VARCHAR(1024) NOT NULL,
+    status VARCHAR(50) NOT NULL DEFAULT 'pending',
+    page_count INTEGER,
+    is_scanned BOOLEAN,
+    document_metadata JSONB,
+    error_message TEXT,
+    uploaded_by UUID REFERENCES admin.users(id) ON DELETE SET NULL,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE TABLE IF NOT EXISTS admin.document_pages (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    document_id UUID NOT NULL REFERENCES admin.documents(id) ON DELETE CASCADE,
+    page_number INTEGER NOT NULL,
+    width DOUBLE PRECISION NOT NULL,
+    height DOUBLE PRECISION NOT NULL,
+    image_path VARCHAR(1024),
+    text_content TEXT,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE TABLE IF NOT EXISTS admin.document_text_blocks (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    page_id UUID NOT NULL REFERENCES admin.document_pages(id) ON DELETE CASCADE,
+    text TEXT NOT NULL,
+    x DOUBLE PRECISION NOT NULL,
+    y DOUBLE PRECISION NOT NULL,
+    width DOUBLE PRECISION NOT NULL,
+    height DOUBLE PRECISION NOT NULL,
+    confidence DOUBLE PRECISION,
+    font_family VARCHAR(255),
+    font_size DOUBLE PRECISION,
+    font_color VARCHAR(50),
+    font_style VARCHAR(50),
+    block_type VARCHAR(50) NOT NULL DEFAULT 'text',
+    sequence INTEGER NOT NULL DEFAULT 0,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE TABLE IF NOT EXISTS admin.document_images (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    page_id UUID NOT NULL REFERENCES admin.document_pages(id) ON DELETE CASCADE,
+    x DOUBLE PRECISION NOT NULL,
+    y DOUBLE PRECISION NOT NULL,
+    width DOUBLE PRECISION NOT NULL,
+    height DOUBLE PRECISION NOT NULL,
+    image_path VARCHAR(1024) NOT NULL,
+    image_type VARCHAR(50) NOT NULL DEFAULT 'figure',
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE TABLE IF NOT EXISTS admin.document_tables (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    page_id UUID NOT NULL REFERENCES admin.document_pages(id) ON DELETE CASCADE,
+    x DOUBLE PRECISION NOT NULL,
+    y DOUBLE PRECISION NOT NULL,
+    width DOUBLE PRECISION NOT NULL,
+    height DOUBLE PRECISION NOT NULL,
+    rows INTEGER NOT NULL,
+    columns INTEGER NOT NULL,
+    data JSONB,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+-- ============================================
+-- Templates
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS admin.document_formats (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    name VARCHAR(255) NOT NULL,
+    description TEXT,
+    page_width DOUBLE PRECISION NOT NULL,
+    page_height DOUBLE PRECISION NOT NULL,
+    page_count INTEGER NOT NULL DEFAULT 1,
+    margin_top DOUBLE PRECISION NOT NULL DEFAULT 72.0,
+    margin_right DOUBLE PRECISION NOT NULL DEFAULT 72.0,
+    margin_bottom DOUBLE PRECISION NOT NULL DEFAULT 72.0,
+    margin_left DOUBLE PRECISION NOT NULL DEFAULT 72.0,
+    fingerprint JSONB,
+    source_document_id UUID REFERENCES admin.documents(id) ON DELETE SET NULL,
+    version INTEGER NOT NULL DEFAULT 1,
+    is_active BOOLEAN NOT NULL DEFAULT TRUE,
+    created_by UUID REFERENCES admin.users(id) ON DELETE SET NULL,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE TABLE IF NOT EXISTS admin.document_cells (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    format_id UUID NOT NULL REFERENCES admin.document_formats(id) ON DELETE CASCADE,
+    page_number INTEGER NOT NULL,
+    x DOUBLE PRECISION NOT NULL,
+    y DOUBLE PRECISION NOT NULL,
+    width DOUBLE PRECISION NOT NULL,
+    height DOUBLE PRECISION NOT NULL,
+    row_no INTEGER NOT NULL DEFAULT 0,
+    column_no INTEGER NOT NULL DEFAULT 0,
+    data_type VARCHAR(50) NOT NULL DEFAULT 'text',
+    font_family VARCHAR(255),
+    font_size DOUBLE PRECISION,
+    font_style VARCHAR(50),
+    font_color VARCHAR(50),
+    background_color VARCHAR(50),
+    border_top VARCHAR(100),
+    border_right VARCHAR(100),
+    border_bottom VARCHAR(100),
+    border_left VARCHAR(100),
+    padding_top DOUBLE PRECISION NOT NULL DEFAULT 0.0,
+    padding_right DOUBLE PRECISION NOT NULL DEFAULT 0.0,
+    padding_bottom DOUBLE PRECISION NOT NULL DEFAULT 0.0,
+    padding_left DOUBLE PRECISION NOT NULL DEFAULT 0.0,
+    alignment VARCHAR(20) NOT NULL DEFAULT 'left',
+    vertical_alignment VARCHAR(20) NOT NULL DEFAULT 'top',
+    rowspan INTEGER NOT NULL DEFAULT 1,
+    colspan INTEGER NOT NULL DEFAULT 1,
+    static_text TEXT,
+    field_name VARCHAR(255),
+    sequence INTEGER NOT NULL DEFAULT 0,
+    is_dynamic BOOLEAN NOT NULL DEFAULT FALSE,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE TABLE IF NOT EXISTS admin.document_regions (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    format_id UUID NOT NULL REFERENCES admin.document_formats(id) ON DELETE CASCADE,
+    page_number INTEGER NOT NULL,
+    region_type VARCHAR(50) NOT NULL,
+    x DOUBLE PRECISION NOT NULL,
+    y DOUBLE PRECISION NOT NULL,
+    width DOUBLE PRECISION NOT NULL,
+    height DOUBLE PRECISION NOT NULL,
+    content JSONB,
+    sequence INTEGER NOT NULL DEFAULT 0,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE TABLE IF NOT EXISTS admin.table_formats (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    format_id UUID NOT NULL REFERENCES admin.document_formats(id) ON DELETE CASCADE,
+    page_number INTEGER NOT NULL,
+    x DOUBLE PRECISION NOT NULL,
+    y DOUBLE PRECISION NOT NULL,
+    width DOUBLE PRECISION NOT NULL,
+    height DOUBLE PRECISION NOT NULL,
+    rows INTEGER NOT NULL,
+    columns INTEGER NOT NULL,
+    border_style VARCHAR(50) NOT NULL DEFAULT 'solid',
+    border_width DOUBLE PRECISION NOT NULL DEFAULT 1.0,
+    border_color VARCHAR(50) NOT NULL DEFAULT '#000000',
+    header_rows INTEGER NOT NULL DEFAULT 1,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE TABLE IF NOT EXISTS admin.table_columns (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    table_format_id UUID NOT NULL REFERENCES admin.table_formats(id) ON DELETE CASCADE,
+    column_index INTEGER NOT NULL,
+    width DOUBLE PRECISION NOT NULL,
+    header_text VARCHAR(500),
+    data_type VARCHAR(50) NOT NULL DEFAULT 'text',
+    alignment VARCHAR(20) NOT NULL DEFAULT 'left',
+    font_family VARCHAR(255),
+    font_size DOUBLE PRECISION,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE TABLE IF NOT EXISTS admin.table_rows (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    table_format_id UUID NOT NULL REFERENCES admin.table_formats(id) ON DELETE CASCADE,
+    row_index INTEGER NOT NULL,
+    height DOUBLE PRECISION NOT NULL DEFAULT 20.0,
+    is_header BOOLEAN NOT NULL DEFAULT FALSE,
+    background_color VARCHAR(50),
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE TABLE IF NOT EXISTS admin.watermarks (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    format_id UUID NOT NULL REFERENCES admin.document_formats(id) ON DELETE CASCADE,
+    page_number INTEGER,
+    text VARCHAR(500),
+    image_path VARCHAR(1024),
+    x DOUBLE PRECISION NOT NULL,
+    y DOUBLE PRECISION NOT NULL,
+    width DOUBLE PRECISION NOT NULL,
+    height DOUBLE PRECISION NOT NULL,
+    opacity DOUBLE PRECISION NOT NULL DEFAULT 0.3,
+    rotation DOUBLE PRECISION NOT NULL DEFAULT 0.0,
+    font_family VARCHAR(255),
+    font_size DOUBLE PRECISION,
+    font_color VARCHAR(50),
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE TABLE IF NOT EXISTS admin.image_regions (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    format_id UUID NOT NULL REFERENCES admin.document_formats(id) ON DELETE CASCADE,
+    page_number INTEGER NOT NULL,
+    x DOUBLE PRECISION NOT NULL,
+    y DOUBLE PRECISION NOT NULL,
+    width DOUBLE PRECISION NOT NULL,
+    height DOUBLE PRECISION NOT NULL,
+    image_path VARCHAR(1024),
+    image_type VARCHAR(50) NOT NULL DEFAULT 'figure',
+    is_static BOOLEAN NOT NULL DEFAULT TRUE,
+    field_name VARCHAR(255),
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE TABLE IF NOT EXISTS admin.template_fingerprints (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    format_id UUID NOT NULL UNIQUE REFERENCES admin.document_formats(id) ON DELETE CASCADE,
+    page_dimensions JSONB,
+    logo_coordinates JSONB,
+    header_coordinates JSONB,
+    footer_coordinates JSONB,
+    table_coordinates JSONB,
+    cell_coordinates JSONB,
+    fingerprint_hash VARCHAR(256) NOT NULL,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE TABLE IF NOT EXISTS admin.template_matches (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    document_id UUID NOT NULL REFERENCES admin.documents(id) ON DELETE CASCADE,
+    format_id UUID NOT NULL REFERENCES admin.document_formats(id) ON DELETE CASCADE,
+    confidence_score DOUBLE PRECISION NOT NULL,
+    match_details JSONB,
+    selected BOOLEAN NOT NULL DEFAULT FALSE,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);