Changes committed

docengine/app/services/document_service.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import uuid
+
+from sqlalchemy.orm import Session
+
+from app.core.logging_config import get_logger
+from app.models.document import Document
+from app.repositories.document_repository import DocumentRepository
+from app.services.layout_service import LayoutService
+from app.services.ocr_service import OCRService
+from app.services.pdf_service import NativePDFService
+from app.services.template_service import TemplateService
+
+logger = get_logger(__name__)
+
+
+class DocumentProcessingService:
+    """Orchestrates the full document processing pipeline."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+        self.doc_repo = DocumentRepository(db)
+        self.pdf_service = NativePDFService(db)
+        self.ocr_service = OCRService(db)
+        self.layout_service = LayoutService(db)
+        self.template_service = TemplateService(db)
+
+    def process_document(self, document_id: str | uuid.UUID) -> Document:
+        """Process a document through the full pipeline."""
+        if isinstance(document_id, str):
+            document_id = uuid.UUID(document_id)
+
+        document = self.doc_repo.get_by_id(document_id)
+        if not document:
+            raise ValueError(f"Document '{document_id}' not found")
+
+        logger.info(
+            "processing_started",
+            document_id=str(document_id),
+            content_type=document.content_type,
+        )
+
+        # Update status to processing
+        self.doc_repo.update_status(document_id, "processing")
+        self.db.commit()
+
+        try:
+            # Step 1: Extract content based on document type
+            if document.content_type == "application/pdf":
+                document = self._process_pdf(document)
+            else:
+                document = self._process_image(document)
+
+            # Step 2: Analyze layout
+            layout_results = self.layout_service.analyze_document_layout(document)
+            document.document_metadata = document.document_metadata or {}
+            document.document_metadata["layout"] = layout_results
+
+            # Step 3: Generate template
+            template = self.template_service.generate_template(document)
+
+            # Step 4: Update document status
+            self.doc_repo.update_status(document_id, "completed")
+            self.db.commit()
+
+            logger.info(
+                "processing_completed",
+                document_id=str(document_id),
+                pages=document.page_count,
+                template_id=str(template.id),
+            )
+
+            return document
+
+        except Exception as e:
+            logger.exception(
+                "processing_failed",
+                document_id=str(document_id),
+                error=str(e),
+            )
+            self.doc_repo.update_status(document_id, "failed", error_message=str(e))
+            self.db.commit()
+            raise
+
+    def _process_pdf(self, document: Document) -> Document:
+        """Process a PDF document - either native or scanned."""
+        # First, try native PDF extraction
+        document = self.pdf_service.process_pdf(document)
+        self.db.flush()
+
+        # If scanned, also run OCR
+        if document.is_scanned:
+            logger.info(
+                "scanned_pdf_detected",
+                document_id=str(document.id),
+            )
+            document = self.ocr_service.process_scanned_pdf(document)
+            self.db.flush()
+
+        return document
+
+    def _process_image(self, document: Document) -> Document:
+        """Process an image document with OCR."""
+        document = self.ocr_service.process_image(document)
+        self.db.flush()
+        return document

docengine/app/services/fingerprint_service.py

@@ -0,0 +1,343 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import uuid
+from typing import Any
+
+from sqlalchemy.orm import Session
+
+from app.core.logging_config import get_logger
+from app.models.template import DocumentFormat, TemplateFingerprint
+from app.repositories.template_repository import TemplateFingerprintRepository, TemplateRepository
+
+logger = get_logger(__name__)
+
+
+class FingerprintService:
+    """Generate and manage layout fingerprints for template matching."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+        self.fingerprint_repo = TemplateFingerprintRepository(db)
+        self.template_repo = TemplateRepository(db)
+
+    def generate_fingerprint(self, template: DocumentFormat) -> TemplateFingerprint:
+        """Generate a layout fingerprint for a template."""
+        # Collect page dimensions
+        page_dimensions = {
+            "width": template.page_width,
+            "height": template.page_height,
+            "page_count": template.page_count,
+            "margins": {
+                "top": template.margin_top,
+                "right": template.margin_right,
+                "bottom": template.margin_bottom,
+                "left": template.margin_left,
+            },
+        }
+
+        # Collect logo coordinates
+        logo_coordinates = self._extract_logo_coordinates(template)
+
+        # Collect header coordinates
+        header_coordinates = self._extract_region_coordinates(template, "header")
+
+        # Collect footer coordinates
+        footer_coordinates = self._extract_region_coordinates(template, "footer")
+
+        # Collect table coordinates
+        table_coordinates = self._extract_table_coordinates(template)
+
+        # Collect cell coordinates
+        cell_coordinates = self._extract_cell_coordinates(template)
+
+        # Compute fingerprint hash
+        fingerprint_data = {
+            "page_dimensions": page_dimensions,
+            "logo_coordinates": logo_coordinates,
+            "header_coordinates": header_coordinates,
+            "footer_coordinates": footer_coordinates,
+            "table_coordinates": table_coordinates,
+            "cell_coordinates": cell_coordinates,
+        }
+        fingerprint_hash = self._compute_hash(fingerprint_data)
+
+        # Check for existing fingerprint
+        existing = self.fingerprint_repo.get_by_format_id(template.id)
+        if existing:
+            # Update existing
+            existing.page_dimensions = page_dimensions
+            existing.logo_coordinates = logo_coordinates
+            existing.header_coordinates = header_coordinates
+            existing.footer_coordinates = footer_coordinates
+            existing.table_coordinates = table_coordinates
+            existing.cell_coordinates = cell_coordinates
+            existing.fingerprint_hash = fingerprint_hash
+            self.db.flush()
+            self.db.refresh(existing)
+            return existing
+
+        # Create new fingerprint
+        fingerprint = self.fingerprint_repo.create_fingerprint(
+            format_id=template.id,
+            fingerprint_hash=fingerprint_hash,
+            page_dimensions=page_dimensions,
+            logo_coordinates=logo_coordinates,
+            header_coordinates=header_coordinates,
+            footer_coordinates=footer_coordinates,
+            table_coordinates=table_coordinates,
+            cell_coordinates=cell_coordinates,
+        )
+
+        logger.info(
+            "fingerprint_generated",
+            template_id=str(template.id),
+            hash=fingerprint_hash[:16],
+        )
+
+        return fingerprint
+
+    def _extract_logo_coordinates(self, template: DocumentFormat) -> dict[str, Any] | None:
+        """Extract logo image coordinates from template."""
+        logos = [ir for ir in template.image_regions if ir.image_type == "logo"]
+        if not logos:
+            return None
+
+        return {
+            "items": [
+                {
+                    "page": ir.page_number,
+                    "x": ir.x,
+                    "y": ir.y,
+                    "width": ir.width,
+                    "height": ir.height,
+                }
+                for ir in logos
+            ]
+        }
+
+    def _extract_region_coordinates(
+        self,
+        template: DocumentFormat,
+        region_type: str,
+    ) -> dict[str, Any] | None:
+        """Extract coordinates for a specific region type."""
+        regions = [r for r in template.regions if r.region_type == region_type]
+        if not regions:
+            return None
+
+        return {
+            "items": [
+                {
+                    "page": r.page_number,
+                    "x": r.x,
+                    "y": r.y,
+                    "width": r.width,
+                    "height": r.height,
+                }
+                for r in regions
+            ]
+        }
+
+    def _extract_table_coordinates(self, template: DocumentFormat) -> dict[str, Any] | None:
+        """Extract table coordinates from template."""
+        if not template.table_formats:
+            return None
+
+        return {
+            "items": [
+                {
+                    "page": tf.page_number,
+                    "x": tf.x,
+                    "y": tf.y,
+                    "width": tf.width,
+                    "height": tf.height,
+                    "rows": tf.rows,
+                    "columns": tf.columns,
+                }
+                for tf in template.table_formats
+            ]
+        }
+
+    def _extract_cell_coordinates(self, template: DocumentFormat) -> dict[str, Any] | None:
+        """Extract cell coordinates from template."""
+        if not template.cells:
+            return None
+
+        return {
+            "items": [
+                {
+                    "page": c.page_number,
+                    "x": c.x,
+                    "y": c.y,
+                    "width": c.width,
+                    "height": c.height,
+                    "row": c.row_no,
+                    "col": c.column_no,
+                }
+                for c in template.cells
+            ]
+        }
+
+    def _compute_hash(self, data: dict[str, Any]) -> str:
+        """Compute a deterministic hash of the fingerprint data."""
+        # Normalize coordinates to reduce sensitivity to minor variations
+        normalized = self._normalize_coordinates(data)
+        serialized = json.dumps(normalized, sort_keys=True, default=str)
+        return hashlib.sha256(serialized.encode()).hexdigest()
+
+    def _normalize_coordinates(self, data: dict[str, Any]) -> dict[str, Any]:
+        """Normalize coordinates by rounding to reduce sensitivity to small variations."""
+        if isinstance(data, dict):
+            return {k: self._normalize_coordinates(v) for k, v in data.items()}
+        elif isinstance(data, list):
+            return [self._normalize_coordinates(item) for item in data]
+        elif isinstance(data, float):
+            return round(data, 1)
+        return data
+
+    def compute_similarity(
+        self,
+        fingerprint1: TemplateFingerprint,
+        fingerprint2_data: dict[str, Any],
+    ) -> float:
+        """Compute similarity score between a stored fingerprint and new document data."""
+        scores: list[float] = []
+        weights: list[float] = []
+
+        # Page dimensions similarity (high weight)
+        dim_score = self._compare_dimensions(
+            fingerprint1.page_dimensions,
+            fingerprint2_data.get("page_dimensions"),
+        )
+        scores.append(dim_score)
+        weights.append(3.0)
+
+        # Logo coordinates similarity
+        logo_score = self._compare_coordinates(
+            fingerprint1.logo_coordinates,
+            fingerprint2_data.get("logo_coordinates"),
+        )
+        scores.append(logo_score)
+        weights.append(2.0)
+
+        # Header coordinates similarity
+        header_score = self._compare_coordinates(
+            fingerprint1.header_coordinates,
+            fingerprint2_data.get("header_coordinates"),
+        )
+        scores.append(header_score)
+        weights.append(2.0)
+
+        # Footer coordinates similarity
+        footer_score = self._compare_coordinates(
+            fingerprint1.footer_coordinates,
+            fingerprint2_data.get("footer_coordinates"),
+        )
+        scores.append(footer_score)
+        weights.append(1.5)
+
+        # Table coordinates similarity
+        table_score = self._compare_coordinates(
+            fingerprint1.table_coordinates,
+            fingerprint2_data.get("table_coordinates"),
+        )
+        scores.append(table_score)
+        weights.append(2.5)
+
+        # Cell coordinates similarity
+        cell_score = self._compare_coordinates(
+            fingerprint1.cell_coordinates,
+            fingerprint2_data.get("cell_coordinates"),
+        )
+        scores.append(cell_score)
+        weights.append(1.5)
+
+        # Weighted average
+        total_weight = sum(weights)
+        if total_weight == 0:
+            return 0.0
+
+        weighted_sum = sum(s * w for s, w in zip(scores, weights))
+        return weighted_sum / total_weight
+
+    def _compare_dimensions(
+        self,
+        dims1: dict[str, Any] | None,
+        dims2: dict[str, Any] | None,
+    ) -> float:
+        """Compare page dimensions similarity."""
+        if not dims1 or not dims2:
+            return 0.0 if (dims1 or dims2) else 1.0
+
+        width_ratio = min(dims1.get("width", 0), dims2.get("width", 0)) / max(
+            dims1.get("width", 1), dims2.get("width", 1)
+        )
+        height_ratio = min(dims1.get("height", 0), dims2.get("height", 0)) / max(
+            dims1.get("height", 1), dims2.get("height", 1)
+        )
+        page_count_match = 1.0 if dims1.get("page_count") == dims2.get("page_count") else 0.5
+
+        return (width_ratio + height_ratio + page_count_match) / 3.0
+
+    def _compare_coordinates(
+        self,
+        coords1: dict[str, Any] | None,
+        coords2: dict[str, Any] | None,
+    ) -> float:
+        """Compare coordinate sets for similarity."""
+        if not coords1 and not coords2:
+            return 1.0
+        if not coords1 or not coords2:
+            return 0.0
+
+        items1 = coords1.get("items", [])
+        items2 = coords2.get("items", [])
+
+        if not items1 and not items2:
+            return 1.0
+        if not items1 or not items2:
+            return 0.0
+
+        # Compare number of items
+        count_ratio = min(len(items1), len(items2)) / max(len(items1), len(items2))
+
+        # Compare positions of matched items
+        position_scores = []
+        for item1 in items1:
+            best_match = 0.0
+            for item2 in items2:
+                if item1.get("page") != item2.get("page"):
+                    continue
+                score = self._compute_bbox_iou(item1, item2)
+                best_match = max(best_match, score)
+            position_scores.append(best_match)
+
+        avg_position_score = sum(position_scores) / len(position_scores) if position_scores else 0.0
+
+        return (count_ratio + avg_position_score) / 2.0
+
+    def _compute_bbox_iou(self, bbox1: dict[str, Any], bbox2: dict[str, Any]) -> float:
+        """Compute Intersection over Union for two bounding boxes."""
+        x1 = max(bbox1.get("x", 0), bbox2.get("x", 0))
+        y1 = max(bbox1.get("y", 0), bbox2.get("y", 0))
+        x2 = min(
+            bbox1.get("x", 0) + bbox1.get("width", 0),
+            bbox2.get("x", 0) + bbox2.get("width", 0),
+        )
+        y2 = min(
+            bbox1.get("y", 0) + bbox1.get("height", 0),
+            bbox2.get("y", 0) + bbox2.get("height", 0),
+        )
+
+        intersection = max(0, x2 - x1) * max(0, y2 - y1)
+
+        area1 = bbox1.get("width", 0) * bbox1.get("height", 0)
+        area2 = bbox2.get("width", 0) * bbox2.get("height", 0)
+        union = area1 + area2 - intersection
+
+        if union == 0:
+            return 0.0
+
+        return intersection / union

docengine/app/services/layout_service.py

@@ -0,0 +1,317 @@
+from __future__ import annotations
+
+import uuid
+from pathlib import Path
+from typing import Any
+
+import cv2
+import numpy as np
+from sqlalchemy.orm import Session
+
+from app.core.logging_config import get_logger
+from app.models.document import Document, DocumentPage
+from app.repositories.document_repository import (
+    DocumentPageRepository,
+    DocumentTableRepository,
+    DocumentTextBlockRepository,
+)
+from app.storage.provider import get_storage_provider
+
+logger = get_logger(__name__)
+
+
+class LayoutService:
+    """Document layout analysis service using OpenCV-based detection."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+        self.storage = get_storage_provider()
+        self.page_repo = DocumentPageRepository(db)
+        self.text_block_repo = DocumentTextBlockRepository(db)
+        self.table_repo = DocumentTableRepository(db)
+
+    def analyze_document_layout(self, document: Document) -> dict[str, Any]:
+        """Analyze the layout of all pages in a document."""
+        layout_results: dict[str, Any] = {"pages": []}
+
+        for page in document.pages:
+            page_layout = self._analyze_page_layout(page)
+            layout_results["pages"].append(page_layout)
+
+        return layout_results
+
+    def _analyze_page_layout(self, page: DocumentPage) -> dict[str, Any]:
+        """Analyze layout of a single page."""
+        result: dict[str, Any] = {
+            "page_number": page.page_number,
+            "width": page.width,
+            "height": page.height,
+            "tables": [],
+            "lines": [],
+            "rectangles": [],
+            "text_regions": [],
+            "image_regions": [],
+        }
+
+        if not page.image_path:
+            return result
+
+        image_path = self.storage.get_absolute_path(page.image_path)
+        image = cv2.imread(image_path)
+        if image is None:
+            logger.warning("layout_image_read_failed", page_id=str(page.id))
+            return result
+
+        gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+
+        # Detect lines
+        result["lines"] = self._detect_lines(gray)
+
+        # Detect rectangles (potential table cells/borders)
+        result["rectangles"] = self._detect_rectangles(gray)
+
+        # Detect tables
+        tables = self._detect_tables(gray, image.shape)
+        result["tables"] = tables
+
+        # Store detected tables in the database
+        for table_data in tables:
+            self.table_repo.create_table(
+                page_id=page.id,
+                x=table_data["x"],
+                y=table_data["y"],
+                width=table_data["width"],
+                height=table_data["height"],
+                rows=table_data["rows"],
+                columns=table_data["columns"],
+                data=table_data.get("cells"),
+            )
+
+        # Detect watermarks
+        watermark = self._detect_watermark(gray, image.shape)
+        if watermark:
+            result["watermark"] = watermark
+
+        return result
+
+    def _detect_lines(self, gray: np.ndarray) -> list[dict[str, Any]]:
+        """Detect horizontal and vertical lines in the image."""
+        lines_detected: list[dict[str, Any]] = []
+
+        # Apply edge detection
+        edges = cv2.Canny(gray, 50, 150, apertureSize=3)
+
+        # Detect lines using Hough transform
+        lines = cv2.HoughLinesP(edges, 1, np.pi / 180, threshold=100, minLineLength=50, maxLineGap=10)
+
+        if lines is not None:
+            for line in lines:
+                x1, y1, x2, y2 = line[0]
+                length = np.sqrt((x2 - x1) ** 2 + (y2 - y1) ** 2)
+
+                # Classify as horizontal or vertical
+                angle = np.degrees(np.arctan2(y2 - y1, x2 - x1))
+                if abs(angle) < 5 or abs(angle - 180) < 5:
+                    orientation = "horizontal"
+                elif abs(angle - 90) < 5 or abs(angle + 90) < 5:
+                    orientation = "vertical"
+                else:
+                    orientation = "diagonal"
+
+                lines_detected.append({
+                    "x1": float(x1),
+                    "y1": float(y1),
+                    "x2": float(x2),
+                    "y2": float(y2),
+                    "length": float(length),
+                    "orientation": orientation,
+                })
+
+        return lines_detected
+
+    def _detect_rectangles(self, gray: np.ndarray) -> list[dict[str, Any]]:
+        """Detect rectangular regions in the image."""
+        rectangles: list[dict[str, Any]] = []
+
+        # Binary threshold
+        _, binary = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY_INV)
+
+        # Find contours
+        contours, _ = cv2.findContours(binary, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
+
+        for contour in contours:
+            # Approximate the contour
+            peri = cv2.arcLength(contour, True)
+            approx = cv2.approxPolyDP(contour, 0.02 * peri, True)
+
+            # If approximation has 4 vertices, it's likely a rectangle
+            if len(approx) == 4:
+                x, y, w, h = cv2.boundingRect(approx)
+                # Filter out very small or very large rectangles
+                area = w * h
+                if area > 500 and w > 10 and h > 10:
+                    rectangles.append({
+                        "x": float(x),
+                        "y": float(y),
+                        "width": float(w),
+                        "height": float(h),
+                        "area": float(area),
+                    })
+
+        return rectangles
+
+    def _detect_tables(self, gray: np.ndarray, image_shape: tuple) -> list[dict[str, Any]]:
+        """Detect table structures using morphological operations."""
+        tables: list[dict[str, Any]] = []
+        h, w = image_shape[:2]
+
+        # Binary threshold
+        _, binary = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY_INV)
+
+        # Detect horizontal lines
+        horizontal_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (max(w // 30, 1), 1))
+        horizontal = cv2.morphologyEx(binary, cv2.MORPH_OPEN, horizontal_kernel, iterations=2)
+
+        # Detect vertical lines
+        vertical_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1, max(h // 30, 1)))
+        vertical = cv2.morphologyEx(binary, cv2.MORPH_OPEN, vertical_kernel, iterations=2)
+
+        # Combine horizontal and vertical lines to find intersections
+        table_mask = cv2.add(horizontal, vertical)
+
+        # Find contours of table regions
+        contours, _ = cv2.findContours(table_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+        for contour in contours:
+            x, y, cw, ch = cv2.boundingRect(contour)
+            area = cw * ch
+
+            # Filter: table should be reasonably sized
+            if area < 5000 or cw < 50 or ch < 30:
+                continue
+
+            # Estimate rows and columns
+            rows, columns = self._estimate_table_dimensions(
+                table_mask[y:y+ch, x:x+cw], cw, ch
+            )
+
+            if rows >= 1 and columns >= 1:
+                # Extract cell contents
+                cells = self._extract_table_cells(
+                    gray[y:y+ch, x:x+cw], rows, columns, cw, ch
+                )
+
+                tables.append({
+                    "x": float(x),
+                    "y": float(y),
+                    "width": float(cw),
+                    "height": float(ch),
+                    "rows": rows,
+                    "columns": columns,
+                    "cells": cells,
+                })
+
+        return tables
+
+    def _estimate_table_dimensions(
+        self,
+        table_region: np.ndarray,
+        width: int,
+        height: int,
+    ) -> tuple[int, int]:
+        """Estimate the number of rows and columns in a table region."""
+        # Project horizontal lines
+        h_projection = np.sum(table_region, axis=1)
+        h_peaks = self._count_peaks(h_projection, height)
+
+        # Project vertical lines
+        v_projection = np.sum(table_region, axis=0)
+        v_peaks = self._count_peaks(v_projection, width)
+
+        rows = max(1, h_peaks - 1)
+        columns = max(1, v_peaks - 1)
+
+        return rows, columns
+
+    def _count_peaks(self, projection: np.ndarray, total_length: int) -> int:
+        """Count significant peaks in a projection array."""
+        if len(projection) == 0:
+            return 0
+
+        threshold = np.max(projection) * 0.3
+        above_threshold = projection > threshold
+
+        # Count transitions from below to above threshold
+        peaks = 0
+        in_peak = False
+        for val in above_threshold:
+            if val and not in_peak:
+                peaks += 1
+                in_peak = True
+            elif not val:
+                in_peak = False
+
+        return peaks
+
+    def _extract_table_cells(
+        self,
+        table_gray: np.ndarray,
+        rows: int,
+        columns: int,
+        width: int,
+        height: int,
+    ) -> dict[str, Any]:
+        """Extract cell structure data from a table region."""
+        cell_height = height / max(rows, 1)
+        cell_width = width / max(columns, 1)
+
+        cells: dict[str, Any] = {"rows": rows, "columns": columns, "data": []}
+
+        for r in range(rows):
+            row_data = []
+            for c in range(columns):
+                cell_x = int(c * cell_width)
+                cell_y = int(r * cell_height)
+                cell_w = int(cell_width)
+                cell_h = int(cell_height)
+
+                row_data.append({
+                    "row": r,
+                    "col": c,
+                    "x": cell_x,
+                    "y": cell_y,
+                    "width": cell_w,
+                    "height": cell_h,
+                })
+            cells["data"].append(row_data)
+
+        return cells
+
+    def _detect_watermark(self, gray: np.ndarray, image_shape: tuple) -> dict[str, Any] | None:
+        """Detect potential watermark regions."""
+        h, w = image_shape[:2]
+
+        # Look for semi-transparent or light text in the center region
+        center_region = gray[h // 4 : 3 * h // 4, w // 4 : 3 * w // 4]
+
+        # Apply adaptive threshold to find light text
+        _, binary = cv2.threshold(center_region, 230, 255, cv2.THRESH_BINARY)
+
+        # Find contours in the center region
+        contours, _ = cv2.findContours(binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+        for contour in contours:
+            x, y, cw, ch = cv2.boundingRect(contour)
+            area = cw * ch
+            # Watermark typically covers a significant portion of the center
+            center_area = (w // 2) * (h // 2)
+            if area > center_area * 0.1:
+                return {
+                    "x": float(x + w // 4),
+                    "y": float(y + h // 4),
+                    "width": float(cw),
+                    "height": float(ch),
+                    "detected": True,
+                }
+
+        return None

docengine/app/services/matching_service.py

@@ -0,0 +1,217 @@
+from __future__ import annotations
+
+import uuid
+from typing import Any
+
+from sqlalchemy.orm import Session
+
+from app.core.logging_config import get_logger
+from app.models.document import Document, TemplateMatch
+from app.models.template import DocumentFormat
+from app.repositories.document_repository import DocumentRepository, TemplateMatchRepository
+from app.repositories.template_repository import TemplateFingerprintRepository, TemplateRepository
+from app.services.fingerprint_service import FingerprintService
+
+logger = get_logger(__name__)
+
+
+class MatchingService:
+    """Match documents against existing templates using fingerprint comparison."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+        self.doc_repo = DocumentRepository(db)
+        self.template_repo = TemplateRepository(db)
+        self.match_repo = TemplateMatchRepository(db)
+        self.fingerprint_repo = TemplateFingerprintRepository(db)
+        self.fingerprint_service = FingerprintService(db)
+
+    def match_document(
+        self,
+        document_id: uuid.UUID,
+        min_confidence: float = 0.5,
+        max_results: int = 5,
+    ) -> list[TemplateMatch]:
+        """Match a document against all existing templates."""
+        document = self.doc_repo.get_with_pages(document_id)
+        if not document:
+            raise ValueError(f"Document '{document_id}' not found")
+
+        if not document.pages:
+            raise ValueError(f"Document '{document_id}' has no processed pages")
+
+        # Generate document fingerprint data
+        doc_fingerprint_data = self._build_document_fingerprint(document)
+
+        # Get all templates with fingerprints
+        templates = self.template_repo.get_all_with_fingerprints()
+        fingerprints = self.fingerprint_repo.get_all_fingerprints()
+
+        # Map format_id -> fingerprint
+        fp_map = {fp.format_id: fp for fp in fingerprints}
+
+        matches: list[tuple[DocumentFormat, float, dict[str, Any]]] = []
+
+        for template in templates:
+            fp = fp_map.get(template.id)
+            if not fp:
+                continue
+
+            score = self.fingerprint_service.compute_similarity(fp, doc_fingerprint_data)
+            if score >= min_confidence:
+                match_details = {
+                    "page_dimensions_score": self.fingerprint_service._compare_dimensions(
+                        fp.page_dimensions, doc_fingerprint_data.get("page_dimensions")
+                    ),
+                    "logo_score": self.fingerprint_service._compare_coordinates(
+                        fp.logo_coordinates, doc_fingerprint_data.get("logo_coordinates")
+                    ),
+                    "header_score": self.fingerprint_service._compare_coordinates(
+                        fp.header_coordinates, doc_fingerprint_data.get("header_coordinates")
+                    ),
+                    "footer_score": self.fingerprint_service._compare_coordinates(
+                        fp.footer_coordinates, doc_fingerprint_data.get("footer_coordinates")
+                    ),
+                    "table_score": self.fingerprint_service._compare_coordinates(
+                        fp.table_coordinates, doc_fingerprint_data.get("table_coordinates")
+                    ),
+                    "cell_score": self.fingerprint_service._compare_coordinates(
+                        fp.cell_coordinates, doc_fingerprint_data.get("cell_coordinates")
+                    ),
+                }
+                matches.append((template, score, match_details))
+
+        # Sort by score descending
+        matches.sort(key=lambda x: x[1], reverse=True)
+        matches = matches[:max_results]
+
+        # Store match results
+        result_matches: list[TemplateMatch] = []
+        for idx, (template, score, details) in enumerate(matches):
+            template_match = self.match_repo.create_match(
+                document_id=document_id,
+                format_id=template.id,
+                confidence_score=score,
+                match_details=details,
+                selected=(idx == 0),  # Auto-select best match
+            )
+            result_matches.append(template_match)
+
+        logger.info(
+            "document_matched",
+            document_id=str(document_id),
+            matches_found=len(result_matches),
+            best_score=result_matches[0].confidence_score if result_matches else 0.0,
+        )
+
+        return result_matches
+
+    def _build_document_fingerprint(self, document: Document) -> dict[str, Any]:
+        """Build fingerprint data from a document for comparison."""
+        first_page = document.pages[0] if document.pages else None
+
+        page_dimensions = None
+        if first_page:
+            page_dimensions = {
+                "width": first_page.width,
+                "height": first_page.height,
+                "page_count": document.page_count or len(document.pages),
+            }
+
+        # Extract logo coordinates from images
+        logo_coordinates = None
+        logos = []
+        for page in document.pages:
+            for img in page.images:
+                if img.image_type == "logo":
+                    logos.append({
+                        "page": page.page_number,
+                        "x": img.x,
+                        "y": img.y,
+                        "width": img.width,
+                        "height": img.height,
+                    })
+        if logos:
+            logo_coordinates = {"items": logos}
+
+        # Extract header coordinates
+        header_coordinates = None
+        headers = []
+        for page in document.pages:
+            header_blocks = [b for b in page.text_blocks if b.block_type == "header"]
+            if header_blocks:
+                min_x = min(b.x for b in header_blocks)
+                min_y = min(b.y for b in header_blocks)
+                max_x = max(b.x + b.width for b in header_blocks)
+                max_y = max(b.y + b.height for b in header_blocks)
+                headers.append({
+                    "page": page.page_number,
+                    "x": min_x,
+                    "y": min_y,
+                    "width": max_x - min_x,
+                    "height": max_y - min_y,
+                })
+        if headers:
+            header_coordinates = {"items": headers}
+
+        # Extract footer coordinates
+        footer_coordinates = None
+        footers = []
+        for page in document.pages:
+            footer_blocks = [b for b in page.text_blocks if b.block_type == "footer"]
+            if footer_blocks:
+                min_x = min(b.x for b in footer_blocks)
+                min_y = min(b.y for b in footer_blocks)
+                max_x = max(b.x + b.width for b in footer_blocks)
+                max_y = max(b.y + b.height for b in footer_blocks)
+                footers.append({
+                    "page": page.page_number,
+                    "x": min_x,
+                    "y": min_y,
+                    "width": max_x - min_x,
+                    "height": max_y - min_y,
+                })
+        if footers:
+            footer_coordinates = {"items": footers}
+
+        # Extract table coordinates
+        table_coordinates = None
+        tables = []
+        for page in document.pages:
+            for table in page.tables:
+                tables.append({
+                    "page": page.page_number,
+                    "x": table.x,
+                    "y": table.y,
+                    "width": table.width,
+                    "height": table.height,
+                    "rows": table.rows,
+                    "columns": table.columns,
+                })
+        if tables:
+            table_coordinates = {"items": tables}
+
+        # Extract cell coordinates from text blocks
+        cell_coordinates = None
+        cells = []
+        for page in document.pages:
+            for block in page.text_blocks:
+                if block.block_type == "text":
+                    cells.append({
+                        "page": page.page_number,
+                        "x": block.x,
+                        "y": block.y,
+                        "width": block.width,
+                        "height": block.height,
+                    })
+        if cells:
+            cell_coordinates = {"items": cells}
+
+        return {
+            "page_dimensions": page_dimensions,
+            "logo_coordinates": logo_coordinates,
+            "header_coordinates": header_coordinates,
+            "footer_coordinates": footer_coordinates,
+            "table_coordinates": table_coordinates,
+            "cell_coordinates": cell_coordinates,
+        }

docengine/app/services/ocr_service.py

@@ -0,0 +1,210 @@
+from __future__ import annotations
+
+import uuid
+from pathlib import Path
+
+import cv2
+import numpy as np
+from paddleocr import PaddleOCR
+from sqlalchemy.orm import Session
+
+from app.core.config import settings
+from app.core.logging_config import get_logger
+from app.models.document import Document, DocumentPage
+from app.repositories.document_repository import (
+    DocumentPageRepository,
+    DocumentRepository,
+    DocumentTextBlockRepository,
+)
+from app.storage.provider import get_storage_provider
+
+logger = get_logger(__name__)
+
+_ocr_instance: PaddleOCR | None = None
+
+
+def get_ocr_engine() -> PaddleOCR:
+    """Get or create singleton PaddleOCR instance."""
+    global _ocr_instance
+    if _ocr_instance is None:
+        _ocr_instance = PaddleOCR(
+            use_angle_cls=True,
+            lang=settings.ocr_language,
+            use_gpu=settings.ocr_use_gpu,
+            show_log=False,
+            det_db_thresh=0.3,
+            det_db_box_thresh=0.5,
+            rec_batch_num=6,
+        )
+    return _ocr_instance
+
+
+class OCRService:
+    """OCR processing service using PaddleOCR for scanned documents."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+        self.storage = get_storage_provider()
+        self.doc_repo = DocumentRepository(db)
+        self.page_repo = DocumentPageRepository(db)
+        self.text_block_repo = DocumentTextBlockRepository(db)
+
+    def process_image(self, document: Document) -> Document:
+        """Process a scanned image document with OCR."""
+        file_path = self.storage.get_absolute_path(document.storage_path)
+        image = cv2.imread(file_path)
+        if image is None:
+            raise ValueError(f"Failed to read image: {file_path}")
+
+        height, width = image.shape[:2]
+        document.page_count = 1
+        document.is_scanned = True
+
+        # Save page image
+        image_filename = f"{document.id}_page_1.png"
+        image_bytes = cv2.imencode(".png", image)[1].tobytes()
+        image_path = self.storage.save_file(image_bytes, "images", image_filename)
+
+        # Create page record
+        doc_page = self.page_repo.create_page(
+            document_id=document.id,
+            page_number=1,
+            width=float(width),
+            height=float(height),
+            image_path=image_path,
+        )
+
+        # Run OCR
+        self._run_ocr_on_page(doc_page, file_path)
+
+        return document
+
+    def process_scanned_pdf(self, document: Document) -> Document:
+        """Process a scanned PDF document - convert pages to images and OCR each."""
+        file_path = self.storage.get_absolute_path(document.storage_path)
+
+        try:
+            from pdf2image import convert_from_path
+            images = convert_from_path(file_path, dpi=300)
+        except Exception as e:
+            logger.error("pdf_to_image_failed", document_id=str(document.id), error=str(e))
+            raise
+
+        document.page_count = len(images)
+        document.is_scanned = True
+
+        for page_num, pil_image in enumerate(images, start=1):
+            # Convert PIL to OpenCV format
+            np_image = np.array(pil_image)
+            cv_image = cv2.cvtColor(np_image, cv2.COLOR_RGB2BGR)
+            height, width = cv_image.shape[:2]
+
+            # Save page image
+            image_filename = f"{document.id}_page_{page_num}.png"
+            image_bytes = cv2.imencode(".png", cv_image)[1].tobytes()
+            image_path = self.storage.save_file(image_bytes, "images", image_filename)
+
+            # Create page record
+            doc_page = self.page_repo.create_page(
+                document_id=document.id,
+                page_number=page_num,
+                width=float(width),
+                height=float(height),
+                image_path=image_path,
+            )
+
+            # Run OCR on saved image
+            temp_path = self.storage.get_absolute_path(image_path)
+            self._run_ocr_on_page(doc_page, temp_path)
+
+        return document
+
+    def _run_ocr_on_page(self, doc_page: DocumentPage, image_path: str) -> None:
+        """Run PaddleOCR on a single page image and store results."""
+        ocr = get_ocr_engine()
+
+        try:
+            results = ocr.ocr(image_path, cls=True)
+        except Exception as e:
+            logger.error("ocr_failed", page_id=str(doc_page.id), error=str(e))
+            return
+
+        if not results or not results[0]:
+            logger.info("ocr_no_results", page_id=str(doc_page.id))
+            return
+
+        full_text_parts = []
+        sequence = 0
+
+        for line in results[0]:
+            if not line or len(line) < 2:
+                continue
+
+            bbox_points = line[0]  # List of 4 corner points
+            text_info = line[1]    # (text, confidence)
+
+            text = text_info[0] if isinstance(text_info, (list, tuple)) else str(text_info)
+            confidence = float(text_info[1]) if isinstance(text_info, (list, tuple)) and len(text_info) > 1 else 0.0
+
+            if not text.strip():
+                continue
+
+            # Convert bbox points to x, y, width, height
+            xs = [p[0] for p in bbox_points]
+            ys = [p[1] for p in bbox_points]
+            x = min(xs)
+            y = min(ys)
+            width = max(xs) - x
+            height = max(ys) - y
+
+            # Determine block type based on position
+            block_type = self._classify_text_block(
+                y, height, doc_page.height, text
+            )
+
+            self.text_block_repo.create_text_block(
+                page_id=doc_page.id,
+                text=text,
+                x=x,
+                y=y,
+                width=width,
+                height=height,
+                confidence=confidence,
+                block_type=block_type,
+                sequence=sequence,
+            )
+            full_text_parts.append(text)
+            sequence += 1
+
+        # Update page text content
+        doc_page.text_content = "\n".join(full_text_parts)
+
+    def _classify_text_block(
+        self,
+        y: float,
+        height: float,
+        page_height: float,
+        text: str,
+    ) -> str:
+        """Classify a text block as header, footer, watermark, or regular text."""
+        if page_height <= 0:
+            return "text"
+
+        relative_y = y / page_height
+
+        # Header: top 10%
+        if relative_y < 0.10:
+            return "header"
+
+        # Footer: bottom 10%
+        if relative_y > 0.90:
+            return "footer"
+
+        # Watermark detection heuristic: large text in center
+        if 0.3 < relative_y < 0.7 and height > page_height * 0.05:
+            # Check for common watermark words
+            watermark_keywords = {"confidential", "draft", "copy", "sample", "watermark", "void"}
+            if text.strip().lower() in watermark_keywords:
+                return "watermark"
+
+        return "text"

docengine/app/services/pdf_service.py

@@ -0,0 +1,292 @@
+from __future__ import annotations
+
+import uuid
+from pathlib import Path
+
+import cv2
+import fitz  # PyMuPDF
+import numpy as np
+from sqlalchemy.orm import Session
+
+from app.core.config import settings
+from app.core.logging_config import get_logger
+from app.models.document import Document, DocumentImage, DocumentPage, DocumentTable, DocumentTextBlock
+from app.repositories.document_repository import (
+    DocumentImageRepository,
+    DocumentPageRepository,
+    DocumentRepository,
+    DocumentTableRepository,
+    DocumentTextBlockRepository,
+)
+from app.storage.provider import get_storage_provider
+
+logger = get_logger(__name__)
+
+
+class NativePDFService:
+    """Extract text, fonts, images, and layout from native (non-scanned) PDFs using PyMuPDF."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+        self.storage = get_storage_provider()
+        self.doc_repo = DocumentRepository(db)
+        self.page_repo = DocumentPageRepository(db)
+        self.text_block_repo = DocumentTextBlockRepository(db)
+        self.image_repo = DocumentImageRepository(db)
+        self.table_repo = DocumentTableRepository(db)
+
+    def process_pdf(self, document: Document) -> Document:
+        """Process a native PDF document, extracting all content."""
+        file_path = self.storage.get_absolute_path(document.storage_path)
+
+        try:
+            pdf_doc = fitz.open(file_path)
+        except Exception as e:
+            logger.error("pdf_open_failed", document_id=str(document.id), error=str(e))
+            raise
+
+        document.page_count = len(pdf_doc)
+        document.is_scanned = self._is_scanned_pdf(pdf_doc)
+
+        for page_num in range(len(pdf_doc)):
+            page = pdf_doc[page_num]
+            self._process_page(document, page, page_num + 1)
+
+        pdf_doc.close()
+        return document
+
+    def _is_scanned_pdf(self, pdf_doc: fitz.Document) -> bool:
+        """Determine if a PDF is scanned (image-based) or native."""
+        total_text_chars = 0
+        total_images = 0
+        for page_num in range(min(len(pdf_doc), 3)):
+            page = pdf_doc[page_num]
+            text = page.get_text("text")
+            total_text_chars += len(text.strip())
+            total_images += len(page.get_images(full=True))
+
+        # If very little text but has images, likely scanned
+        if total_text_chars < 50 and total_images > 0:
+            return True
+        return False
+
+    def _process_page(self, document: Document, page: fitz.Page, page_number: int) -> DocumentPage:
+        """Process a single PDF page."""
+        rect = page.rect
+        width = rect.width
+        height = rect.height
+
+        # Save page as image for reference
+        pix = page.get_pixmap(dpi=150)
+        image_filename = f"{document.id}_page_{page_number}.png"
+        image_data = pix.tobytes("png")
+        image_path = self.storage.save_file(image_data, "images", image_filename)
+
+        # Get full text content
+        text_content = page.get_text("text")
+
+        doc_page = self.page_repo.create_page(
+            document_id=document.id,
+            page_number=page_number,
+            width=width,
+            height=height,
+            image_path=image_path,
+            text_content=text_content,
+        )
+
+        # Extract text blocks with font information
+        self._extract_text_blocks(doc_page, page)
+
+        # Extract images
+        self._extract_images(doc_page, page, document)
+
+        # Detect headers and footers
+        self._detect_headers_footers(doc_page, page)
+
+        return doc_page
+
+    def _extract_text_blocks(self, doc_page: DocumentPage, page: fitz.Page) -> None:
+        """Extract text blocks with positioning and font information."""
+        blocks = page.get_text("dict", flags=fitz.TEXT_PRESERVE_WHITESPACE)["blocks"]
+        sequence = 0
+
+        for block in blocks:
+            if block["type"] != 0:  # Skip non-text blocks
+                continue
+
+            block_text_parts = []
+            font_info = {"family": None, "size": None, "color": None, "style": None}
+
+            for line in block.get("lines", []):
+                for span in line.get("spans", []):
+                    text = span.get("text", "").strip()
+                    if text:
+                        block_text_parts.append(text)
+                        # Capture font info from the first non-empty span
+                        if font_info["family"] is None:
+                            font_info["family"] = span.get("font", None)
+                            font_info["size"] = span.get("size", None)
+                            color_int = span.get("color", 0)
+                            font_info["color"] = f"#{color_int:06x}" if isinstance(color_int, int) else None
+                            flags = span.get("flags", 0)
+                            styles = []
+                            if flags & 1:
+                                styles.append("superscript")
+                            if flags & 2:
+                                styles.append("italic")
+                            if flags & 4:
+                                styles.append("serif")
+                            if flags & 8:
+                                styles.append("monospace")
+                            if flags & 16:
+                                styles.append("bold")
+                            font_info["style"] = ",".join(styles) if styles else "regular"
+
+            full_text = " ".join(block_text_parts)
+            if not full_text.strip():
+                continue
+
+            bbox = block["bbox"]
+            self.text_block_repo.create_text_block(
+                page_id=doc_page.id,
+                text=full_text,
+                x=bbox[0],
+                y=bbox[1],
+                width=bbox[2] - bbox[0],
+                height=bbox[3] - bbox[1],
+                font_family=font_info["family"],
+                font_size=font_info["size"],
+                font_color=font_info["color"],
+                font_style=font_info["style"],
+                block_type="text",
+                sequence=sequence,
+            )
+            sequence += 1
+
+    def _extract_images(self, doc_page: DocumentPage, page: fitz.Page, document: Document) -> None:
+        """Extract embedded images from a PDF page."""
+        image_list = page.get_images(full=True)
+
+        for img_index, img_info in enumerate(image_list):
+            xref = img_info[0]
+            try:
+                base_image = page.parent.extract_image(xref)
+                if not base_image:
+                    continue
+
+                image_bytes = base_image["image"]
+                ext = base_image.get("ext", "png")
+                img_filename = f"{document.id}_page_{doc_page.page_number}_img_{img_index}.{ext}"
+                img_storage_path = self.storage.save_file(image_bytes, "images", img_filename)
+
+                # Try to get image position on page
+                img_rects = page.get_image_rects(xref)
+                if img_rects:
+                    rect = img_rects[0]
+                    x, y, x1, y1 = rect.x0, rect.y0, rect.x1, rect.y1
+                else:
+                    x, y, x1, y1 = 0, 0, base_image.get("width", 100), base_image.get("height", 100)
+
+                # Determine image type based on position
+                page_height = doc_page.height
+                page_width = doc_page.width
+                image_type = self._classify_image_type(x, y, x1, y1, page_width, page_height)
+
+                self.image_repo.create_image(
+                    page_id=doc_page.id,
+                    x=x,
+                    y=y,
+                    width=x1 - x,
+                    height=y1 - y,
+                    image_path=img_storage_path,
+                    image_type=image_type,
+                )
+
+            except Exception as e:
+                logger.warning(
+                    "image_extraction_failed",
+                    page_id=str(doc_page.id),
+                    img_index=img_index,
+                    error=str(e),
+                )
+
+    def _classify_image_type(
+        self,
+        x: float,
+        y: float,
+        x1: float,
+        y1: float,
+        page_width: float,
+        page_height: float,
+    ) -> str:
+        """Classify an image as logo, figure, background, or stamp based on position and size."""
+        width = x1 - x
+        height = y1 - y
+        area_ratio = (width * height) / (page_width * page_height) if page_width > 0 and page_height > 0 else 0
+
+        # Background: covers most of the page
+        if area_ratio > 0.8:
+            return "background"
+
+        # Logo: small image in top portion
+        if y < page_height * 0.15 and area_ratio < 0.05:
+            return "logo"
+
+        # Stamp: small image in bottom-right
+        if x > page_width * 0.6 and y > page_height * 0.7 and area_ratio < 0.05:
+            return "stamp"
+
+        return "figure"
+
+    def _detect_headers_footers(self, doc_page: DocumentPage, page: fitz.Page) -> None:
+        """Detect header and footer regions based on vertical position."""
+        page_height = page.rect.height
+        header_threshold = page_height * 0.1
+        footer_threshold = page_height * 0.9
+
+        blocks = page.get_text("dict")["blocks"]
+        header_seq = 0
+        footer_seq = 0
+
+        for block in blocks:
+            if block["type"] != 0:
+                continue
+
+            bbox = block["bbox"]
+            block_y = bbox[1]
+
+            text_parts = []
+            for line in block.get("lines", []):
+                for span in line.get("spans", []):
+                    t = span.get("text", "").strip()
+                    if t:
+                        text_parts.append(t)
+
+            full_text = " ".join(text_parts)
+            if not full_text.strip():
+                continue
+
+            if block_y < header_threshold:
+                self.text_block_repo.create_text_block(
+                    page_id=doc_page.id,
+                    text=full_text,
+                    x=bbox[0],
+                    y=bbox[1],
+                    width=bbox[2] - bbox[0],
+                    height=bbox[3] - bbox[1],
+                    block_type="header",
+                    sequence=header_seq,
+                )
+                header_seq += 1
+            elif block_y > footer_threshold:
+                self.text_block_repo.create_text_block(
+                    page_id=doc_page.id,
+                    text=full_text,
+                    x=bbox[0],
+                    y=bbox[1],
+                    width=bbox[2] - bbox[0],
+                    height=bbox[3] - bbox[1],
+                    block_type="footer",
+                    sequence=footer_seq,
+                )
+                footer_seq += 1

docengine/app/services/reconstruction_service.py

@@ -0,0 +1,455 @@
+from __future__ import annotations
+
+import os
+import uuid
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from reportlab.lib import colors
+from reportlab.lib.pagesizes import letter
+from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
+from reportlab.lib.units import inch, mm
+from reportlab.pdfbase import pdfmetrics
+from reportlab.pdfbase.ttfonts import TTFont
+from reportlab.platypus import (
+    BaseDocTemplate,
+    Frame,
+    Image,
+    NextPageTemplate,
+    PageBreak,
+    PageTemplate,
+    Paragraph,
+    SimpleDocTemplate,
+    Spacer,
+    Table,
+    TableStyle,
+)
+from sqlalchemy.orm import Session
+
+from app.core.logging_config import get_logger
+from app.models.template import DocumentFormat
+from app.schemas.template import TemplateRenderResponse
+from app.storage.provider import get_storage_provider
+
+logger = get_logger(__name__)
+
+
+class ReconstructionService:
+    """Reconstruct documents from templates using ReportLab."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+        self.storage = get_storage_provider()
+        self.styles = getSampleStyleSheet()
+        self._register_fonts()
+
+    def _register_fonts(self) -> None:
+        """Register additional fonts if available."""
+        # ReportLab includes Helvetica, Times-Roman, Courier by default
+        # Custom fonts can be registered here
+        pass
+
+    def render_template(
+        self,
+        template: DocumentFormat,
+        data: dict[str, Any],
+        output_filename: str | None = None,
+        images: dict[str, str] | None = None,
+    ) -> TemplateRenderResponse:
+        """Render a template to PDF with supplied data."""
+        if output_filename is None:
+            output_filename = f"{template.name}_{uuid.uuid4().hex[:8]}.pdf"
+
+        if not output_filename.endswith(".pdf"):
+            output_filename += ".pdf"
+
+        # Determine output path
+        output_storage_path = f"rendered/{output_filename}"
+        absolute_output_path = self.storage.get_absolute_path(output_storage_path)
+
+        # Ensure the rendered directory exists
+        os.makedirs(os.path.dirname(absolute_output_path), exist_ok=True)
+
+        # Build the PDF
+        self._build_pdf(template, data, absolute_output_path, images)
+
+        file_size = os.path.getsize(absolute_output_path)
+
+        logger.info(
+            "template_rendered",
+            template_id=str(template.id),
+            output=output_filename,
+            size=file_size,
+        )
+
+        return TemplateRenderResponse(
+            output_path=output_storage_path,
+            filename=output_filename,
+            file_size=file_size,
+            page_count=template.page_count,
+            rendered_at=datetime.now(UTC),
+        )
+
+    def _build_pdf(
+        self,
+        template: DocumentFormat,
+        data: dict[str, Any],
+        output_path: str,
+        images: dict[str, str] | None = None,
+    ) -> None:
+        """Build a PDF document from template definition."""
+        page_width = template.page_width
+        page_height = template.page_height
+
+        doc = SimpleDocTemplate(
+            output_path,
+            pagesize=(page_width, page_height),
+            topMargin=template.margin_top,
+            rightMargin=template.margin_right,
+            bottomMargin=template.margin_bottom,
+            leftMargin=template.margin_left,
+        )
+
+        # Build story (content elements)
+        story: list[Any] = []
+
+        for page_num in range(1, template.page_count + 1):
+            if page_num > 1:
+                story.append(PageBreak())
+
+            # Add page content
+            page_elements = self._build_page_content(template, page_num, data, images)
+            story.extend(page_elements)
+
+        # Build with watermark/header/footer callbacks
+        def on_page(canvas, doc_obj):  # noqa: ANN001, ANN202
+            self._draw_watermarks(canvas, template, doc_obj.page)
+            self._draw_headers_footers(canvas, template, doc_obj.page, page_width, page_height)
+
+        def on_page_later(canvas, doc_obj):  # noqa: ANN001, ANN202
+            self._draw_watermarks(canvas, template, doc_obj.page)
+            self._draw_headers_footers(canvas, template, doc_obj.page, page_width, page_height)
+
+        doc.build(story, onFirstPage=on_page, onLaterPages=on_page_later)
+
+    def _build_page_content(
+        self,
+        template: DocumentFormat,
+        page_number: int,
+        data: dict[str, Any],
+        images: dict[str, str] | None = None,
+    ) -> list[Any]:
+        """Build content elements for a specific page."""
+        elements: list[Any] = []
+
+        # Get cells for this page, sorted by sequence
+        page_cells = sorted(
+            [c for c in template.cells if c.page_number == page_number],
+            key=lambda c: c.sequence,
+        )
+
+        # Get table formats for this page
+        page_tables = [t for t in template.table_formats if t.page_number == page_number]
+
+        # Get image regions for this page
+        page_images = [i for i in template.image_regions if i.page_number == page_number]
+
+        # Add static and dynamic text cells
+        for cell in page_cells:
+            text = self._resolve_cell_text(cell, data)
+            if text:
+                style = self._create_cell_style(cell)
+                para = Paragraph(text, style)
+                elements.append(para)
+                elements.append(Spacer(1, 2))
+
+        # Add tables
+        for table_format in page_tables:
+            table_element = self._build_table(table_format, data)
+            if table_element:
+                elements.append(table_element)
+                elements.append(Spacer(1, 6))
+
+        # Add images
+        for img_region in page_images:
+            img_element = self._build_image(img_region, images)
+            if img_element:
+                elements.append(img_element)
+                elements.append(Spacer(1, 6))
+
+        if not elements:
+            elements.append(Spacer(1, 12))
+
+        return elements
+
+    def _resolve_cell_text(self, cell: Any, data: dict[str, Any]) -> str:
+        """Resolve cell text from static content or dynamic data."""
+        if cell.is_dynamic and cell.field_name:
+            value = data.get(cell.field_name, "")
+            return str(value) if value else ""
+        return cell.static_text or ""
+
+    def _create_cell_style(self, cell: Any) -> ParagraphStyle:
+        """Create a ReportLab paragraph style from cell properties."""
+        font_name = "Helvetica"
+        if cell.font_family:
+            family = cell.font_family.lower()
+            if "times" in family or "serif" in family:
+                font_name = "Times-Roman"
+            elif "courier" in family or "mono" in family:
+                font_name = "Courier"
+
+        font_size = cell.font_size or 10
+        if cell.font_style and "bold" in (cell.font_style or ""):
+            if font_name == "Helvetica":
+                font_name = "Helvetica-Bold"
+            elif font_name == "Times-Roman":
+                font_name = "Times-Bold"
+            elif font_name == "Courier":
+                font_name = "Courier-Bold"
+
+        text_color = colors.black
+        if cell.font_color:
+            try:
+                text_color = colors.HexColor(cell.font_color)
+            except (ValueError, TypeError):
+                text_color = colors.black
+
+        alignment_map = {"left": 0, "center": 1, "right": 2, "justify": 4}
+        alignment = alignment_map.get(cell.alignment, 0)
+
+        style = ParagraphStyle(
+            name=f"cell_{cell.id}",
+            parent=self.styles["Normal"],
+            fontName=font_name,
+            fontSize=font_size,
+            textColor=text_color,
+            alignment=alignment,
+            leading=font_size * 1.2,
+            spaceBefore=cell.padding_top,
+            spaceAfter=cell.padding_bottom,
+            leftIndent=cell.padding_left,
+            rightIndent=cell.padding_right,
+        )
+
+        return style
+
+    def _build_table(self, table_format: Any, data: dict[str, Any]) -> Table | None:
+        """Build a ReportLab table from a table format definition."""
+        rows = table_format.rows
+        columns = table_format.columns
+
+        if rows <= 0 or columns <= 0:
+            return None
+
+        # Build table data
+        table_data: list[list[str]] = []
+
+        # Header row
+        if table_format.table_columns:
+            header_row = [col.header_text or f"Col {col.column_index + 1}" for col in table_format.table_columns]
+            table_data.append(header_row)
+        else:
+            table_data.append([f"Column {i + 1}" for i in range(columns)])
+
+        # Data rows from supplied data
+        table_field_name = f"table_{table_format.id}"
+        table_rows_data = data.get(table_field_name, data.get("table_data", []))
+
+        if isinstance(table_rows_data, list):
+            for row_data in table_rows_data:
+                if isinstance(row_data, list):
+                    # Pad or trim to match column count
+                    row = row_data[:columns]
+                    while len(row) < columns:
+                        row.append("")
+                    table_data.append([str(v) for v in row])
+                elif isinstance(row_data, dict):
+                    row = []
+                    for col in table_format.table_columns:
+                        key = col.header_text or f"col_{col.column_index}"
+                        row.append(str(row_data.get(key, "")))
+                    table_data.append(row)
+
+        # If no data rows, add empty rows
+        if len(table_data) <= 1:
+            for _ in range(max(rows - 1, 1)):
+                table_data.append([""] * columns)
+
+        # Determine column widths
+        col_widths = []
+        if table_format.table_columns:
+            col_widths = [col.width for col in table_format.table_columns]
+        else:
+            col_width = table_format.width / columns
+            col_widths = [col_width] * columns
+
+        # Build table
+        table = Table(table_data, colWidths=col_widths)
+
+        # Apply table style
+        border_color = colors.black
+        if table_format.border_color:
+            try:
+                border_color = colors.HexColor(table_format.border_color)
+            except (ValueError, TypeError):
+                pass
+
+        style_commands = [
+            ("GRID", (0, 0), (-1, -1), table_format.border_width, border_color),
+            ("FONTNAME", (0, 0), (-1, 0), "Helvetica-Bold"),
+            ("FONTSIZE", (0, 0), (-1, -1), 9),
+            ("ALIGN", (0, 0), (-1, -1), "LEFT"),
+            ("VALIGN", (0, 0), (-1, -1), "MIDDLE"),
+            ("TOPPADDING", (0, 0), (-1, -1), 4),
+            ("BOTTOMPADDING", (0, 0), (-1, -1), 4),
+            ("LEFTPADDING", (0, 0), (-1, -1), 4),
+            ("RIGHTPADDING", (0, 0), (-1, -1), 4),
+        ]
+
+        # Header row background
+        if table_format.table_rows:
+            for row in table_format.table_rows:
+                if row.is_header and row.background_color:
+                    try:
+                        bg_color = colors.HexColor(row.background_color)
+                        style_commands.append(
+                            ("BACKGROUND", (0, row.row_index), (-1, row.row_index), bg_color)
+                        )
+                    except (ValueError, TypeError):
+                        pass
+        else:
+            style_commands.append(("BACKGROUND", (0, 0), (-1, 0), colors.HexColor("#E0E0E0")))
+
+        table.setStyle(TableStyle(style_commands))
+
+        return table
+
+    def _build_image(self, img_region: Any, images: dict[str, str] | None = None) -> Image | None:
+        """Build a ReportLab image from an image region definition."""
+        image_path = None
+
+        # Check dynamic images first
+        if not img_region.is_static and img_region.field_name and images:
+            image_path = images.get(img_region.field_name)
+
+        # Fall back to stored image
+        if not image_path and img_region.image_path:
+            try:
+                image_path = self.storage.get_absolute_path(img_region.image_path)
+            except Exception:
+                image_path = None
+
+        if not image_path or not os.path.exists(image_path):
+            return None
+
+        try:
+            img = Image(image_path, width=img_region.width, height=img_region.height)
+            return img
+        except Exception as e:
+            logger.warning("image_build_failed", error=str(e), path=image_path)
+            return None
+
+    def _draw_watermarks(self, canvas: Any, template: DocumentFormat, current_page: int) -> None:
+        """Draw watermarks on the canvas."""
+        for watermark in template.watermarks:
+            # Apply to all pages if page_number is None, or specific page
+            if watermark.page_number is not None and watermark.page_number != current_page:
+                continue
+
+            canvas.saveState()
+
+            # Set opacity
+            canvas.setFillAlpha(watermark.opacity)
+
+            if watermark.text:
+                # Text watermark
+                font_name = "Helvetica"
+                if watermark.font_family:
+                    family = watermark.font_family.lower()
+                    if "times" in family:
+                        font_name = "Times-Roman"
+                    elif "courier" in family:
+                        font_name = "Courier"
+
+                font_size = watermark.font_size or 48
+
+                if watermark.font_color:
+                    try:
+                        canvas.setFillColor(colors.HexColor(watermark.font_color))
+                    except (ValueError, TypeError):
+                        canvas.setFillColor(colors.grey)
+                else:
+                    canvas.setFillColor(colors.grey)
+
+                canvas.setFont(font_name, font_size)
+
+                # Position and rotate
+                canvas.translate(
+                    watermark.x + watermark.width / 2,
+                    watermark.y + watermark.height / 2,
+                )
+                canvas.rotate(watermark.rotation)
+                canvas.drawCentredString(0, 0, watermark.text)
+
+            elif watermark.image_path:
+                # Image watermark
+                try:
+                    img_path = self.storage.get_absolute_path(watermark.image_path)
+                    if os.path.exists(img_path):
+                        canvas.drawImage(
+                            img_path,
+                            watermark.x,
+                            watermark.y,
+                            width=watermark.width,
+                            height=watermark.height,
+                            mask="auto",
+                        )
+                except Exception as e:
+                    logger.warning("watermark_image_failed", error=str(e))
+
+            canvas.restoreState()
+
+    def _draw_headers_footers(
+        self,
+        canvas: Any,
+        template: DocumentFormat,
+        current_page: int,
+        page_width: float,
+        page_height: float,
+    ) -> None:
+        """Draw header and footer regions on the canvas."""
+        for region in template.regions:
+            if region.page_number != current_page:
+                continue
+
+            content = region.content or {}
+            blocks = content.get("blocks", [])
+
+            canvas.saveState()
+
+            for block in blocks:
+                text = block.get("text", "")
+                if not text:
+                    continue
+
+                x = block.get("x", region.x)
+                y = page_height - block.get("y", region.y) - block.get("height", 12)
+                font_family = block.get("font_family", "Helvetica")
+                font_size = block.get("font_size", 10)
+
+                # Map font family
+                font_name = "Helvetica"
+                if font_family:
+                    fl = font_family.lower()
+                    if "times" in fl or "serif" in fl:
+                        font_name = "Times-Roman"
+                    elif "courier" in fl or "mono" in fl:
+                        font_name = "Courier"
+
+                canvas.setFont(font_name, font_size)
+                canvas.setFillColor(colors.black)
+                canvas.drawString(x, y, text)
+
+            canvas.restoreState()

docengine/app/services/template_service.py

@@ -0,0 +1,401 @@
+from __future__ import annotations
+
+import uuid
+from typing import Any
+
+from sqlalchemy.orm import Session
+
+from app.core.logging_config import get_logger
+from app.models.document import Document
+from app.models.template import (
+    DocumentCell,
+    DocumentFormat,
+    DocumentRegion,
+    ImageRegion,
+    TableColumn,
+    TableFormat,
+    TableRow,
+    Watermark,
+)
+from app.repositories.document_repository import DocumentRepository
+from app.repositories.template_repository import (
+    DocumentCellRepository,
+    DocumentRegionRepository,
+    ImageRegionRepository,
+    TableColumnRepository,
+    TableFormatRepository,
+    TableRowRepository,
+    TemplateRepository,
+    WatermarkRepository,
+)
+from app.services.fingerprint_service import FingerprintService
+
+logger = get_logger(__name__)
+
+
+class TemplateService:
+    """Generate reusable document templates from processed documents."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+        self.doc_repo = DocumentRepository(db)
+        self.template_repo = TemplateRepository(db)
+        self.cell_repo = DocumentCellRepository(db)
+        self.region_repo = DocumentRegionRepository(db)
+        self.table_format_repo = TableFormatRepository(db)
+        self.table_column_repo = TableColumnRepository(db)
+        self.table_row_repo = TableRowRepository(db)
+        self.watermark_repo = WatermarkRepository(db)
+        self.image_region_repo = ImageRegionRepository(db)
+        self.fingerprint_service = FingerprintService(db)
+
+    def generate_template(
+        self,
+        document: Document,
+        user_id: uuid.UUID | None = None,
+    ) -> DocumentFormat:
+        """Generate a reusable template from a processed document."""
+        if not document.pages:
+            raise ValueError(f"Document '{document.id}' has no processed pages")
+
+        # Check if template already exists for this document
+        existing = self.template_repo.get_by_source_document(document.id)
+        if existing:
+            logger.info(
+                "template_already_exists",
+                document_id=str(document.id),
+                template_id=str(existing.id),
+            )
+            return existing
+
+        first_page = document.pages[0]
+        template_name = f"Template_{document.original_filename}_{uuid.uuid4().hex[:8]}"
+
+        # Create template
+        template = self.template_repo.create_template(
+            name=template_name,
+            page_width=first_page.width,
+            page_height=first_page.height,
+            page_count=document.page_count or len(document.pages),
+            description=f"Auto-generated template from {document.original_filename}",
+            source_document_id=document.id,
+            created_by=user_id,
+        )
+
+        # Process each page
+        for page in document.pages:
+            self._process_page_for_template(template, page)
+
+        # Generate fingerprint
+        self.fingerprint_service.generate_fingerprint(template)
+
+        logger.info(
+            "template_generated",
+            template_id=str(template.id),
+            document_id=str(document.id),
+            cells=len(template.cells),
+            regions=len(template.regions),
+        )
+
+        return template
+
+    def _process_page_for_template(
+        self,
+        template: DocumentFormat,
+        page: Any,
+    ) -> None:
+        """Process a document page and create template components."""
+        page_number = page.page_number
+
+        # Create cells from text blocks
+        self._create_cells_from_text_blocks(template, page, page_number)
+
+        # Create regions from headers, footers
+        self._create_regions(template, page, page_number)
+
+        # Create table formats
+        self._create_table_formats(template, page, page_number)
+
+        # Create image regions
+        self._create_image_regions(template, page, page_number)
+
+        # Detect watermarks from text blocks
+        self._create_watermarks(template, page, page_number)
+
+    def _create_cells_from_text_blocks(
+        self,
+        template: DocumentFormat,
+        page: Any,
+        page_number: int,
+    ) -> None:
+        """Create template cells from extracted text blocks."""
+        for seq, block in enumerate(page.text_blocks):
+            if block.block_type in ("header", "footer", "watermark"):
+                continue
+
+            # Determine if this is a dynamic field
+            is_dynamic = self._is_dynamic_field(block.text)
+            field_name = self._generate_field_name(block.text, seq) if is_dynamic else None
+
+            self.cell_repo.create_cell(
+                format_id=template.id,
+                page_number=page_number,
+                x=block.x,
+                y=block.y,
+                width=block.width,
+                height=block.height,
+                data_type=self._infer_data_type(block.text),
+                font_family=block.font_family,
+                font_size=block.font_size,
+                font_style=block.font_style,
+                font_color=block.font_color,
+                alignment=self._infer_alignment(block.x, template.page_width),
+                static_text=block.text if not is_dynamic else None,
+                field_name=field_name,
+                sequence=seq,
+                is_dynamic=is_dynamic,
+            )
+
+    def _create_regions(
+        self,
+        template: DocumentFormat,
+        page: Any,
+        page_number: int,
+    ) -> None:
+        """Create template regions from headers and footers."""
+        header_blocks = [b for b in page.text_blocks if b.block_type == "header"]
+        footer_blocks = [b for b in page.text_blocks if b.block_type == "footer"]
+
+        if header_blocks:
+            # Compute bounding box for all header blocks
+            min_x = min(b.x for b in header_blocks)
+            min_y = min(b.y for b in header_blocks)
+            max_x = max(b.x + b.width for b in header_blocks)
+            max_y = max(b.y + b.height for b in header_blocks)
+
+            content = {
+                "blocks": [
+                    {
+                        "text": b.text,
+                        "x": b.x,
+                        "y": b.y,
+                        "width": b.width,
+                        "height": b.height,
+                        "font_family": b.font_family,
+                        "font_size": b.font_size,
+                    }
+                    for b in header_blocks
+                ]
+            }
+
+            self.region_repo.create_region(
+                format_id=template.id,
+                page_number=page_number,
+                region_type="header",
+                x=min_x,
+                y=min_y,
+                width=max_x - min_x,
+                height=max_y - min_y,
+                content=content,
+                sequence=0,
+            )
+
+        if footer_blocks:
+            min_x = min(b.x for b in footer_blocks)
+            min_y = min(b.y for b in footer_blocks)
+            max_x = max(b.x + b.width for b in footer_blocks)
+            max_y = max(b.y + b.height for b in footer_blocks)
+
+            content = {
+                "blocks": [
+                    {
+                        "text": b.text,
+                        "x": b.x,
+                        "y": b.y,
+                        "width": b.width,
+                        "height": b.height,
+                        "font_family": b.font_family,
+                        "font_size": b.font_size,
+                    }
+                    for b in footer_blocks
+                ]
+            }
+
+            self.region_repo.create_region(
+                format_id=template.id,
+                page_number=page_number,
+                region_type="footer",
+                x=min_x,
+                y=min_y,
+                width=max_x - min_x,
+                height=max_y - min_y,
+                content=content,
+                sequence=1,
+            )
+
+    def _create_table_formats(
+        self,
+        template: DocumentFormat,
+        page: Any,
+        page_number: int,
+    ) -> None:
+        """Create table format definitions from detected tables."""
+        for table in page.tables:
+            table_format = self.table_format_repo.create_table_format(
+                format_id=template.id,
+                page_number=page_number,
+                x=table.x,
+                y=table.y,
+                width=table.width,
+                height=table.height,
+                rows=table.rows,
+                columns=table.columns,
+            )
+
+            # Create columns
+            col_width = table.width / max(table.columns, 1)
+            for col_idx in range(table.columns):
+                self.table_column_repo.create_column(
+                    table_format_id=table_format.id,
+                    column_index=col_idx,
+                    width=col_width,
+                    data_type="text",
+                    alignment="left",
+                )
+
+            # Create rows
+            row_height = table.height / max(table.rows, 1)
+            for row_idx in range(table.rows):
+                self.table_row_repo.create_row(
+                    table_format_id=table_format.id,
+                    row_index=row_idx,
+                    height=row_height,
+                    is_header=(row_idx == 0),
+                )
+
+    def _create_image_regions(
+        self,
+        template: DocumentFormat,
+        page: Any,
+        page_number: int,
+    ) -> None:
+        """Create image region definitions from detected images."""
+        for img in page.images:
+            self.image_region_repo.create_image_region(
+                format_id=template.id,
+                page_number=page_number,
+                x=img.x,
+                y=img.y,
+                width=img.width,
+                height=img.height,
+                image_path=img.image_path,
+                image_type=img.image_type,
+                is_static=True,
+            )
+
+    def _create_watermarks(
+        self,
+        template: DocumentFormat,
+        page: Any,
+        page_number: int,
+    ) -> None:
+        """Create watermark definitions from detected watermark text blocks."""
+        watermark_blocks = [b for b in page.text_blocks if b.block_type == "watermark"]
+        for block in watermark_blocks:
+            self.watermark_repo.create_watermark(
+                format_id=template.id,
+                page_number=page_number,
+                text=block.text,
+                x=block.x,
+                y=block.y,
+                width=block.width,
+                height=block.height,
+                opacity=0.3,
+                rotation=0.0,
+                font_family=block.font_family,
+                font_size=block.font_size,
+                font_color=block.font_color or "#CCCCCC",
+            )
+
+    def _is_dynamic_field(self, text: str) -> bool:
+        """Determine if a text block represents a dynamic (variable) field."""
+        if not text:
+            return False
+
+        # Common patterns indicating dynamic content
+        dynamic_patterns = [
+            "{{", "}}", "${", "##",
+            "__________", "___", "...........",
+        ]
+        for pattern in dynamic_patterns:
+            if pattern in text:
+                return True
+
+        # Short single-word values that might be labels are static
+        # Longer values with numbers/dates tend to be dynamic
+        import re
+        # Date patterns
+        if re.search(r"\d{1,2}[/\-\.]\d{1,2}[/\-\.]\d{2,4}", text):
+            return True
+        # Currency patterns
+        if re.search(r"[$€£¥]\s*[\d,]+\.?\d*", text):
+            return True
+        # Phone patterns
+        if re.search(r"\+?\d[\d\s\-()]{7,}", text):
+            return True
+
+        return False
+
+    def _generate_field_name(self, text: str, sequence: int) -> str:
+        """Generate a field name from text content."""
+        import re
+        # Clean text
+        clean = re.sub(r"[^a-zA-Z0-9\s]", "", text)
+        clean = clean.strip().lower()
+        words = clean.split()[:3]
+        if words:
+            return "_".join(words)
+        return f"field_{sequence}"
+
+    def _infer_data_type(self, text: str) -> str:
+        """Infer the data type from text content."""
+        import re
+
+        if not text:
+            return "text"
+
+        stripped = text.strip()
+
+        # Number
+        if re.match(r"^-?[\d,]+\.?\d*$", stripped.replace(",", "")):
+            return "number"
+
+        # Date
+        if re.search(r"\d{1,2}[/\-\.]\d{1,2}[/\-\.]\d{2,4}", stripped):
+            return "date"
+
+        # Currency
+        if re.search(r"^[$€£¥]\s*[\d,]+\.?\d*$", stripped):
+            return "currency"
+
+        # Email
+        if re.search(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", stripped):
+            return "email"
+
+        return "text"
+
+    def _infer_alignment(self, x: float, page_width: float) -> str:
+        """Infer text alignment based on horizontal position."""
+        if page_width <= 0:
+            return "left"
+
+        relative_x = x / page_width
+
+        if relative_x < 0.15:
+            return "left"
+        elif relative_x > 0.6:
+            return "right"
+        elif 0.35 < relative_x < 0.65:
+            return "center"
+
+        return "left"