commit

check_matches.py

@@ -0,0 +1,16 @@
+import sys
+import os
+sys.path.append(os.path.join(os.getcwd(), 'docengine'))
+
+from app.core.database import SessionLocal
+from app.models.document import TemplateMatch
+from app.models.template import DocumentFormat
+
+db = SessionLocal()
+matches = db.query(TemplateMatch).order_by(TemplateMatch.created_at.desc()).limit(10).all()
+print(f"Found {len(matches)} matches.")
+for m in matches:
+    fmt = db.query(DocumentFormat).filter(DocumentFormat.id == m.format_id).first()
+    fmt_name = fmt.name if fmt else 'Unknown'
+    print(f"Match: doc_id={m.document_id}, format_id={m.format_id}, name={fmt_name}, score={m.confidence_score}")
+    print(f"Details: {m.match_details}")

debug_match.py

@@ -0,0 +1,26 @@
+import sys
+import os
+sys.path.append(os.path.join(os.getcwd(), 'docengine'))
+
+from app.core.database import SessionLocal
+from app.models.document import Document
+from app.services.matching_service import MatchingService
+
+db = SessionLocal()
+doc = db.query(Document).order_by(Document.created_at.desc()).first()
+if not doc:
+    print("No documents found.")
+    sys.exit(0)
+
+print(f"Latest document: {doc.id} (status: {doc.status})")
+
+svc = MatchingService(db)
+try:
+    # Match with min_confidence=0.0 so it returns EVERYTHING
+    matches = svc.match_document(doc.id, min_confidence=0.0)
+    print(f"Returned {len(matches)} matches.")
+    for m in matches:
+        print(f"Match: format_id={m.format_id}, score={m.confidence_score}")
+        print(f"Details: {m.match_details}")
+except Exception as e:
+    print(f"Error matching: {e}")

docengine/.gitignore

@@ -58,7 +58,7 @@ ENV/
 *~
 .DS_Store
 Thumbs.db
-storage/
+/storage/
 *.pid
 celerybeat-schedule
 celerybeat.pid

docengine/.pids/server.pid

@@ -0,0 +1 @@
+3173

docengine/.pids/worker.pid

@@ -0,0 +1 @@
+3176

docengine/app/api/v1/documents.py

@@ -235,6 +235,45 @@ def get_document_template_matches(
     return results
 
 
+@router.post(
+    "/{document_id}/extraction",
+    response_model=dict,
+    summary="Extract Document Data",
+    description="Match a document against templates and extract key-value & table data.",
+)
+def extract_document_data(
+    document_id: uuid.UUID,
+    current_user: CurrentUser = None,
+    db: Session = Depends(get_db),
+) -> dict:
+    """Extract document data using matched template mappings."""
+    doc_repo = DocumentRepository(db)
+    document = doc_repo.get_by_id(document_id)
+    if not document:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Document '{document_id}' not found",
+        )
+
+    if document.status != "completed":
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"Document must be in 'completed' status. Current status: '{document.status}'",
+        )
+
+    from app.services.extraction_service import ExtractionService
+    extraction_service = ExtractionService(db)
+    try:
+        result = extraction_service.extract_document_data(document_id)
+        return result
+    except Exception as e:
+        logger.exception("extraction_endpoint_failed", document_id=str(document_id), error=str(e))
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Extraction failed: {str(e)}",
+        )
+
+
 @router.delete(
     "/{document_id}",
     response_model=SuccessResponse,

docengine/app/api/v1/templates.py

@@ -10,7 +10,13 @@ from app.core.database import get_db
 from app.core.dependencies import CurrentUser
 from app.core.logging_config import get_logger
 from app.repositories.document_repository import DocumentRepository, TemplateMatchRepository
-from app.repositories.template_repository import TemplateRepository
+from app.repositories.template_repository import (
+    TemplateRepository, 
+    DocumentRegionRepository, 
+    DocumentCellRepository,
+    TableFormatRepository,
+    TableColumnRepository
+)
 from app.schemas.common import PaginatedResponse, SuccessResponse
 from app.schemas.document import TemplateMatchRequest, TemplateMatchResponse
 from app.schemas.template import (
@@ -18,6 +24,8 @@ from app.schemas.template import (
     TemplateRenderRequest,
     TemplateRenderResponse,
     TemplateResponse,
+    TemplateCreateRequest,
+    TemplateMappingSaveRequest,
 )
 
 logger = get_logger(__name__)
@@ -25,6 +33,198 @@ logger = get_logger(__name__)
 router = APIRouter(prefix="/templates", tags=["Templates"])
 
 
+@router.post(
+    "",
+    response_model=dict,
+    summary="Create Template",
+    description="Create a new template.",
+)
+def create_template(
+    payload: TemplateCreateRequest,
+    current_user: CurrentUser,
+    db: Session = Depends(get_db),
+) -> dict:
+    """Create a template."""
+    template_repo = TemplateRepository(db)
+    
+    template = template_repo.create_template(
+        name=payload.template_name,
+        page_width=1000.0,
+        page_height=1000.0,
+        source_document_id=uuid.UUID(payload.source_document_id) if payload.source_document_id else None
+    )
+    cell_repo = DocumentCellRepository(db)
+    saved_fields = []
+    
+    for index, field in enumerate(payload.fields):
+        cell = cell_repo.create_cell(
+            format_id=template.id,
+            page_number=1,
+            x=0.0,
+            y=0.0,
+            width=0.0,
+            height=0.0,
+            data_type=field.field_type,
+            field_name=field.field_label,
+            sequence=field.display_order or index,
+            is_dynamic=True
+        )
+        saved_fields.append({
+            "field_label": cell.field_name,
+            "field_type": cell.data_type
+        })
+        
+    db.commit()
+
+    return {
+        "pk_template_id": str(template.id),
+        "template_name": template.name,
+        "fields": saved_fields
+    }
+
+
+@router.put(
+    "/{template_id}",
+    response_model=dict,
+    summary="Update Template",
+    description="Update an existing template name and its fields.",
+)
+def update_template(
+    template_id: uuid.UUID,
+    payload: TemplateCreateRequest,
+    current_user: CurrentUser,
+    db: Session = Depends(get_db),
+) -> dict:
+    """Update a template."""
+    template_repo = TemplateRepository(db)
+    template = template_repo.get_by_id(template_id)
+    if not template:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Template '{template_id}' not found",
+        )
+
+    if template.name != payload.template_name:
+        template.name = payload.template_name
+
+    from app.models.template import DocumentCell
+    db.query(DocumentCell).filter(DocumentCell.format_id == template_id).delete()
+    
+    saved_fields = []
+    cell_repo = DocumentCellRepository(db)
+    for index, field in enumerate(payload.fields):
+        cell = cell_repo.create_cell(
+            format_id=template.id,
+            page_number=1,
+            row_no=0,
+            column_no=0,
+            field_name=field.field_label,
+            data_type=field.field_type,
+            x=0.0,
+            y=0.0,
+            width=0.0,
+            height=0.0,
+            is_dynamic=True
+        )
+        saved_fields.append({
+            "field_label": cell.field_name,
+            "field_type": cell.data_type
+        })
+        
+    db.commit()
+
+    return {
+        "pk_template_id": str(template.id),
+        "template_name": template.name,
+        "fields": saved_fields
+    }
+
+
+@router.post(
+    "/{template_id}/mappings/save",
+    response_model=SuccessResponse,
+    summary="Save Template Mappings",
+    description="Save the field mappings for a template.",
+)
+def save_mappings(
+    template_id: uuid.UUID,
+    payload: list[TemplateMappingSaveRequest],
+    current_user: CurrentUser,
+    db: Session = Depends(get_db),
+) -> SuccessResponse:
+    """Save template mappings."""
+    template_repo = TemplateRepository(db)
+    template = template_repo.get_by_id(template_id)
+    if not template:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Template '{template_id}' not found",
+        )
+
+    region_repo = DocumentRegionRepository(db)
+    cell_repo = DocumentCellRepository(db)
+    table_repo = TableFormatRepository(db)
+    table_col_repo = TableColumnRepository(db)
+    
+    # Delete existing field mappings and table formats for this template
+    from app.models.template import DocumentRegion, TableFormat
+    db.query(DocumentRegion).filter(
+        DocumentRegion.format_id == template_id, 
+        DocumentRegion.region_type == "field_mapping"
+    ).delete()
+    db.query(TableFormat).filter(TableFormat.format_id == template_id).delete()
+    db.flush()
+
+    cells = cell_repo.get_template_cells(template_id)
+    cell_map = {c.field_name: c for c in cells}
+    table_format = None
+    
+    # Save each mapped node
+    for mapping in payload:
+        cell = cell_map.get(mapping.field_name)
+        is_table_column = cell and cell.data_type == 'TABLE_COLUMN'
+        
+        for node_idx, node in enumerate(mapping.mapped_nodes):
+            if is_table_column:
+                if not table_format:
+                    table_format = table_repo.create_table_format(
+                        format_id=template_id,
+                        page_number=node.page_no or 1,
+                        x=0.0, y=0.0, width=1000.0, height=1000.0,
+                        rows=1, columns=10
+                    )
+                table_col_repo.create_column(
+                    table_format_id=table_format.id,
+                    column_index=node_idx,
+                    width=node.width,
+                    header_text=mapping.field_name,
+                    data_type="text"
+                )
+
+            region_repo.create_region(
+                format_id=template_id,
+                page_number=node.page_no or 1,
+                region_type="field_mapping",
+                x=node.x_coordinate,
+                y=node.y_coordinate,
+                width=node.width,
+                height=node.height,
+                content={
+                    "field_name": mapping.field_name,
+                    "text_value": node.text_value
+                }
+            )
+            
+    db.commit()
+    
+    # Generate fingerprint now that mappings are populated
+    from app.services.fingerprint_service import FingerprintService
+    FingerprintService(db).generate_fingerprint(template)
+    db.commit()
+    
+    return SuccessResponse(message="Mappings saved successfully")
+
+
 @router.get(
     "",
     response_model=PaginatedResponse[TemplateListResponse],
@@ -32,9 +232,9 @@ router = APIRouter(prefix="/templates", tags=["Templates"])
     description="List all active templates with pagination.",
 )
 def list_templates(
+    current_user: CurrentUser,
     page: int = Query(default=1, ge=1),
     page_size: int = Query(default=20, ge=1, le=100),
-    current_user: CurrentUser = None,
     db: Session = Depends(get_db),
 ) -> PaginatedResponse[TemplateListResponse]:
     """List all active templates."""
@@ -61,7 +261,7 @@ def list_templates(
 )
 def get_template(
     template_id: uuid.UUID,
-    current_user: CurrentUser = None,
+    current_user: CurrentUser,
     db: Session = Depends(get_db),
 ) -> TemplateResponse:
     """Get a template by ID."""
@@ -83,7 +283,7 @@ def get_template(
 )
 def delete_template(
     template_id: uuid.UUID,
-    current_user: CurrentUser = None,
+    current_user: CurrentUser,
     db: Session = Depends(get_db),
 ) -> SuccessResponse:
     """Soft-delete a template."""
@@ -109,7 +309,7 @@ def delete_template(
 )
 def match_template(
     payload: TemplateMatchRequest,
-    current_user: CurrentUser = None,
+    current_user: CurrentUser,
     db: Session = Depends(get_db),
 ) -> list[TemplateMatchResponse]:
     """Match a document against existing templates."""
@@ -162,7 +362,7 @@ def match_template(
 )
 def render_template(
     payload: TemplateRenderRequest,
-    current_user: CurrentUser = None,
+    current_user: CurrentUser,
     db: Session = Depends(get_db),
 ) -> TemplateRenderResponse:
     """Render a template to PDF."""
@@ -199,8 +399,8 @@ def render_template(
 )
 def download_rendered_pdf(
     template_id: uuid.UUID,
+    current_user: CurrentUser,
     filename: str = Query(..., description="Filename of the rendered PDF"),
-    current_user: CurrentUser = None,
     db: Session = Depends(get_db),
 ) -> FileResponse:
     """Download a rendered PDF."""

docengine/app/core/config.py

@@ -83,13 +83,20 @@ class Settings(BaseSettings):
     @classmethod
     def parse_cors_origins(cls, v: Any) -> list[str]:
         if isinstance(v, str):
+            if not v.strip():
+                return []
             try:
                 parsed = json.loads(v)
                 if isinstance(parsed, list):
-                    return parsed
+                    return [str(item).strip() for item in parsed]
+                elif isinstance(parsed, str):
+                    return [parsed.strip()]
             except (json.JSONDecodeError, TypeError):
+                pass
             return [origin.strip() for origin in v.split(",") if origin.strip()]
-        return v
+        if isinstance(v, list):
+            return [str(item).strip() for item in v]
+        return []
 
     @property
     def database_url(self) -> str:

docengine/app/repositories/template_repository.py

@@ -70,6 +70,7 @@ class TemplateRepository(BaseRepository[DocumentFormat]):
         fingerprint: dict | None = None,
         source_document_id: uuid.UUID | None = None,
         created_by: uuid.UUID | None = None,
+        is_active: bool = True,
     ) -> DocumentFormat:
         """Create a new template."""
         template = DocumentFormat(
@@ -85,6 +86,7 @@ class TemplateRepository(BaseRepository[DocumentFormat]):
             fingerprint=fingerprint,
             source_document_id=source_document_id,
             created_by=created_by,
+            is_active=is_active,
         )
         return self.create(template)
 

docengine/app/schemas/document.py

@@ -130,5 +130,5 @@ class TemplateMatchRequest(BaseSchema):
     """Request to match a document against templates."""
 
     document_id: uuid.UUID
-    min_confidence: float = Field(default=0.5, ge=0.0, le=1.0)
+    min_confidence: float = Field(default=0.75, ge=0.0, le=1.0)
     max_results: int = Field(default=5, ge=1, le=20)

docengine/app/schemas/template.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import uuid
 from datetime import datetime
-from typing import Any
+from typing import Any, List, Optional, Union
 
 from pydantic import Field
 
@@ -248,3 +248,30 @@ class TemplateRenderResponse(BaseSchema):
     file_size: int
     page_count: int
     rendered_at: datetime
+
+
+class TemplateFieldCreate(BaseSchema):
+    field_label: str
+    field_type: str = "text"
+    display_order: int = 0
+    required_flag: bool = False
+
+class TemplateCreateRequest(BaseSchema):
+    template_name: str
+    source_document_id: Optional[str] = None
+    fields: List[TemplateFieldCreate] = Field(default_factory=list)
+
+class MappingNodeRequest(BaseSchema):
+    pk_document_data_id: Union[int, str, None] = None
+    x_coordinate: float
+    y_coordinate: float
+    width: float
+    height: float
+    text_value: Optional[str] = None
+    page_width: Optional[float] = None
+    page_height: Optional[float] = None
+    page_no: Optional[int] = None
+
+class TemplateMappingSaveRequest(BaseSchema):
+    field_name: str
+    mapped_nodes: List[MappingNodeRequest] = Field(default_factory=list)

docengine/app/services/document_service.py

@@ -54,13 +54,11 @@ class DocumentProcessingService:
 
             # Step 2: Analyze layout
             layout_results = self.layout_service.analyze_document_layout(document)
-            document.document_metadata = document.document_metadata or {}
+            metadata = document.document_metadata or {}
-            document.document_metadata["layout"] = layout_results
+            metadata["layout"] = layout_results
+            document.document_metadata = metadata
 
-            # Step 3: Generate template
+            # Step 3: Update document status
-            template = self.template_service.generate_template(document)
-
-            # Step 4: Update document status
             self.doc_repo.update_status(document_id, "completed")
             self.db.commit()
 
@@ -68,7 +66,6 @@ class DocumentProcessingService:
                 "processing_completed",
                 document_id=str(document_id),
                 pages=document.page_count,
-                template_id=str(template.id),
             )
 
             return document

docengine/app/services/extraction_service.py

@@ -0,0 +1,295 @@
+from __future__ import annotations
+
+import uuid
+from typing import Any
+
+from sqlalchemy.orm import Session
+
+from app.core.logging_config import get_logger
+from app.repositories.document_repository import DocumentRepository
+from app.repositories.template_repository import TemplateRepository
+from app.services.matching_service import MatchingService
+
+logger = get_logger(__name__)
+
+
+class ExtractionService:
+    """Extract document values using matched template coordinates and structures."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+        self.matching_service = MatchingService(db)
+        self.doc_repo = DocumentRepository(db)
+        self.template_repo = TemplateRepository(db)
+
+    def extract_document_data(self, document_id: uuid.UUID) -> dict[str, Any]:
+        """Perform template matching on the document and extract structured field values."""
+        document = self.doc_repo.get_with_pages(document_id)
+        if not document:
+            raise ValueError(f"Document '{document_id}' not found")
+
+        # 1. Match document against existing templates
+        matches = self.matching_service.match_document(document_id, min_confidence=0.75)
+        if not matches:
+            logger.info("extraction_failed_no_match", document_id=str(document_id))
+            return {
+                "template_matched": False,
+                "template_id": None,
+                "template_name": None,
+                "confidence_score": 0.0,
+                "extracted_data": {},
+            }
+
+        best_match = matches[0]
+        template = self.template_repo.get_by_id(best_match.format_id)
+        if not template:
+            raise ValueError(f"Matched template '{best_match.format_id}' not found")
+
+        logger.info(
+            "extraction_matched_template",
+            document_id=str(document_id),
+            template_id=str(template.id),
+            template_name=template.name,
+            score=best_match.confidence_score,
+        )
+
+        extracted_data: dict[str, Any] = {}
+
+        # Load all mapped regions for the template
+        regions = [r for r in template.regions if r.region_type == "field_mapping"]
+        regions_by_field: dict[str, list[Any]] = {}
+        for r in regions:
+            field_name = r.content.get("field_name") if r.content else None
+            if field_name:
+                regions_by_field.setdefault(field_name, []).append(r)
+
+        # 2. Divide fields into Scalar vs Table Column types
+        scalar_cells = [cell for cell in template.cells if cell.data_type != "TABLE_COLUMN"]
+        table_column_cells = [cell for cell in template.cells if cell.data_type == "TABLE_COLUMN"]
+
+        # 3. Extract Scalar Fields
+        for cell in scalar_cells:
+            field_name = cell.field_name
+            if not field_name:
+                continue
+
+            field_regions = regions_by_field.get(field_name, [])
+            if not field_regions:
+                extracted_data[field_name] = ""
+                continue
+
+            extracted_values = []
+            extracted_block_ids = set()
+            sorted_regions = sorted(field_regions, key=lambda r: (r.page_number, r.sequence, r.y, r.x))
+
+            for region in sorted_regions:
+                page = next((p for p in document.pages if p.page_number == region.page_number), None)
+                if not page:
+                    continue
+                best_block = self._find_best_overlapping_block(page, region)
+                if best_block and best_block.id not in extracted_block_ids:
+                    extracted_block_ids.add(best_block.id)
+                    val = best_block.text.strip()
+                    if val:
+                        extracted_values.append(val)
+
+            extracted_data[field_name] = " ".join(extracted_values)
+
+        # 4. Extract Table Column Fields
+        if table_column_cells:
+            table_column_names = {cell.field_name for cell in table_column_cells if cell.field_name}
+            table_regions = [
+                r for r in regions 
+                if r.content and r.content.get("field_name") in table_column_names
+            ]
+
+            if table_regions:
+                # Determine vertical boundaries of the table area
+                table_start_y = min((r.y for r in table_regions), default=0.0)
+                table_start_y = max(0.0, table_start_y - 10.0)  # subtract buffer
+
+                # Process page where the table coordinates are mapped
+                page_number = min((r.page_number for r in table_regions), default=1)
+                page = next((p for p in document.pages if p.page_number == page_number), None)
+
+                # Determine table vertical end Y by finding any summary scalar fields below the table
+                summary_keywords = {"tax", "total", "shipping", "discount", "vat", "handling", "duty", "subtotal", "grand"}
+                summary_regions = []
+                for col_name, regs in regions_by_field.items():
+                    if col_name in table_column_names:
+                        continue
+                    for r in regs:
+                        if r.y > table_start_y and any(kw in col_name.lower() for kw in summary_keywords):
+                            # Skip left-aligned metadata fields (like Shipping Method)
+                            if page and r.x < page.width * 0.4:
+                                continue
+                            summary_regions.append(r)
+
+                table_end_y = min((r.y for r in summary_regions), default=99999.0)
+
+                # Check if template has a footer to define the end vertical boundary
+                footer_regions = [r for r in template.regions if r.region_type == "footer"]
+                if footer_regions:
+                    table_end_y = min(table_end_y, min(r.y for r in footer_regions))
+
+                # Determine horizontal ranges (X span) for each column in the template
+                col_x_spans: dict[str, tuple[float, float]] = {}
+                for col_name in table_column_names:
+                    col_regs = [r for r in table_regions if r.content and r.content.get("field_name") == col_name]
+                    if col_regs:
+                        x_min = min(r.x for r in col_regs)
+                        x_max = max(r.x + r.width for r in col_regs)
+                        # Add a 15px margin to accommodate layout differences
+                        col_x_spans[col_name] = (max(0.0, x_min - 15.0), x_max + 15.0)
+                    else:
+                        col_x_spans[col_name] = (0.0, 0.0)
+
+                if page:
+                    # Check for summary keyword text blocks (e.g. Total, Tax) as vertical boundary fallback
+                    summary_labels = {"subtotal", "total", "grand total", "tax", "shipping & handling", "discount"}
+                    for block in page.text_blocks:
+                        if block.y > table_start_y:
+                            # Skip left-aligned text blocks
+                            if block.x < page.width * 0.4:
+                                continue
+                            block_text_clean = block.text.strip().lower()
+                            if any(label in block_text_clean for label in summary_labels):
+                                table_end_y = min(table_end_y, block.y)
+
+                    # Collect all document blocks inside Y range
+                    candidate_blocks = [
+                        b for b in page.text_blocks
+                        if b.y >= table_start_y and b.y < table_end_y
+                    ]
+                    # Sort blocks by Y coordinate
+                    sorted_blocks = sorted(candidate_blocks, key=lambda b: b.y)
+
+                    # Group blocks into rows by vertical center alignment (15px threshold)
+                    rows: list[list[Any]] = []
+                    current_row: list[Any] = []
+                    current_y_center = None
+
+                    for block in sorted_blocks:
+                        if not block.text.strip():
+                            continue
+                        block_y_center = block.y + block.height / 2
+                        if current_y_center is None:
+                            current_row.append(block)
+                            current_y_center = block_y_center
+                        elif abs(block_y_center - current_y_center) < 15.0:
+                            current_row.append(block)
+                        else:
+                            rows.append(current_row)
+                            current_row = [block]
+                            current_y_center = block_y_center
+                    if current_row:
+                        rows.append(current_row)
+
+                    # Map candidate blocks in each row to columns
+                    rows_data: list[dict[str, str]] = []
+                    for row_blocks in rows:
+                        row_dict: dict[str, list[Any]] = {name: [] for name in table_column_names}
+                        for block in row_blocks:
+                            best_col = None
+                            best_overlap = 0.0
+                            for col_name, (x_min, x_max) in col_x_spans.items():
+                                overlap = max(
+                                    0.0, 
+                                    min(x_max, block.x + block.width) - max(x_min, block.x)
+                                )
+                                overlap_ratio = overlap / block.width if block.width > 0 else 0.0
+                                if overlap_ratio > 0.2 and overlap_ratio > best_overlap:
+                                    best_overlap = overlap_ratio
+                                    best_col = col_name
+                            if best_col:
+                                row_dict[best_col].append(block)
+
+                        # Construct row values
+                        row_values: dict[str, str] = {}
+                        for col_name, blocks in row_dict.items():
+                            sorted_blocks_in_col = sorted(blocks, key=lambda b: b.x)
+                            row_values[col_name] = " ".join(
+                                b.text.strip() for b in sorted_blocks_in_col
+                            )
+
+                        # Filter out table header rows
+                        is_header = False
+                        for col_name, val in row_values.items():
+                            val_lower = val.lower()
+                            if (
+                                val_lower == col_name.lower() or 
+                                val_lower in [
+                                    "item", "items", "qty", "quantity", "price", 
+                                    "amount", "total", "subtotal", "description"
+                                ]
+                            ):
+                                is_header = True
+                                break
+
+                        # Append if not header and at least one cell has a value
+                        if not is_header and any(row_values.values()):
+                            rows_data.append(row_values)
+
+                    # Merge continuation lines (descriptions spanning multiple rows with empty sibling columns)
+                    merged_rows: list[dict[str, str]] = []
+                    for row in rows_data:
+                        non_empty_cols = [k for k, v in row.items() if v.strip()]
+                        if len(merged_rows) > 0 and len(non_empty_cols) == 1:
+                            col_name = non_empty_cols[0]
+                            last_row = merged_rows[-1]
+                            if last_row.get(col_name):
+                                last_row[col_name] = last_row[col_name] + " " + row[col_name]
+                            else:
+                                last_row[col_name] = row[col_name]
+                        else:
+                            merged_rows.append(row.copy())
+                    
+                    rows_data = merged_rows
+
+                    # Pivot list of rows into parallel arrays under column names
+                    for col_name in table_column_names:
+                        extracted_data[col_name] = []
+                    for row in rows_data:
+                        for col_name in table_column_names:
+                            extracted_data[col_name].append(row.get(col_name, ""))
+
+        return {
+            "template_matched": True,
+            "template_id": str(template.id),
+            "template_name": template.name,
+            "confidence_score": best_match.confidence_score,
+            "extracted_data": extracted_data,
+        }
+
+    def _find_best_overlapping_block(self, page: Any, region: Any) -> Any | None:
+        """Find the document text block overlapping most with the template region."""
+        best_block = None
+        best_overlap = 0.0
+        for block in page.text_blocks:
+            x_overlap = max(
+                0.0, 
+                min(region.x + region.width, block.x + block.width) - max(region.x, block.x)
+            )
+            y_overlap = max(
+                0.0, 
+                min(region.y + region.height, block.y + block.height) - max(region.y, block.y)
+            )
+            overlap = x_overlap * y_overlap
+            if overlap > best_overlap:
+                best_overlap = overlap
+                best_block = block
+
+        # Fallback: if no overlapping block, find the closest center-to-center
+        if not best_block:
+            min_dist = 100.0  # max 100px center-to-center distance
+            for block in page.text_blocks:
+                c_rx = region.x + region.width / 2
+                c_ry = region.y + region.height / 2
+                c_bx = block.x + block.width / 2
+                c_by = block.y + block.height / 2
+                dist = ((c_rx - c_bx) ** 2 + (c_ry - c_by) ** 2) ** 0.5
+                if dist < min_dist:
+                    min_dist = dist
+                    best_block = block
+
+        return best_block

docengine/app/services/fingerprint_service.py

@@ -145,6 +145,11 @@ class FingerprintService:
         if not template.table_formats:
             return None
 
+        # Filter out dummy tables created by manual mappings (width=1000, height=1000)
+        valid_tables = [tf for tf in template.table_formats if tf.width != 1000.0 and tf.height != 1000.0]
+        if not valid_tables:
+            return None
+
         return {
             "items": [
                 {
@@ -156,15 +161,36 @@ class FingerprintService:
                     "rows": tf.rows,
                     "columns": tf.columns,
                 }
-                for tf in template.table_formats
+                for tf in valid_tables
             ]
         }
 
     def _extract_cell_coordinates(self, template: DocumentFormat) -> dict[str, Any] | None:
         """Extract cell coordinates from template."""
+        # For manually mapped templates, prefer the actual mapped regions
+        field_regions = [r for r in template.regions if r.region_type == "field_mapping"]
+        if field_regions:
+            return {
+                "items": [
+                    {
+                        "page": r.page_number,
+                        "x": r.x,
+                        "y": r.y,
+                        "width": r.width,
+                        "height": r.height,
+                    }
+                    for r in field_regions
+                ]
+            }
+
+        # Otherwise fallback to template cells, but skip if they are all 0,0 (unmapped placeholders)
         if not template.cells:
             return None
             
+        has_real_coords = any(c.width > 0 or c.height > 0 for c in template.cells)
+        if not has_real_coords:
+            return None
+
         return {
             "items": [
                 {
@@ -187,7 +213,7 @@ class FingerprintService:
         serialized = json.dumps(normalized, sort_keys=True, default=str)
         return hashlib.sha256(serialized.encode()).hexdigest()
 
-    def _normalize_coordinates(self, data: dict[str, Any]) -> dict[str, Any]:
+    def _normalize_coordinates(self, data: Any) -> Any:
         """Normalize coordinates by rounding to reduce sensitivity to small variations."""
         if isinstance(data, dict):
             return {k: self._normalize_coordinates(v) for k, v in data.items()}
@@ -215,6 +241,7 @@ class FingerprintService:
         weights.append(3.0)
 
         # Logo coordinates similarity
+        if fingerprint1.logo_coordinates and fingerprint1.logo_coordinates.get("items"):
             logo_score = self._compare_coordinates(
                 fingerprint1.logo_coordinates,
                 fingerprint2_data.get("logo_coordinates"),
@@ -223,6 +250,7 @@ class FingerprintService:
             weights.append(2.0)
 
         # Header coordinates similarity
+        if fingerprint1.header_coordinates and fingerprint1.header_coordinates.get("items"):
             header_score = self._compare_coordinates(
                 fingerprint1.header_coordinates,
                 fingerprint2_data.get("header_coordinates"),
@@ -231,6 +259,7 @@ class FingerprintService:
             weights.append(2.0)
 
         # Footer coordinates similarity
+        if fingerprint1.footer_coordinates and fingerprint1.footer_coordinates.get("items"):
             footer_score = self._compare_coordinates(
                 fingerprint1.footer_coordinates,
                 fingerprint2_data.get("footer_coordinates"),
@@ -239,14 +268,16 @@ class FingerprintService:
             weights.append(1.5)
 
         # Table coordinates similarity
+        if fingerprint1.table_coordinates and fingerprint1.table_coordinates.get("items"):
             table_score = self._compare_coordinates(
                 fingerprint1.table_coordinates,
                 fingerprint2_data.get("table_coordinates"),
             )
             scores.append(table_score)
-        weights.append(2.5)
+            weights.append(2.0)
 
         # Cell coordinates similarity
+        if fingerprint1.cell_coordinates and fingerprint1.cell_coordinates.get("items"):
             cell_score = self._compare_coordinates(
                 fingerprint1.cell_coordinates,
                 fingerprint2_data.get("cell_coordinates"),

docengine/app/services/matching_service.py

@@ -29,7 +29,7 @@ class MatchingService:
     def match_document(
         self,
         document_id: uuid.UUID,
-        min_confidence: float = 0.5,
+        min_confidence: float = 0.75,
         max_results: int = 5,
     ) -> list[TemplateMatch]:
         """Match a document against all existing templates."""

docengine/app/services/pdf_service.py

@@ -107,49 +107,94 @@ class NativePDFService:
 
     def _extract_text_blocks(self, doc_page: DocumentPage, page: fitz.Page) -> None:
         """Extract text blocks with positioning and font information."""
-        blocks = page.get_text("dict", flags=fitz.TEXT_PRESERVE_WHITESPACE)["blocks"]
+        blocks = page.get_text("rawdict")["blocks"]
         sequence = 0
+        chunks = []
 
         for block in blocks:
-            if block["type"] != 0:  # Skip non-text blocks
+            if block["type"] != 0:
                 continue
 
-            block_text_parts = []
+            current_chunk = None
-            font_info = {"family": None, "size": None, "color": None, "style": None}
+            space_count = 0
 
             for line in block.get("lines", []):
                 for span in line.get("spans", []):
-                    text = span.get("text", "").strip()
+                    font_size = span.get("size", 12.0)
-                    if text:
-                        block_text_parts.append(text)
-                        # Capture font info from the first non-empty span
-                        if font_info["family"] is None:
-                            font_info["family"] = span.get("font", None)
-                            font_info["size"] = span.get("size", None)
-                            color_int = span.get("color", 0)
-                            font_info["color"] = f"#{color_int:06x}" if isinstance(color_int, int) else None
-                            flags = span.get("flags", 0)
-                            styles = []
-                            if flags & 1:
-                                styles.append("superscript")
-                            if flags & 2:
-                                styles.append("italic")
-                            if flags & 4:
-                                styles.append("serif")
-                            if flags & 8:
-                                styles.append("monospace")
-                            if flags & 16:
-                                styles.append("bold")
-                            font_info["style"] = ",".join(styles) if styles else "regular"
                     
-            full_text = " ".join(block_text_parts)
+                    # Ignore massive text (like diagonal watermarks)
-            if not full_text.strip():
+                    if font_size > 60:
                         continue
                         
-            bbox = block["bbox"]
+                    space_threshold = font_size * 1.5
+                    
+                    font_family = span.get("font", None)
+                    color_int = span.get("color", 0)
+                    font_color = f"#{color_int:06x}" if isinstance(color_int, int) else None
+                    
+                    flags = span.get("flags", 0)
+                    styles = []
+                    if flags & 1: styles.append("superscript")
+                    if flags & 2: styles.append("italic")
+                    if flags & 4: styles.append("serif")
+                    if flags & 8: styles.append("monospace")
+                    if flags & 16: styles.append("bold")
+                    font_style = ",".join(styles) if styles else "regular"
+                    
+                    font_info = {
+                        "family": font_family,
+                        "size": font_size,
+                        "color": font_color,
+                        "style": font_style
+                    }
+                    
+                    for char in span.get("chars", []):
+                        c = char["c"]
+                        bbox = char["bbox"]
+                        
+                        if c == ' ':
+                            space_count += 1
+                            if space_count >= 2:
+                                if current_chunk and current_chunk["text"].strip():
+                                    chunks.append(current_chunk)
+                                    current_chunk = None
+                            elif current_chunk:
+                                current_chunk["text"] += c
+                                current_chunk["bbox"][2] = max(current_chunk["bbox"][2], bbox[2])
+                                current_chunk["bbox"][3] = max(current_chunk["bbox"][3], bbox[3])
+                            continue
+                        else:
+                            space_count = 0
+                        
+                        if current_chunk is None:
+                            current_chunk = {"text": c, "bbox": list(bbox), "font_info": font_info}
+                            continue
+                        
+                        prev_x1 = current_chunk["bbox"][2]
+                        distance = bbox[0] - prev_x1
+                        
+                        if distance > space_threshold:
+                            if current_chunk["text"].strip():
+                                chunks.append(current_chunk)
+                            current_chunk = {"text": c, "bbox": list(bbox), "font_info": font_info}
+                        else:
+                            current_chunk["text"] += c
+                            current_chunk["bbox"][2] = max(current_chunk["bbox"][2], bbox[2])
+                            current_chunk["bbox"][3] = max(current_chunk["bbox"][3], bbox[3])
+                            current_chunk["bbox"][1] = min(current_chunk["bbox"][1], bbox[1])
+                            current_chunk["bbox"][0] = min(current_chunk["bbox"][0], bbox[0])
+
+                if current_chunk and current_chunk["text"].strip():
+                    chunks.append(current_chunk)
+                    current_chunk = None
+
+        for chunk in chunks:
+            bbox = chunk["bbox"]
+            font_info = chunk["font_info"]
+            
             self.text_block_repo.create_text_block(
                 page_id=doc_page.id,
-                text=full_text,
+                text=chunk["text"].strip(),
                 x=bbox[0],
                 y=bbox[1],
                 width=bbox[2] - bbox[0],

docengine/app/services/template_service.py

@@ -80,6 +80,7 @@ class TemplateService:
             description=f"Auto-generated template from {document.original_filename}",
             source_document_id=document.id,
             created_by=user_id,
+            is_active=False,
         )
 
         # Process each page

docengine/app/storage/__init__.py

@@ -0,0 +1 @@
+# Storage module

docengine/app/storage/provider.py

@@ -0,0 +1,163 @@
+from __future__ import annotations
+
+import hashlib
+import os
+import shutil
+import uuid
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+from app.core.config import settings
+from app.core.exceptions import StorageError
+from app.core.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+class StorageProvider(ABC):
+    """Abstract base class for storage providers."""
+
+    @abstractmethod
+    def save_file(self, file_data: bytes, directory: str, filename: str | None = None) -> str:
+        """Save file data and return the storage path."""
+        ...
+
+    @abstractmethod
+    def read_file(self, storage_path: str) -> bytes:
+        """Read file data from storage."""
+        ...
+
+    @abstractmethod
+    def delete_file(self, storage_path: str) -> bool:
+        """Delete a file from storage. Returns True if successful."""
+        ...
+
+    @abstractmethod
+    def file_exists(self, storage_path: str) -> bool:
+        """Check if a file exists in storage."""
+        ...
+
+    @abstractmethod
+    def get_file_size(self, storage_path: str) -> int:
+        """Get the file size in bytes."""
+        ...
+
+    @abstractmethod
+    def get_absolute_path(self, storage_path: str) -> str:
+        """Get the absolute filesystem path for a storage path."""
+        ...
+
+    @staticmethod
+    def compute_checksum(data: bytes, algorithm: str = "sha256") -> str:
+        """Compute checksum of file data."""
+        hasher = hashlib.new(algorithm)
+        hasher.update(data)
+        return hasher.hexdigest()
+
+    @staticmethod
+    def generate_filename(original_filename: str) -> str:
+        """Generate a unique filename preserving the original extension."""
+        ext = Path(original_filename).suffix.lower()
+        return f"{uuid.uuid4().hex}{ext}"
+
+
+class LocalStorageProvider(StorageProvider):
+    """Local filesystem storage provider."""
+
+    def __init__(self, base_path: str | None = None) -> None:
+        self.base_path = Path(base_path or settings.storage_local_path).resolve()
+        self._ensure_directories()
+
+    def _ensure_directories(self) -> None:
+        """Create required storage directories."""
+        for subdir in ("documents", "templates", "images", "temp", "rendered"):
+            (self.base_path / subdir).mkdir(parents=True, exist_ok=True)
+
+    def _resolve_path(self, storage_path: str) -> Path:
+        """Resolve a storage path to an absolute path."""
+        resolved = (self.base_path / storage_path).resolve()
+        if not str(resolved).startswith(str(self.base_path)):
+            raise StorageError(f"Path traversal detected: {storage_path}")
+        return resolved
+
+    def save_file(self, file_data: bytes, directory: str, filename: str | None = None) -> str:
+        """Save file data to local storage."""
+        if filename is None:
+            filename = f"{uuid.uuid4().hex}.bin"
+
+        dir_path = self.base_path / directory
+        dir_path.mkdir(parents=True, exist_ok=True)
+
+        file_path = dir_path / filename
+        try:
+            file_path.write_bytes(file_data)
+            storage_path = str(file_path.relative_to(self.base_path))
+            logger.info("file_saved", storage_path=storage_path, size=len(file_data))
+            return storage_path
+        except OSError as e:
+            raise StorageError(f"Failed to save file: {e}") from e
+
+    def read_file(self, storage_path: str) -> bytes:
+        """Read file data from local storage."""
+        file_path = self._resolve_path(storage_path)
+        if not file_path.exists():
+            raise StorageError(f"File not found: {storage_path}")
+        try:
+            return file_path.read_bytes()
+        except OSError as e:
+            raise StorageError(f"Failed to read file: {e}") from e
+
+    def delete_file(self, storage_path: str) -> bool:
+        """Delete a file from local storage."""
+        file_path = self._resolve_path(storage_path)
+        if not file_path.exists():
+            return False
+        try:
+            file_path.unlink()
+            logger.info("file_deleted", storage_path=storage_path)
+            return True
+        except OSError as e:
+            logger.error("file_delete_failed", storage_path=storage_path, error=str(e))
+            raise StorageError(f"Failed to delete file: {e}") from e
+
+    def file_exists(self, storage_path: str) -> bool:
+        """Check if a file exists in local storage."""
+        file_path = self._resolve_path(storage_path)
+        return file_path.exists()
+
+    def get_file_size(self, storage_path: str) -> int:
+        """Get the file size in bytes."""
+        file_path = self._resolve_path(storage_path)
+        if not file_path.exists():
+            raise StorageError(f"File not found: {storage_path}")
+        return file_path.stat().st_size
+
+    def get_absolute_path(self, storage_path: str) -> str:
+        """Get the absolute filesystem path."""
+        return str(self._resolve_path(storage_path))
+
+    def save_temp_file(self, file_data: bytes, filename: str) -> str:
+        """Save a temporary file."""
+        return self.save_file(file_data, "temp", filename)
+
+    def cleanup_temp(self) -> int:
+        """Remove all files in the temp directory."""
+        temp_dir = self.base_path / "temp"
+        count = 0
+        if temp_dir.exists():
+            for item in temp_dir.iterdir():
+                if item.is_file():
+                    item.unlink()
+                    count += 1
+                elif item.is_dir():
+                    shutil.rmtree(item)
+                    count += 1
+        logger.info("temp_cleanup", files_removed=count)
+        return count
+
+
+def get_storage_provider() -> StorageProvider:
+    """Factory function to get the configured storage provider."""
+    if settings.storage_provider == "local":
+        return LocalStorageProvider()
+    raise StorageError(f"Unknown storage provider: {settings.storage_provider}")

docengine/app/tasks/document_tasks.py

@@ -57,7 +57,7 @@ def process_document_task(self, document_id: str) -> dict:  # noqa: ANN001
 def match_document_task(
     self,  # noqa: ANN001
     document_id: str,
-    min_confidence: float = 0.5,
+    min_confidence: float = 0.75,
     max_results: int = 5,
 ) -> dict:
     """Celery task to match a document against templates."""

docengine/app/utils/sanitizers.py

@@ -0,0 +1,62 @@
+import re
+from typing import Optional
+from datetime import datetime
+from dateutil import parser
+
+def sanitize_amount(text: str) -> Optional[float]:
+    """
+    Sanitizes a string representing an amount/number (e.g., "$1,234.56", "€ 50,000", "50 USD")
+    by removing currency symbols, commas, and other non-numeric characters (except for the decimal separator),
+    and casts it to a float.
+    
+    Args:
+        text (str): The raw extracted text from the document.
+        
+    Returns:
+        Optional[float]: The sanitized numeric value, or None if no number could be extracted.
+    """
+    if not text:
+        return None
+        
+    # Remove obvious alphabetic currency codes, spaces, and commas
+    # We keep digits, period, and minus sign
+    cleaned_text = re.sub(r'[^\d\.-]', '', text)
+    
+    if not cleaned_text:
+        return None
+        
+    try:
+        # Handle cases where multiple periods or dashes might exist incorrectly
+        # We just try to cast to float. If the OCR produced something like "1.23.45", this will fail.
+        # A more robust regex can handle exact capture if needed.
+        # But this basic cast covers 95% of standard sanitized strings.
+        return float(cleaned_text)
+    except ValueError:
+        return None
+
+def sanitize_date(text: str, field_type: str = 'DATE') -> Optional[str]:
+    """
+    Sanitizes a string representing a date or datetime and converts it to ISO 8601 format.
+    
+    Args:
+        text (str): The raw extracted text from the document.
+        field_type (str): 'DATE' or 'DATETIME'. Determines the output format.
+        
+    Returns:
+        Optional[str]: The sanitized date string in ISO format, or None if it could not be parsed.
+    """
+    if not text:
+        return None
+        
+    try:
+        # dateutil.parser is very robust at handling formats like "Dec 11, 2020", "11/12/2020", etc.
+        # fuzzy=True allows it to ignore extra words/characters around the date
+        parsed_date = parser.parse(text, fuzzy=True)
+        
+        if field_type == 'DATETIME':
+            return parsed_date.strftime('%Y-%m-%dT%H:%M:%S')
+        else:
+            return parsed_date.strftime('%Y-%m-%d')
+    except (ValueError, TypeError, OverflowError):
+        return None
+

docengine/app/workers/celery_app.py

@@ -25,6 +25,7 @@ celery_app.conf.update(
     task_reject_on_worker_lost=True,
     broker_connection_retry_on_startup=True,
     result_expires=86400,
+    task_default_queue="docengine_default",
     task_routes={
         "app.tasks.document_tasks.*": {"queue": "document_processing"},
     },