maddy/OCR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed ui and template mapping issue

Browse Source
This commit is contained in:
Narayanan Madaswamy
2026-07-14 12:43:09 +05:30
parent fb0a78a405
commit 460a1c5c51
11 changed files with 383 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

148
docengine/app/api/v1/templates.py
View File

@@ -10,7 +10,13 @@ from app.core.database import get_db
from app.core.dependencies import CurrentUser
from app.core.logging_config import get_logger
from app.repositories.document_repository import DocumentRepository, TemplateMatchRepository
from app.repositories.template_repository import TemplateRepository
from app.repositories.template_repository import (
TemplateRepository,
DocumentRegionRepository,
DocumentCellRepository,
TableFormatRepository,
TableColumnRepository
)
from app.schemas.common import PaginatedResponse, SuccessResponse
from app.schemas.document import TemplateMatchRequest, TemplateMatchResponse
from app.schemas.template import (
@@ -18,6 +24,8 @@ from app.schemas.template import (
TemplateRenderRequest,
TemplateRenderResponse,
TemplateResponse,
TemplateCreateRequest,
TemplateMappingSaveRequest,
)
logger = get_logger(__name__)
@@ -25,6 +33,132 @@ logger = get_logger(__name__)
router = APIRouter(prefix="/templates", tags=["Templates"])
@router.post(
"",
response_model=dict,
summary="Create Template",
description="Create a new template.",
)
def create_template(
payload: TemplateCreateRequest,
current_user: CurrentUser,
db: Session = Depends(get_db),
) -> dict:
"""Create a template."""
template_repo = TemplateRepository(db)
template = template_repo.create_template(
name=payload.template_name,
page_width=1000.0,
page_height=1000.0,
source_document_id=uuid.UUID(payload.source_document_id) if payload.source_document_id else None
)
cell_repo = DocumentCellRepository(db)
saved_fields = []
for index, field in enumerate(payload.fields):
cell = cell_repo.create_cell(
format_id=template.id,
page_number=1,
x=0.0,
y=0.0,
width=0.0,
height=0.0,
data_type=field.field_type,
field_name=field.field_label,
sequence=field.display_order or index,
is_dynamic=True
)
saved_fields.append({
"field_label": cell.field_name,
"field_type": cell.data_type
})
db.commit()
return {
"pk_template_id": str(template.id),
"template_name": template.name,
"fields": saved_fields
}
@router.post(
"/{template_id}/mappings/save",
response_model=SuccessResponse,
summary="Save Template Mappings",
description="Save the field mappings for a template.",
)
def save_mappings(
template_id: uuid.UUID,
payload: list[TemplateMappingSaveRequest],
current_user: CurrentUser,
db: Session = Depends(get_db),
) -> SuccessResponse:
"""Save template mappings."""
template_repo = TemplateRepository(db)
template = template_repo.get_by_id(template_id)
if not template:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Template '{template_id}' not found",
)
region_repo = DocumentRegionRepository(db)
cell_repo = DocumentCellRepository(db)
table_repo = TableFormatRepository(db)
table_col_repo = TableColumnRepository(db)
cells = cell_repo.get_template_cells(template_id)
cell_map = {c.field_name: c for c in cells}
table_format = None
# Save each mapped node
for mapping in payload:
cell = cell_map.get(mapping.field_name)
is_table_column = cell and cell.data_type == 'TABLE_COLUMN'
for node_idx, node in enumerate(mapping.mapped_nodes):
if is_table_column:
if not table_format:
table_format = table_repo.create_table_format(
format_id=template_id,
page_number=node.page_no or 1,
x=0.0, y=0.0, width=1000.0, height=1000.0,
rows=1, columns=10
)
table_col_repo.create_column(
table_format_id=table_format.id,
column_index=node_idx,
width=node.width,
header_text=mapping.field_name,
data_type="text"
)
region_repo.create_region(
format_id=template_id,
page_number=node.page_no or 1,
region_type="field_mapping",
x=node.x_coordinate,
y=node.y_coordinate,
width=node.width,
height=node.height,
content={
"field_name": mapping.field_name,
"text_value": node.text_value
}
)
db.commit()
# Generate fingerprint now that mappings are populated
from app.services.fingerprint_service import FingerprintService
FingerprintService(db).generate_fingerprint(template)
db.commit()
return SuccessResponse(message="Mappings saved successfully")
@router.get(
"",
response_model=PaginatedResponse[TemplateListResponse],
@@ -32,9 +166,9 @@ router = APIRouter(prefix="/templates", tags=["Templates"])
description="List all active templates with pagination.",
)
def list_templates(
current_user: CurrentUser,
page: int = Query(default=1, ge=1),
page_size: int = Query(default=20, ge=1, le=100),
current_user: CurrentUser = None,
db: Session = Depends(get_db),
) -> PaginatedResponse[TemplateListResponse]:
"""List all active templates."""
@@ -61,7 +195,7 @@ def list_templates(
)
def get_template(
template_id: uuid.UUID,
current_user: CurrentUser = None,
current_user: CurrentUser,
db: Session = Depends(get_db),
) -> TemplateResponse:
"""Get a template by ID."""
@@ -83,7 +217,7 @@ def get_template(
)
def delete_template(
template_id: uuid.UUID,
current_user: CurrentUser = None,
current_user: CurrentUser,
db: Session = Depends(get_db),
) -> SuccessResponse:
"""Soft-delete a template."""
@@ -109,7 +243,7 @@ def delete_template(
)
def match_template(
payload: TemplateMatchRequest,
current_user: CurrentUser = None,
current_user: CurrentUser,
db: Session = Depends(get_db),
) -> list[TemplateMatchResponse]:
"""Match a document against existing templates."""
@@ -162,7 +296,7 @@ def match_template(
)
def render_template(
payload: TemplateRenderRequest,
current_user: CurrentUser = None,
current_user: CurrentUser,
db: Session = Depends(get_db),
) -> TemplateRenderResponse:
"""Render a template to PDF."""
@@ -199,8 +333,8 @@ def render_template(
)
def download_rendered_pdf(
template_id: uuid.UUID,
current_user: CurrentUser,
filename: str = Query(..., description="Filename of the rendered PDF"),
current_user: CurrentUser = None,
db: Session = Depends(get_db),
) -> FileResponse:
"""Download a rendered PDF."""

2
docengine/app/repositories/template_repository.py
View File

@@ -70,6 +70,7 @@ class TemplateRepository(BaseRepository[DocumentFormat]):
fingerprint: dict | None = None,
source_document_id: uuid.UUID | None = None,
created_by: uuid.UUID | None = None,
is_active: bool = True,
) -> DocumentFormat:
"""Create a new template."""
template = DocumentFormat(
@@ -85,6 +86,7 @@ class TemplateRepository(BaseRepository[DocumentFormat]):
fingerprint=fingerprint,
source_document_id=source_document_id,
created_by=created_by,
is_active=is_active,
)
return self.create(template)

29
docengine/app/schemas/template.py
View File

@@ -2,7 +2,7 @@ from __future__ import annotations
import uuid
from datetime import datetime
from typing import Any
from typing import Any, List, Optional, Union
from pydantic import Field
@@ -248,3 +248,30 @@ class TemplateRenderResponse(BaseSchema):
file_size: int
page_count: int
rendered_at: datetime
class TemplateFieldCreate(BaseSchema):
field_label: str
field_type: str = "text"
display_order: int = 0
required_flag: bool = False
class TemplateCreateRequest(BaseSchema):
template_name: str
source_document_id: Optional[str] = None
fields: List[TemplateFieldCreate] = Field(default_factory=list)
class MappingNodeRequest(BaseSchema):
pk_document_data_id: Union[int, str, None] = None
x_coordinate: float
y_coordinate: float
width: float
height: float
text_value: Optional[str] = None
page_width: Optional[float] = None
page_height: Optional[float] = None
page_no: Optional[int] = None
class TemplateMappingSaveRequest(BaseSchema):
field_name: str
mapped_nodes: List[MappingNodeRequest] = Field(default_factory=list)

7
docengine/app/services/document_service.py
View File

@@ -58,11 +58,7 @@ class DocumentProcessingService:
metadata["layout"] = layout_results
document.document_metadata = metadata
# Step 3: Generate template
self.db.refresh(document)
template = self.template_service.generate_template(document)
# Step 4: Update document status
# Step 3: Update document status
self.doc_repo.update_status(document_id, "completed")
self.db.commit()
@@ -70,7 +66,6 @@ class DocumentProcessingService:
"processing_completed",
document_id=str(document_id),
pages=document.page_count,
template_id=str(template.id),
)
return document

2
docengine/app/services/fingerprint_service.py
View File

@@ -187,7 +187,7 @@ class FingerprintService:
serialized = json.dumps(normalized, sort_keys=True, default=str)
return hashlib.sha256(serialized.encode()).hexdigest()
def _normalize_coordinates(self, data: dict[str, Any]) -> dict[str, Any]:
def _normalize_coordinates(self, data: Any) -> Any:
"""Normalize coordinates by rounding to reduce sensitivity to small variations."""
if isinstance(data, dict):
return {k: self._normalize_coordinates(v) for k, v in data.items()}

1
docengine/app/services/template_service.py
View File

@@ -80,6 +80,7 @@ class TemplateService:
description=f"Auto-generated template from {document.original_filename}",
source_document_id=document.id,
created_by=user_id,
is_active=False,
)
# Process each page

12
frontend/src/app/services/template.service.ts
View File

@@ -81,15 +81,19 @@ export class TemplateService {
);
}
createTemplate(data: { template_name: string, fields: TemplateField[] }): Observable<Template> {
createTemplate(data: { template_name: string, source_document_id?: string, fields: TemplateField[] }): Observable<Template> {
return this.http.post<Template>(`${this.apiUrl}/templates`, data);
}
saveMappings(templateId: number, mappings: any[]): Observable<any> {
saveMappings(templateId: string, mappings: any[]): Observable<any> {
return this.http.post(`${this.apiUrl}/templates/${templateId}/mappings/save`, mappings);
}
recognizeTemplate(documentId: string): Observable<any> {
return this.http.post(`${this.apiUrl}/documents/${documentId}/recognize`, {});
getTemplate(templateId: string): Observable<any> {
return this.http.get(`${this.apiUrl}/templates/${templateId}`);
}
recognizeTemplate(documentId: string): Observable<any[]> {
return this.http.post<any[]>(`${this.apiUrl}/templates/match`, { document_id: documentId });
}
}

17
frontend/src/app/templates/templates.component.html
View File

@@ -32,7 +32,9 @@
<div *ngFor="let node of page.nodes; let j = index"
cdkDrag
[cdkDragData]="node"
class="layout-node"
(mouseup)="onNodeMouseUp(node, $event)"
[ngClass]="node.block_type.toLowerCase()"
[style.left.%]="(node.x_coordinate / node.page_width) * 100"
[style.top.%]="(node.y_coordinate / node.page_height) * 100"
@@ -71,9 +73,12 @@
<div class="field-list">
<div *ngFor="let field of templateFields" class="template-field-container">
<div class="field-header">
<strong>{{ field.field_label }}</strong>
<span class="badge">{{ field.field_type }}</span>
<div class="field-header flex justify-content-between align-items-center">
<div>
<strong>{{ field.field_label }}</strong>
<span class="badge ml-2">{{ field.field_type }}</span>
</div>
<p-button icon="pi pi-trash" styleClass="p-button-rounded p-button-danger p-button-text p-button-sm" (onClick)="removeField(field.pk_template_field_id)"></p-button>
</div>
<div class="drop-zone"
@@ -87,8 +92,10 @@
Drop value here...
</div>
<div *ngFor="let mappedNode of mappings[field.pk_template_field_id]" cdkDrag class="mapped-node">
<i class="pi pi-arrow-left text-xs mr-2"></i> {{ mappedNode.text_value }}
<div *ngFor="let mappedNode of mappings[field.pk_template_field_id]; let k = index" cdkDrag [cdkDragData]="mappedNode" class="mapped-node">
<i class="pi pi-arrow-left text-xs mr-2"></i>
<span class="flex-1 overflow-hidden white-space-nowrap text-overflow-ellipsis">{{ mappedNode.text_value }}</span>
<i class="pi pi-times cursor-pointer text-red-500 hover:text-red-700 ml-2" (click)="removeFromField(field.pk_template_field_id!, k)" title="Remove"></i>
</div>
</div>
</div>

175
frontend/src/app/templates/templates.component.ts
View File

@@ -1,7 +1,7 @@
import { Component, OnInit } from '@angular/core';
import { CommonModule } from '@angular/common';
import { FormsModule } from '@angular/forms';
import { DragDropModule, CdkDragDrop, moveItemInArray, transferArrayItem } from '@angular/cdk/drag-drop';
import { DragDropModule, CdkDragDrop, moveItemInArray, transferArrayItem, copyArrayItem } from '@angular/cdk/drag-drop';
import { SplitterModule } from 'primeng/splitter';
import { ButtonModule } from 'primeng/button';
import { DialogModule } from 'primeng/dialog';
@@ -121,10 +121,71 @@ export class TemplatesComponent implements OnInit {
autoRecognize() {
if (!this.documentId) return;
this.templateService.recognizeTemplate(this.documentId).subscribe({
next: (res) => {
if (res.templateMatched) {
this.messageService.add({ severity: 'info', summary: 'Template Recognized', detail: `Confidence: ${res.confidence}%` });
// In a full implementation, we'd load the template fields and populate the mappings.
next: (matches) => {
if (matches && matches.length > 0 && matches[0].confidence_score > 0.5) {
const match = matches[0];
this.messageService.add({ severity: 'info', summary: 'Template Recognized', detail: `Confidence: ${(match.confidence_score * 100).toFixed(1)}%` });
this.templateService.getTemplate(match.format_id).subscribe({
next: (templateData) => {
this.templateName = templateData.name;
this.templateFields = [];
this.mappings = {};
if (templateData.cells) {
templateData.cells.forEach((cell: any) => {
if (cell.is_dynamic) {
const fieldId = new Date().getTime() + Math.random();
this.templateFields.push({
pk_template_field_id: fieldId as any,
field_label: cell.field_name,
field_type: cell.data_type,
display_order: cell.sequence,
required_flag: false
});
this.mappings[fieldId] = [];
}
});
}
if (templateData.regions) {
templateData.regions.forEach((region: any) => {
if (region.region_type === 'field_mapping' && region.content && region.content.field_name) {
const field = this.templateFields.find(f => f.field_label === region.content.field_name);
if (field && field.pk_template_field_id) {
// Find the corresponding page in the current document
const page = this.pages.find(p => p.page_number === region.page_number);
if (page) {
let bestMatchIdx = -1;
let bestOverlap = 0;
// Find the text node on the canvas that overlaps most with this region's bounding box
for (let i = 0; i < page.nodes.length; i++) {
const node = page.nodes[i];
const x_overlap = Math.max(0, Math.min(region.x + region.width, node.x_coordinate + node.width) - Math.max(region.x, node.x_coordinate));
const y_overlap = Math.max(0, Math.min(region.y + region.height, node.y_coordinate + node.height) - Math.max(region.y, node.y_coordinate));
const overlapArea = x_overlap * y_overlap;
if (overlapArea > bestOverlap) {
bestOverlap = overlapArea;
bestMatchIdx = i;
}
}
// If we found a matching node (at least some overlap), move it from canvas to mappings
if (bestMatchIdx !== -1) {
const matchedNode = page.nodes.splice(bestMatchIdx, 1)[0];
// Because *ngFor tracks objects by reference, we clone it to force Angular to render it in the right panel cleanly
this.mappings[field.pk_template_field_id].push({...matchedNode});
}
}
}
}
});
}
}
});
}
}
});
@@ -143,6 +204,12 @@ export class TemplatesComponent implements OnInit {
this.displayAddField = false;
}
removeField(fieldId?: number) {
if (!fieldId) return;
this.templateFields = this.templateFields.filter(f => f.pk_template_field_id !== fieldId);
delete this.mappings[fieldId];
}
saveTemplateAndMappings() {
if (!this.templateName || this.templateName.trim() === '') {
this.messageService.add({ severity: 'warn', summary: 'Warning', detail: 'Please provide a template name.' });
@@ -156,11 +223,39 @@ export class TemplatesComponent implements OnInit {
this.templateService.createTemplate({
template_name: this.templateName,
source_document_id: this.documentId || undefined,
fields: this.templateFields
}).subscribe({
next: (res) => {
// In a real scenario, we pass res.templateId and map the layoutNodes
this.messageService.add({ severity: 'success', summary: 'Success', detail: 'Template & Mappings saved successfully.' });
next: (res: any) => {
const templateId = res.pk_template_id;
// Construct the mappings payload
const mappingsPayload: any[] = [];
this.templateFields.forEach(field => {
const fieldId = field.pk_template_field_id;
if (fieldId && this.mappings[fieldId] && this.mappings[fieldId].length > 0) {
mappingsPayload.push({
field_name: field.field_label,
mapped_nodes: this.mappings[fieldId]
});
}
});
if (mappingsPayload.length > 0) {
this.templateService.saveMappings(templateId, mappingsPayload).subscribe({
next: () => {
this.messageService.add({ severity: 'success', summary: 'Success', detail: 'Template & Mappings saved successfully.' });
},
error: (err) => {
this.messageService.add({ severity: 'error', summary: 'Error', detail: 'Failed to save mappings.' });
}
});
} else {
this.messageService.add({ severity: 'success', summary: 'Success', detail: 'Template saved successfully (No mappings).' });
}
},
error: (err) => {
this.messageService.add({ severity: 'error', summary: 'Error', detail: 'Failed to save template.' });
}
});
}
@@ -168,14 +263,58 @@ export class TemplatesComponent implements OnInit {
// Drag and Drop Logic
drop(event: CdkDragDrop<DocumentLayout[]>, fieldId?: number) {
if (event.previousContainer === event.container) {
moveItemInArray(event.container.data, event.previousIndex, event.currentIndex);
// Do nothing! This allows the item to naturally snap back to its original position
// since it's a failed drop (didn't land in a mapping field).
} else {
transferArrayItem(
event.previousContainer.data,
event.container.data,
event.previousIndex,
event.currentIndex,
);
const isFromCanvas = event.previousContainer.id.startsWith('document-layout-list');
const isToCanvas = event.container.id.startsWith('document-layout-list');
if (isFromCanvas && !isToCanvas) {
// Drag from Canvas -> Field
// We use event.item.data which perfectly tracks the dragged object regardless of DOM indexes
const clonedNode = JSON.parse(JSON.stringify(event.item.data));
// Insert the clone into the destination mapping field
event.container.data.splice(event.currentIndex, 0, clonedNode);
// Force Angular to completely recreate the DOM elements for this specific canvas page.
// By using .map(node => ({...node})), we change every object's identity.
// This forces Angular to destroy the corrupted DOM element (which CDK moved and left a translate3d on)
// and recreate it fresh with its original absolute coordinates!
const pageIndex = parseInt(event.previousContainer.id.split('-').pop() || '0');
if (!isNaN(pageIndex) && this.pages[pageIndex]) {
this.pages[pageIndex].nodes = this.pages[pageIndex].nodes.map(node => ({...node}));
// Also explicitly clear the transform on the dragged element just in case CDK holds a ref to it
event.item.element.nativeElement.style.transform = '';
}
} else if (!isFromCanvas && isToCanvas) {
// Drag from Field -> Canvas (Delete from Field)
event.previousContainer.data.splice(event.previousIndex, 1);
} else {
// Drag from Field -> Field (Move)
transferArrayItem(
event.previousContainer.data,
event.container.data,
event.previousIndex,
event.currentIndex,
);
}
}
}
onNodeMouseUp(node: DocumentLayout, event: MouseEvent) {
// If the user resized the node using the native CSS resize handle, save the new size
const el = event.currentTarget as HTMLElement;
if (el.style.width && el.style.width.endsWith('px')) {
const parentRect = el.parentElement!.getBoundingClientRect();
node.width = (el.offsetWidth / parentRect.width) * node.page_width;
node.height = (el.offsetHeight / parentRect.height) * node.page_height;
// Clear the inline pixel styles so Angular bindings take over smoothly
el.style.width = '';
el.style.height = '';
}
}
@@ -192,4 +331,10 @@ export class TemplatesComponent implements OnInit {
event.stopPropagation();
this.pages[pageIndex].nodes.splice(nodeIndex, 1);
}
removeFromField(fieldId: number, nodeIndex: number) {
if (this.mappings[fieldId]) {
this.mappings[fieldId].splice(nodeIndex, 1);
}
}
}

11
list_templates.py Normal file
View File

@@ -0,0 +1,11 @@
import sys
import os
sys.path.append(os.path.join(os.getcwd(), 'docengine'))
from app.core.database import SessionLocal
from app.models.template import DocumentFormat
db = SessionLocal()
templates = db.query(DocumentFormat).all()
for t in templates:
print(f"ID: {t.id}, Name: {t.name}, Source: {t.source_document_id}, Active: {t.is_active}")

18
update_templates.py Normal file
View File

@@ -0,0 +1,18 @@
import sys
import os
sys.path.append(os.path.join(os.getcwd(), 'docengine'))
from app.core.database import SessionLocal
from app.models.template import DocumentFormat
db = SessionLocal()
# Deactivate all templates starting with 'Template_' that have a UUID-like suffix (or just all where description starts with 'Auto-generated')
templates = db.query(DocumentFormat).filter(
DocumentFormat.description.startswith("Auto-generated")
).all()
for t in templates:
t.is_active = False
db.commit()
print(f"Deactivated {len(templates)} auto-generated templates.")