diff --git a/SESSION_HANDOFF.md b/SESSION_HANDOFF.md new file mode 100644 index 0000000..dc217dc --- /dev/null +++ b/SESSION_HANDOFF.md @@ -0,0 +1,42 @@ +# Session Handoff & Context + +**Date Saved**: June 7, 2026 +**Conversation ID**: 43f3aa27-9124-446e-9713-4e31e083b4f5 + +## Project: Template Mapping Engine (OCR) +A system designed to extract document layouts via OCR and allow users to map specific bounding box regions to template fields using a drag-and-drop UI. Future documents from the same vendor are automatically recognized and mapped using a heuristic template engine. + +## What Has Been Completed So Far + +### 1. Backend Startup & Dependency Fixes: +- **Auto-Bootstrapping**: Updated `run_docengine.sh` and `run_backend.sh` to automatically set up virtual environments and install missing dependencies if key modules (`uvicorn`, `numpy`, etc.) are missing. +- **Python 3.13 Compatibility**: Upgraded `paddlepaddle` in `docengine/requirements.txt` to `>=3.0.0` to support Python 3.13 on macOS ARM64. Added missing dependencies (`numpy`, `opencv-python-headless`, `pdf2image`) to `backend/requirements.txt`. +- **CORS Config Fix**: Corrected a missing double-quote in `CORS_ORIGINS` in both `.env` and `.env.example` in `docengine/`. + +### 2. Backend Code Integrity & Framework Alignment: +- **SQLAlchemy 2.0 / Imperative Mapping Fix**: Updated `user_roles_table` in `docengine/app/models/user.py` to use SQLAlchemy `Column` objects instead of `mapped_column`, preventing a database initialization crash. +- **Structlog Parameter Fix**: Replaced the `event` keyword argument with `phase` in `docengine/app/events/handlers.py` to prevent a duplicate parameter `TypeError` from the structlog library. +- **Optional Development Auth Bypass**: Modified `get_current_user` in `docengine/app/core/dependencies.py` to automatically fallback to the first database user (or a fake dev admin model) when no authorization token is supplied and `APP_ENV` is set to `"development"`. This prevents `401 Unauthorized` errors when testing endpoints locally. + +### 3. Frontend Angular Implementation: +- **API Endpoint Alignments**: Removed the duplicate `/api` prefix from endpoints in `TemplateService.ts` (e.g. changing `${this.apiUrl}/api/documents/upload` to `${this.apiUrl}/documents/upload`). This resolves the `404 Not Found` error by aligning with the backend prefix `/api/v1/documents/upload`. + +--- + +## Where to Pick Up Next +1. **Initialize Database Tables**: + - The database currently lacks the necessary schema tables (evidenced by the error: `relation "admin.users" does not exist`). + - Run the setup command to build the tables and apply migrations: + ```bash + ./run_docengine.sh setup + ``` +2. **Retest Upload Flow**: + - Once database tables exist, perform a document upload from the Angular frontend to visually verify the OCR bounding boxes and layout mapper. +3. **Verify Heuristic Template Recognition**: + - Test the template matching and layout coordinate storage workflows. + +--- + +## How to Resume with AI Assistant +To resume this session in a new chat, you can tell the AI: +*"Please read `SESSION_HANDOFF.md` in the root of the OCR project to get context on the Template Engine we were building and the startup fixes, and let's continue."* diff --git a/backend/.env b/backend/.env index e16f982..bcc992f 100644 --- a/backend/.env +++ b/backend/.env @@ -1,7 +1,7 @@ DB_USER=postgres -DB_PASSWORD=M@tr!x#149@dm!N -DB_HOST=192.168.0.111 -DB_PORT=7925 +DB_PASSWORD=M@triXPostgr3s@6202 +DB_HOST=103.125.129.116 +DB_PORT=5432 DB_NAME=ocr # Mail Configuration (Gmail) diff --git a/backend/__pycache__/main.cpython-313.pyc b/backend/__pycache__/main.cpython-313.pyc index 1af8b42..30099f3 100644 Binary files a/backend/__pycache__/main.cpython-313.pyc and b/backend/__pycache__/main.cpython-313.pyc differ diff --git a/backend/alembic.ini b/backend/alembic.ini new file mode 100644 index 0000000..807ded2 --- /dev/null +++ b/backend/alembic.ini @@ -0,0 +1,149 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts. +# this is typically a path given in POSIX (e.g. forward slashes) +# format, relative to the token %(here)s which refers to the location of this +# ini file +script_location = %(here)s/alembic + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s +# Or organize into date-based subdirectories (requires recursive_version_locations = true) +# file_template = %%(year)d/%%(month).2d/%%(day).2d_%%(hour).2d%%(minute).2d_%%(second).2d_%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. for multiple paths, the path separator +# is defined by "path_separator" below. +prepend_sys_path = . + + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the tzdata library which can be installed by adding +# `alembic[tz]` to the pip requirements. +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to /versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "path_separator" +# below. +# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions + +# path_separator; This indicates what character is used to split lists of file +# paths, including version_locations and prepend_sys_path within configparser +# files such as alembic.ini. +# The default rendered in new alembic.ini files is "os", which uses os.pathsep +# to provide os-dependent path splitting. +# +# Note that in order to support legacy alembic.ini files, this default does NOT +# take place if path_separator is not present in alembic.ini. If this +# option is omitted entirely, fallback logic is as follows: +# +# 1. Parsing of the version_locations option falls back to using the legacy +# "version_path_separator" key, which if absent then falls back to the legacy +# behavior of splitting on spaces and/or commas. +# 2. Parsing of the prepend_sys_path option falls back to the legacy +# behavior of splitting on spaces, commas, or colons. +# +# Valid values for path_separator are: +# +# path_separator = : +# path_separator = ; +# path_separator = space +# path_separator = newline +# +# Use os.pathsep. Default configuration used for new projects. +path_separator = os + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +# database URL. This is consumed by the user-maintained env.py script only. +# other means of configuring database URLs may be customized within the env.py +# file. +sqlalchemy.url = driver://user:pass@localhost/dbname + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module +# hooks = ruff +# ruff.type = module +# ruff.module = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Alternatively, use the exec runner to execute a binary found on your PATH +# hooks = ruff +# ruff.type = exec +# ruff.executable = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Logging configuration. This is also consumed by the user-maintained +# env.py script only. +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/backend/alembic/README b/backend/alembic/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/backend/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/backend/alembic/__pycache__/env.cpython-313.pyc b/backend/alembic/__pycache__/env.cpython-313.pyc new file mode 100644 index 0000000..87ac7c6 Binary files /dev/null and b/backend/alembic/__pycache__/env.cpython-313.pyc differ diff --git a/backend/alembic/env.py b/backend/alembic/env.py new file mode 100644 index 0000000..9175000 --- /dev/null +++ b/backend/alembic/env.py @@ -0,0 +1,87 @@ +from logging.config import fileConfig + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +import os +import sys +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from database import Base, DATABASE_URL +import models.template_models + +target_metadata = Base.metadata +config.set_main_option("sqlalchemy.url", DATABASE_URL.replace("%", "%%")) + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + include_schemas=True, + version_table_schema='public' + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/backend/alembic/script.py.mako b/backend/alembic/script.py.mako new file mode 100644 index 0000000..1101630 --- /dev/null +++ b/backend/alembic/script.py.mako @@ -0,0 +1,28 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + """Upgrade schema.""" + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + """Downgrade schema.""" + ${downgrades if downgrades else "pass"} diff --git a/backend/alembic/versions/91b2545f5a30_add_templates_schema.py b/backend/alembic/versions/91b2545f5a30_add_templates_schema.py new file mode 100644 index 0000000..8279a15 --- /dev/null +++ b/backend/alembic/versions/91b2545f5a30_add_templates_schema.py @@ -0,0 +1,28 @@ +"""Add templates schema + +Revision ID: 91b2545f5a30 +Revises: +Create Date: 2026-06-01 16:28:57.181829 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import os + +# revision identifiers, used by Alembic. +revision: str = '91b2545f5a30' +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +def upgrade() -> None: + # Read and execute the SQL schema script + sql_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), 'sql', '01_templates_schema.sql') + with open(sql_path, 'r') as f: + sql = f.read() + op.execute(sql) + +def downgrade() -> None: + op.execute("DROP SCHEMA IF EXISTS templates CASCADE;") diff --git a/backend/alembic/versions/__pycache__/91b2545f5a30_add_templates_schema.cpython-313.pyc b/backend/alembic/versions/__pycache__/91b2545f5a30_add_templates_schema.cpython-313.pyc new file mode 100644 index 0000000..db28e36 Binary files /dev/null and b/backend/alembic/versions/__pycache__/91b2545f5a30_add_templates_schema.cpython-313.pyc differ diff --git a/backend/api/__pycache__/documents.cpython-313.pyc b/backend/api/__pycache__/documents.cpython-313.pyc new file mode 100644 index 0000000..c607e16 Binary files /dev/null and b/backend/api/__pycache__/documents.cpython-313.pyc differ diff --git a/backend/api/__pycache__/templates.cpython-313.pyc b/backend/api/__pycache__/templates.cpython-313.pyc new file mode 100644 index 0000000..a7d2d21 Binary files /dev/null and b/backend/api/__pycache__/templates.cpython-313.pyc differ diff --git a/backend/api/documents.py b/backend/api/documents.py new file mode 100644 index 0000000..5b934da --- /dev/null +++ b/backend/api/documents.py @@ -0,0 +1,54 @@ +import json +from fastapi import APIRouter, Depends, HTTPException, status, UploadFile, File +from sqlalchemy.orm import Session +from typing import List + +from database import get_db +from models.template_models import TemplateDocument, DocumentLayout +from schemas.template_schemas import DocumentSchema, DocumentLayoutSchema, TemplateRecognitionResult +from engine.ocr.document_processor import DocumentProcessor +from engine.recognition.template_engine import TemplateRecognitionEngine + +router = APIRouter(prefix="/api/documents", tags=["Documents"]) + +@router.post("/upload", response_model=DocumentSchema, status_code=status.HTTP_201_CREATED) +async def upload_document(file: UploadFile = File(...), db: Session = Depends(get_db)): + # 1. Create document record + content = await file.read() + file_hash = hash(content) # Simple hash for demo + + db_doc = TemplateDocument( + document_name=file.filename, + document_type=file.content_type, + file_name=file.filename, + file_hash=str(file_hash), + status="processing" + ) + db.add(db_doc) + db.commit() + db.refresh(db_doc) + + # 2. Process File (OCR & Layout) + processor = DocumentProcessor(db) + processor.process_file(db_doc.pk_document_id, file.filename, content) + + db.refresh(db_doc) + return db_doc + +@router.get("/{document_id}", response_model=DocumentSchema) +def get_document(document_id: int, db: Session = Depends(get_db)): + doc = db.query(TemplateDocument).filter(TemplateDocument.pk_document_id == document_id).first() + if not doc: + raise HTTPException(status_code=404, detail="Document not found") + return doc + +@router.get("/{document_id}/layout", response_model=List[DocumentLayoutSchema]) +def get_document_layout(document_id: int, db: Session = Depends(get_db)): + layouts = db.query(DocumentLayout).filter(DocumentLayout.fk_document_id == document_id).order_by(DocumentLayout.sequence_no).all() + return layouts + +@router.post("/{document_id}/recognize", response_model=TemplateRecognitionResult) +def recognize_template(document_id: int, db: Session = Depends(get_db)): + engine = TemplateRecognitionEngine(db) + return engine.recognize_template(document_id) + diff --git a/backend/api/templates.py b/backend/api/templates.py new file mode 100644 index 0000000..bc859cd --- /dev/null +++ b/backend/api/templates.py @@ -0,0 +1,53 @@ +from fastapi import APIRouter, Depends, HTTPException, status +from sqlalchemy.orm import Session +from typing import List + +from database import get_db +from schemas.template_schemas import ( + TemplateCreate, TemplateSchema, + TemplateFieldCreate, TemplateFieldSchema, + TemplateFieldMappingCreate, TemplateFieldMappingSchema +) +from services.template_service import TemplateService + +router = APIRouter(prefix="/api/templates", tags=["Templates"]) + +@router.post("", response_model=TemplateSchema, status_code=status.HTTP_201_CREATED) +def create_template(template_data: TemplateCreate, db: Session = Depends(get_db)): + service = TemplateService(db) + return service.create_template(template_data) + +@router.get("", response_model=List[TemplateSchema]) +def get_templates(skip: int = 0, limit: int = 100, db: Session = Depends(get_db)): + service = TemplateService(db) + return service.get_all_templates(skip, limit) + +@router.get("/{template_id}", response_model=TemplateSchema) +def get_template(template_id: int, db: Session = Depends(get_db)): + service = TemplateService(db) + template = service.get_template(template_id) + if not template: + raise HTTPException(status_code=404, detail="Template not found") + return template + +@router.delete("/{template_id}", status_code=status.HTTP_204_NO_CONTENT) +def delete_template(template_id: int, db: Session = Depends(get_db)): + service = TemplateService(db) + if not service.delete_template(template_id): + raise HTTPException(status_code=404, detail="Template not found") + return None + +@router.post("/{template_id}/fields", response_model=TemplateFieldSchema, status_code=status.HTTP_201_CREATED) +def add_template_field(template_id: int, field_data: TemplateFieldCreate, db: Session = Depends(get_db)): + service = TemplateService(db) + # Check if template exists + if not service.get_template(template_id): + raise HTTPException(status_code=404, detail="Template not found") + return service.add_template_field(template_id, field_data) + +@router.post("/{template_id}/mappings/save", response_model=List[TemplateFieldMappingSchema]) +def save_mappings(template_id: int, mappings: List[TemplateFieldMappingCreate], db: Session = Depends(get_db)): + service = TemplateService(db) + if not service.get_template(template_id): + raise HTTPException(status_code=404, detail="Template not found") + return service.save_mapping(template_id, mappings) diff --git a/backend/engine/ocr/__pycache__/document_processor.cpython-313.pyc b/backend/engine/ocr/__pycache__/document_processor.cpython-313.pyc new file mode 100644 index 0000000..3c8b5c2 Binary files /dev/null and b/backend/engine/ocr/__pycache__/document_processor.cpython-313.pyc differ diff --git a/backend/engine/ocr/document_processor.py b/backend/engine/ocr/document_processor.py new file mode 100644 index 0000000..c12a129 --- /dev/null +++ b/backend/engine/ocr/document_processor.py @@ -0,0 +1,217 @@ +import io +import re +from typing import List, Dict, Any +import numpy as np +import cv2 +import pdfplumber +import pytesseract +from PIL import Image +from pdf2image import convert_from_bytes +from sqlalchemy.orm import Session +from models.template_models import DocumentLayout, TemplateDocument +from schemas.template_schemas import DocumentLayoutSchema + +class DocumentProcessor: + def __init__(self, db: Session): + self.db = db + + def process_file(self, document_id: int, filename: str, file_bytes: bytes) -> List[DocumentLayout]: + document = self.db.query(TemplateDocument).filter(TemplateDocument.pk_document_id == document_id).first() + if not document: + raise ValueError("Document not found") + + layouts = [] + if filename.lower().endswith('.pdf'): + layouts = self._process_pdf(document_id, file_bytes) + elif filename.lower().endswith(('.png', '.jpg', '.jpeg', '.tiff')): + layouts = self._process_image(document_id, file_bytes) + + # Update page count + if layouts: + document.page_count = max([l.page_no for l in layouts]) + document.status = "processed" + self.db.commit() + + return layouts + + def _process_pdf(self, document_id: int, file_bytes: bytes) -> List[DocumentLayout]: + layouts = [] + try: + with pdfplumber.open(io.BytesIO(file_bytes)) as pdf: + sequence_no = 1 + for page_no, page in enumerate(pdf.pages, start=1): + # Check if it has text + text = page.extract_text() + if text and text.strip(): + # Extract words with bounding boxes + words = page.extract_words() + + # Group words into blocks using simple proximity heuristics + blocks = self._group_words_to_blocks(words, page.width, page.height) + + for block in blocks: + db_layout = DocumentLayout( + fk_document_id=document_id, + page_no=page_no, + text_value=block['text'], + block_type=block['type'], + x_coordinate=block['x0'], + y_coordinate=block['top'], + width=block['x1'] - block['x0'], + height=block['bottom'] - block['top'], + confidence=100.0, + sequence_no=sequence_no + ) + self.db.add(db_layout) + layouts.append(db_layout) + sequence_no += 1 + else: + # Scanned PDF page -> convert to image and process + # Handled separately to avoid complexity in this mock + pass + except Exception as e: + print(f"Error processing PDF with pdfplumber: {e}") + + if not layouts: + # Fallback to image-based processing for scanned PDFs + images = convert_from_bytes(file_bytes, dpi=300) + sequence_no = 1 + for page_no, img in enumerate(images, start=1): + img_byte_arr = io.BytesIO() + img.save(img_byte_arr, format='PNG') + page_layouts = self._process_image(document_id, img_byte_arr.getvalue(), page_no=page_no, start_sequence=sequence_no) + layouts.extend(page_layouts) + sequence_no += len(page_layouts) + + self.db.commit() + return layouts + + def _process_image(self, document_id: int, file_bytes: bytes, page_no: int = 1, start_sequence: int = 1) -> List[DocumentLayout]: + layouts = [] + try: + # Decode image + nparr = np.frombuffer(file_bytes, np.uint8) + img = cv2.imdecode(nparr, cv2.IMREAD_COLOR) + + # Preprocessing + gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) + # Denoise & Threshold + gray = cv2.medianBlur(gray, 3) + gray = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)[1] + + # OCR using PyTesseract with bounding boxes + data = pytesseract.image_to_data(gray, output_type=pytesseract.Output.DICT) + + sequence_no = start_sequence + + # Grouping words into lines/blocks simplified + height, width = gray.shape + blocks = self._tesseract_data_to_blocks(data, width, height) + + for block in blocks: + db_layout = DocumentLayout( + fk_document_id=document_id, + page_no=page_no, + text_value=block['text'], + block_type=block['type'], + x_coordinate=block['x'], + y_coordinate=block['y'], + width=block['w'], + height=block['h'], + confidence=block['conf'], + sequence_no=sequence_no + ) + self.db.add(db_layout) + layouts.append(db_layout) + sequence_no += 1 + + except Exception as e: + print(f"Error processing image with tesseract: {e}") + + self.db.commit() + return layouts + + def _group_words_to_blocks(self, words: List[Dict], page_width: float, page_height: float) -> List[Dict]: + """Group nearby words into logical blocks and classify them.""" + # Extremely simplified grouping by Y coordinate proximity + blocks = [] + if not words: + return blocks + + # Sort words by Y then X + words.sort(key=lambda w: (w['top'], w['x0'])) + + current_block = { + 'text': words[0]['text'], + 'x0': words[0]['x0'], + 'top': words[0]['top'], + 'x1': words[0]['x1'], + 'bottom': words[0]['bottom'] + } + + for word in words[1:]: + # If word is roughly on the same line (y diff is small) and close horizontally + if abs(word['top'] - current_block['top']) < 10 and (word['x0'] - current_block['x1']) < 50: + current_block['text'] += ' ' + word['text'] + current_block['x1'] = word['x1'] + current_block['top'] = min(current_block['top'], word['top']) + current_block['bottom'] = max(current_block['bottom'], word['bottom']) + else: + current_block['type'] = self._classify_block(current_block, page_height) + blocks.append(current_block) + current_block = { + 'text': word['text'], + 'x0': word['x0'], + 'top': word['top'], + 'x1': word['x1'], + 'bottom': word['bottom'] + } + + current_block['type'] = self._classify_block(current_block, page_height) + blocks.append(current_block) + return blocks + + def _tesseract_data_to_blocks(self, data: Dict, page_width: float, page_height: float) -> List[Dict]: + blocks = [] + n_boxes = len(data['text']) + current_line = [] + + for i in range(n_boxes): + if int(data['conf'][i]) > 10 and data['text'][i].strip(): + # We can group by line_num + blocks.append({ + 'text': data['text'][i], + 'x': data['left'][i], + 'y': data['top'][i], + 'w': data['width'][i], + 'h': data['height'][i], + 'conf': float(data['conf'][i]), + 'type': self._classify_block({'top': data['top'][i], 'text': data['text'][i]}, page_height) + }) + return blocks + + def _classify_block(self, block: Dict, page_height: float) -> str: + y = block.get('top') or block.get('y') or 0 + text = block.get('text', '').lower() + + if y < page_height * 0.15: + return "HEADER" + elif y > page_height * 0.85: + return "FOOTER" + + if "total" in text: + return "TOTAL" + if "tax" in text or "gst" in text or "vat" in text: + return "TAX" + if re.search(r'\b(vendor|from)\b', text): + return "VENDOR" + if re.search(r'\b(bill to|sold to)\b', text): + return "BILL_TO" + if re.search(r'\b(ship to)\b', text): + return "SHIP_TO" + + # Very naive fallback for table check + if re.search(r'\b(qty|rate|amount|price|item)\b', text): + return "TABLE_HEADER" + + return "TEXT" diff --git a/backend/engine/recognition/__pycache__/template_engine.cpython-313.pyc b/backend/engine/recognition/__pycache__/template_engine.cpython-313.pyc new file mode 100644 index 0000000..dc9b72b Binary files /dev/null and b/backend/engine/recognition/__pycache__/template_engine.cpython-313.pyc differ diff --git a/backend/engine/recognition/template_engine.py b/backend/engine/recognition/template_engine.py new file mode 100644 index 0000000..4c482ab --- /dev/null +++ b/backend/engine/recognition/template_engine.py @@ -0,0 +1,174 @@ +import hashlib +import json +from typing import List, Dict, Any, Tuple +from sqlalchemy.orm import Session +from models.template_models import Template, TemplateDocument, DocumentLayout, TemplateRecognitionHistory +from schemas.template_schemas import TemplateRecognitionResult + +class TemplateRecognitionEngine: + def __init__(self, db: Session): + self.db = db + + def generate_fingerprint(self, layouts: List[DocumentLayout]) -> str: + """ + Generate a structural fingerprint based on the document's layout. + We use relative positions of HEADER, VENDOR, TABLE_HEADER blocks. + """ + key_blocks = [] + for layout in layouts: + if layout.block_type in ["HEADER", "VENDOR", "TABLE_HEADER", "TOTAL"]: + key_blocks.append({ + "type": layout.block_type, + "text": layout.text_value[:50] if layout.text_value else "", # first 50 chars + "rx": round(float(layout.x_coordinate) / 100) if layout.x_coordinate else 0, # relative bucket x + "ry": round(float(layout.y_coordinate) / 100) if layout.y_coordinate else 0 # relative bucket y + }) + + # Sort top to bottom, left to right + key_blocks.sort(key=lambda b: (b['ry'], b['rx'])) + + fingerprint_data = json.dumps(key_blocks) + return hashlib.sha256(fingerprint_data.encode('utf-8')).hexdigest() + + def recognize_template(self, document_id: int) -> TemplateRecognitionResult: + """ + Matches a document against existing templates using a 4-level weighted score: + Level 1: Vendor Match (40%) + Level 2: Header Similarity (20%) + Level 3: Layout/Fingerprint Similarity (20%) + Level 4: Coordinate Similarity (20%) + Threshold: 85% + """ + layouts = self.db.query(DocumentLayout).filter(DocumentLayout.fk_document_id == document_id).all() + if not layouts: + return TemplateRecognitionResult(templateMatched=False) + + doc_fingerprint = self.generate_fingerprint(layouts) + + # Extract features for scoring + doc_vendor = self._extract_vendor_name(layouts) + doc_headers = self._extract_headers(layouts) + + templates = self.db.query(Template).filter(Template.active_flag == True).all() + best_match = None + highest_score = 0 + + for template in templates: + score = 0.0 + + # Level 1: Fingerprint Exact Match (counts for Layout + Vendor + Header if exact) + if template.template_fingerprint == doc_fingerprint: + score += 100.0 + else: + # Need to load template sample mapping to compare heuristics + # In a real system, we'd compare against the mapped fields' coordinates + score += self._calculate_heuristic_score(template, doc_vendor, doc_headers, layouts) + + if score > highest_score: + highest_score = score + best_match = template + + # Store history + if best_match: + history = TemplateRecognitionHistory( + fk_template_id=best_match.pk_template_id, + fk_document_id=document_id, + recognition_score=highest_score, + matched_flag=(highest_score >= 85.0) + ) + self.db.add(history) + self.db.commit() + + if best_match and highest_score >= 85.0: + extracted_fields = self._auto_map_fields(best_match, layouts) + return TemplateRecognitionResult( + templateMatched=True, + templateId=best_match.pk_template_id, + confidence=highest_score, + extractedFields=extracted_fields + ) + + return TemplateRecognitionResult(templateMatched=False) + + def _extract_vendor_name(self, layouts: List[DocumentLayout]) -> str: + for layout in layouts: + if layout.block_type == "VENDOR": + return layout.text_value.lower() + return "" + + def _extract_headers(self, layouts: List[DocumentLayout]) -> List[str]: + headers = [] + for layout in layouts: + if layout.block_type == "TABLE_HEADER" and layout.text_value: + headers.append(layout.text_value.lower()) + return headers + + def _calculate_heuristic_score(self, template: Template, doc_vendor: str, doc_headers: List[str], layouts: List[DocumentLayout]) -> float: + score = 0.0 + + # This requires the template to have some stored metadata or we check its fields + # E.g., if template has a field "Vendor Name" mapped to a specific text + + # 1. Vendor Match (40%) + # For this mockup, we check if the template name matches the vendor + if template.template_name.lower() in doc_vendor or doc_vendor in template.template_name.lower(): + score += 40.0 + + # 2. Header Similarity (20%) + # Check if template fields exist that match doc_headers + field_labels = [f.field_label.lower() for f in template.fields] + header_matches = sum(1 for h in doc_headers if any(h in fl or fl in h for fl in field_labels)) + if len(doc_headers) > 0: + score += (header_matches / len(doc_headers)) * 20.0 + + # 3. Layout / Coordinate Similarity + # We can check the template's previous mappings coordinates against current document + # If coordinates are within a tolerance, we add score + mapping_count = 0 + match_count = 0 + for mapping in template.mappings: + mapping_count += 1 + # Find a layout block in current document that is near mapping coordinates + for layout in layouts: + if layout.page_no == mapping.page_no: + # check distance + if mapping.x_coordinate and mapping.y_coordinate and layout.x_coordinate and layout.y_coordinate: + dx = abs(float(mapping.x_coordinate) - float(layout.x_coordinate)) + dy = abs(float(mapping.y_coordinate) - float(layout.y_coordinate)) + if dx < 50 and dy < 20: # arbitrary tolerance + match_count += 1 + break + + if mapping_count > 0: + coord_score = (match_count / mapping_count) * 40.0 # Layout 20% + Coord 20% + score += coord_score + + return score + + def _auto_map_fields(self, template: Template, layouts: List[DocumentLayout]) -> List[dict]: + extracted = [] + for mapping in template.mappings: + # Find closest block in new document + best_block = None + min_dist = float('inf') + + for layout in layouts: + if layout.page_no == mapping.page_no: + if mapping.x_coordinate and mapping.y_coordinate and layout.x_coordinate and layout.y_coordinate: + dx = abs(float(mapping.x_coordinate) - float(layout.x_coordinate)) + dy = abs(float(mapping.y_coordinate) - float(layout.y_coordinate)) + dist = dx**2 + dy**2 + if dist < min_dist and dist < 5000: # tolerance squared + min_dist = dist + best_block = layout + + if best_block: + extracted.append({ + "field_id": mapping.fk_template_field_id, + "field_label": mapping.field.field_label if mapping.field else "", + "value": best_block.text_value, + "confidence": 95.0, # arbitrary high confidence for coordinate match + "layout_id": best_block.pk_document_data_id + }) + + return extracted diff --git a/backend/main.py b/backend/main.py index 5ac7444..1789f21 100644 --- a/backend/main.py +++ b/backend/main.py @@ -48,6 +48,10 @@ from database import get_db, Email, Vendor, Document from scheduler import start_scheduler, stop_scheduler from mail_service import fetch_and_store_emails +# Import the new routers +from api.templates import router as templates_router +from api.documents import router as documents_router + # Lifespan for Scheduler @asynccontextmanager async def lifespan(app: FastAPI): @@ -59,6 +63,10 @@ async def lifespan(app: FastAPI): app = FastAPI(lifespan=lifespan) +# Include Routers +app.include_router(templates_router) +app.include_router(documents_router) + # CORS configuration origins = [ "http://localhost", diff --git a/backend/models/__pycache__/template_models.cpython-313.pyc b/backend/models/__pycache__/template_models.cpython-313.pyc new file mode 100644 index 0000000..b92120c Binary files /dev/null and b/backend/models/__pycache__/template_models.cpython-313.pyc differ diff --git a/backend/models/template_models.py b/backend/models/template_models.py new file mode 100644 index 0000000..b924681 --- /dev/null +++ b/backend/models/template_models.py @@ -0,0 +1,105 @@ +from sqlalchemy import Column, Integer, String, Text, DateTime, Boolean, ForeignKey, Numeric +from sqlalchemy.sql import func +from sqlalchemy.orm import relationship +from database import Base + +class TemplateDocument(Base): + __tablename__ = "documents" + __table_args__ = {"schema": "templates"} + + pk_document_id = Column(Integer, primary_key=True, index=True) + document_name = Column(String(500)) + document_type = Column(String(100)) + file_name = Column(String(500)) + file_hash = Column(String(500), index=True) + page_count = Column(Integer) + status = Column(String(50)) + created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + + # Relationships + layouts = relationship("DocumentLayout", back_populates="document", cascade="all, delete") + +class DocumentLayout(Base): + __tablename__ = "document_layout" + __table_args__ = {"schema": "templates"} + + pk_document_data_id = Column(Integer, primary_key=True, index=True) + fk_document_id = Column(Integer, ForeignKey("templates.documents.pk_document_id", ondelete="CASCADE"), nullable=False, index=True) + page_no = Column(Integer, nullable=False, index=True) + text_value = Column(Text) + block_type = Column(String(100)) + parent_block_id = Column(Integer, ForeignKey("templates.document_layout.pk_document_data_id", ondelete="SET NULL")) + x_coordinate = Column(Numeric) + y_coordinate = Column(Numeric) + width = Column(Numeric) + height = Column(Numeric) + confidence = Column(Numeric) + sequence_no = Column(Integer) + layout_path = Column(Text) + created_at = Column(DateTime, server_default=func.now()) + + document = relationship("TemplateDocument", back_populates="layouts") + parent = relationship("DocumentLayout", remote_side=[pk_document_data_id]) + mappings = relationship("TemplateFieldMapping", back_populates="document_layout", cascade="all, delete") + +class Template(Base): + __tablename__ = "templates" + __table_args__ = {"schema": "templates"} + + pk_template_id = Column(Integer, primary_key=True, index=True) + template_name = Column(String(255), nullable=False, index=True) + template_fingerprint = Column(Text) + active_flag = Column(Boolean, default=True) + created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + + fields = relationship("TemplateField", back_populates="template", cascade="all, delete") + mappings = relationship("TemplateFieldMapping", back_populates="template", cascade="all, delete") + +class TemplateField(Base): + __tablename__ = "template_fields" + __table_args__ = {"schema": "templates"} + + pk_template_field_id = Column(Integer, primary_key=True, index=True) + fk_template_id = Column(Integer, ForeignKey("templates.templates.pk_template_id", ondelete="CASCADE"), nullable=False, index=True) + field_label = Column(String(255), nullable=False) + field_type = Column(String(100), nullable=False) + display_order = Column(Integer) + required_flag = Column(Boolean, default=False) + created_at = Column(DateTime, server_default=func.now()) + + template = relationship("Template", back_populates="fields") + mappings = relationship("TemplateFieldMapping", back_populates="field", cascade="all, delete") + +class TemplateFieldMapping(Base): + __tablename__ = "template_fields_mapping" + __table_args__ = {"schema": "templates"} + + pk_mapping_id = Column(Integer, primary_key=True, index=True) + fk_template_id = Column(Integer, ForeignKey("templates.templates.pk_template_id", ondelete="CASCADE"), nullable=False, index=True) + fk_template_field_id = Column(Integer, ForeignKey("templates.template_fields.pk_template_field_id", ondelete="CASCADE"), nullable=False, index=True) + fk_document_data_id = Column(Integer, ForeignKey("templates.document_layout.pk_document_data_id", ondelete="SET NULL")) + page_no = Column(Integer) + x_coordinate = Column(Numeric) + y_coordinate = Column(Numeric) + width = Column(Numeric) + height = Column(Numeric) + mapping_confidence = Column(Numeric) + layout_path = Column(Text) + created_at = Column(DateTime, server_default=func.now()) + + template = relationship("Template", back_populates="mappings") + field = relationship("TemplateField", back_populates="mappings") + document_layout = relationship("DocumentLayout", back_populates="mappings") + +class TemplateRecognitionHistory(Base): + __tablename__ = "template_recognition_history" + __table_args__ = {"schema": "templates"} + + pk_history_id = Column(Integer, primary_key=True, index=True) + fk_template_id = Column(Integer, ForeignKey("templates.templates.pk_template_id", ondelete="CASCADE"), nullable=False) + fk_document_id = Column(Integer, ForeignKey("templates.documents.pk_document_id", ondelete="CASCADE"), nullable=False, index=True) + recognition_score = Column(Numeric) + matched_flag = Column(Boolean) + created_at = Column(DateTime, server_default=func.now()) diff --git a/backend/repositories/__pycache__/template_repository.cpython-313.pyc b/backend/repositories/__pycache__/template_repository.cpython-313.pyc new file mode 100644 index 0000000..fcb3482 Binary files /dev/null and b/backend/repositories/__pycache__/template_repository.cpython-313.pyc differ diff --git a/backend/repositories/template_repository.py b/backend/repositories/template_repository.py new file mode 100644 index 0000000..0f8dd35 --- /dev/null +++ b/backend/repositories/template_repository.py @@ -0,0 +1,75 @@ +from sqlalchemy.orm import Session +from sqlalchemy import desc +from models.template_models import Template, TemplateField, TemplateFieldMapping, TemplateRecognitionHistory +from schemas.template_schemas import TemplateCreate, TemplateFieldCreate, TemplateFieldMappingCreate + +class TemplateRepository: + def __init__(self, db: Session): + self.db = db + + def get_template(self, template_id: int): + return self.db.query(Template).filter(Template.pk_template_id == template_id).first() + + def get_templates(self, skip: int = 0, limit: int = 100): + return self.db.query(Template).offset(skip).limit(limit).all() + + def create_template(self, template_data: TemplateCreate): + db_template = Template(template_name=template_data.template_name) + self.db.add(db_template) + self.db.commit() + self.db.refresh(db_template) + + # Create fields if provided + for field in template_data.fields: + db_field = TemplateField( + fk_template_id=db_template.pk_template_id, + field_label=field.field_label, + field_type=field.field_type, + display_order=field.display_order, + required_flag=field.required_flag + ) + self.db.add(db_field) + + self.db.commit() + self.db.refresh(db_template) + return db_template + + def add_template_field(self, template_id: int, field_data: TemplateFieldCreate): + db_field = TemplateField( + fk_template_id=template_id, + field_label=field_data.field_label, + field_type=field_data.field_type, + display_order=field_data.display_order, + required_flag=field_data.required_flag + ) + self.db.add(db_field) + self.db.commit() + self.db.refresh(db_field) + return db_field + + def delete_template(self, template_id: int): + template = self.get_template(template_id) + if template: + self.db.delete(template) + self.db.commit() + return True + return False + + def save_mapping(self, template_id: int, mappings: list[TemplateFieldMappingCreate]): + saved_mappings = [] + for mapping in mappings: + db_mapping = TemplateFieldMapping( + fk_template_id=template_id, + fk_template_field_id=mapping.fk_template_field_id, + fk_document_data_id=mapping.fk_document_data_id, + page_no=mapping.page_no, + x_coordinate=mapping.x_coordinate, + y_coordinate=mapping.y_coordinate, + width=mapping.width, + height=mapping.height, + mapping_confidence=mapping.mapping_confidence + ) + self.db.add(db_mapping) + saved_mappings.append(db_mapping) + self.db.commit() + return saved_mappings diff --git a/backend/requirements.txt b/backend/requirements.txt index db6102a..fbf9d2c 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -11,3 +11,7 @@ apscheduler python-dotenv pdfplumber ollama +numpy +opencv-python-headless +pdf2image + diff --git a/backend/schemas/__pycache__/template_schemas.cpython-313.pyc b/backend/schemas/__pycache__/template_schemas.cpython-313.pyc new file mode 100644 index 0000000..6448164 Binary files /dev/null and b/backend/schemas/__pycache__/template_schemas.cpython-313.pyc differ diff --git a/backend/schemas/template_schemas.py b/backend/schemas/template_schemas.py new file mode 100644 index 0000000..176816f --- /dev/null +++ b/backend/schemas/template_schemas.py @@ -0,0 +1,80 @@ +from pydantic import BaseModel, ConfigDict +from typing import List, Optional +from datetime import datetime + +# Common +class BaseSchema(BaseModel): + model_config = ConfigDict(from_attributes=True) + +# Document +class DocumentLayoutSchema(BaseSchema): + pk_document_data_id: int + fk_document_id: int + page_no: int + text_value: Optional[str] + block_type: Optional[str] + parent_block_id: Optional[int] + x_coordinate: Optional[float] + y_coordinate: Optional[float] + width: Optional[float] + height: Optional[float] + confidence: Optional[float] + sequence_no: Optional[int] + +class DocumentSchema(BaseSchema): + pk_document_id: int + document_name: Optional[str] + document_type: Optional[str] + file_name: Optional[str] + page_count: Optional[int] + status: Optional[str] + created_at: datetime + layouts: List[DocumentLayoutSchema] = [] + +# Template Field +class TemplateFieldCreate(BaseSchema): + field_label: str + field_type: str + display_order: Optional[int] = 0 + required_flag: Optional[bool] = False + +class TemplateFieldSchema(TemplateFieldCreate): + pk_template_field_id: int + fk_template_id: int + created_at: datetime + +# Template Field Mapping +class TemplateFieldMappingCreate(BaseSchema): + fk_template_field_id: int + fk_document_data_id: Optional[int] = None + page_no: Optional[int] = None + x_coordinate: Optional[float] = None + y_coordinate: Optional[float] = None + width: Optional[float] = None + height: Optional[float] = None + mapping_confidence: Optional[float] = None + +class TemplateFieldMappingSchema(TemplateFieldMappingCreate): + pk_mapping_id: int + fk_template_id: int + created_at: datetime + +# Template +class TemplateCreate(BaseSchema): + template_name: str + fields: Optional[List[TemplateFieldCreate]] = [] + +class TemplateSchema(BaseSchema): + pk_template_id: int + template_name: str + template_fingerprint: Optional[str] = None + active_flag: bool + created_at: datetime + fields: List[TemplateFieldSchema] = [] + mappings: List[TemplateFieldMappingSchema] = [] + +class TemplateRecognitionResult(BaseSchema): + templateMatched: bool + templateId: Optional[int] = None + confidence: Optional[float] = None + extractedFields: List[dict] = [] diff --git a/backend/services/__pycache__/template_service.cpython-313.pyc b/backend/services/__pycache__/template_service.cpython-313.pyc new file mode 100644 index 0000000..2879896 Binary files /dev/null and b/backend/services/__pycache__/template_service.cpython-313.pyc differ diff --git a/backend/services/template_service.py b/backend/services/template_service.py new file mode 100644 index 0000000..809aa86 --- /dev/null +++ b/backend/services/template_service.py @@ -0,0 +1,25 @@ +from sqlalchemy.orm import Session +from schemas.template_schemas import TemplateCreate, TemplateFieldCreate, TemplateFieldMappingCreate +from repositories.template_repository import TemplateRepository + +class TemplateService: + def __init__(self, db: Session): + self.repository = TemplateRepository(db) + + def get_template(self, template_id: int): + return self.repository.get_template(template_id) + + def get_all_templates(self, skip: int = 0, limit: int = 100): + return self.repository.get_templates(skip, limit) + + def create_template(self, template_data: TemplateCreate): + return self.repository.create_template(template_data) + + def add_template_field(self, template_id: int, field_data: TemplateFieldCreate): + return self.repository.add_template_field(template_id, field_data) + + def delete_template(self, template_id: int): + return self.repository.delete_template(template_id) + + def save_mapping(self, template_id: int, mappings: list[TemplateFieldMappingCreate]): + return self.repository.save_mapping(template_id, mappings) diff --git a/backend/sql/01_templates_schema.sql b/backend/sql/01_templates_schema.sql new file mode 100644 index 0000000..f17313d --- /dev/null +++ b/backend/sql/01_templates_schema.sql @@ -0,0 +1,107 @@ +-- Schema for Template Engine +CREATE SCHEMA IF NOT EXISTS templates; + +-- 1. documents table +CREATE TABLE IF NOT EXISTS templates.documents ( + pk_document_id BIGSERIAL PRIMARY KEY, + document_name VARCHAR(500), + document_type VARCHAR(100), + file_name VARCHAR(500), + file_hash VARCHAR(500), + page_count INTEGER, + status VARCHAR(50), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +-- Index for quick lookups +CREATE INDEX idx_documents_hash ON templates.documents(file_hash); + +-- 2. document_layout table +CREATE TABLE IF NOT EXISTS templates.document_layout ( + pk_document_data_id BIGSERIAL PRIMARY KEY, + fk_document_id BIGINT NOT NULL, + page_no INTEGER NOT NULL, + text_value TEXT, + block_type VARCHAR(100), + parent_block_id BIGINT, + x_coordinate NUMERIC, + y_coordinate NUMERIC, + width NUMERIC, + height NUMERIC, + confidence NUMERIC, + sequence_no INTEGER, + layout_path TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT fk_dl_document FOREIGN KEY (fk_document_id) REFERENCES templates.documents(pk_document_id) ON DELETE CASCADE, + CONSTRAINT fk_dl_parent FOREIGN KEY (parent_block_id) REFERENCES templates.document_layout(pk_document_data_id) ON DELETE SET NULL +); + +CREATE INDEX idx_doc_layout_doc_id ON templates.document_layout(fk_document_id); +CREATE INDEX idx_doc_layout_page ON templates.document_layout(page_no); + +-- 3. templates table +CREATE TABLE IF NOT EXISTS templates.templates ( + pk_template_id BIGSERIAL PRIMARY KEY, + template_name VARCHAR(255) NOT NULL, + template_fingerprint TEXT, + active_flag BOOLEAN DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX idx_templates_name ON templates.templates(template_name); + +-- 4. template_fields table +CREATE TABLE IF NOT EXISTS templates.template_fields ( + pk_template_field_id BIGSERIAL PRIMARY KEY, + fk_template_id BIGINT NOT NULL, + field_label VARCHAR(255) NOT NULL, + field_type VARCHAR(100) NOT NULL, + display_order INTEGER, + required_flag BOOLEAN DEFAULT FALSE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT fk_tf_template FOREIGN KEY (fk_template_id) REFERENCES templates.templates(pk_template_id) ON DELETE CASCADE +); + +CREATE INDEX idx_template_fields_tmpl_id ON templates.template_fields(fk_template_id); + +-- 5. template_fields_mapping table +CREATE TABLE IF NOT EXISTS templates.template_fields_mapping ( + pk_mapping_id BIGSERIAL PRIMARY KEY, + fk_template_id BIGINT NOT NULL, + fk_template_field_id BIGINT NOT NULL, + fk_document_data_id BIGINT, + page_no INTEGER, + x_coordinate NUMERIC, + y_coordinate NUMERIC, + width NUMERIC, + height NUMERIC, + mapping_confidence NUMERIC, + layout_path TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT fk_tfm_template FOREIGN KEY (fk_template_id) REFERENCES templates.templates(pk_template_id) ON DELETE CASCADE, + CONSTRAINT fk_tfm_field FOREIGN KEY (fk_template_field_id) REFERENCES templates.template_fields(pk_template_field_id) ON DELETE CASCADE, + CONSTRAINT fk_tfm_doc_data FOREIGN KEY (fk_document_data_id) REFERENCES templates.document_layout(pk_document_data_id) ON DELETE SET NULL +); + +CREATE INDEX idx_mapping_tmpl_id ON templates.template_fields_mapping(fk_template_id); +CREATE INDEX idx_mapping_field_id ON templates.template_fields_mapping(fk_template_field_id); + +-- 6. template_recognition_history table +CREATE TABLE IF NOT EXISTS templates.template_recognition_history ( + pk_history_id BIGSERIAL PRIMARY KEY, + fk_template_id BIGINT NOT NULL, + fk_document_id BIGINT NOT NULL, + recognition_score NUMERIC, + matched_flag BOOLEAN, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT fk_trh_template FOREIGN KEY (fk_template_id) REFERENCES templates.templates(pk_template_id) ON DELETE CASCADE, + CONSTRAINT fk_trh_document FOREIGN KEY (fk_document_id) REFERENCES templates.documents(pk_document_id) ON DELETE CASCADE +); + +CREATE INDEX idx_recognition_history_doc ON templates.template_recognition_history(fk_document_id); diff --git a/backend/uploads/Invoice For Mar 2026.pdf b/backend/uploads/Invoice For Mar 2026.pdf new file mode 100644 index 0000000..65e0311 Binary files /dev/null and b/backend/uploads/Invoice For Mar 2026.pdf differ diff --git a/backend/uploads/Invoice For Mar 2026.pdf.jpg b/backend/uploads/Invoice For Mar 2026.pdf.jpg new file mode 100644 index 0000000..1bffb55 Binary files /dev/null and b/backend/uploads/Invoice For Mar 2026.pdf.jpg differ diff --git a/backend/uploads/sample-pdf-invoice.pdf.jpg b/backend/uploads/sample-pdf-invoice.pdf.jpg new file mode 100644 index 0000000..ea280cc Binary files /dev/null and b/backend/uploads/sample-pdf-invoice.pdf.jpg differ diff --git a/docengine/.env.example b/docengine/.env.example index e300811..cd2a2fa 100644 --- a/docengine/.env.example +++ b/docengine/.env.example @@ -10,7 +10,7 @@ APP_WORKERS=4 # Database DB_HOST=localhost DB_PORT=5432 -DB_NAME=document_engine +DB_NAME=ocr DB_USER=postgres DB_PASSWORD=changeme DB_SCHEMA=admin @@ -48,7 +48,7 @@ LOG_LEVEL=INFO LOG_FORMAT=json # CORS -CORS_ORIGINS=["http://localhost:3000","http://localhost:8080"] +CORS_ORIGINS=["http://localhost:3000","http://localhost:8080","http://localhost:4200"] CORS_ALLOW_CREDENTIALS=true # Rate Limiting diff --git a/docengine/README.md b/docengine/README.md index 46b36f7..373a614 100644 --- a/docengine/README.md +++ b/docengine/README.md @@ -13,7 +13,7 @@ A production-ready system for scanning documents, detecting layouts, extracting ▼ ▼ ┌───────────────────────────────────────┐ │ PostgreSQL (Schema: admin) │ -│ 192.168.0.111:7925 │ +│ 192.168.0.111:5432 │ └───────────────────────────────────────┘ ``` @@ -92,7 +92,7 @@ docengine/ ### Prerequisites - Python 3.12+ -- PostgreSQL 16 (running at `192.168.0.111:7925`) +- PostgreSQL 16 (running at `192.168.0.111:5432`) - Redis (for Celery) - `poppler-utils` and `ghostscript` (for pdf2image/camelot) @@ -116,8 +116,8 @@ mkdir -p storage/{documents,templates,images,temp,rendered} alembic upgrade head # (Optional) Seed default data -psql -h 192.168.0.111 -p 7925 -U postgres -d document_engine -f sql/003_seed_data.sql -psql -h 192.168.0.111 -p 7925 -U postgres -d document_engine -f sql/004_indexes.sql +psql -h 192.168.0.111 -p 5432 -U postgres -d ocr -f sql/003_seed_data.sql +psql -h 192.168.0.111 -p 5432 -U postgres -d ocr -f sql/004_indexes.sql # Start the application python -m app.main @@ -142,7 +142,7 @@ docker compose up --build -d docker compose exec app alembic upgrade head # Seed data -docker compose exec app bash -c "psql -h db -U postgres -d document_engine -f sql/003_seed_data.sql" +docker compose exec app bash -c "psql -h db -U postgres -d ocr -f sql/003_seed_data.sql" ``` --- @@ -403,7 +403,7 @@ When a document is uploaded, the following Celery task pipeline executes asynchr ## Database -**Connection**: `postgresql://postgres:***@192.168.0.111:7925/document_engine` +**Connection**: `postgresql://postgres:***@192.168.0.111:5432/ocr` **Schema**: `admin` ### Migrations @@ -474,11 +474,11 @@ All configuration is via environment variables (`.env` file). Key settings: |------------------------------------|------------------------|---------------------------------| | `APP_PORT` | `7989` | Application port | | `DB_HOST` | `192.168.0.111` | PostgreSQL host | -| `DB_PORT` | `7925` | PostgreSQL port | -| `DB_NAME` | `document_engine` | Database name | +| `DB_PORT` | `5432` | PostgreSQL port | +| `DB_NAME` | `ocr` | Database name | | `DB_SCHEMA` | `admin` | PostgreSQL schema | -| `REDIS_HOST` | `localhost` | Redis host | -| `CELERY_BROKER_URL` | `redis://localhost:6379/0` | Celery broker | +| `REDIS_HOST` | `192.168.0.111` | Redis host | +| `CELERY_BROKER_URL` | `redis://:***@192.168.0.111:7901/0` | Celery broker | | `JWT_SECRET_KEY` | *(see .env)* | JWT signing key | | `JWT_ACCESS_TOKEN_EXPIRE_MINUTES` | `30` | Access token TTL | | `STORAGE_LOCAL_PATH` | `./storage` | Local file storage path | diff --git a/docengine/alembic.ini b/docengine/alembic.ini index c427555..21d448c 100644 --- a/docengine/alembic.ini +++ b/docengine/alembic.ini @@ -2,7 +2,7 @@ script_location = alembic prepend_sys_path = . version_path_separator = os -sqlalchemy.url = postgresql+psycopg2://postgres:M%%40tr%%21x%%23149%%40dm%%21N@192.168.0.111:7925/document_engine +sqlalchemy.url = postgresql+psycopg2://postgres:M%%40triXPostgr3s%%406202@192.168.0.111:5432/ocr [post_write_hooks] diff --git a/docengine/alembic/env.py b/docengine/alembic/env.py index 35de6c4..c5fe0fd 100644 --- a/docengine/alembic/env.py +++ b/docengine/alembic/env.py @@ -17,7 +17,7 @@ if config.config_file_name is not None: target_metadata = Base.metadata # Override the database URL from settings -config.set_main_option("sqlalchemy.url", settings.database_url) +config.set_main_option("sqlalchemy.url", settings.database_url.replace('%', '%%')) def run_migrations_offline() -> None: diff --git a/docengine/app/core/aes.py b/docengine/app/core/aes.py new file mode 100644 index 0000000..aacc416 --- /dev/null +++ b/docengine/app/core/aes.py @@ -0,0 +1,38 @@ +import base64 +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + +def decrypt(base64_input: str, secret: bytes) -> str | None: + try: + combined = base64.b64decode(base64_input) + + # IV length is 12 bytes in the Java implementation + iv_length = 12 + if len(combined) <= iv_length + 16: + return None + + iv = combined[:iv_length] + + # Java AES/GCM appends a 16-byte authentication tag at the end of cipherBytes + # cryptography library expects it to be passed into the modes.GCM(iv, tag) + cipher_bytes_with_tag = combined[iv_length:] + actual_ciphertext = cipher_bytes_with_tag[:-16] + tag = cipher_bytes_with_tag[-16:] + + cipher = Cipher(algorithms.AES(secret), modes.GCM(iv, tag)) + decryptor = cipher.decryptor() + + plain_bytes = decryptor.update(actual_ciphertext) + decryptor.finalize() + return plain_bytes.decode('utf-8') + except Exception: + return None + +def get_string_value(encrypted_id: str, secret: str, secret_key_internal: str) -> str | None: + if not secret or not secret_key_internal: + return None + try: + # Replicate Java's logic: (s + secretKeyInternal).substring(0, Math.min(..., 16)) + combined_str = secret + secret_key_internal + combined_secret = combined_str[:16].encode('utf-8') + return decrypt(encrypted_id, combined_secret) + except Exception: + return None diff --git a/docengine/app/core/config.py b/docengine/app/core/config.py index 52fef4e..e6ebb6e 100644 --- a/docengine/app/core/config.py +++ b/docengine/app/core/config.py @@ -28,30 +28,32 @@ class Settings(BaseSettings): # Database db_host: str = "192.168.0.111" - db_port: int = 7925 - db_name: str = "document_engine" + db_port: int = 5432 + db_name: str = "ocr" db_user: str = "postgres" - db_password: str = "M@tr!x#149@dm!N" + db_password: str = "M@triXPostgr3s@6202" db_schema: str = "admin" db_pool_size: int = 20 db_max_overflow: int = 10 db_echo: bool = False # Redis - redis_host: str = "localhost" - redis_port: int = 6379 + redis_host: str = "192.168.0.111" + redis_port: int = 7901 redis_db: int = 0 - redis_password: str = "" + redis_password: str = "M@triXR3d1s@6202" # Celery - celery_broker_url: str = "redis://localhost:6379/0" - celery_result_backend: str = "redis://localhost:6379/1" + celery_broker_url: str = "redis://:M@triXR3d1s@6202@192.168.0.111:7901/0" + celery_result_backend: str = "redis://:M@triXR3d1s@6202@192.168.0.111:7901/1" # JWT jwt_secret_key: str = "a7f3c9e1d4b8f2a6c0e5d7b3a9f1c4e8d2b6a0f5c3e7d1b9a4f8c2e6d0b5a3" jwt_algorithm: str = "HS256" jwt_access_token_expire_minutes: int = 30 jwt_refresh_token_expire_days: int = 7 + session_encryption_secret: str = "" + session_encryption_secret_internal: str = "" # Storage storage_provider: str = "local" @@ -67,7 +69,7 @@ class Settings(BaseSettings): log_format: str = "json" # CORS - cors_origins: list[str] = ["http://localhost:3000", "http://localhost:8080"] + cors_origins: list[str] = ["http://localhost:3000", "http://localhost:8080", "http://localhost:4200"] cors_allow_credentials: bool = True # Rate Limiting diff --git a/docengine/app/core/dependencies.py b/docengine/app/core/dependencies.py index 102723e..5ecc07e 100644 --- a/docengine/app/core/dependencies.py +++ b/docengine/app/core/dependencies.py @@ -11,17 +11,63 @@ from app.core.security import InvalidTokenError, decode_token from app.models.user import User from app.repositories.user_repository import UserRepository -security_scheme = HTTPBearer(auto_error=True) +import uuid +from app.core.config import settings + +security_scheme = HTTPBearer(auto_error=False) def get_current_user( - credentials: Annotated[HTTPAuthorizationCredentials, Depends(security_scheme)], + credentials: Annotated[HTTPAuthorizationCredentials | None, Depends(security_scheme)], db: Annotated[Session, Depends(get_db)], ) -> User: """Extract and validate the current user from the JWT token.""" + if credentials is None: + if settings.app_env == "development": + # Auto-login as default dev admin user if no token provided in dev + user_repo = UserRepository(db) + user = db.query(User).first() + if user: + return user + dev_user = User( + id=uuid.uuid4(), + username="dev_admin", + email="admin@docengine.local", + hashed_password="mock_password", + is_active=True, + is_superuser=True, + ) + db.add(dev_user) + db.commit() + db.refresh(dev_user) + return dev_user + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Not authenticated", + headers={"WWW-Authenticate": "Bearer"}, + ) + try: payload = decode_token(credentials.credentials) except InvalidTokenError: + if settings.app_env == "development": + # Fallback to dev admin user on token decode failure in dev + user_repo = UserRepository(db) + user = db.query(User).first() + if user: + return user + dev_user = User( + id=uuid.uuid4(), + username="dev_admin", + email="admin@docengine.local", + hashed_password="mock_password", + is_active=True, + is_superuser=True, + ) + db.add(dev_user) + db.commit() + db.refresh(dev_user) + return dev_user raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid or expired token", diff --git a/docengine/app/core/security.py b/docengine/app/core/security.py index 3b68389..6c080da 100644 --- a/docengine/app/core/security.py +++ b/docengine/app/core/security.py @@ -8,6 +8,7 @@ from jose import JWTError, jwt from passlib.context import CryptContext from app.core.config import settings +from app.core.aes import get_string_value pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto") @@ -44,8 +45,14 @@ def create_refresh_token(data: dict[str, Any], expires_delta: timedelta | None = def decode_token(token: str) -> dict[str, Any]: """Decode and validate a JWT token.""" + decrypted_token = get_string_value( + token, + settings.session_encryption_secret, + settings.session_encryption_secret_internal + ) + final_token = decrypted_token if decrypted_token else token try: - payload = jwt.decode(token, settings.jwt_secret_key, algorithms=[settings.jwt_algorithm]) + payload = jwt.decode(final_token, settings.jwt_secret_key, algorithms=[settings.jwt_algorithm]) return payload except JWTError as e: raise InvalidTokenError(str(e)) from e diff --git a/docengine/app/events/handlers.py b/docengine/app/events/handlers.py index 23464ad..0ae2df1 100644 --- a/docengine/app/events/handlers.py +++ b/docengine/app/events/handlers.py @@ -8,7 +8,7 @@ logger = get_logger(__name__) def on_startup() -> None: """Application startup event handler.""" setup_logging() - logger.info("application_starting", event="startup") + logger.info("application_starting", phase="startup") # Ensure storage directories exist from app.storage.provider import get_storage_provider @@ -25,15 +25,15 @@ def on_startup() -> None: else: logger.error("database_connection_failed") - logger.info("application_started", event="startup_complete") + logger.info("application_started", phase="startup_complete") def on_shutdown() -> None: """Application shutdown event handler.""" - logger.info("application_shutting_down", event="shutdown") + logger.info("application_shutting_down", phase="shutdown") # Cleanup resources from app.core.database import engine engine.dispose() - logger.info("application_stopped", event="shutdown_complete") + logger.info("application_stopped", phase="shutdown_complete") diff --git a/docengine/app/main.py b/docengine/app/main.py index e6684f4..683fcbc 100644 --- a/docengine/app/main.py +++ b/docengine/app/main.py @@ -37,9 +37,9 @@ app = FastAPI( ) # Setup middleware (order matters: last added = first executed) -setup_cors(app) app.add_middleware(AuditMiddleware) app.add_middleware(RateLimitMiddleware) +setup_cors(app) # Setup Prometheus metrics setup_metrics(app) diff --git a/docengine/app/middleware/cors.py b/docengine/app/middleware/cors.py index f85d200..5c1146a 100644 --- a/docengine/app/middleware/cors.py +++ b/docengine/app/middleware/cors.py @@ -13,13 +13,7 @@ def setup_cors(app: FastAPI) -> None: allow_origins=settings.cors_origins, allow_credentials=settings.cors_allow_credentials, allow_methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"], - allow_headers=[ - "Authorization", - "Content-Type", - "Accept", - "X-Request-ID", - "X-Requested-With", - ], + allow_headers=["*"], expose_headers=[ "X-Request-ID", "X-Process-Time", diff --git a/docengine/app/models/user.py b/docengine/app/models/user.py index 06c59ea..e0aa4de 100644 --- a/docengine/app/models/user.py +++ b/docengine/app/models/user.py @@ -3,7 +3,7 @@ from __future__ import annotations import uuid from datetime import datetime -from sqlalchemy import Boolean, DateTime, ForeignKey, String, Table, Text, func +from sqlalchemy import Boolean, Column, DateTime, ForeignKey, String, Table, Text, func from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -13,8 +13,8 @@ from app.models.base import TimestampMixin, UUIDPrimaryKeyMixin user_roles_table = Table( "user_roles", Base.metadata, - mapped_column("user_id", UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True), - mapped_column("role_id", UUID(as_uuid=True), ForeignKey("roles.id", ondelete="CASCADE"), primary_key=True), + Column("user_id", UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True), + Column("role_id", UUID(as_uuid=True), ForeignKey("roles.id", ondelete="CASCADE"), primary_key=True), ) diff --git a/docengine/app/workers/celery_app.py b/docengine/app/workers/celery_app.py index 2e0457a..fde23ae 100644 --- a/docengine/app/workers/celery_app.py +++ b/docengine/app/workers/celery_app.py @@ -40,4 +40,9 @@ celery_app.conf.update( }, ) -celery_app.autodiscover_tasks(["app.tasks"]) +celery_app.conf.update( + imports=[ + "app.tasks.document_tasks", + "app.tasks.maintenance_tasks" + ] +) diff --git a/docengine/application.properties b/docengine/application.properties index 2668960..4ad7a3c 100644 --- a/docengine/application.properties +++ b/docengine/application.properties @@ -2,7 +2,7 @@ server.port=7989 db.host=192.168.0.111 -db.port=7925 +db.port=5432 db.user=postgres -db.password=M@tr!x#149@dm!N +db.password=M@triXPostgr3s@6202 db.schema=admin diff --git a/docengine/docker-compose.prod.yml b/docengine/docker-compose.prod.yml index 4eaf60a..d78aeb9 100644 --- a/docengine/docker-compose.prod.yml +++ b/docengine/docker-compose.prod.yml @@ -7,14 +7,14 @@ services: environment: POSTGRES_USER: postgres POSTGRES_PASSWORD: "${DB_PASSWORD}" - POSTGRES_DB: document_engine + POSTGRES_DB: ocr ports: - - "7925:5432" + - "5432:5432" volumes: - docengine_pgdata_prod:/var/lib/postgresql/data - ./sql/001_create_schema.sql:/docker-entrypoint-initdb.d/001_create_schema.sql healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres -d document_engine"] + test: ["CMD-SHELL", "pg_isready -U postgres -d ocr"] interval: 10s timeout: 5s retries: 5 diff --git a/docengine/docker-compose.yml b/docengine/docker-compose.yml index 0c0fefa..340ff12 100644 --- a/docengine/docker-compose.yml +++ b/docengine/docker-compose.yml @@ -6,15 +6,15 @@ services: container_name: docengine_db environment: POSTGRES_USER: postgres - POSTGRES_PASSWORD: "M@tr!x#149@dm!N" - POSTGRES_DB: document_engine + POSTGRES_PASSWORD: "M@triXPostgr3s@6202" + POSTGRES_DB: ocr ports: - - "7925:5432" + - "5432:5432" volumes: - docengine_pgdata:/var/lib/postgresql/data - ./sql/001_create_schema.sql:/docker-entrypoint-initdb.d/001_create_schema.sql healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres -d document_engine"] + test: ["CMD-SHELL", "pg_isready -U postgres -d ocr"] interval: 10s timeout: 5s retries: 5 diff --git a/docengine/requirements.txt b/docengine/requirements.txt index 024e4c0..8e6a283 100644 --- a/docengine/requirements.txt +++ b/docengine/requirements.txt @@ -21,7 +21,7 @@ bcrypt==4.2.1 # Document Processing PyMuPDF==1.25.3 paddleocr==2.9.1 -paddlepaddle==3.0.0b1 +paddlepaddle>=3.0.0 layoutparser==0.3.4 opencv-python-headless==4.10.0.84 camelot-py[cv]==0.11.0 diff --git a/docengine/sql/001_create_schema.sql b/docengine/sql/001_create_schema.sql index 17e0751..1ba0050 100644 --- a/docengine/sql/001_create_schema.sql +++ b/docengine/sql/001_create_schema.sql @@ -2,7 +2,7 @@ CREATE SCHEMA IF NOT EXISTS admin; -- Set the default search path -ALTER DATABASE document_engine SET search_path TO admin, public; +ALTER DATABASE ocr SET search_path TO admin, public; -- Grant privileges GRANT ALL ON SCHEMA admin TO postgres; diff --git a/docengine/sql/002_create_tables.sql b/docengine/sql/002_create_tables.sql index 4a309d2..dea724e 100644 --- a/docengine/sql/002_create_tables.sql +++ b/docengine/sql/002_create_tables.sql @@ -1,6 +1,6 @@ -- DocEngine: Complete table creation script -- Schema: admin --- Database: document_engine +-- Database: ocr SET search_path TO admin, public; diff --git a/frontend/src/app/app.routes.ts b/frontend/src/app/app.routes.ts index 49e8453..0f67e5e 100644 --- a/frontend/src/app/app.routes.ts +++ b/frontend/src/app/app.routes.ts @@ -3,6 +3,7 @@ import { LoginComponent } from './login/login.component'; import { DashboardComponent } from './pages/dashboard/dashboard.component'; import { OcrComponent } from './ocr/ocr.component'; import { MailboxComponent } from './mailbox/mailbox.component'; +import { TemplatesComponent } from './templates/templates.component'; import { AuthorizeComponent } from './pages/session/auth/authorize.component'; import { AuthorizeGuard } from './interceptors/authorize.guard'; @@ -23,7 +24,8 @@ export const routes: Routes = [ children: [ { path: 'profile', component: ProfileComponent }, { path: 'mailbox', component: MailboxComponent }, - { path: 'ocr', component: OcrComponent } + { path: 'ocr', component: OcrComponent }, + { path: 'templates', component: TemplatesComponent } ] }, { path: 'account', diff --git a/frontend/src/app/services/template.service.ts b/frontend/src/app/services/template.service.ts new file mode 100644 index 0000000..71936d9 --- /dev/null +++ b/frontend/src/app/services/template.service.ts @@ -0,0 +1,91 @@ +import { Injectable } from '@angular/core'; +import { HttpClient } from '@angular/common/http'; +import { Observable } from 'rxjs'; +import { map } from 'rxjs/operators'; +import { environment } from '../../environments/environment'; + +export interface DocumentLayout { + pk_document_data_id: number; + fk_document_id: number; + page_no: number; + text_value: string; + block_type: string; + parent_block_id?: number; + x_coordinate: number; + y_coordinate: number; + width: number; + height: number; + confidence: number; + sequence_no: number; +} + +export interface TemplateField { + pk_template_field_id?: number; + field_label: string; + field_type: string; + display_order: number; + required_flag: boolean; +} + +export interface Template { + pk_template_id: number; + template_name: string; + fields: TemplateField[]; +} + +@Injectable({ + providedIn: 'root' +}) +export class TemplateService { + private apiUrl = environment.docEngineService; + + constructor(private http: HttpClient) {} + + uploadDocument(file: File): Observable { + const formData = new FormData(); + formData.append('file', file); + return this.http.post(`${this.apiUrl}/documents/upload`, formData); + } + + getDocumentLayout(documentId: string): Observable { + return this.http.get(`${this.apiUrl}/documents/${documentId}`).pipe( + map(doc => { + const layouts: DocumentLayout[] = []; + if (doc.pages) { + doc.pages.forEach((page: any) => { + if (page.text_blocks) { + page.text_blocks.forEach((tb: any) => { + layouts.push({ + pk_document_data_id: tb.id, + fk_document_id: doc.id, + page_no: page.page_number, + text_value: tb.text, + block_type: tb.block_type, + x_coordinate: tb.x, + y_coordinate: tb.y, + width: tb.width, + height: tb.height, + confidence: tb.confidence || 0, + sequence_no: tb.sequence || 0 + }); + }); + } + }); + } + return layouts; + }) + ); + } + + createTemplate(data: { template_name: string, fields: TemplateField[] }): Observable