Text Extraction Logic

.env.prod

@@ -0,0 +1,13 @@
+# Database Configuration
+# Connecting to existing 'postgres-db' container in 'arbit-app_arbit-network'
+DB_USER=postgres
+DB_PASSWORD='M@tr!x#149@dm!N'
+DB_NAME=ocr
+DB_HOST=postgres-db
+DB_PORT=5432
+
+# Mail Configuration (IMAP)
+MAIL_SERVER=imap.gmail.com
+MAIL_USERNAME=matrixinfotech.it@gmail.com
+MAIL_PASSWORD=qtxsthbxbisgcmqu
+MAIL_PORT=993

DEPLOYMENT.md

@@ -0,0 +1,95 @@
+# OCR Application Deployment Guide
+
+This guide describes how to deploy the OCR application on a Linux host (e.g., Ubuntu/Debian).
+
+## Architecture
+- **Backend**: Containerized (FastAPI, Python 3.9).
+- **Database**: Containerized (PostgreSQL 15).
+- **Frontend**: Static files (Angular) served by Host Nginx.
+- **Reverse Proxy**: Host Nginx proxies requests to Frontend (Static) and Backend (API).
+
+## Prerequisites
+- **Deployment Host**: Docker & Docker Compose, Nginx.
+- **Build Machine**: Node.js & NPM (to run the package script).
+
+---
+
+## 1. Full Stack Deployment (Docker)
+
+The app connects to your **existing Postgres container** (`postgres-db`) in the `arbit-app_arbit-network`.
+
+1.  **Create the Database**:
+    Since we are using an existing postgres instance, we must manually create the `ocr_db`.
+    ```bash
+    # Run this on your host to create the DB inside the existing container
+    docker exec -it postgres-db psql -U postgres -c "CREATE DATABASE ocr_db;"
+    ```
+
+2.  **Navigate to the packaged directory**:
+    ```bash
+    cd ocr_build
+    ```
+
+3.  **Verify Environment**:
+    Ensure `.env.prod` exists and points to `DB_HOST=postgres-db`.
+    **Also configure your Email credentials** in `.env.prod` if you want the Mailbox feature to work (Gmail requires an App Password).
+    ```bash
+    cat .env.prod
+    ```
+
+4.  **Start App Containers**:
+    This will start `ocr_backend` and `ocr_frontend`.
+    ```bash
+    # Build and start in detached mode
+    docker-compose up -d --build
+    ```
+
+5.  **Verify Status**:
+    ```bash
+    docker-compose ps
+    ```
+    You should see `ocr_backend` and `ocr_frontend` running.
+
+---
+
+## 2. Nginx Configuration (Host Reverse Proxy)
+
+Since the Frontend is now running in a container on port 8080 (serving `/ocrf/`), we configure the Host Nginx to proxy traffic to it.
+
+1.  **Create Config File**:
+    Copy the provided config to `/etc/nginx/sites-available/ocr`.
+    ```bash
+    sudo cp nginx_host.conf /etc/nginx/sites-available/ocr
+    ```
+
+2.  **Enable Site**:
+    ```bash
+    sudo ln -s /etc/nginx/sites-available/ocr /etc/nginx/sites-enabled/
+    ```
+
+3.  **Test & Reload**:
+    ```bash
+    sudo nginx -t
+    sudo systemctl reload nginx
+    ```
+
+---
+
+## 3. Verification
+
+- **Url**: `http://app.technobeesolutions.in/ocrf/` (Proxies to Frontend Container)
+- **API**: `http://app.technobeesolutions.in/ocrb/` (Proxies to Backend Container)
+
+---
+
+## 4. Troubleshooting
+
+- **Logs**:
+    ```bash
+    docker-compose logs -f backend
+    ```
+- **Database**:
+    Connect via the existing container:
+    ```bash
+    docker exec -it postgres-db psql -U postgres -d ocr_db
+    ```

backend/Dockerfile

@@ -0,0 +1,30 @@
+# Use official lightweight Python image
+FROM python:3.9-slim
+
+# Install system dependencies
+# tesseract-ocr: for pytesseract
+# poppler-utils: for pdf2image
+# libtesseract-dev: development headers
+RUN apt-get update && apt-get install -y \
+    tesseract-ocr \
+    poppler-utils \
+    libtesseract-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set working directory
+WORKDIR /app
+
+# Copy requirements first to leverage Docker cache
+COPY requirements.txt .
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy the rest of the application code
+COPY . .
+
+# Expose port (default for Uvicorn)
+EXPOSE 8000
+
+# Run the application
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]

backend/database.py

@@ -1,5 +1,7 @@
 import os
-from sqlalchemy import create_engine, Column, Integer, String, Text, DateTime, Boolean, ForeignKey, LargeBinary
+import urllib.parse
+from sqlalchemy import create_engine, Column, Integer, String, Text, DateTime, Boolean, ForeignKey, LargeBinary, Enum as SqlEnum
+from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker, relationship
 from dotenv import load_dotenv
@@ -7,22 +9,11 @@ from dotenv import load_dotenv
 # Load environment variables
 load_dotenv()
 
-DB_USER = os.getenv("DB_USER")
-DB_PASSWORD = os.getenv("DB_PASSWORD")
-DB_HOST = os.getenv("DB_HOST")
-DB_PORT = os.getenv("DB_PORT")
-import urllib.parse
-
-# ... (imports)
-
-# Load environment variables
-load_dotenv()
-
-DB_USER = os.getenv("DB_USER")
-DB_PASSWORD = os.getenv("DB_PASSWORD")
-DB_HOST = os.getenv("DB_HOST")
-DB_PORT = os.getenv("DB_PORT")
-DB_NAME = os.getenv("DB_NAME")
+DB_USER = os.getenv("DB_USER", "postgres")
+DB_PASSWORD = os.getenv("DB_PASSWORD", "password")
+DB_HOST = os.getenv("DB_HOST", "localhost")
+DB_PORT = os.getenv("DB_PORT", "5432")
+DB_NAME = os.getenv("DB_NAME", "ocr_db")
 
 encoded_user = urllib.parse.quote_plus(DB_USER)
 encoded_password = urllib.parse.quote_plus(DB_PASSWORD)
@@ -35,6 +26,26 @@ SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 
 Base = declarative_base()
 
+class Vendor(Base):
+    __tablename__ = "vendors"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    name = Column(String, unique=True, index=True)
+    default_model = Column(String) # 'text' or 'vision'
+    created_at = Column(DateTime)
+
+class Document(Base):
+    __tablename__ = "documents"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    vendor_id = Column(Integer, ForeignKey("vendors.id"), nullable=True)
+    filename = Column(String)
+    upload_date = Column(DateTime)
+    status = Column(String) # 'pending', 'verified'
+    processed_data = Column(JSONB) # Store the verified extraction results
+    
+    vendor = relationship("Vendor")
+
 class Email(Base):
     __tablename__ = "emails"
 
@@ -54,7 +65,7 @@ class Attachment(Base):
     email_id = Column(Integer, ForeignKey("emails.id"))
     filename = Column(String)
     content_type = Column(String)
-    file_content = Column(LargeBinary) # Storing content directly in DB as requested
+    file_content = Column(LargeBinary)
     
     email = relationship("Email", back_populates="attachments")
 

backend/main.py

@@ -85,6 +85,12 @@ class LoginResponse(BaseModel):
     token: str
     message: str
 
+@app.post("/api/login", response_model=LoginResponse)
+def login(request: LoginRequest):
+    if request.username == "admin" and request.password == "admin":
+        return LoginResponse(token="fake-super-secret-token", message="Success")
+    raise HTTPException(status_code=401, detail="Invalid username or password")
+
 class NERResponse(BaseModel):
     text: str
 

backend/requirements.txt

@@ -10,3 +10,4 @@ imap-tools
 apscheduler
 python-dotenv
 pdfplumber
+pdf2image

deployment/nginx.conf

@@ -0,0 +1,22 @@
+server {
+    listen 80;
+    server_name app.technobeesolutions.in;
+
+    # Proxy Angular Frontend Container
+    location /ocrf/ {
+        proxy_pass http://localhost:8080/;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
+
+    # Proxy API requests to the Docker Backend
+    location /ocrb/ {
+        proxy_pass http://localhost:8000/;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
+}

docengine/.env

@@ -0,0 +1,59 @@
+# Application
+APP_NAME=DocEngine
+APP_VERSION=1.0.0
+APP_ENV=development
+APP_DEBUG=true
+APP_HOST=0.0.0.0
+APP_PORT=7989
+APP_WORKERS=4
+
+# Database
+DB_HOST=103.125.129.116
+DB_PORT=5432
+DB_NAME=ocr
+DB_USER=postgres
+DB_PASSWORD=M@triXPostgr3s@6202
+DB_SCHEMA=admin
+DB_POOL_SIZE=20
+DB_MAX_OVERFLOW=10
+DB_ECHO=false
+
+# Redis
+REDIS_HOST=103.125.129.116
+REDIS_PORT=7901
+REDIS_DB=0
+REDIS_PASSWORD=M@triXR3d1s@6202
+
+# Celery
+CELERY_BROKER_URL=redis://:M%40triXR3d1s%406202@103.125.129.116:7901/0
+CELERY_RESULT_BACKEND=redis://:M%40triXR3d1s%406202@103.125.129.116:7901/1
+
+# JWT
+JWT_SECRET_KEY=a7f3c9e1d4b8f2a6c0e5d7b3a9f1c4e8d2b6a0f5c3e7d1b9a4f8c2e6d0b5a3
+JWT_ALGORITHM=HS256
+JWT_ACCESS_TOKEN_EXPIRE_MINUTES=30
+JWT_REFRESH_TOKEN_EXPIRE_DAYS=7
+
+# Storage
+STORAGE_PROVIDER=local
+STORAGE_LOCAL_PATH=./storage
+STORAGE_MAX_FILE_SIZE_MB=100
+
+# OCR
+OCR_LANGUAGE=en
+OCR_USE_GPU=false
+
+# Logging
+LOG_LEVEL=INFO
+LOG_FORMAT=json
+
+# CORS
+CORS_ORIGINS=["http://localhost:3000","http://localhost:8080","http://localhost:4200"]
+CORS_ALLOW_CREDENTIALS=true
+
+# Rate Limiting
+RATE_LIMIT_REQUESTS=100
+RATE_LIMIT_WINDOW_SECONDS=60
+
+# Prometheus
+PROMETHEUS_ENABLED=true

docengine/.pids/server.pid

@@ -1 +0,0 @@
-3173

docengine/.pids/worker.pid

@@ -1 +0,0 @@
-3176

docengine/app/storage/provider.py

@@ -0,0 +1,163 @@
+from __future__ import annotations
+
+import hashlib
+import os
+import shutil
+import uuid
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+from app.core.config import settings
+from app.core.exceptions import StorageError
+from app.core.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+class StorageProvider(ABC):
+    """Abstract base class for storage providers."""
+
+    @abstractmethod
+    def save_file(self, file_data: bytes, directory: str, filename: str | None = None) -> str:
+        """Save file data and return the storage path."""
+        ...
+
+    @abstractmethod
+    def read_file(self, storage_path: str) -> bytes:
+        """Read file data from storage."""
+        ...
+
+    @abstractmethod
+    def delete_file(self, storage_path: str) -> bool:
+        """Delete a file from storage. Returns True if successful."""
+        ...
+
+    @abstractmethod
+    def file_exists(self, storage_path: str) -> bool:
+        """Check if a file exists in storage."""
+        ...
+
+    @abstractmethod
+    def get_file_size(self, storage_path: str) -> int:
+        """Get the file size in bytes."""
+        ...
+
+    @abstractmethod
+    def get_absolute_path(self, storage_path: str) -> str:
+        """Get the absolute filesystem path for a storage path."""
+        ...
+
+    @staticmethod
+    def compute_checksum(data: bytes, algorithm: str = "sha256") -> str:
+        """Compute checksum of file data."""
+        hasher = hashlib.new(algorithm)
+        hasher.update(data)
+        return hasher.hexdigest()
+
+    @staticmethod
+    def generate_filename(original_filename: str) -> str:
+        """Generate a unique filename preserving the original extension."""
+        ext = Path(original_filename).suffix.lower()
+        return f"{uuid.uuid4().hex}{ext}"
+
+
+class LocalStorageProvider(StorageProvider):
+    """Local filesystem storage provider."""
+
+    def __init__(self, base_path: str | None = None) -> None:
+        self.base_path = Path(base_path or settings.storage_local_path).resolve()
+        self._ensure_directories()
+
+    def _ensure_directories(self) -> None:
+        """Create required storage directories."""
+        for subdir in ("documents", "templates", "images", "temp", "rendered"):
+            (self.base_path / subdir).mkdir(parents=True, exist_ok=True)
+
+    def _resolve_path(self, storage_path: str) -> Path:
+        """Resolve a storage path to an absolute path."""
+        resolved = (self.base_path / storage_path).resolve()
+        if not str(resolved).startswith(str(self.base_path)):
+            raise StorageError(f"Path traversal detected: {storage_path}")
+        return resolved
+
+    def save_file(self, file_data: bytes, directory: str, filename: str | None = None) -> str:
+        """Save file data to local storage."""
+        if filename is None:
+            filename = f"{uuid.uuid4().hex}.bin"
+
+        dir_path = self.base_path / directory
+        dir_path.mkdir(parents=True, exist_ok=True)
+
+        file_path = dir_path / filename
+        try:
+            file_path.write_bytes(file_data)
+            storage_path = str(file_path.relative_to(self.base_path))
+            logger.info("file_saved", storage_path=storage_path, size=len(file_data))
+            return storage_path
+        except OSError as e:
+            raise StorageError(f"Failed to save file: {e}") from e
+
+    def read_file(self, storage_path: str) -> bytes:
+        """Read file data from local storage."""
+        file_path = self._resolve_path(storage_path)
+        if not file_path.exists():
+            raise StorageError(f"File not found: {storage_path}")
+        try:
+            return file_path.read_bytes()
+        except OSError as e:
+            raise StorageError(f"Failed to read file: {e}") from e
+
+    def delete_file(self, storage_path: str) -> bool:
+        """Delete a file from local storage."""
+        file_path = self._resolve_path(storage_path)
+        if not file_path.exists():
+            return False
+        try:
+            file_path.unlink()
+            logger.info("file_deleted", storage_path=storage_path)
+            return True
+        except OSError as e:
+            logger.error("file_delete_failed", storage_path=storage_path, error=str(e))
+            raise StorageError(f"Failed to delete file: {e}") from e
+
+    def file_exists(self, storage_path: str) -> bool:
+        """Check if a file exists in local storage."""
+        file_path = self._resolve_path(storage_path)
+        return file_path.exists()
+
+    def get_file_size(self, storage_path: str) -> int:
+        """Get the file size in bytes."""
+        file_path = self._resolve_path(storage_path)
+        if not file_path.exists():
+            raise StorageError(f"File not found: {storage_path}")
+        return file_path.stat().st_size
+
+    def get_absolute_path(self, storage_path: str) -> str:
+        """Get the absolute filesystem path."""
+        return str(self._resolve_path(storage_path))
+
+    def save_temp_file(self, file_data: bytes, filename: str) -> str:
+        """Save a temporary file."""
+        return self.save_file(file_data, "temp", filename)
+
+    def cleanup_temp(self) -> int:
+        """Remove all files in the temp directory."""
+        temp_dir = self.base_path / "temp"
+        count = 0
+        if temp_dir.exists():
+            for item in temp_dir.iterdir():
+                if item.is_file():
+                    item.unlink()
+                    count += 1
+                elif item.is_dir():
+                    shutil.rmtree(item)
+                    count += 1
+        logger.info("temp_cleanup", files_removed=count)
+        return count
+
+
+def get_storage_provider() -> StorageProvider:
+    """Factory function to get the configured storage provider."""
+    if settings.storage_provider == "local":
+        return LocalStorageProvider()
+    raise StorageError(f"Unknown storage provider: {settings.storage_provider}")