maddy/OCR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Text Extraction Done, UI Fixes Done, Extracted Template Layout Saved in DB

Browse Source
This commit is contained in:
Narayanan Madaswamy
2026-07-12 14:24:17 +05:30
parent 3163bb213e
commit 3d6614f408
79 changed files with 3139 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
docengine/.env.example
View File

@@ -10,7 +10,7 @@ APP_WORKERS=4
# Database
DB_HOST=localhost
DB_PORT=5432
DB_NAME=document_engine
DB_NAME=ocr
DB_USER=postgres
DB_PASSWORD=changeme
DB_SCHEMA=admin
@@ -48,7 +48,7 @@ LOG_LEVEL=INFO
LOG_FORMAT=json
# CORS
CORS_ORIGINS=["http://localhost:3000","http://localhost:8080"]
CORS_ORIGINS=["http://localhost:3000","http://localhost:8080","http://localhost:4200"]
CORS_ALLOW_CREDENTIALS=true
# Rate Limiting

20
docengine/README.md
View File

@@ -13,7 +13,7 @@ A production-ready system for scanning documents, detecting layouts, extracting
▼ ▼
┌───────────────────────────────────────┐
│ PostgreSQL (Schema: admin) │
│ 192.168.0.111:7925 │
│ 192.168.0.111:5432 │
└───────────────────────────────────────┘
```
@@ -92,7 +92,7 @@ docengine/
### Prerequisites
- Python 3.12+
- PostgreSQL 16 (running at `192.168.0.111:7925`)
- PostgreSQL 16 (running at `192.168.0.111:5432`)
- Redis (for Celery)
- `poppler-utils` and `ghostscript` (for pdf2image/camelot)
@@ -116,8 +116,8 @@ mkdir -p storage/{documents,templates,images,temp,rendered}
alembic upgrade head
# (Optional) Seed default data
psql -h 192.168.0.111 -p 7925 -U postgres -d document_engine -f sql/003_seed_data.sql
psql -h 192.168.0.111 -p 7925 -U postgres -d document_engine -f sql/004_indexes.sql
psql -h 192.168.0.111 -p 5432 -U postgres -d ocr -f sql/003_seed_data.sql
psql -h 192.168.0.111 -p 5432 -U postgres -d ocr -f sql/004_indexes.sql
# Start the application
python -m app.main
@@ -142,7 +142,7 @@ docker compose up --build -d
docker compose exec app alembic upgrade head
# Seed data
docker compose exec app bash -c "psql -h db -U postgres -d document_engine -f sql/003_seed_data.sql"
docker compose exec app bash -c "psql -h db -U postgres -d ocr -f sql/003_seed_data.sql"
```
---
@@ -403,7 +403,7 @@ When a document is uploaded, the following Celery task pipeline executes asynchr
## Database
**Connection**: `postgresql://postgres:***@192.168.0.111:7925/document_engine`
**Connection**: `postgresql://postgres:***@192.168.0.111:5432/ocr`
**Schema**: `admin`
### Migrations
@@ -474,11 +474,11 @@ All configuration is via environment variables (`.env` file). Key settings:
|------------------------------------|------------------------|---------------------------------|
| `APP_PORT` | `7989` | Application port |
| `DB_HOST` | `192.168.0.111` | PostgreSQL host |
| `DB_PORT` | `7925` | PostgreSQL port |
| `DB_NAME` | `document_engine` | Database name |
| `DB_PORT` | `5432` | PostgreSQL port |
| `DB_NAME` | `ocr` | Database name |
| `DB_SCHEMA` | `admin` | PostgreSQL schema |
| `REDIS_HOST` | `localhost` | Redis host |
| `CELERY_BROKER_URL` | `redis://localhost:6379/0` | Celery broker |
| `REDIS_HOST` | `192.168.0.111` | Redis host |
| `CELERY_BROKER_URL` | `redis://:***@192.168.0.111:7901/0` | Celery broker |
| `JWT_SECRET_KEY` | *(see .env)* | JWT signing key |
| `JWT_ACCESS_TOKEN_EXPIRE_MINUTES` | `30` | Access token TTL |
| `STORAGE_LOCAL_PATH` | `./storage` | Local file storage path |

2
docengine/alembic.ini
View File

@@ -2,7 +2,7 @@
script_location = alembic
prepend_sys_path = .
version_path_separator = os
sqlalchemy.url = postgresql+psycopg2://postgres:M%%40tr%%21x%%23149%%40dm%%21N@192.168.0.111:7925/document_engine
sqlalchemy.url = postgresql+psycopg2://postgres:M%%40triXPostgr3s%%406202@192.168.0.111:5432/ocr
[post_write_hooks]

2
docengine/alembic/env.py
View File

@@ -17,7 +17,7 @@ if config.config_file_name is not None:
target_metadata = Base.metadata
# Override the database URL from settings
config.set_main_option("sqlalchemy.url", settings.database_url)
config.set_main_option("sqlalchemy.url", settings.database_url.replace('%', '%%'))
def run_migrations_offline() -> None:

38
docengine/app/core/aes.py Normal file
View File

@@ -0,0 +1,38 @@
import base64
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
def decrypt(base64_input: str, secret: bytes) -> str | None:
try:
combined = base64.b64decode(base64_input)
# IV length is 12 bytes in the Java implementation
iv_length = 12
if len(combined) <= iv_length + 16:
return None
iv = combined[:iv_length]
# Java AES/GCM appends a 16-byte authentication tag at the end of cipherBytes
# cryptography library expects it to be passed into the modes.GCM(iv, tag)
cipher_bytes_with_tag = combined[iv_length:]
actual_ciphertext = cipher_bytes_with_tag[:-16]
tag = cipher_bytes_with_tag[-16:]
cipher = Cipher(algorithms.AES(secret), modes.GCM(iv, tag))
decryptor = cipher.decryptor()
plain_bytes = decryptor.update(actual_ciphertext) + decryptor.finalize()
return plain_bytes.decode('utf-8')
except Exception:
return None
def get_string_value(encrypted_id: str, secret: str, secret_key_internal: str) -> str | None:
if not secret or not secret_key_internal:
return None
try:
# Replicate Java's logic: (s + secretKeyInternal).substring(0, Math.min(..., 16))
combined_str = secret + secret_key_internal
combined_secret = combined_str[:16].encode('utf-8')
return decrypt(encrypted_id, combined_secret)
except Exception:
return None

20
docengine/app/core/config.py
View File

@@ -28,30 +28,32 @@ class Settings(BaseSettings):
# Database
db_host: str = "192.168.0.111"
db_port: int = 7925
db_name: str = "document_engine"
db_port: int = 5432
db_name: str = "ocr"
db_user: str = "postgres"
db_password: str = "M@tr!x#149@dm!N"
db_password: str = "M@triXPostgr3s@6202"
db_schema: str = "admin"
db_pool_size: int = 20
db_max_overflow: int = 10
db_echo: bool = False
# Redis
redis_host: str = "localhost"
redis_port: int = 6379
redis_host: str = "192.168.0.111"
redis_port: int = 7901
redis_db: int = 0
redis_password: str = ""
redis_password: str = "M@triXR3d1s@6202"
# Celery
celery_broker_url: str = "redis://localhost:6379/0"
celery_result_backend: str = "redis://localhost:6379/1"
celery_broker_url: str = "redis://:M@triXR3d1s@6202@192.168.0.111:7901/0"
celery_result_backend: str = "redis://:M@triXR3d1s@6202@192.168.0.111:7901/1"
# JWT
jwt_secret_key: str = "a7f3c9e1d4b8f2a6c0e5d7b3a9f1c4e8d2b6a0f5c3e7d1b9a4f8c2e6d0b5a3"
jwt_algorithm: str = "HS256"
jwt_access_token_expire_minutes: int = 30
jwt_refresh_token_expire_days: int = 7
session_encryption_secret: str = ""
session_encryption_secret_internal: str = ""
# Storage
storage_provider: str = "local"
@@ -67,7 +69,7 @@ class Settings(BaseSettings):
log_format: str = "json"
# CORS
cors_origins: list[str] = ["http://localhost:3000", "http://localhost:8080"]
cors_origins: list[str] = ["http://localhost:3000", "http://localhost:8080", "http://localhost:4200"]
cors_allow_credentials: bool = True
# Rate Limiting

50
docengine/app/core/dependencies.py
View File

@@ -11,17 +11,63 @@ from app.core.security import InvalidTokenError, decode_token
from app.models.user import User
from app.repositories.user_repository import UserRepository
security_scheme = HTTPBearer(auto_error=True)
import uuid
from app.core.config import settings
security_scheme = HTTPBearer(auto_error=False)
def get_current_user(
credentials: Annotated[HTTPAuthorizationCredentials, Depends(security_scheme)],
credentials: Annotated[HTTPAuthorizationCredentials | None, Depends(security_scheme)],
db: Annotated[Session, Depends(get_db)],
) -> User:
"""Extract and validate the current user from the JWT token."""
if credentials is None:
if settings.app_env == "development":
# Auto-login as default dev admin user if no token provided in dev
user_repo = UserRepository(db)
user = db.query(User).first()
if user:
return user
dev_user = User(
id=uuid.uuid4(),
username="dev_admin",
email="admin@docengine.local",
hashed_password="mock_password",
is_active=True,
is_superuser=True,
)
db.add(dev_user)
db.commit()
db.refresh(dev_user)
return dev_user
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Not authenticated",
headers={"WWW-Authenticate": "Bearer"},
)
try:
payload = decode_token(credentials.credentials)
except InvalidTokenError:
if settings.app_env == "development":
# Fallback to dev admin user on token decode failure in dev
user_repo = UserRepository(db)
user = db.query(User).first()
if user:
return user
dev_user = User(
id=uuid.uuid4(),
username="dev_admin",
email="admin@docengine.local",
hashed_password="mock_password",
is_active=True,
is_superuser=True,
)
db.add(dev_user)
db.commit()
db.refresh(dev_user)
return dev_user
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid or expired token",

9
docengine/app/core/security.py
View File

@@ -8,6 +8,7 @@ from jose import JWTError, jwt
from passlib.context import CryptContext
from app.core.config import settings
from app.core.aes import get_string_value
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
@@ -44,8 +45,14 @@ def create_refresh_token(data: dict[str, Any], expires_delta: timedelta | None =
def decode_token(token: str) -> dict[str, Any]:
"""Decode and validate a JWT token."""
decrypted_token = get_string_value(
token,
settings.session_encryption_secret,
settings.session_encryption_secret_internal
)
final_token = decrypted_token if decrypted_token else token
try:
payload = jwt.decode(token, settings.jwt_secret_key, algorithms=[settings.jwt_algorithm])
payload = jwt.decode(final_token, settings.jwt_secret_key, algorithms=[settings.jwt_algorithm])
return payload
except JWTError as e:
raise InvalidTokenError(str(e)) from e

8
docengine/app/events/handlers.py
View File

@@ -8,7 +8,7 @@ logger = get_logger(__name__)
def on_startup() -> None:
"""Application startup event handler."""
setup_logging()
logger.info("application_starting", event="startup")
logger.info("application_starting", phase="startup")
# Ensure storage directories exist
from app.storage.provider import get_storage_provider
@@ -25,15 +25,15 @@ def on_startup() -> None:
else:
logger.error("database_connection_failed")
logger.info("application_started", event="startup_complete")
logger.info("application_started", phase="startup_complete")
def on_shutdown() -> None:
"""Application shutdown event handler."""
logger.info("application_shutting_down", event="shutdown")
logger.info("application_shutting_down", phase="shutdown")
# Cleanup resources
from app.core.database import engine
engine.dispose()
logger.info("application_stopped", event="shutdown_complete")
logger.info("application_stopped", phase="shutdown_complete")

2
docengine/app/main.py
View File

@@ -37,9 +37,9 @@ app = FastAPI(
)
# Setup middleware (order matters: last added = first executed)
setup_cors(app)
app.add_middleware(AuditMiddleware)
app.add_middleware(RateLimitMiddleware)
setup_cors(app)
# Setup Prometheus metrics
setup_metrics(app)

8
docengine/app/middleware/cors.py
View File

@@ -13,13 +13,7 @@ def setup_cors(app: FastAPI) -> None:
allow_origins=settings.cors_origins,
allow_credentials=settings.cors_allow_credentials,
allow_methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"],
allow_headers=[
"Authorization",
"Content-Type",
"Accept",
"X-Request-ID",
"X-Requested-With",
],
allow_headers=["*"],
expose_headers=[
"X-Request-ID",
"X-Process-Time",

6
docengine/app/models/user.py
View File

@@ -3,7 +3,7 @@ from __future__ import annotations
import uuid
from datetime import datetime
from sqlalchemy import Boolean, DateTime, ForeignKey, String, Table, Text, func
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, String, Table, Text, func
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import Mapped, mapped_column, relationship
@@ -13,8 +13,8 @@ from app.models.base import TimestampMixin, UUIDPrimaryKeyMixin
user_roles_table = Table(
"user_roles",
Base.metadata,
mapped_column("user_id", UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True),
mapped_column("role_id", UUID(as_uuid=True), ForeignKey("roles.id", ondelete="CASCADE"), primary_key=True),
Column("user_id", UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True),
Column("role_id", UUID(as_uuid=True), ForeignKey("roles.id", ondelete="CASCADE"), primary_key=True),
)

7
docengine/app/workers/celery_app.py
View File

@@ -40,4 +40,9 @@ celery_app.conf.update(
},
)
celery_app.autodiscover_tasks(["app.tasks"])
celery_app.conf.update(
imports=[
"app.tasks.document_tasks",
"app.tasks.maintenance_tasks"
]
)

4
docengine/application.properties
View File

@@ -2,7 +2,7 @@
server.port=7989
db.host=192.168.0.111
db.port=7925
db.port=5432
db.user=postgres
db.password=M@tr!x#149@dm!N
db.password=M@triXPostgr3s@6202
db.schema=admin

6
docengine/docker-compose.prod.yml
View File

@@ -7,14 +7,14 @@ services:
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: "${DB_PASSWORD}"
POSTGRES_DB: document_engine
POSTGRES_DB: ocr
ports:
- "7925:5432"
- "5432:5432"
volumes:
- docengine_pgdata_prod:/var/lib/postgresql/data
- ./sql/001_create_schema.sql:/docker-entrypoint-initdb.d/001_create_schema.sql
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres -d document_engine"]
test: ["CMD-SHELL", "pg_isready -U postgres -d ocr"]
interval: 10s
timeout: 5s
retries: 5

8
docengine/docker-compose.yml
View File

@@ -6,15 +6,15 @@ services:
container_name: docengine_db
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: "M@tr!x#149@dm!N"
POSTGRES_DB: document_engine
POSTGRES_PASSWORD: "M@triXPostgr3s@6202"
POSTGRES_DB: ocr
ports:
- "7925:5432"
- "5432:5432"
volumes:
- docengine_pgdata:/var/lib/postgresql/data
- ./sql/001_create_schema.sql:/docker-entrypoint-initdb.d/001_create_schema.sql
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres -d document_engine"]
test: ["CMD-SHELL", "pg_isready -U postgres -d ocr"]
interval: 10s
timeout: 5s
retries: 5

2
docengine/requirements.txt
View File

@@ -21,7 +21,7 @@ bcrypt==4.2.1
# Document Processing
PyMuPDF==1.25.3
paddleocr==2.9.1
paddlepaddle==3.0.0b1
paddlepaddle>=3.0.0
layoutparser==0.3.4
opencv-python-headless==4.10.0.84
camelot-py[cv]==0.11.0

2
docengine/sql/001_create_schema.sql
View File

@@ -2,7 +2,7 @@
CREATE SCHEMA IF NOT EXISTS admin;
-- Set the default search path
ALTER DATABASE document_engine SET search_path TO admin, public;
ALTER DATABASE ocr SET search_path TO admin, public;
-- Grant privileges
GRANT ALL ON SCHEMA admin TO postgres;

2
docengine/sql/002_create_tables.sql
View File

@@ -1,6 +1,6 @@
-- DocEngine: Complete table creation script
-- Schema: admin
-- Database: document_engine
-- Database: ocr
SET search_path TO admin, public;