Changes committed

This commit is contained in:
2026-06-01 21:49:53 +05:30
parent 8537c653c1
commit 3163bb213e
387 changed files with 21940 additions and 107 deletions

67
docengine/alembic/env.py Normal file
View File

@@ -0,0 +1,67 @@
from logging.config import fileConfig
from alembic import context
from sqlalchemy import engine_from_config, pool, text
from app.core.config import settings
from app.core.database import Base
# Import all models so Alembic can detect them
import app.models # noqa: F401
config = context.config
if config.config_file_name is not None:
fileConfig(config.config_file_name)
target_metadata = Base.metadata
# Override the database URL from settings
config.set_main_option("sqlalchemy.url", settings.database_url)
def run_migrations_offline() -> None:
"""Run migrations in 'offline' mode."""
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url,
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
version_table_schema=settings.db_schema,
include_schemas=True,
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online() -> None:
"""Run migrations in 'online' mode."""
connectable = engine_from_config(
config.get_section(config.config_ini_section, {}),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
# Ensure schema exists
connection.execute(text(f"CREATE SCHEMA IF NOT EXISTS {settings.db_schema}"))
connection.execute(text(f"SET search_path TO {settings.db_schema}, public"))
connection.commit()
context.configure(
connection=connection,
target_metadata=target_metadata,
version_table_schema=settings.db_schema,
include_schemas=True,
)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()

View File

@@ -0,0 +1,26 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
# revision identifiers, used by Alembic.
revision: str = ${repr(up_revision)}
down_revision: Union[str, None] = ${repr(down_revision)}
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}

View File

@@ -0,0 +1,398 @@
"""initial schema
Revision ID: 001_initial
Revises:
Create Date: 2026-05-31 18:00:00.000000
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
revision: str = "001_initial"
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
SCHEMA = "admin"
def upgrade() -> None:
# Create schema
op.execute(f"CREATE SCHEMA IF NOT EXISTS {SCHEMA}")
# Users table
op.create_table(
"users",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("username", sa.String(150), unique=True, nullable=False, index=True),
sa.Column("email", sa.String(255), unique=True, nullable=False, index=True),
sa.Column("hashed_password", sa.String(255), nullable=False),
sa.Column("full_name", sa.String(255), nullable=True),
sa.Column("is_active", sa.Boolean, default=True, nullable=False),
sa.Column("is_superuser", sa.Boolean, default=False, nullable=False),
sa.Column("last_login", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
schema=SCHEMA,
)
# Roles table
op.create_table(
"roles",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("name", sa.String(50), unique=True, nullable=False, index=True),
sa.Column("description", sa.Text, nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
schema=SCHEMA,
)
# User roles (many-to-many)
op.create_table(
"user_roles",
sa.Column("user_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.users.id", ondelete="CASCADE"), primary_key=True),
sa.Column("role_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.roles.id", ondelete="CASCADE"), primary_key=True),
schema=SCHEMA,
)
# Refresh tokens
op.create_table(
"refresh_tokens",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("user_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.users.id", ondelete="CASCADE"), nullable=False, index=True),
sa.Column("token", sa.String(512), unique=True, nullable=False, index=True),
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("revoked", sa.Boolean, default=False, nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
schema=SCHEMA,
)
# Audit logs
op.create_table(
"audit_logs",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("user_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.users.id", ondelete="SET NULL"), nullable=True, index=True),
sa.Column("action", sa.String(100), nullable=False, index=True),
sa.Column("resource_type", sa.String(100), nullable=False, index=True),
sa.Column("resource_id", sa.String(255), nullable=True),
sa.Column("details", sa.Text, nullable=True),
sa.Column("ip_address", sa.String(45), nullable=True),
sa.Column("user_agent", sa.String(512), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False, index=True),
schema=SCHEMA,
)
# Documents
op.create_table(
"documents",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("filename", sa.String(500), nullable=False),
sa.Column("original_filename", sa.String(500), nullable=False),
sa.Column("content_type", sa.String(100), nullable=False),
sa.Column("file_size", sa.BigInteger, nullable=False),
sa.Column("checksum", sa.String(128), nullable=False, index=True),
sa.Column("storage_path", sa.String(1024), nullable=False),
sa.Column("status", sa.String(50), default="pending", nullable=False, index=True),
sa.Column("page_count", sa.Integer, nullable=True),
sa.Column("is_scanned", sa.Boolean, nullable=True),
sa.Column("document_metadata", postgresql.JSONB, nullable=True),
sa.Column("error_message", sa.Text, nullable=True),
sa.Column("uploaded_by", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.users.id", ondelete="SET NULL"), nullable=True, index=True),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
schema=SCHEMA,
)
# Document pages
op.create_table(
"document_pages",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("document_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.documents.id", ondelete="CASCADE"), nullable=False, index=True),
sa.Column("page_number", sa.Integer, nullable=False),
sa.Column("width", sa.Float, nullable=False),
sa.Column("height", sa.Float, nullable=False),
sa.Column("image_path", sa.String(1024), nullable=True),
sa.Column("text_content", sa.Text, nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
schema=SCHEMA,
)
# Document text blocks
op.create_table(
"document_text_blocks",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("page_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_pages.id", ondelete="CASCADE"), nullable=False, index=True),
sa.Column("text", sa.Text, nullable=False),
sa.Column("x", sa.Float, nullable=False),
sa.Column("y", sa.Float, nullable=False),
sa.Column("width", sa.Float, nullable=False),
sa.Column("height", sa.Float, nullable=False),
sa.Column("confidence", sa.Float, nullable=True),
sa.Column("font_family", sa.String(255), nullable=True),
sa.Column("font_size", sa.Float, nullable=True),
sa.Column("font_color", sa.String(50), nullable=True),
sa.Column("font_style", sa.String(50), nullable=True),
sa.Column("block_type", sa.String(50), default="text", nullable=False),
sa.Column("sequence", sa.Integer, default=0, nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
schema=SCHEMA,
)
# Document images
op.create_table(
"document_images",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("page_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_pages.id", ondelete="CASCADE"), nullable=False, index=True),
sa.Column("x", sa.Float, nullable=False),
sa.Column("y", sa.Float, nullable=False),
sa.Column("width", sa.Float, nullable=False),
sa.Column("height", sa.Float, nullable=False),
sa.Column("image_path", sa.String(1024), nullable=False),
sa.Column("image_type", sa.String(50), default="figure", nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
schema=SCHEMA,
)
# Document tables
op.create_table(
"document_tables",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("page_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_pages.id", ondelete="CASCADE"), nullable=False, index=True),
sa.Column("x", sa.Float, nullable=False),
sa.Column("y", sa.Float, nullable=False),
sa.Column("width", sa.Float, nullable=False),
sa.Column("height", sa.Float, nullable=False),
sa.Column("rows", sa.Integer, nullable=False),
sa.Column("columns", sa.Integer, nullable=False),
sa.Column("data", postgresql.JSONB, nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
schema=SCHEMA,
)
# Document formats (templates)
op.create_table(
"document_formats",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("name", sa.String(255), nullable=False, index=True),
sa.Column("description", sa.Text, nullable=True),
sa.Column("page_width", sa.Float, nullable=False),
sa.Column("page_height", sa.Float, nullable=False),
sa.Column("page_count", sa.Integer, default=1, nullable=False),
sa.Column("margin_top", sa.Float, default=72.0, nullable=False),
sa.Column("margin_right", sa.Float, default=72.0, nullable=False),
sa.Column("margin_bottom", sa.Float, default=72.0, nullable=False),
sa.Column("margin_left", sa.Float, default=72.0, nullable=False),
sa.Column("fingerprint", postgresql.JSONB, nullable=True),
sa.Column("source_document_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.documents.id", ondelete="SET NULL"), nullable=True, index=True),
sa.Column("version", sa.Integer, default=1, nullable=False),
sa.Column("is_active", sa.Boolean, default=True, nullable=False, index=True),
sa.Column("created_by", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.users.id", ondelete="SET NULL"), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
schema=SCHEMA,
)
# Document cells
op.create_table(
"document_cells",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_formats.id", ondelete="CASCADE"), nullable=False, index=True),
sa.Column("page_number", sa.Integer, nullable=False),
sa.Column("x", sa.Float, nullable=False),
sa.Column("y", sa.Float, nullable=False),
sa.Column("width", sa.Float, nullable=False),
sa.Column("height", sa.Float, nullable=False),
sa.Column("row_no", sa.Integer, default=0, nullable=False),
sa.Column("column_no", sa.Integer, default=0, nullable=False),
sa.Column("data_type", sa.String(50), default="text", nullable=False),
sa.Column("font_family", sa.String(255), nullable=True),
sa.Column("font_size", sa.Float, nullable=True),
sa.Column("font_style", sa.String(50), nullable=True),
sa.Column("font_color", sa.String(50), nullable=True),
sa.Column("background_color", sa.String(50), nullable=True),
sa.Column("border_top", sa.String(100), nullable=True),
sa.Column("border_right", sa.String(100), nullable=True),
sa.Column("border_bottom", sa.String(100), nullable=True),
sa.Column("border_left", sa.String(100), nullable=True),
sa.Column("padding_top", sa.Float, default=0.0, nullable=False),
sa.Column("padding_right", sa.Float, default=0.0, nullable=False),
sa.Column("padding_bottom", sa.Float, default=0.0, nullable=False),
sa.Column("padding_left", sa.Float, default=0.0, nullable=False),
sa.Column("alignment", sa.String(20), default="left", nullable=False),
sa.Column("vertical_alignment", sa.String(20), default="top", nullable=False),
sa.Column("rowspan", sa.Integer, default=1, nullable=False),
sa.Column("colspan", sa.Integer, default=1, nullable=False),
sa.Column("static_text", sa.Text, nullable=True),
sa.Column("field_name", sa.String(255), nullable=True),
sa.Column("sequence", sa.Integer, default=0, nullable=False),
sa.Column("is_dynamic", sa.Boolean, default=False, nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
schema=SCHEMA,
)
# Document regions
op.create_table(
"document_regions",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_formats.id", ondelete="CASCADE"), nullable=False, index=True),
sa.Column("page_number", sa.Integer, nullable=False),
sa.Column("region_type", sa.String(50), nullable=False),
sa.Column("x", sa.Float, nullable=False),
sa.Column("y", sa.Float, nullable=False),
sa.Column("width", sa.Float, nullable=False),
sa.Column("height", sa.Float, nullable=False),
sa.Column("content", postgresql.JSONB, nullable=True),
sa.Column("sequence", sa.Integer, default=0, nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
schema=SCHEMA,
)
# Table formats
op.create_table(
"table_formats",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_formats.id", ondelete="CASCADE"), nullable=False, index=True),
sa.Column("page_number", sa.Integer, nullable=False),
sa.Column("x", sa.Float, nullable=False),
sa.Column("y", sa.Float, nullable=False),
sa.Column("width", sa.Float, nullable=False),
sa.Column("height", sa.Float, nullable=False),
sa.Column("rows", sa.Integer, nullable=False),
sa.Column("columns", sa.Integer, nullable=False),
sa.Column("border_style", sa.String(50), default="solid", nullable=False),
sa.Column("border_width", sa.Float, default=1.0, nullable=False),
sa.Column("border_color", sa.String(50), default="#000000", nullable=False),
sa.Column("header_rows", sa.Integer, default=1, nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
schema=SCHEMA,
)
# Table columns
op.create_table(
"table_columns",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("table_format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.table_formats.id", ondelete="CASCADE"), nullable=False, index=True),
sa.Column("column_index", sa.Integer, nullable=False),
sa.Column("width", sa.Float, nullable=False),
sa.Column("header_text", sa.String(500), nullable=True),
sa.Column("data_type", sa.String(50), default="text", nullable=False),
sa.Column("alignment", sa.String(20), default="left", nullable=False),
sa.Column("font_family", sa.String(255), nullable=True),
sa.Column("font_size", sa.Float, nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
schema=SCHEMA,
)
# Table rows
op.create_table(
"table_rows",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("table_format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.table_formats.id", ondelete="CASCADE"), nullable=False, index=True),
sa.Column("row_index", sa.Integer, nullable=False),
sa.Column("height", sa.Float, default=20.0, nullable=False),
sa.Column("is_header", sa.Boolean, default=False, nullable=False),
sa.Column("background_color", sa.String(50), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
schema=SCHEMA,
)
# Watermarks
op.create_table(
"watermarks",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_formats.id", ondelete="CASCADE"), nullable=False, index=True),
sa.Column("page_number", sa.Integer, nullable=True),
sa.Column("text", sa.String(500), nullable=True),
sa.Column("image_path", sa.String(1024), nullable=True),
sa.Column("x", sa.Float, nullable=False),
sa.Column("y", sa.Float, nullable=False),
sa.Column("width", sa.Float, nullable=False),
sa.Column("height", sa.Float, nullable=False),
sa.Column("opacity", sa.Float, default=0.3, nullable=False),
sa.Column("rotation", sa.Float, default=0.0, nullable=False),
sa.Column("font_family", sa.String(255), nullable=True),
sa.Column("font_size", sa.Float, nullable=True),
sa.Column("font_color", sa.String(50), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
schema=SCHEMA,
)
# Image regions
op.create_table(
"image_regions",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_formats.id", ondelete="CASCADE"), nullable=False, index=True),
sa.Column("page_number", sa.Integer, nullable=False),
sa.Column("x", sa.Float, nullable=False),
sa.Column("y", sa.Float, nullable=False),
sa.Column("width", sa.Float, nullable=False),
sa.Column("height", sa.Float, nullable=False),
sa.Column("image_path", sa.String(1024), nullable=True),
sa.Column("image_type", sa.String(50), default="figure", nullable=False),
sa.Column("is_static", sa.Boolean, default=True, nullable=False),
sa.Column("field_name", sa.String(255), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
schema=SCHEMA,
)
# Template fingerprints
op.create_table(
"template_fingerprints",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_formats.id", ondelete="CASCADE"), nullable=False, unique=True, index=True),
sa.Column("page_dimensions", postgresql.JSONB, nullable=True),
sa.Column("logo_coordinates", postgresql.JSONB, nullable=True),
sa.Column("header_coordinates", postgresql.JSONB, nullable=True),
sa.Column("footer_coordinates", postgresql.JSONB, nullable=True),
sa.Column("table_coordinates", postgresql.JSONB, nullable=True),
sa.Column("cell_coordinates", postgresql.JSONB, nullable=True),
sa.Column("fingerprint_hash", sa.String(256), nullable=False, index=True),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
schema=SCHEMA,
)
# Template matches
op.create_table(
"template_matches",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("document_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.documents.id", ondelete="CASCADE"), nullable=False, index=True),
sa.Column("format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_formats.id", ondelete="CASCADE"), nullable=False, index=True),
sa.Column("confidence_score", sa.Float, nullable=False),
sa.Column("match_details", postgresql.JSONB, nullable=True),
sa.Column("selected", sa.Boolean, default=False, nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
schema=SCHEMA,
)
# Additional indexes
op.create_index("ix_documents_status_created", "documents", ["status", "created_at"], schema=SCHEMA)
op.create_index("ix_document_pages_doc_page", "document_pages", ["document_id", "page_number"], schema=SCHEMA)
op.create_index("ix_document_text_blocks_type", "document_text_blocks", ["page_id", "block_type"], schema=SCHEMA)
op.create_index("ix_document_cells_format_page", "document_cells", ["format_id", "page_number"], schema=SCHEMA)
op.create_index("ix_template_matches_doc_score", "template_matches", ["document_id", "confidence_score"], schema=SCHEMA)
op.create_index("ix_audit_logs_resource", "audit_logs", ["resource_type", "resource_id"], schema=SCHEMA)
def downgrade() -> None:
op.drop_table("template_matches", schema=SCHEMA)
op.drop_table("template_fingerprints", schema=SCHEMA)
op.drop_table("image_regions", schema=SCHEMA)
op.drop_table("watermarks", schema=SCHEMA)
op.drop_table("table_rows", schema=SCHEMA)
op.drop_table("table_columns", schema=SCHEMA)
op.drop_table("table_formats", schema=SCHEMA)
op.drop_table("document_regions", schema=SCHEMA)
op.drop_table("document_cells", schema=SCHEMA)
op.drop_table("document_formats", schema=SCHEMA)
op.drop_table("document_tables", schema=SCHEMA)
op.drop_table("document_images", schema=SCHEMA)
op.drop_table("document_text_blocks", schema=SCHEMA)
op.drop_table("document_pages", schema=SCHEMA)
op.drop_table("documents", schema=SCHEMA)
op.drop_table("audit_logs", schema=SCHEMA)
op.drop_table("refresh_tokens", schema=SCHEMA)
op.drop_table("user_roles", schema=SCHEMA)
op.drop_table("roles", schema=SCHEMA)
op.drop_table("users", schema=SCHEMA)