Changes committed

2026-06-01 21:49:53 +05:30
parent 8537c653c1
commit 3163bb213e
387 changed files with 21940 additions and 107 deletions
--- a/docengine/alembic/versions/001_initial.py
+++ b/docengine/alembic/versions/001_initial.py
@@ -0,0 +1,398 @@
+"""initial schema
+
+Revision ID: 001_initial
+Revises:
+Create Date: 2026-05-31 18:00:00.000000
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+revision: str = "001_initial"
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+SCHEMA = "admin"
+
+
+def upgrade() -> None:
+    # Create schema
+    op.execute(f"CREATE SCHEMA IF NOT EXISTS {SCHEMA}")
+
+    # Users table
+    op.create_table(
+        "users",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("username", sa.String(150), unique=True, nullable=False, index=True),
+        sa.Column("email", sa.String(255), unique=True, nullable=False, index=True),
+        sa.Column("hashed_password", sa.String(255), nullable=False),
+        sa.Column("full_name", sa.String(255), nullable=True),
+        sa.Column("is_active", sa.Boolean, default=True, nullable=False),
+        sa.Column("is_superuser", sa.Boolean, default=False, nullable=False),
+        sa.Column("last_login", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Roles table
+    op.create_table(
+        "roles",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("name", sa.String(50), unique=True, nullable=False, index=True),
+        sa.Column("description", sa.Text, nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # User roles (many-to-many)
+    op.create_table(
+        "user_roles",
+        sa.Column("user_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.users.id", ondelete="CASCADE"), primary_key=True),
+        sa.Column("role_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.roles.id", ondelete="CASCADE"), primary_key=True),
+        schema=SCHEMA,
+    )
+
+    # Refresh tokens
+    op.create_table(
+        "refresh_tokens",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("user_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.users.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("token", sa.String(512), unique=True, nullable=False, index=True),
+        sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("revoked", sa.Boolean, default=False, nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Audit logs
+    op.create_table(
+        "audit_logs",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("user_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.users.id", ondelete="SET NULL"), nullable=True, index=True),
+        sa.Column("action", sa.String(100), nullable=False, index=True),
+        sa.Column("resource_type", sa.String(100), nullable=False, index=True),
+        sa.Column("resource_id", sa.String(255), nullable=True),
+        sa.Column("details", sa.Text, nullable=True),
+        sa.Column("ip_address", sa.String(45), nullable=True),
+        sa.Column("user_agent", sa.String(512), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False, index=True),
+        schema=SCHEMA,
+    )
+
+    # Documents
+    op.create_table(
+        "documents",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("filename", sa.String(500), nullable=False),
+        sa.Column("original_filename", sa.String(500), nullable=False),
+        sa.Column("content_type", sa.String(100), nullable=False),
+        sa.Column("file_size", sa.BigInteger, nullable=False),
+        sa.Column("checksum", sa.String(128), nullable=False, index=True),
+        sa.Column("storage_path", sa.String(1024), nullable=False),
+        sa.Column("status", sa.String(50), default="pending", nullable=False, index=True),
+        sa.Column("page_count", sa.Integer, nullable=True),
+        sa.Column("is_scanned", sa.Boolean, nullable=True),
+        sa.Column("document_metadata", postgresql.JSONB, nullable=True),
+        sa.Column("error_message", sa.Text, nullable=True),
+        sa.Column("uploaded_by", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.users.id", ondelete="SET NULL"), nullable=True, index=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Document pages
+    op.create_table(
+        "document_pages",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("document_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.documents.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("page_number", sa.Integer, nullable=False),
+        sa.Column("width", sa.Float, nullable=False),
+        sa.Column("height", sa.Float, nullable=False),
+        sa.Column("image_path", sa.String(1024), nullable=True),
+        sa.Column("text_content", sa.Text, nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Document text blocks
+    op.create_table(
+        "document_text_blocks",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("page_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_pages.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("text", sa.Text, nullable=False),
+        sa.Column("x", sa.Float, nullable=False),
+        sa.Column("y", sa.Float, nullable=False),
+        sa.Column("width", sa.Float, nullable=False),
+        sa.Column("height", sa.Float, nullable=False),
+        sa.Column("confidence", sa.Float, nullable=True),
+        sa.Column("font_family", sa.String(255), nullable=True),
+        sa.Column("font_size", sa.Float, nullable=True),
+        sa.Column("font_color", sa.String(50), nullable=True),
+        sa.Column("font_style", sa.String(50), nullable=True),
+        sa.Column("block_type", sa.String(50), default="text", nullable=False),
+        sa.Column("sequence", sa.Integer, default=0, nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Document images
+    op.create_table(
+        "document_images",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("page_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_pages.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("x", sa.Float, nullable=False),
+        sa.Column("y", sa.Float, nullable=False),
+        sa.Column("width", sa.Float, nullable=False),
+        sa.Column("height", sa.Float, nullable=False),
+        sa.Column("image_path", sa.String(1024), nullable=False),
+        sa.Column("image_type", sa.String(50), default="figure", nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Document tables
+    op.create_table(
+        "document_tables",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("page_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_pages.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("x", sa.Float, nullable=False),
+        sa.Column("y", sa.Float, nullable=False),
+        sa.Column("width", sa.Float, nullable=False),
+        sa.Column("height", sa.Float, nullable=False),
+        sa.Column("rows", sa.Integer, nullable=False),
+        sa.Column("columns", sa.Integer, nullable=False),
+        sa.Column("data", postgresql.JSONB, nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Document formats (templates)
+    op.create_table(
+        "document_formats",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("name", sa.String(255), nullable=False, index=True),
+        sa.Column("description", sa.Text, nullable=True),
+        sa.Column("page_width", sa.Float, nullable=False),
+        sa.Column("page_height", sa.Float, nullable=False),
+        sa.Column("page_count", sa.Integer, default=1, nullable=False),
+        sa.Column("margin_top", sa.Float, default=72.0, nullable=False),
+        sa.Column("margin_right", sa.Float, default=72.0, nullable=False),
+        sa.Column("margin_bottom", sa.Float, default=72.0, nullable=False),
+        sa.Column("margin_left", sa.Float, default=72.0, nullable=False),
+        sa.Column("fingerprint", postgresql.JSONB, nullable=True),
+        sa.Column("source_document_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.documents.id", ondelete="SET NULL"), nullable=True, index=True),
+        sa.Column("version", sa.Integer, default=1, nullable=False),
+        sa.Column("is_active", sa.Boolean, default=True, nullable=False, index=True),
+        sa.Column("created_by", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.users.id", ondelete="SET NULL"), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Document cells
+    op.create_table(
+        "document_cells",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_formats.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("page_number", sa.Integer, nullable=False),
+        sa.Column("x", sa.Float, nullable=False),
+        sa.Column("y", sa.Float, nullable=False),
+        sa.Column("width", sa.Float, nullable=False),
+        sa.Column("height", sa.Float, nullable=False),
+        sa.Column("row_no", sa.Integer, default=0, nullable=False),
+        sa.Column("column_no", sa.Integer, default=0, nullable=False),
+        sa.Column("data_type", sa.String(50), default="text", nullable=False),
+        sa.Column("font_family", sa.String(255), nullable=True),
+        sa.Column("font_size", sa.Float, nullable=True),
+        sa.Column("font_style", sa.String(50), nullable=True),
+        sa.Column("font_color", sa.String(50), nullable=True),
+        sa.Column("background_color", sa.String(50), nullable=True),
+        sa.Column("border_top", sa.String(100), nullable=True),
+        sa.Column("border_right", sa.String(100), nullable=True),
+        sa.Column("border_bottom", sa.String(100), nullable=True),
+        sa.Column("border_left", sa.String(100), nullable=True),
+        sa.Column("padding_top", sa.Float, default=0.0, nullable=False),
+        sa.Column("padding_right", sa.Float, default=0.0, nullable=False),
+        sa.Column("padding_bottom", sa.Float, default=0.0, nullable=False),
+        sa.Column("padding_left", sa.Float, default=0.0, nullable=False),
+        sa.Column("alignment", sa.String(20), default="left", nullable=False),
+        sa.Column("vertical_alignment", sa.String(20), default="top", nullable=False),
+        sa.Column("rowspan", sa.Integer, default=1, nullable=False),
+        sa.Column("colspan", sa.Integer, default=1, nullable=False),
+        sa.Column("static_text", sa.Text, nullable=True),
+        sa.Column("field_name", sa.String(255), nullable=True),
+        sa.Column("sequence", sa.Integer, default=0, nullable=False),
+        sa.Column("is_dynamic", sa.Boolean, default=False, nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Document regions
+    op.create_table(
+        "document_regions",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_formats.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("page_number", sa.Integer, nullable=False),
+        sa.Column("region_type", sa.String(50), nullable=False),
+        sa.Column("x", sa.Float, nullable=False),
+        sa.Column("y", sa.Float, nullable=False),
+        sa.Column("width", sa.Float, nullable=False),
+        sa.Column("height", sa.Float, nullable=False),
+        sa.Column("content", postgresql.JSONB, nullable=True),
+        sa.Column("sequence", sa.Integer, default=0, nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Table formats
+    op.create_table(
+        "table_formats",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_formats.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("page_number", sa.Integer, nullable=False),
+        sa.Column("x", sa.Float, nullable=False),
+        sa.Column("y", sa.Float, nullable=False),
+        sa.Column("width", sa.Float, nullable=False),
+        sa.Column("height", sa.Float, nullable=False),
+        sa.Column("rows", sa.Integer, nullable=False),
+        sa.Column("columns", sa.Integer, nullable=False),
+        sa.Column("border_style", sa.String(50), default="solid", nullable=False),
+        sa.Column("border_width", sa.Float, default=1.0, nullable=False),
+        sa.Column("border_color", sa.String(50), default="#000000", nullable=False),
+        sa.Column("header_rows", sa.Integer, default=1, nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Table columns
+    op.create_table(
+        "table_columns",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("table_format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.table_formats.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("column_index", sa.Integer, nullable=False),
+        sa.Column("width", sa.Float, nullable=False),
+        sa.Column("header_text", sa.String(500), nullable=True),
+        sa.Column("data_type", sa.String(50), default="text", nullable=False),
+        sa.Column("alignment", sa.String(20), default="left", nullable=False),
+        sa.Column("font_family", sa.String(255), nullable=True),
+        sa.Column("font_size", sa.Float, nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Table rows
+    op.create_table(
+        "table_rows",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("table_format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.table_formats.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("row_index", sa.Integer, nullable=False),
+        sa.Column("height", sa.Float, default=20.0, nullable=False),
+        sa.Column("is_header", sa.Boolean, default=False, nullable=False),
+        sa.Column("background_color", sa.String(50), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Watermarks
+    op.create_table(
+        "watermarks",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_formats.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("page_number", sa.Integer, nullable=True),
+        sa.Column("text", sa.String(500), nullable=True),
+        sa.Column("image_path", sa.String(1024), nullable=True),
+        sa.Column("x", sa.Float, nullable=False),
+        sa.Column("y", sa.Float, nullable=False),
+        sa.Column("width", sa.Float, nullable=False),
+        sa.Column("height", sa.Float, nullable=False),
+        sa.Column("opacity", sa.Float, default=0.3, nullable=False),
+        sa.Column("rotation", sa.Float, default=0.0, nullable=False),
+        sa.Column("font_family", sa.String(255), nullable=True),
+        sa.Column("font_size", sa.Float, nullable=True),
+        sa.Column("font_color", sa.String(50), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Image regions
+    op.create_table(
+        "image_regions",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_formats.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("page_number", sa.Integer, nullable=False),
+        sa.Column("x", sa.Float, nullable=False),
+        sa.Column("y", sa.Float, nullable=False),
+        sa.Column("width", sa.Float, nullable=False),
+        sa.Column("height", sa.Float, nullable=False),
+        sa.Column("image_path", sa.String(1024), nullable=True),
+        sa.Column("image_type", sa.String(50), default="figure", nullable=False),
+        sa.Column("is_static", sa.Boolean, default=True, nullable=False),
+        sa.Column("field_name", sa.String(255), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Template fingerprints
+    op.create_table(
+        "template_fingerprints",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_formats.id", ondelete="CASCADE"), nullable=False, unique=True, index=True),
+        sa.Column("page_dimensions", postgresql.JSONB, nullable=True),
+        sa.Column("logo_coordinates", postgresql.JSONB, nullable=True),
+        sa.Column("header_coordinates", postgresql.JSONB, nullable=True),
+        sa.Column("footer_coordinates", postgresql.JSONB, nullable=True),
+        sa.Column("table_coordinates", postgresql.JSONB, nullable=True),
+        sa.Column("cell_coordinates", postgresql.JSONB, nullable=True),
+        sa.Column("fingerprint_hash", sa.String(256), nullable=False, index=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Template matches
+    op.create_table(
+        "template_matches",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column("document_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.documents.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("format_id", postgresql.UUID(as_uuid=True), sa.ForeignKey(f"{SCHEMA}.document_formats.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("confidence_score", sa.Float, nullable=False),
+        sa.Column("match_details", postgresql.JSONB, nullable=True),
+        sa.Column("selected", sa.Boolean, default=False, nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        schema=SCHEMA,
+    )
+
+    # Additional indexes
+    op.create_index("ix_documents_status_created", "documents", ["status", "created_at"], schema=SCHEMA)
+    op.create_index("ix_document_pages_doc_page", "document_pages", ["document_id", "page_number"], schema=SCHEMA)
+    op.create_index("ix_document_text_blocks_type", "document_text_blocks", ["page_id", "block_type"], schema=SCHEMA)
+    op.create_index("ix_document_cells_format_page", "document_cells", ["format_id", "page_number"], schema=SCHEMA)
+    op.create_index("ix_template_matches_doc_score", "template_matches", ["document_id", "confidence_score"], schema=SCHEMA)
+    op.create_index("ix_audit_logs_resource", "audit_logs", ["resource_type", "resource_id"], schema=SCHEMA)
+
+
+def downgrade() -> None:
+    op.drop_table("template_matches", schema=SCHEMA)
+    op.drop_table("template_fingerprints", schema=SCHEMA)
+    op.drop_table("image_regions", schema=SCHEMA)
+    op.drop_table("watermarks", schema=SCHEMA)
+    op.drop_table("table_rows", schema=SCHEMA)
+    op.drop_table("table_columns", schema=SCHEMA)
+    op.drop_table("table_formats", schema=SCHEMA)
+    op.drop_table("document_regions", schema=SCHEMA)
+    op.drop_table("document_cells", schema=SCHEMA)
+    op.drop_table("document_formats", schema=SCHEMA)
+    op.drop_table("document_tables", schema=SCHEMA)
+    op.drop_table("document_images", schema=SCHEMA)
+    op.drop_table("document_text_blocks", schema=SCHEMA)
+    op.drop_table("document_pages", schema=SCHEMA)
+    op.drop_table("documents", schema=SCHEMA)
+    op.drop_table("audit_logs", schema=SCHEMA)
+    op.drop_table("refresh_tokens", schema=SCHEMA)
+    op.drop_table("user_roles", schema=SCHEMA)
+    op.drop_table("roles", schema=SCHEMA)
+    op.drop_table("users", schema=SCHEMA)