Mail box and OCR done

This commit is contained in:
2026-01-22 20:56:35 +05:30
parent 2148c4a61c
commit d662aea7e4
16 changed files with 644 additions and 14 deletions

11
backend/.env Normal file
View File

@@ -0,0 +1,11 @@
DB_USER=postgres
DB_PASSWORD=M@tr!x#149@dm!N
DB_HOST=192.168.0.111
DB_PORT=7925
DB_NAME=ocr
# Mail Configuration (Gmail)
MAIL_SERVER=imap.gmail.com
MAIL_USERNAME=matrixinfotech.it@gmail.com
MAIL_PASSWORD=qtxsthbxbisgcmqu
MAIL_PORT=993

Binary file not shown.

Binary file not shown.

Binary file not shown.

12
backend/create_tables.py Normal file
View File

@@ -0,0 +1,12 @@
from database import engine, Base
def create_tables():
print("Creating tables in database...")
try:
Base.metadata.create_all(bind=engine)
print("Tables created successfully!")
except Exception as e:
print(f"Error creating tables: {e}")
if __name__ == "__main__":
create_tables()

66
backend/database.py Normal file
View File

@@ -0,0 +1,66 @@
import os
from sqlalchemy import create_engine, Column, Integer, String, Text, DateTime, Boolean, ForeignKey, LargeBinary
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker, relationship
from dotenv import load_dotenv
# Load environment variables
load_dotenv()
DB_USER = os.getenv("DB_USER")
DB_PASSWORD = os.getenv("DB_PASSWORD")
DB_HOST = os.getenv("DB_HOST")
DB_PORT = os.getenv("DB_PORT")
import urllib.parse
# ... (imports)
# Load environment variables
load_dotenv()
DB_USER = os.getenv("DB_USER")
DB_PASSWORD = os.getenv("DB_PASSWORD")
DB_HOST = os.getenv("DB_HOST")
DB_PORT = os.getenv("DB_PORT")
DB_NAME = os.getenv("DB_NAME")
encoded_user = urllib.parse.quote_plus(DB_USER)
encoded_password = urllib.parse.quote_plus(DB_PASSWORD)
# SQLAlchemy Database URL
DATABASE_URL = f"postgresql://{encoded_user}:{encoded_password}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
engine = create_engine(DATABASE_URL)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
Base = declarative_base()
class Email(Base):
__tablename__ = "emails"
id = Column(Integer, primary_key=True, index=True)
subject = Column(String, index=True)
sender = Column(String, index=True)
body = Column(Text)
received_date = Column(DateTime)
is_read = Column(Boolean, default=False)
attachments = relationship("Attachment", back_populates="email")
class Attachment(Base):
__tablename__ = "attachments"
id = Column(Integer, primary_key=True, index=True)
email_id = Column(Integer, ForeignKey("emails.id"))
filename = Column(String)
content_type = Column(String)
file_content = Column(LargeBinary) # Storing content directly in DB as requested
email = relationship("Email", back_populates="attachments")
def get_db():
db = SessionLocal()
try:
yield db
finally:
db.close()

95
backend/mail_service.py Normal file
View File

@@ -0,0 +1,95 @@
import os
import datetime
import mimetypes
import time
from imap_tools import MailBox, AND
from sqlalchemy.orm import Session
from database import SessionLocal, Email, Attachment
from dotenv import load_dotenv
load_dotenv()
MAIL_SERVER = os.getenv("MAIL_SERVER")
MAIL_USERNAME = os.getenv("MAIL_USERNAME")
MAIL_PASSWORD = os.getenv("MAIL_PASSWORD")
MAIL_PORT = int(os.getenv("MAIL_PORT", 993))
def fetch_and_store_emails():
"""
Connects to Gmail, fetches UNREAD emails, stores them in DB, and marks as READ.
"""
print(f"[{datetime.datetime.now()}] Checking for new emails...")
if not MAIL_USERNAME or not MAIL_PASSWORD or MAIL_USERNAME == "your-email@gmail.com":
print("Skipping Email Sync: Credentials not configured.")
return {"status": "skipped", "message": "Credentials not configured"}
db: Session = SessionLocal()
count = 0
try:
# Connect to Inbox
# imap-tools MailBox defaults to SSL (port 993) if not specified, but we make it explicit here.
with MailBox(MAIL_SERVER, port=MAIL_PORT).login(MAIL_USERNAME, MAIL_PASSWORD, initial_folder='INBOX') as mailbox:
# Fetch UNREAD emails
for msg in mailbox.fetch(AND(seen=False)):
print(f"Processing Email: {msg.subject}")
# Check if already exists (optional, but good practice if marking read fails)
# For now using simplistic approach: if we fetch it, we save it.
new_email = Email(
subject=msg.subject,
sender=msg.from_,
body=msg.text or msg.html,
received_date=msg.date,
is_read=False # Consumed by app, but 'seen' on server
)
db.add(new_email)
db.commit() # Commit to get ID
db.refresh(new_email)
# Process Attachments
for i, att in enumerate(msg.attachments):
filename = att.filename
if not filename:
# Fallback for missing filename
ext = mimetypes.guess_extension(att.content_type) or ".bin"
timestamp = int(time.time())
filename = f"attachment_{timestamp}_{i}{ext}"
print(f" - Attachment: {filename}")
new_attachment = Attachment(
email_id=new_email.id,
filename=filename,
content_type=att.content_type,
file_content=att.payload # raw bytes
)
db.add(new_attachment)
db.commit()
# Mark as Read on Server (Auto-done by fetch unless bulk=True or mark_seen=False)
# imap-tools fetches are auto-seen by default.
count += 1
print(f"Sync Complete. {count} new emails.")
return {"status": "success", "count": count}
except Exception as e:
error_msg = str(e)
print(f"Error fetching emails: {error_msg}")
if "AUTHENTICATIONFAILED" in error_msg or "Invalid credentials" in error_msg:
return {
"status": "error",
"message": f"Authentication Failed: {error_msg}. \n[HINT]: If using Gmail, you MUST use an 'App Password', not your login password. Also ensure IMAP is enabled in Gmail Settings."
}
return {"status": "error", "message": error_msg}
finally:
db.close()
if __name__ == "__main__":
fetch_and_store_emails()

View File

@@ -1,13 +1,32 @@
import io
from typing import Optional
import datetime
from typing import Optional, List
from contextlib import asynccontextmanager
from fastapi import FastAPI, File, UploadFile, HTTPException, Depends
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel
from pypdf import PdfReader
import pytesseract
from PIL import Image
from pdf2image import convert_from_bytes
from sqlalchemy.orm import Session
app = FastAPI()
# Internal modules
from database import get_db, Email
from scheduler import start_scheduler, stop_scheduler
from mail_service import fetch_and_store_emails
# Lifespan for Scheduler
@asynccontextmanager
async def lifespan(app: FastAPI):
# Startup
start_scheduler()
yield
# Shutdown
stop_scheduler()
app = FastAPI(lifespan=lifespan)
# CORS configuration
origins = [
@@ -38,13 +57,6 @@ class LoginResponse(BaseModel):
class NERResponse(BaseModel):
text: str
# 1. User Login Module (Dummy)
@app.post("/api/login", response_model=LoginResponse)
async def login(request: LoginRequest):
if request.username == "admin" and request.password == "admin":
return LoginResponse(token="dummy-jwt-token-123", message="Login Successful")
raise HTTPException(status_code=401, detail="Invalid credentials")
# 2. OCR Module
def extract_text_from_pdf(file_bytes: bytes) -> str:
try:
@@ -68,10 +80,6 @@ def extract_text_from_image(file_bytes: bytes) -> str:
print(f"Error reading Image: {e}")
return ""
from pdf2image import convert_from_bytes
# ... (imports)
@app.post("/api/ocr/extract", response_model=NERResponse)
async def extract_text(file: UploadFile = File(...)):
content = await file.read()
@@ -100,6 +108,131 @@ async def extract_text(file: UploadFile = File(...)):
return NERResponse(text=extracted_text)
import zipfile
import mimetypes
from fastapi.responses import StreamingResponse
class EmailDTO(BaseModel):
id: int
subject: Optional[str]
sender: Optional[str]
received_date: Optional[datetime.datetime]
is_read: bool
has_attachments: bool = False
class AttachmentDTO(BaseModel):
id: int
filename: str
content_type: str
class EmailListResponse(BaseModel):
items: List[EmailDTO]
total: int
class EmailDetailDTO(EmailDTO):
body: Optional[str]
attachments: List[AttachmentDTO]
@app.get("/api/emails", response_model=EmailListResponse)
def get_emails(skip: int = 0, limit: int = 50, db: Session = Depends(get_db)):
total = db.query(Email).count()
emails = db.query(Email).order_by(Email.received_date.desc()).offset(skip).limit(limit).all()
items = [
EmailDTO(
id=e.id,
subject=e.subject,
sender=e.sender,
received_date=e.received_date,
is_read=e.is_read,
has_attachments=bool(e.attachments) # Check if list is not empty
) for e in emails
]
return EmailListResponse(items=items, total=total)
# ...
@app.get("/api/emails/{email_id}", response_model=EmailDetailDTO)
def get_email_details(email_id: int, db: Session = Depends(get_db)):
email = db.query(Email).filter(Email.id == email_id).first()
if not email:
raise HTTPException(status_code=404, detail="Email not found")
# Mark as read if opened
if not email.is_read:
email.is_read = True
db.commit()
# Safely handle missing content_type and filename
attachments = []
for a in email.attachments:
fname = a.filename
if not fname or fname == "unknown":
ext = mimetypes.guess_extension(a.content_type or "") or ".bin"
fname = f"attachment_{a.id}{ext}"
attachments.append(
AttachmentDTO(
id=a.id,
filename=fname,
content_type=a.content_type or "application/octet-stream"
)
)
return EmailDetailDTO(
id=email.id,
subject=email.subject,
sender=email.sender,
received_date=email.received_date,
is_read=email.is_read,
has_attachments=bool(email.attachments),
body=email.body,
attachments=attachments
)
@app.get("/api/emails/{email_id}/download-all")
def download_all_attachments(email_id: int, db: Session = Depends(get_db)):
email = db.query(Email).filter(Email.id == email_id).first()
if not email:
raise HTTPException(status_code=404, detail="Email not found")
if not email.attachments:
raise HTTPException(status_code=400, detail="No attachments found")
# Create a zip file in memory
zip_buffer = io.BytesIO()
with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zip_file:
for att in email.attachments:
# Use filename from DB, ensure unique names if duplicates exist (simple append here)
zip_file.writestr(att.filename, att.file_content)
zip_buffer.seek(0)
return StreamingResponse(
zip_buffer,
media_type="application/zip",
headers={"Content-Disposition": f"attachment; filename=email-{email_id}-attachments.zip"}
)
@app.get("/api/attachments/{attachment_id}")
def download_attachment(attachment_id: int, view: bool = False, db: Session = Depends(get_db)):
attachment = db.query(DBAttachment).filter(DBAttachment.id == attachment_id).first()
if not attachment:
raise HTTPException(status_code=404, detail="Attachment not found")
disposition = "inline" if view else "attachment"
return StreamingResponse(
io.BytesIO(attachment.file_content),
media_type=attachment.content_type,
headers={"Content-Disposition": f"{disposition}; filename={attachment.filename}"}
)
@app.post("/api/emails/sync")
def sync_emails():
result = fetch_and_store_emails()
return result
@app.get("/")
def read_root():
return {"message": "OCR Backend API is running"}

View File

@@ -4,3 +4,8 @@ python-multipart
pypdf
pytesseract
Pillow
sqlalchemy
psycopg2-binary
imap-tools
apscheduler
python-dotenv

14
backend/scheduler.py Normal file
View File

@@ -0,0 +1,14 @@
from apscheduler.schedulers.background import BackgroundScheduler
from mail_service import fetch_and_store_emails
scheduler = BackgroundScheduler()
def start_scheduler():
# Add job to run every 10 seconds
scheduler.add_job(fetch_and_store_emails, 'interval', seconds=10, id='mail_sync_job', replace_existing=True)
scheduler.start()
print("Mail Scheduler Started (Interval: 10s)")
def stop_scheduler():
scheduler.shutdown()
print("Mail Scheduler Stopped")

View File

@@ -51,6 +51,12 @@ export class AdminLayoutComponent implements OnInit {
icon: 'pi pi-history'
}
]
},
{
label: 'Mailbox',
icon: 'pi pi-envelope',
routerLink: '/mailbox'
},
{
label: 'Settings',

View File

@@ -2,6 +2,7 @@ import { Routes } from '@angular/router';
import { LoginComponent } from './login/login.component';
import { AdminLayoutComponent } from './admin-layout/admin-layout.component';
import { OcrComponent } from './ocr/ocr.component';
import { MailboxComponent } from './mailbox/mailbox.component';
import { AuthGuard } from './auth.guard';
export const routes: Routes = [
@@ -12,7 +13,8 @@ export const routes: Routes = [
canActivate: [AuthGuard],
children: [
{ path: '', redirectTo: 'ocr', pathMatch: 'full' },
{ path: 'ocr', component: OcrComponent }
{ path: 'ocr', component: OcrComponent },
{ path: 'mailbox', component: MailboxComponent }
]
},
{ path: '**', redirectTo: '' }

View File

@@ -0,0 +1,59 @@
import { Injectable } from '@angular/core';
import { HttpClient } from '@angular/common/http';
import { Observable } from 'rxjs';
export interface AttachmentDTO {
id: number;
filename: string;
content_type: string;
}
export interface EmailDTO {
id: number;
subject: string;
sender: string;
received_date: string;
is_read: boolean;
has_attachments?: boolean; // New flag from backend
body?: string;
attachments?: AttachmentDTO[];
}
export interface EmailListResponse {
items: EmailDTO[];
total: number;
}
@Injectable({
providedIn: 'root'
})
export class MailboxService {
private apiUrl = 'http://localhost:8000/api/emails';
private attachmentUrl = 'http://localhost:8000/api/attachments';
constructor(private http: HttpClient) {}
getEmails(skip: number = 0, limit: number = 50): Observable<EmailListResponse> {
return this.http.get<EmailListResponse>(`${this.apiUrl}?skip=${skip}&limit=${limit}`);
}
getEmailDetails(id: number): Observable<EmailDTO> {
return this.http.get<EmailDTO>(`${this.apiUrl}/${id}`);
}
getAttachmentUrl(id: number): string {
return `${this.attachmentUrl}/${id}`;
}
getPreviewUrl(id: number): string {
return `${this.attachmentUrl}/${id}?view=true`;
}
getZipDownloadUrl(emailId: number): string {
return `${this.apiUrl}/${emailId}/download-all`;
}
syncEmails(): Observable<any> {
return this.http.post<any>(`${this.apiUrl}/sync`, {});
}
}

View File

@@ -0,0 +1,222 @@
import { Component, OnInit } from '@angular/core';
import { CommonModule } from '@angular/common';
import { TableModule, TableLazyLoadEvent } from 'primeng/table';
import { ButtonModule } from 'primeng/button';
import { ToastModule } from 'primeng/toast';
import { ProgressBarModule } from 'primeng/progressbar';
import { DialogModule } from 'primeng/dialog';
import { MessageService } from 'primeng/api';
import { MailboxService, EmailDTO } from '../mailbox.service';
@Component({
selector: 'app-mailbox',
standalone: true,
imports: [
CommonModule,
TableModule,
ButtonModule,
ToastModule,
ProgressBarModule,
DialogModule
],
providers: [MessageService],
template: `
<div class="card">
<div class="flex justify-content-between align-items-center mb-4">
<h2>Mailbox</h2>
<button pButton
label="Sync Now"
icon="pi pi-refresh"
(click)="sync()"
[loading]="syncing">
</button>
</div>
<p-progressBar *ngIf="syncing" mode="indeterminate" [style]="{'height': '4px'}"></p-progressBar>
<p-table [value]="emails"
[paginator]="true"
[rows]="50"
[totalRecords]="totalRecords"
[lazy]="true"
(onLazyLoad)="loadEmails($event)"
[loading]="loading"
[showCurrentPageReport]="true"
currentPageReportTemplate="Showing {first} to {last} of {totalRecords} entries"
styleClass="p-datatable-sm"
[tableStyle]="{'min-width': '50rem'}">
<ng-template pTemplate="header">
<tr>
<th style="width: 5%">ID</th>
<th style="width: 5%"></th> <!-- Attachment Icon -->
<th style="width: 25%">Sender</th>
<th style="width: 40%">Subject</th>
<th style="width: 20%">Date</th>
<th style="width: 5%">Status</th>
<th style="width: 5%">Action</th>
</tr>
</ng-template>
<ng-template pTemplate="body" let-email>
<tr [ngClass]="{'font-bold': !email.is_read}">
<td>{{email.id}}</td>
<td>
<i *ngIf="email.has_attachments" class="pi pi-paperclip text-gray-500"></i>
</td>
<td>{{email.sender}}</td>
<td>{{email.subject}}</td>
<td>{{email.received_date | date:'short'}}</td>
<td>
<i class="pi" [ngClass]="email.is_read ? 'pi-envelope-open' : 'pi-envelope'"></i>
</td>
<td>
<button pButton icon="pi pi-eye" class="p-button-rounded p-button-text" (click)="viewEmail(email.id)"></button>
</td>
</tr>
</ng-template>
<ng-template pTemplate="emptymessage">
<tr>
<td colspan="7">No emails found. Click Sync to fetch from Gmail.</td>
</tr>
</ng-template>
</p-table>
<p-toast></p-toast>
<!-- Email Detail Dialog -->
<p-dialog [(visible)]="displayDialog" [style]="{width: '60vw'}" header="Email Details" [modal]="true">
<div *ngIf="selectedEmail">
<div class="mb-3">
<strong>From:</strong> {{selectedEmail.sender}}<br>
<strong>Subject:</strong> {{selectedEmail.subject}}<br>
<strong>Date:</strong> {{selectedEmail.received_date | date:'medium'}}<br>
</div>
<hr>
<div class="email-body mb-4" [innerHTML]="selectedEmail.body || '<i>No Content</i>'"></div>
<div *ngIf="selectedEmail.attachments && selectedEmail.attachments.length > 0">
<div class="flex justify-content-between align-items-center mb-2">
<h4>Attachments ({{selectedEmail.attachments.length}})</h4>
<a [href]="getDownloadAllLink(selectedEmail.id)" target="_blank" class="p-button p-button-sm p-button-outlined">
<i class="pi pi-download mr-2"></i> Download All
</a>
</div>
<div class="flex flex-wrap gap-3">
<div *ngFor="let att of selectedEmail.attachments" class="attachment-card relative border-1 surface-border border-round p-2 flex flex-column align-items-center justify-content-center" style="width: 120px; height: 120px;">
<!-- Icon Only View -->
<div class="flex align-items-center justify-content-center h-full w-full bg-gray-50 border-round">
<i *ngIf="isImage(att.content_type)" class="pi pi-image text-purple-500 text-5xl"></i>
<i *ngIf="!isImage(att.content_type) && isPdf(att.content_type)" class="pi pi-file-pdf text-red-500 text-5xl"></i>
<i *ngIf="!isImage(att.content_type) && !isPdf(att.content_type)" class="pi pi-file text-gray-500 text-5xl"></i>
</div>
<!-- Hover Overlay -->
<a [href]="getDownloadLink(att.id)" target="_blank" class="download-overlay absolute top-0 left-0 w-full h-full flex align-items-center justify-content-center bg-black-alpha-50 border-round hover:opacity-100 opacity-0 transition-duration-200 cursor-pointer">
<i class="pi pi-download text-white text-3xl"></i>
</a>
<!-- Filename Tooltip/Label (Optional, maybe shortened) -->
<span class="text-xs text-center mt-1 white-space-nowrap overflow-hidden text-overflow-ellipsis w-full" [title]="att.filename">
{{att.filename}}
</span>
</div>
</div>
</div>
</div>
</p-dialog>
</div>
`,
styles: [`
.font-bold { font-weight: 700; }
.email-body { max-height: 400px; overflow-y: auto; background: #f9f9f9; padding: 1rem; border-radius: 4px; }
.attachment-card:hover .download-overlay { opacity: 1; }
.download-overlay { transition: opacity 0.2s; }
`]
})
export class MailboxComponent implements OnInit {
emails: EmailDTO[] = [];
totalRecords: number = 0;
loading: boolean = true;
syncing: boolean = false;
displayDialog: boolean = false;
selectedEmail: EmailDTO | null = null;
constructor(
private mailboxService: MailboxService,
private messageService: MessageService
) {}
ngOnInit() {
// Initial load handled by p-table lazy load
}
loadEmails(event: TableLazyLoadEvent) {
this.loading = true;
const skip = event.first || 0;
const limit = event.rows || 50;
this.mailboxService.getEmails(skip, limit).subscribe({
next: (res) => {
this.emails = res.items;
this.totalRecords = res.total;
this.loading = false;
},
error: (err) => {
console.error(err);
this.loading = false;
}
});
}
viewEmail(id: number) {
this.mailboxService.getEmailDetails(id).subscribe({
next: (email) => {
this.selectedEmail = email;
this.displayDialog = true;
// Update local read status
const idx = this.emails.findIndex(e => e.id === id);
if(idx !== -1) {
this.emails[idx].is_read = true;
}
},
error: (err) => this.messageService.add({severity:'error', summary:'Error', detail:'Could not load email details'})
});
}
getDownloadLink(id: number): string {
return this.mailboxService.getAttachmentUrl(id);
}
getPreviewLink(id: number): string {
return this.mailboxService.getPreviewUrl(id);
}
getDownloadAllLink(emailId: number): string {
return this.mailboxService.getZipDownloadUrl(emailId);
}
isImage(contentType: string): boolean {
return contentType.startsWith('image/');
}
isPdf(contentType: string): boolean {
return contentType === 'application/pdf';
}
sync() {
this.syncing = true;
this.mailboxService.syncEmails().subscribe({
next: (res) => {
this.syncing = false;
this.messageService.add({severity:'success', summary:'Sync Complete', detail: res.message || 'Emails synced successfully'});
// Refresh table
this.loadEmails({first: 0, rows: 50});
},
error: (err) => {
this.syncing = false;
this.messageService.add({severity:'error', summary:'Sync Failed', detail: 'Could not fetch emails'});
}
});
}
}

View File

@@ -35,6 +35,7 @@ import { ToastModule } from 'primeng/toast';
cancelLabel="Clear"
[customUpload]="true"
(uploadHandler)="onUpload($event)"
(onClear)="onClear()"
accept=".pdf,image/*"
maxFileSize="10000000">
</p-fileUpload>
@@ -84,4 +85,8 @@ export class OcrComponent {
}
});
}
onClear() {
this.extractedText = null;
}
}