first commit

files/modules/excel/queue_manager.py

@@ -0,0 +1,113 @@
+from queue import Queue
+import threading
+import logging
+
+logger = logging.getLogger(__name__)
+
+# =========================================
+# CONFIG
+# =========================================
+
+MAX_CONCURRENT_EXCEL = 10
+
+# =========================================
+# STATE
+# =========================================
+
+EXCEL_QUEUE = Queue()
+EXCEL_LOCK = threading.Lock()
+
+ACTIVE_JOBS = set()   # jobs em execução
+
+# =========================================
+# Helpers
+# =========================================
+
+def get_queue_position(job_id: str) -> int:
+    """
+    Retorna:
+    0 = já está executando
+    1..N = posição na fila
+    -1 = não encontrado
+    """
+    with EXCEL_LOCK:
+
+        if job_id in ACTIVE_JOBS:
+            return 0
+
+        items = list(EXCEL_QUEUE.queue)
+
+        for i, item in enumerate(items):
+            if item["job_id"] == job_id:
+                return i + 1
+
+        return -1
+
+
+# =========================================
+# WORKER
+# =========================================
+
+def _worker(worker_id: int):
+    logger.info(f"🟢 Excel worker-{worker_id} started")
+
+    while True:
+        job = EXCEL_QUEUE.get()
+
+        job_id = job["job_id"]
+
+        try:
+            with EXCEL_LOCK:
+                ACTIVE_JOBS.add(job_id)
+
+            logger.info(f"🚀 [worker-{worker_id}] Processing {job_id}")
+
+            job["fn"](*job["args"], **job["kwargs"])
+
+            logger.info(f"✅ [worker-{worker_id}] Finished {job_id}")
+
+        except Exception as e:
+            logger.exception(f"❌ [worker-{worker_id}] Failed {job_id}: {e}")
+
+        finally:
+            with EXCEL_LOCK:
+                ACTIVE_JOBS.discard(job_id)
+
+            EXCEL_QUEUE.task_done()
+
+
+# =========================================
+# START POOL
+# =========================================
+
+def start_excel_worker():
+    """
+    Inicia pool com N workers simultâneos
+    """
+    for i in range(MAX_CONCURRENT_EXCEL):
+        threading.Thread(
+            target=_worker,
+            args=(i + 1,),
+            daemon=True
+        ).start()
+
+    logger.info(f"🔥 Excel queue started with {MAX_CONCURRENT_EXCEL} workers")
+
+
+# =========================================
+# ENQUEUE
+# =========================================
+
+def enqueue_excel_job(job_id, fn, *args, **kwargs):
+    job = {
+        "job_id": job_id,
+        "fn": fn,
+        "args": args,
+        "kwargs": kwargs
+    }
+
+    with EXCEL_LOCK:
+        EXCEL_QUEUE.put(job)
+        position = EXCEL_QUEUE.qsize()
+
+    return position

files/modules/excel/routes.py

@@ -0,0 +1,110 @@
+from flask import Blueprint, request, jsonify, send_file, render_template
+from pathlib import Path
+import uuid
+import json
+from config_loader import load_config
+from modules.core.audit import audit_log
+from modules.core.security import get_current_user
+
+from modules.core.security import requires_app_auth
+from .service import start_excel_job
+from .store import EXCEL_JOBS, EXCEL_LOCK
+
+excel_bp = Blueprint("excel", __name__)
+config = load_config()
+API_BASE_URL = f"{config.app_base}:{config.service_port}"
+
+UPLOAD_FOLDER = Path("./uploads")
+UPLOAD_FOLDER.mkdir(exist_ok=True)
+
+ALLOWED_EXTENSIONS = {"xlsx"}
+API_URL = API_BASE_URL + "/chat"
+
+
+def allowed_file(filename):
+    return "." in filename and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS
+
+
+# =========================
+# Upload + start processing
+# =========================
+@excel_bp.route("/upload/excel", methods=["POST"])
+@requires_app_auth
+def upload_excel():
+    file = request.files.get("file")
+    email = request.form.get("email")
+
+    if not file or not email:
+        return jsonify({"error": "file and email required"}), 400
+
+    if not allowed_file(file.filename):
+        return jsonify({"error": "invalid file type"}), 400
+
+    job_id = str(uuid.uuid4())
+    audit_log("UPLOAD_EXCEL", f"job_id={job_id}")
+
+    job_dir = UPLOAD_FOLDER / job_id
+    job_dir.mkdir(parents=True, exist_ok=True)
+
+    input_path = job_dir / "input.xlsx"
+    file.save(input_path)
+
+    with EXCEL_LOCK:
+        EXCEL_JOBS[job_id] = {"status": "RUNNING"}
+
+    user = get_current_user()
+
+    start_excel_job(
+        job_id=job_id,
+        input_path=input_path,
+        email=email,
+        auth_user=None,
+        auth_pass=None,
+        api_url=API_URL
+    )
+
+    return jsonify({"status": "STARTED", "job_id": job_id})
+
+
+# =========================
+# Status
+# =========================
+@excel_bp.route("/job/<job_id>/status")
+@requires_app_auth
+def job_status(job_id):
+    status_file = UPLOAD_FOLDER / job_id / "status.json"
+
+    if not status_file.exists():
+        return jsonify({"status": "PROCESSING"})
+
+    return jsonify(json.loads(status_file.read_text()))
+
+
+# =========================
+# Download result
+# =========================
+@excel_bp.route("/download/<job_id>")
+@requires_app_auth
+def download(job_id):
+    result_file = UPLOAD_FOLDER / job_id / "result.xlsx"
+
+    if not result_file.exists():
+        return jsonify({"error": "not ready"}), 404
+
+    return send_file(
+        result_file,
+        as_attachment=True,
+        download_name=f"RFP_result_{job_id}.xlsx"
+    )
+
+@excel_bp.route("/job/<job_id>/logs", methods=["GET"])
+@requires_app_auth
+def excel_logs(job_id):
+    with EXCEL_LOCK:
+        job = EXCEL_JOBS.get(job_id, {})
+    return jsonify({"logs": job.get("logs", [])})
+
+@excel_bp.route("/excel/job/<job_id>")
+@requires_app_auth
+def job_page(job_id):
+    return render_template("excel/job_status.html", job_id=job_id)

files/modules/excel/service.py

@@ -0,0 +1,115 @@
+import threading
+import json
+from pathlib import Path
+from datetime import datetime
+from flask import jsonify
+from .storage import upload_file, generate_download_url
+
+from rfp_process import process_excel_rfp
+from .store import EXCEL_JOBS, EXCEL_LOCK
+from modules.users.email_service import send_completion_email
+from modules.excel.queue_manager import enqueue_excel_job
+
+EXECUTION_METHOD = "QUEUE" # THREAD OR QUEUE
+
+UPLOAD_FOLDER = Path("uploads")
+UPLOAD_FOLDER.mkdir(exist_ok=True)
+
+
+def make_job_logger(job_id: str):
+    """Logger simples: guarda logs na memória (igual ao arquiteto)."""
+    def _log(msg):
+        with EXCEL_LOCK:
+            job = EXCEL_JOBS.get(job_id)
+            if job is not None:
+                job.setdefault("logs", []).append(str(msg))
+    return _log
+
+
+def start_excel_job(job_id: str, input_path: Path, email: str, auth_user: str, auth_pass: str, api_url: str):
+
+    job_dir = UPLOAD_FOLDER / job_id
+    job_dir.mkdir(parents=True, exist_ok=True)
+
+    output_path = job_dir / "result.xlsx"
+    status_file = job_dir / "status.json"
+    object_name = f"{job_id}/result.xlsx"
+
+    logger = make_job_logger(job_id)
+
+    def write_status(state: str, detail: str | None = None):
+        payload = {
+            "status": state,
+            "updated_at": datetime.utcnow().isoformat(),
+        }
+        if detail:
+            payload["detail"] = detail
+
+        status_file.write_text(
+            json.dumps(payload, ensure_ascii=False, indent=2),
+            encoding="utf-8"
+        )
+
+        with EXCEL_LOCK:
+            job = EXCEL_JOBS.get(job_id)
+            if job is not None:
+                job["status"] = state
+                if detail:
+                    job["detail"] = detail
+
+    # garante estrutura do job na memória
+    with EXCEL_LOCK:
+        EXCEL_JOBS.setdefault(job_id, {})
+        EXCEL_JOBS[job_id].setdefault("logs", [])
+        EXCEL_JOBS[job_id]["status"] = "PROCESSING"
+
+    write_status("PROCESSING")
+    logger(f"Starting Excel job {job_id}")
+    logger(f"Input: {input_path}")
+    logger(f"Output: {output_path}")
+
+    def background():
+        try:
+            # processamento principal
+            process_excel_rfp(
+                input_excel=input_path,
+                output_excel=output_path,
+                api_url=api_url,
+                auth_user=auth_user,
+                auth_pass=auth_pass,
+            )
+
+            write_status("DONE")
+            logger("Excel processing DONE")
+
+            upload_file(output_path, object_name)
+            download_url = generate_download_url(object_name)
+
+            write_status("DONE", download_url)
+
+            # email / dev message
+            dev_message = send_completion_email(email, download_url, job_id)
+            if dev_message:
+                logger(f"DEV email message/link: {dev_message}")
+
+        except Exception as e:
+            logger(f"ERROR: {e}")
+            write_status("ERROR", str(e))
+
+            try:
+                dev_message = send_completion_email(
+                    email=email,
+                    download_url=download_url,
+                    job_id=job_id,
+                    status="ERROR",
+                    detail=str(e)
+                )
+                if dev_message:
+                    logger(f"DEV email error message/link: {dev_message}")
+            except Exception as mail_err:
+                logger(f"EMAIL ERROR: {mail_err}")
+
+    if EXECUTION_METHOD == "THREAD":
+        threading.Thread(target=background, daemon=True).start()
+    else:
+        enqueue_excel_job(job_id, background)

files/modules/excel/storage.py

@@ -0,0 +1,67 @@
+import oci
+from datetime import datetime, timedelta
+from config_loader import load_config
+from oci.object_storage.models import CreatePreauthenticatedRequestDetails
+
+config = load_config()
+
+
+oci_config = oci.config.from_file(
+    file_location="~/.oci/config",
+    profile_name=config.bucket_profile
+)
+
+object_storage = oci.object_storage.ObjectStorageClient(oci_config)
+
+
+def _namespace():
+    if config.oci_namespace != "auto":
+        return config.oci_namespace
+
+    return object_storage.get_namespace().data
+
+
+# =========================
+# Upload file
+# =========================
+def upload_file(local_path: str, object_name: str):
+
+    with open(local_path, "rb") as f:
+        object_storage.put_object(
+            namespace_name=_namespace(),
+            bucket_name=config.oci_bucket,
+            object_name=object_name,
+            put_object_body=f
+        )
+    print(f"SUCCESS on Upload {object_name}")
+
+# =========================
+# Pre-authenticated download URL
+# =========================
+def generate_download_url(object_name: str, hours=24):
+
+    expire = datetime.utcnow() + timedelta(hours=hours)
+
+    details = CreatePreauthenticatedRequestDetails(
+        name=f"job-{object_name}",
+        access_type="ObjectRead",
+        object_name=object_name,
+        time_expires=expire
+    )
+
+    response = object_storage.create_preauthenticated_request(
+        namespace_name=_namespace(),
+        bucket_name=config.oci_bucket,
+        create_preauthenticated_request_details=details
+    )
+
+    par = response.data
+
+    download_link = (
+        f"https://objectstorage.{oci_config['region']}.oraclecloud.com{par.access_uri}"
+    )
+
+    print("PAR CREATED OK")
+    print(download_link)
+
+    return download_link

files/modules/excel/store.py

@@ -0,0 +1,4 @@
+from threading import Lock
+
+EXCEL_JOBS = {}
+EXCEL_LOCK = Lock()