[Pipeline C] Form generation: generate endpoint, fill task, retrieval endpoints (#552)

app/api/router.py

@@ -1,11 +1,28 @@
 from fastapi import APIRouter
 
-from app.api.routes import extraction, forms, form_templates, input, jobs, system, weather, zipcode
+from app.api.routes import (
+    extraction,
+    form_generation,
+    forms,
+    form_templates,
+    input,
+    jobs,
+    system,
+    weather,
+    zipcode,
+)
 from app.core.config import API_PREFIX
 
 api_router = APIRouter()
 api_router.include_router(form_templates.router, prefix=API_PREFIX)
 api_router.include_router(forms.router, prefix=API_PREFIX)
+# v1 form generation — same "/forms" prefix as the legacy router above, kept
+# in a separate file/router rather than added to forms.py. Included AFTER
+# `forms` on purpose: the legacy router's literal GET paths (/forms/models,
+# /forms/submissions, ...) must be matched before this router's catch-all
+# GET /forms/{form_id}, same reasoning form_templates.py uses for /pdf vs
+# /{template_id} — otherwise "models"/"submissions" would be read as a form_id.
+api_router.include_router(form_generation.router, prefix=API_PREFIX)
 api_router.include_router(system.router, prefix=API_PREFIX)
 api_router.include_router(jobs.router, prefix=API_PREFIX)
 api_router.include_router(weather.router, prefix=API_PREFIX)

app/api/routes/form_generation.py

@@ -0,0 +1,176 @@
+"""Contract Layer 3 form generation endpoints (contracts/path/forms.yaml).
+
+Serves POST /forms/generate and the retrieval endpoints at /api/v1/forms,
+backed by the v1 Form model. Handlers are thin; business logic lives in
+app/services/form_generation.py (write path) and app/services/form_fill_worker.py
+(the Celery-dispatched fill). Distinct from the legacy prototype routes in
+app/api/routes/forms.py (int template_id, no incident/batch concept), which
+stay mounted at the same "/forms" prefix unchanged.
+
+/batch/{batch_id} is declared before /{form_id} for the same reason
+form_templates.py declares /pdf before /{template_id}: FastAPI matches paths
+in declaration order, so the literal segment has to come first or "batch"
+gets read as a form_id.
+"""
+
+from uuid import UUID
+
+from fastapi import APIRouter, Depends
+from fastapi.responses import FileResponse, JSONResponse
+from sqlmodel import Session
+
+from app.api.deps import get_db
+from app.api.schemas.enums import FormStatus
+from app.api.schemas.form_generation import (
+    BatchFormEntry,
+    BatchGenerateResponse,
+    BatchStatus,
+    FormMappedJson,
+    FormRecord,
+    GenerateFormsRequest,
+    QueuedForm,
+    SkippedForm,
+)
+from app.core.config import (
+    DATA_DIR,
+    ESTIMATED_FORM_GENERATION_SECONDS,
+    FORM_GENERATION_POLL_INTERVAL_SECONDS,
+)
+from app.core.errors.base import AppError
+from app.db.repositories import get_form, list_forms_by_batch
+from app.services.form_generation import FormGenerationService
+
+router = APIRouter(prefix="/forms", tags=["forms"])
+
+
+@router.post("/generate", response_model=BatchGenerateResponse, status_code=202)
+def generate_forms(body: GenerateFormsRequest, db: Session = Depends(get_db)):
+    result = FormGenerationService().start_generation(db, body)
+    return BatchGenerateResponse(
+        batch_id=result.batch_id,
+        incident_id=result.incident_id,
+        forms_queued=[
+            QueuedForm(form_id=f.form_id, template_id=f.template_id, form_type=f.form_type)
+            for f in result.queued
+        ],
+        forms_skipped=[
+            SkippedForm(template_id=s.template_id, form_type=s.form_type, reason=s.reason)
+            for s in result.skipped
+        ],
+        estimated_seconds=ESTIMATED_FORM_GENERATION_SECONDS,
+        poll_url=f"/api/v1/forms/batch/{result.batch_id}",
+    )
+
+
+@router.get("/batch/{batch_id}", response_model=BatchStatus)
+def get_batch_status(batch_id: UUID, db: Session = Depends(get_db)):
+    forms = list_forms_by_batch(db, batch_id)
+    if not forms:
+        raise AppError(f"Batch {batch_id} not found", status_code=404, error_code="BATCH_NOT_FOUND")
+
+    completed = sum(1 for f in forms if f.status == FormStatus.completed)
+    failed = sum(1 for f in forms if f.status == FormStatus.failed)
+    total = len(forms)
+    done = completed + failed
+    if done < total:
+        status = "processing"
+    elif failed == total:
+        status = "failed"
+    else:
+        # Per design, a per-form failure doesn't fail the batch: the Job (and
+        # this status) reads "completed" as long as every form reached a
+        # terminal state and at least one succeeded — the forms list below
+        # still shows exactly which ones failed.
+        status = "completed"
+
+    return BatchStatus(
+        batch_id=batch_id,
+        status=status,
+        total=total,
+        completed=completed,
+        failed=failed,
+        forms=[
+            BatchFormEntry(
+                form_id=f.form_id,
+                template_id=f.template_id,
+                form_type=f.form_type,
+                status=f.status,
+            )
+            for f in forms
+        ],
+        download_url=None,
+    )
+
+
+@router.get("/{form_id}", response_model=FormRecord)
+def get_form_record(form_id: UUID, db: Session = Depends(get_db)):
+    form = get_form(db, form_id)
+    if not form:
+        raise AppError(f"Form {form_id} not found", status_code=404, error_code="FORM_NOT_FOUND")
+
+    return FormRecord(
+        form_id=form.form_id,
+        template_id=form.template_id,
+        form_type=form.form_type,
+        status=form.status,
+        incident_id=form.incident_id,
+        batch_id=form.batch_id,
+        created_at=form.created_at,
+        completed_at=form.completed_at,
+        pdf_ready=form.pdf_ready,
+        json_ready=form.json_ready,
+        field_mapping_summary=form.field_mapping_summary,
+    )
+
+
+@router.get("/{form_id}/pdf", response_class=FileResponse)
+def download_form_pdf(form_id: UUID, db: Session = Depends(get_db)):
+    form = get_form(db, form_id)
+    if not form:
+        raise AppError(f"Form {form_id} not found", status_code=404, error_code="FORM_NOT_FOUND")
+
+    if form.status == FormStatus.failed:
+        raise AppError(
+            f"Form {form_id} failed to generate",
+            status_code=500,
+            error_code="PDF_GENERATION_FAILED",
+            detail={"reason": "Form generation failed"},
+        )
+
+    if not form.pdf_ready or not form.pdf_path:
+        return JSONResponse(
+            status_code=202,
+            content={
+                "message": "Form generation is still in progress",
+                "status": form.status,
+                "retry_after_seconds": FORM_GENERATION_POLL_INTERVAL_SECONDS,
+            },
+        )
+
+    path = (DATA_DIR / form.pdf_path).resolve()
+    if not path.is_relative_to(DATA_DIR) or not path.is_file():
+        raise AppError(f"Form {form_id} not found", status_code=404, error_code="FORM_NOT_FOUND")
+
+    return FileResponse(path, media_type="application/pdf", filename=path.name)
+
+
+@router.get("/{form_id}/json", response_model=FormMappedJson)
+def get_form_json(form_id: UUID, db: Session = Depends(get_db)):
+    form = get_form(db, form_id)
+    if not form:
+        raise AppError(f"Form {form_id} not found", status_code=404, error_code="FORM_NOT_FOUND")
+
+    if not form.json_ready or form.json_data is None:
+        raise AppError(
+            f"Form {form_id} has no JSON output yet",
+            status_code=404,
+            error_code="FORM_JSON_NOT_READY",
+        )
+
+    return FormMappedJson(
+        form_type=form.form_type,
+        form_id=form.form_id,
+        template_id=form.template_id,
+        incident_id=form.incident_id,
+        agency_fields=form.json_data,
+    )

app/api/schemas/form_generation.py

@@ -0,0 +1,137 @@
+"""Contract Layer 3 form generation schemas (contracts/schemas/form-record.yaml).
+
+Separate from app/api/schemas/forms.py, which holds the legacy prototype
+fill-pipeline shapes (int template_id, no incident/batch concept) still served
+by the old routes in app/api/routes/forms.py. This file is the v1 contract
+shape only — mirrors the extraction.py / templates.py split, one file per
+contract domain.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Literal
+from uuid import UUID
+
+from pydantic import BaseModel, Field
+
+from app.api.schemas.enums import FormStatus, OutputFormat
+
+
+# ---------------------------------------------------------------------------
+# Request
+# ---------------------------------------------------------------------------
+
+class GenerateFormsOptions(BaseModel):
+    output_format: OutputFormat | None = None
+    force_partial: bool = False
+
+
+class GenerateFormsRequest(BaseModel):
+    """POST /forms/generate body.
+
+    template_ids is required in this build: omitting it (generate every
+    template the readiness matrix reports as ready) is #554, not built here.
+    """
+
+    incident_id: UUID
+    template_ids: list[UUID] = Field(min_length=1)
+    options: GenerateFormsOptions | None = None
+
+
+# ---------------------------------------------------------------------------
+# Responses
+# ---------------------------------------------------------------------------
+
+class QueuedForm(BaseModel):
+    form_id: UUID
+    template_id: UUID
+    form_type: str
+
+
+class SkippedForm(BaseModel):
+    template_id: UUID
+    form_type: str
+    reason: str
+
+
+class BatchGenerateResponse(BaseModel):
+    """202 body for POST /forms/generate."""
+
+    batch_id: UUID
+    status: Literal["processing"] = "processing"
+    incident_id: UUID
+    forms_queued: list[QueuedForm] = Field(default_factory=list)
+    forms_skipped: list[SkippedForm] = Field(default_factory=list)
+    estimated_seconds: int | None = None
+    poll_url: str
+
+
+class FieldMappingSummary(BaseModel):
+    total_form_fields: int
+    fields_filled: int
+    fields_blank: int
+    coverage_percent: float
+
+
+class FormRecord(BaseModel):
+    """GET /forms/{form_id} response."""
+
+    form_id: UUID
+    template_id: UUID
+    # form_type is an open string on the wire: registries can add form types
+    # the closed FormType enum does not know about yet (see FormTemplate.form_type).
+    form_type: str
+    status: FormStatus
+    incident_id: UUID
+    batch_id: UUID | None = None
+    created_at: datetime
+    completed_at: datetime | None = None
+    pdf_ready: bool
+    json_ready: bool
+    field_mapping_summary: FieldMappingSummary | None = None
+
+
+class FormMappedJson(BaseModel):
+    """GET /forms/{form_id}/json response."""
+
+    form_type: str
+    form_id: UUID
+    template_id: UUID
+    incident_id: UUID
+    agency_fields: dict = Field(default_factory=dict)
+
+
+class BatchFormEntry(BaseModel):
+    form_id: UUID
+    template_id: UUID
+    form_type: str
+    status: FormStatus
+
+
+class BatchStatus(BaseModel):
+    """GET /forms/batch/{batch_id} response, derived on the fly from the
+    batch's Form rows — there is no Batch table."""
+
+    batch_id: UUID
+    status: Literal["processing", "completed", "failed"]
+    total: int
+    completed: int
+    failed: int
+    forms: list[BatchFormEntry] = Field(default_factory=list)
+    # Zip bundling of a batch's PDFs is #554; always null here.
+    download_url: str | None = None
+
+
+__all__ = [
+    "GenerateFormsOptions",
+    "GenerateFormsRequest",
+    "QueuedForm",
+    "SkippedForm",
+    "BatchGenerateResponse",
+    "FieldMappingSummary",
+    "FormRecord",
+    "FormMappedJson",
+    "BatchFormEntry",
+    "BatchStatus",
+]

app/core/celery.py

@@ -43,6 +43,7 @@ def _check_llm_config(**_kwargs):
     "app.tasks.transcribe",
     "app.tasks.extract",
     "app.tasks.detect_fields",
+    "app.tasks.generate_forms",
 ]
 
 # Optional Celery Beat schedule — runs purge_old_submissions once a day.

app/core/config.py

@@ -181,4 +181,17 @@
     "ogg": "audio/ogg",
     "webm": "audio/webm",
 }
-ALLOWED_AUDIO_EXTENSIONS: frozenset[str] = frozenset(AUDIO_CONTENT_TYPES)
+ALLOWED_AUDIO_EXTENSIONS: frozenset[str] = frozenset(AUDIO_CONTENT_TYPES)
+
+# --- Generated form storage -------------------------------------------------
+# Filled form PDFs land here: {FORMS_OUTPUT_DIR}/{form_id}.pdf. Form.pdf_path
+# stores this DATA_DIR-relative, same convention as FormTemplate.pdf_template_ref.
+FORMS_OUTPUT_DIR = DATA_DIR / "forms" / "generated"
+
+# Advisory estimate returned in the 202 body of POST /forms/generate. Filling
+# is pure lookup-and-draw (no LLM), so this is far below the extraction estimate.
+ESTIMATED_FORM_GENERATION_SECONDS = int(os.getenv("ESTIMATED_FORM_GENERATION_SECONDS", "10"))
+
+# Polling hint returned by GET /forms/{id}/pdf while generation is still in
+# progress. Matches the contract example (contracts/path/forms.yaml).
+FORM_GENERATION_POLL_INTERVAL_SECONDS = 5