wbruno/PFTR-Processor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implemented the memory fix.

Browse Source 
Changed app.py:
•
process_pft_pdf() now uses the uploaded file stream directly instead of copying the PDF into bytes.
•
Output now uses SpooledTemporaryFile, so larger generated PDFs spill to disk instead of staying fully in RAM.
•
Explicitly closes PdfReader objects after writing.
•
Removed @app.teardown_request gc.collect(), which was running too early and could break send_file.
•
Added response.call_on_close() cleanup so the generated PDF stream is closed only after Flask/Werkzeug finishes sending it.
Changed Dockerfile:
•
Added Gunicorn --max-requests 50 --max-requests-jitter 10 as a production guardrail against PDF-library or allocator memory growth.
This commit is contained in:
wbruno
2026-06-14 19:04:04 -04:00
parent b8bb31842a
commit de0b3ccdbc
2 changed files with 123 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockerfile
+1 -1
View File
@@ -15,4 +15,4 @@ COPY . .
EXPOSE 5000
# Command to run the application using Gunicorn (production ready)
CMD ["gunicorn", "--bind", "0.0.0.0:5000", "app:app"]
CMD ["gunicorn", "--bind", "0.0.0.0:5000", "--max-requests", "50", "--max-requests-jitter", "10", "app:app"]
app.py
+32 -26
View File
@@ -2,15 +2,15 @@ import sys
import gc
import re
import json
import io
import logging
import tempfile
from pathlib import Path
from datetime import datetime
import pdfplumber
from pypdf import PdfReader, PdfWriter
from pypdf.constants import UserAccessPermissions
from pypdf.generic import NameObject, BooleanObject
from typing import TypedDict, Literal
from typing import BinaryIO, TypedDict, Literal
from flask import Flask, request, send_file, abort
from werkzeug.utils import secure_filename
@@ -45,7 +45,7 @@ class ParsedPdfData(TypedDict):
highlighted_box: InterpretationType
def process_pft_pdf(input_bytes: bytes) -> io.BytesIO:
def process_pft_pdf(input_stream: BinaryIO) -> BinaryIO:
text_mapping: dict[str, InterpretationType] = {
"normal": "normal",
"pv disorders": "pvd",
@@ -70,8 +70,11 @@ def process_pft_pdf(input_bytes: bytes) -> io.BytesIO:
"highlighted_box": "normal"
}
# Use a detached context for pdfplumber to read the file metadata
with pdfplumber.open(io.BytesIO(input_bytes)) as pdf:
input_stream.seek(0)
# Use the upload stream directly so each request does not duplicate the PDF
# in memory before parsing.
with pdfplumber.open(input_stream) as pdf:
if len(pdf.pages) < 2:
logger.error(
"Expected a 2-page PFT report, found %s page(s).", len(pdf.pages))
@@ -82,9 +85,8 @@ def process_pft_pdf(input_bytes: bytes) -> io.BytesIO:
page2 = pdf.pages[1]
# --- 1. FULL TEXT METADATA EXTRACTION (Page 1) ---
full_text_p1 = page1.extract_text()
full_text_p1 = page1.extract_text() or ""
if full_text_p1:
lines = [line.strip()
for line in full_text_p1.split("\n") if line.strip()]
for line in lines:
@@ -106,11 +108,11 @@ def process_pft_pdf(input_bytes: bytes) -> io.BytesIO:
record["smoker"] = smoker_match.group(1).capitalize()
# --- 2. INTERPRETATION DIAGRAM PROCESSING (Page 2) ---
text_p2 = page2.extract_text()
text_p2 = page2.extract_text() or ""
detected_box: InterpretationType = "normal"
box_found = False
if text_p2 and "System Summary Interpretation" in text_p2:
if "System Summary Interpretation" in text_p2:
summary_text = text_p2.split(
"System Summary Interpretation")[-1].strip().lower()
for phrase, token in text_mapping.items():
@@ -154,8 +156,10 @@ def process_pft_pdf(input_bytes: bytes) -> io.BytesIO:
f"Template file '{TEMPLATE_NAME}' missing inside container workspace.")
writer = PdfWriter()
template_reader = PdfReader(TEMPLATE_NAME)
raw_results_reader: PdfReader | None = None
try:
writer.append(template_reader)
patient_data = {"patient_name": "",
@@ -230,7 +234,8 @@ def process_pft_pdf(input_bytes: bytes) -> io.BytesIO:
writer.remove_annotations(subtypes="/Widget")
# Append raw input pages directly onto the end of our current working context
raw_results_reader = PdfReader(io.BytesIO(input_bytes))
input_stream.seek(0)
raw_results_reader = PdfReader(input_stream)
writer.append(raw_results_reader)
# Secure document permissions
@@ -243,10 +248,14 @@ def process_pft_pdf(input_bytes: bytes) -> io.BytesIO:
permissions_flag=permissions
)
# Output final joined stream directly to Flask
output_stream = io.BytesIO()
# Keep small responses in memory, but spill larger PDFs to a temp file.
output_stream = tempfile.SpooledTemporaryFile(max_size=8 * 1024 * 1024, mode="w+b")
writer.write(output_stream)
output_stream.seek(0)
finally:
template_reader.close()
if raw_results_reader is not None:
raw_results_reader.close()
logger.info("PDTR successfully prepended to PFT result PDF.")
return output_stream
@@ -268,20 +277,24 @@ def upload_file():
logger.info("Received PDF: %s", original_filename)
try:
# Read input file directly into memory bytes
raw_input_bytes = file.read()
# Execute logic sequentially
processed_file_stream = process_pft_pdf(raw_input_bytes)
# Execute logic sequentially using the upload stream directly.
processed_file_stream = process_pft_pdf(file.stream)
logger.info("Returning processed PDF file: %s", original_filename)
return send_file(
response = send_file(
processed_file_stream,
mimetype='application/pdf',
as_attachment=True,
download_name=original_filename
)
@response.call_on_close
def close_processed_file_stream() -> None:
processed_file_stream.close()
gc.collect()
return response
except Exception as e:
logger.exception("Error during execution processing pipeline")
return abort(500, f"Error processing file: {str(e)}")
@@ -323,12 +336,5 @@ def index():
"""
@app.teardown_request
def spi_clear_memory(exception=None):
"""Runs automatically after every single request."""
# Force collection of unreferenced objects
gc.collect()
if __name__ == "__main__":
app.run(host='0.0.0.0', port=5000)