From 1e8be242d116d12cfc1b67bbfc6452037d28f57c Mon Sep 17 00:00:00 2001 From: Peter Gorniak Date: Sat, 4 May 2024 14:50:16 -0700 Subject: [PATCH 1/5] file download link instead of direct response --- app.py | 13 ++++++++- assets/lambda/lambda-handlers.py | 45 +++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 93eaf70..b337a03 100644 --- a/app.py +++ b/app.py @@ -108,6 +108,13 @@ def __init__(self, scope, id_, **kwargs): static_website_bucket = s3.Bucket( self, "Dominion Divider Generator Site", + removal_policy=aws_cdk.RemovalPolicy.DESTROY, + auto_delete_objects=True, + lifecycle_rules=[ + s3.LifecycleRule( + expiration=aws_cdk.Duration.days(1), prefix="generated/" + ) + ], ) monitoring_facade.monitor_s3_bucket(bucket=static_website_bucket) @@ -138,10 +145,14 @@ def __init__(self, scope, id_, **kwargs): "GA_CONFIG": self.config.get("GA_CONFIG", ""), "LOG_LEVEL": self.config.get("LOG_LEVEL", "INFO"), "FONT_DIR": self.config.get("FONT_DIR", ""), + "OUTPUT_BUCKET": static_website_bucket.bucket_name, + "OUTPUT_PREFIX": "generated/", + "OUTPUT_URL_PREFIX": f"{static_website_bucket.s3_url_for_object('generated/')}", }, timeout=aws_cdk.Duration.seconds(60), - memory_size=1024, + memory_size=4096, ) + static_website_bucket.grant_write(flask_app) monitoring_facade.monitor_lambda_function(lambda_function=flask_app) api = apig.LambdaRestApi( diff --git a/assets/lambda/lambda-handlers.py b/assets/lambda/lambda-handlers.py index abbfb94..867cd9b 100644 --- a/assets/lambda/lambda-handlers.py +++ b/assets/lambda/lambda-handlers.py @@ -1,8 +1,14 @@ import base64 +import binascii +import functools +import io import json import os +import random +import string import sys +import boto3 from loguru import logger import apig_wsgi import domdiv @@ -37,6 +43,12 @@ apig_wsgi_handler = apig_wsgi.make_lambda_handler(flask_app, binary_support=True) + +@functools.lru_cache(maxsize=1) +def get_client(c): + return boto3.client(c) + + if os.environ.get("DEBUG"): apig_wsgi_handler_helper = apig_wsgi_handler @@ -52,6 +64,19 @@ def get_pages(): return {url_for(p): n for p, n in PAGES.items()} +def upload_to_s3(buf): + s3 = get_client("s3") + tag = "".join(random.choice(string.ascii_letters) for i in range(6)) + fname = f"dominion_dividers_{tag}.pdf" + key = f"{os.environ['OUTPUT_PREFIX'].strip('/')}/{fname}" + s3.upload_fileobj( + buf, + os.environ["OUTPUT_BUCKET"], + key, + ) + return fname + + @flask_app.route("/", methods=["GET", "POST"]) def dominion_dividers(): logger.info(f"root call, request is {request}, form is {request.form}") @@ -67,8 +92,12 @@ def dominion_dividers(): logger.info(f"expansion choices: {domdiv.main.get_expansions()}") if form.validate_on_submit(): buf = form.generate() + # boto3 seems to close the fileobj, so make a view that prevents that + buf_view = io.BytesIO(buf.getvalue()) + upload_to_s3(buf_view) + r = send_file( - buf, + buf_view, mimetype="application/pdf", as_attachment=True, download_name="sumpfork_dominion_dividers.pdf", @@ -107,6 +136,13 @@ def tuckboxes(): if form.validate_on_submit(): logger.info(f"tuckbox files: {request.files}") buf = form.generate(files=request.files) + s3 = get_client("s3") + s3.upload_fileobj( + buf, + os.environ["OUTPUT_BUCKET"], + f"tuckbox_generated/tuckbox_{binascii.b2a_base64(os.urandom(6), newline=False)}.pdf", + ) + r = send_file( buf, mimetype="application/pdf", @@ -134,6 +170,13 @@ def chitboxes(): if form.validate_on_submit(): logger.info(f"chitbox files: {request.files}") buf = form.generate(files=request.files) + s3 = get_client("s3") + s3.upload_fileobj( + buf, + os.environ["OUTPUT_BUCKET"], + f"chitbox_generated/chitbox_{binascii.b2a_base64(os.urandom(6), newline=False)}.pdf", + ) + r = send_file( buf, mimetype="application/pdf", From 90d4c313b77e87a199cc41444819e2f8860682ec Mon Sep 17 00:00:00 2001 From: Peter Gorniak Date: Sat, 13 Sep 2025 11:23:52 -0700 Subject: [PATCH 2/5] comment --- assets/lambda/domdiv_form.py | 1 + 1 file changed, 1 insertion(+) diff --git a/assets/lambda/domdiv_form.py b/assets/lambda/domdiv_form.py index 8ac678f..b8db69c 100644 --- a/assets/lambda/domdiv_form.py +++ b/assets/lambda/domdiv_form.py @@ -43,6 +43,7 @@ class DomDivForm(FlaskForm): "andguilds2ndedition": "and Guilds 2nd Edition", "2ndedition": "2nd Edition", "-bigbox2-de": "(Deutsche Big Box v2)", + # next shouldn't be needed anymore, test and take out "risingsun": "Rising Sun", } for choice in expansion_choices: From d90ad2ce4820e1d64234f2f39d7e79c3c0488845 Mon Sep 17 00:00:00 2001 From: Peter Gorniak Date: Sat, 13 Sep 2025 12:26:07 -0700 Subject: [PATCH 3/5] Fix file closed issue --- app.py | 6 ++- assets/lambda/lambda-handlers.py | 92 ++++++++++++++++---------------- 2 files changed, 49 insertions(+), 49 deletions(-) diff --git a/app.py b/app.py index 7f29a3e..bc367b7 100644 --- a/app.py +++ b/app.py @@ -55,6 +55,8 @@ def invalidate_cloudfront_distribution(_, _2): """.format(t=time.time()) +S3_GENERATED_PREFIX = "generated/" + app = aws_cdk.App() @@ -146,7 +148,7 @@ def __init__(self, scope, id_, **kwargs): auto_delete_objects=True, lifecycle_rules=[ s3.LifecycleRule( - expiration=aws_cdk.Duration.days(1), prefix="generated/" + expiration=aws_cdk.Duration.days(1), prefix=S3_GENERATED_PREFIX ) ], ) @@ -187,7 +189,7 @@ def __init__(self, scope, id_, **kwargs): "LOG_LEVEL": self.config.get("LOG_LEVEL", "INFO"), "FONT_DIR": self.config.get("FONT_DIR", ""), "OUTPUT_BUCKET": static_website_bucket.bucket_name, - "OUTPUT_PREFIX": "generated/", + "OUTPUT_PREFIX": S3_GENERATED_PREFIX, "OUTPUT_URL_PREFIX": f"{static_website_bucket.s3_url_for_object('generated/')}", }, timeout=aws_cdk.Duration.seconds(60), diff --git a/assets/lambda/lambda-handlers.py b/assets/lambda/lambda-handlers.py index a3f6016..fdebd3f 100644 --- a/assets/lambda/lambda-handlers.py +++ b/assets/lambda/lambda-handlers.py @@ -12,10 +12,18 @@ import boto3 import domdiv import domdiv.db -import domdiv.main from chitbox_form import ChitboxForm from domdiv_form import DomDivForm -from flask import Flask, abort, jsonify, render_template, request, send_file, url_for +from flask import ( + Flask, + abort, + jsonify, + redirect, + render_template, + request, + send_file, + url_for, +) from flask_bootstrap import Bootstrap4 from flask_uploads import IMAGES from loguru import logger @@ -63,17 +71,28 @@ def get_pages(): return {url_for(p): n for p, n in PAGES.items()} -def upload_to_s3(buf): +def upload_pdf_to_s3(buf, file_type): + """Upload PDF to S3 and return CloudFront URL""" s3 = get_client("s3") tag = "".join(random.choice(string.ascii_letters) for i in range(6)) - fname = f"dominion_dividers_{tag}.pdf" + fname = f"{file_type}_{tag}.pdf" key = f"{os.environ['OUTPUT_PREFIX'].strip('/')}/{fname}" + s3.upload_fileobj( buf, os.environ["OUTPUT_BUCKET"], key, + ExtraArgs={ + "ContentType": "application/pdf", + "ContentDisposition": f'attachment; filename="{fname}"', + }, ) - return fname + + # Generate the CloudFront URL for download + static_url = os.environ["STATIC_WEB_URL"].rstrip("/") + output_prefix = os.environ["OUTPUT_PREFIX"].strip("/") + download_url = f"{static_url}/{output_prefix}/{fname}" + return download_url @flask_app.route("/", methods=["GET", "POST"]) @@ -91,18 +110,13 @@ def dominion_dividers(): logger.info(f"expansion choices: {domdiv.db.get_expansions()}") if form.validate_on_submit(): buf = form.generate() - # boto3 seems to close the fileobj, so make a view that prevents that - buf_view = io.BytesIO(buf.getvalue()) - upload_to_s3(buf_view) - - r = send_file( - buf_view, - mimetype="application/pdf", - as_attachment=True, - download_name="sumpfork_dominion_dividers.pdf", - ) - logger.info(f"response: {r}") - return r + # Create a copy of the buffer data for S3 upload + buf_data = buf.getvalue() + buf_copy = io.BytesIO(buf_data) + download_url = upload_pdf_to_s3(buf_copy, "dominion_dividers") + + logger.info(f"redirecting to: {download_url}") + return redirect(download_url) # setting the default doesn't seem to work, so override here form.expansions.data = ["dominion2ndEdition"] @@ -135,21 +149,13 @@ def tuckboxes(): if form.validate_on_submit(): logger.info(f"tuckbox files: {request.files}") buf = form.generate(files=request.files) - s3 = get_client("s3") - s3.upload_fileobj( - buf, - os.environ["OUTPUT_BUCKET"], - f"tuckbox_generated/tuckbox_{binascii.b2a_base64(os.urandom(6), newline=False)}.pdf", - ) - - r = send_file( - buf, - mimetype="application/pdf", - as_attachment=True, - download_name="sumpfork_tuckbox.pdf", - ) - logger.info(f"response: {r}") - return r + # Create a copy of the buffer data for S3 upload + buf_data = buf.getvalue() + buf_copy = io.BytesIO(buf_data) + download_url = upload_pdf_to_s3(buf_copy, "tuckbox") + + logger.info(f"redirecting to: {download_url}") + return redirect(download_url) r = render_template( "index.html", pages=PAGES, @@ -169,21 +175,13 @@ def chitboxes(): if form.validate_on_submit(): logger.info(f"chitbox files: {request.files}") buf = form.generate(files=request.files) - s3 = get_client("s3") - s3.upload_fileobj( - buf, - os.environ["OUTPUT_BUCKET"], - f"chitbox_generated/chitbox_{binascii.b2a_base64(os.urandom(6), newline=False)}.pdf", - ) - - r = send_file( - buf, - mimetype="application/pdf", - as_attachment=True, - download_name="sumpfork_chitbox.pdf", - ) - logger.info(f"response: {r}") - return r + # Create a copy of the buffer data for S3 upload + buf_data = buf.getvalue() + buf_copy = io.BytesIO(buf_data) + download_url = upload_pdf_to_s3(buf_copy, "chitbox") + + logger.info(f"redirecting to: {download_url}") + return redirect(download_url) r = render_template( "index.html", pages=PAGES, From 538ef47ceaa5f7eb65fcb484c4de4929f619fa48 Mon Sep 17 00:00:00 2001 From: Peter Gorniak Date: Sat, 13 Sep 2025 12:49:45 -0700 Subject: [PATCH 4/5] Simplify, and bump all versions --- assets/lambda/Dockerfile | 8 +++---- assets/lambda/lambda-handlers.py | 15 +++--------- assets/lambda/requirements.txt | 14 +++++------ requirements.txt | 40 +++++++++++++++++--------------- 4 files changed, 35 insertions(+), 42 deletions(-) diff --git a/assets/lambda/Dockerfile b/assets/lambda/Dockerfile index 4239338..20bb9ea 100644 --- a/assets/lambda/Dockerfile +++ b/assets/lambda/Dockerfile @@ -1,8 +1,8 @@ -FROM public.ecr.aws/lambda/python:3.11-x86_64 +FROM public.ecr.aws/lambda/python:3.13-x86_64 -COPY --from=ghcr.io/astral-sh/uv:0.7.12 /uv /uvx /bin/ +COPY --from=ghcr.io/astral-sh/uv:0.8.17 /uv /uvx /bin/ -RUN yum update -y && yum install -y git libicu-devel gcc gcc-c++ +RUN dnf update -y && dnf install -y git libicu-devel gcc gcc-c++ # Copy requirements.txt COPY requirements.txt . @@ -11,7 +11,7 @@ COPY requirements.txt . RUN uv pip install --system -r requirements.txt # Lots of trouble with newer pyicus on Amazon Linux -RUN uv pip install --system "pyicu<2.9" +RUN uv pip install --system "pyicu" COPY . . diff --git a/assets/lambda/lambda-handlers.py b/assets/lambda/lambda-handlers.py index fdebd3f..8fd3f2a 100644 --- a/assets/lambda/lambda-handlers.py +++ b/assets/lambda/lambda-handlers.py @@ -110,10 +110,7 @@ def dominion_dividers(): logger.info(f"expansion choices: {domdiv.db.get_expansions()}") if form.validate_on_submit(): buf = form.generate() - # Create a copy of the buffer data for S3 upload - buf_data = buf.getvalue() - buf_copy = io.BytesIO(buf_data) - download_url = upload_pdf_to_s3(buf_copy, "dominion_dividers") + download_url = upload_pdf_to_s3(buf, "dominion_dividers") logger.info(f"redirecting to: {download_url}") return redirect(download_url) @@ -149,10 +146,7 @@ def tuckboxes(): if form.validate_on_submit(): logger.info(f"tuckbox files: {request.files}") buf = form.generate(files=request.files) - # Create a copy of the buffer data for S3 upload - buf_data = buf.getvalue() - buf_copy = io.BytesIO(buf_data) - download_url = upload_pdf_to_s3(buf_copy, "tuckbox") + download_url = upload_pdf_to_s3(buf, "tuckbox") logger.info(f"redirecting to: {download_url}") return redirect(download_url) @@ -175,10 +169,7 @@ def chitboxes(): if form.validate_on_submit(): logger.info(f"chitbox files: {request.files}") buf = form.generate(files=request.files) - # Create a copy of the buffer data for S3 upload - buf_data = buf.getvalue() - buf_copy = io.BytesIO(buf_data) - download_url = upload_pdf_to_s3(buf_copy, "chitbox") + download_url = upload_pdf_to_s3(buf, "chitbox") logger.info(f"redirecting to: {download_url}") return redirect(download_url) diff --git a/assets/lambda/requirements.txt b/assets/lambda/requirements.txt index 2e218cc..6281868 100644 --- a/assets/lambda/requirements.txt +++ b/assets/lambda/requirements.txt @@ -1,12 +1,12 @@ # This file was autogenerated by uv via the following command: # uv pip compile requirements.in -o requirements.txt -apig-wsgi==2.19.0 +apig-wsgi==2.20.0 # via -r requirements.in blinker==1.9.0 # via flask bootstrap-flask==2.5.0 # via -r requirements.in -chardet==5.2.0 +charset-normalizer==3.4.3 # via reportlab chitboxes==0.0.1 # via -r requirements.in @@ -14,9 +14,9 @@ click==8.2.1 # via flask configargparse==1.7.1 # via domdiv -domdiv==4.9.2 +domdiv==4.9.3 # via -r requirements.in -flask==3.1.1 +flask==3.1.2 # via # bootstrap-flask # flask-uploads @@ -41,17 +41,17 @@ markupsafe==3.0.2 # jinja2 # werkzeug # wtforms -numpy==2.2.6 +numpy==2.3.3 # via # chitboxes # tuckboxes -pillow==11.2.1 +pillow==11.3.0 # via # chitboxes # domdiv # reportlab # tuckboxes -reportlab==4.4.1 +reportlab==4.4.3 # via # chitboxes # domdiv diff --git a/requirements.txt b/requirements.txt index d1f9f40..c243992 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,39 +4,39 @@ attrs==25.3.0 # via # cattrs # jsii -aws-cdk-asset-awscli-v1==2.2.237 +aws-cdk-asset-awscli-v1==2.2.242 # via aws-cdk-lib aws-cdk-asset-node-proxy-agent-v6==2.1.0 # via aws-cdk-lib -aws-cdk-aws-lambda-python-alpha==2.199.0a0 +aws-cdk-aws-lambda-python-alpha==2.214.0a0 # via -r requirements.in -aws-cdk-cloud-assembly-schema==41.2.0 +aws-cdk-cloud-assembly-schema==48.9.0 # via aws-cdk-lib -aws-cdk-lib==2.199.0 +aws-cdk-lib==2.214.0 # via # -r requirements.in # aws-cdk-aws-lambda-python-alpha # cdk-monitoring-constructs -cattrs==24.1.3 +cattrs==25.2.0 # via jsii -cdk-monitoring-constructs==9.12.0 +cdk-monitoring-constructs==9.15.2 # via -r requirements.in -certifi==2025.4.26 +certifi==2025.8.3 # via requests cfgv==3.4.0 # via pre-commit -charset-normalizer==3.4.2 +charset-normalizer==3.4.3 # via requests constructs==10.4.2 # via # aws-cdk-aws-lambda-python-alpha # aws-cdk-lib # cdk-monitoring-constructs -distlib==0.3.9 +distlib==0.4.0 # via virtualenv -filelock==3.18.0 +filelock==3.19.1 # via virtualenv -identify==2.6.12 +identify==2.6.14 # via pre-commit idna==3.10 # via requests @@ -44,7 +44,7 @@ importlib-resources==6.5.2 # via jsii jinja2==3.1.6 # via -r requirements.in -jsii==1.112.0 +jsii==1.114.1 # via # aws-cdk-asset-awscli-v1 # aws-cdk-asset-node-proxy-agent-v6 @@ -57,9 +57,9 @@ markupsafe==3.0.2 # via jinja2 nodeenv==1.9.1 # via pre-commit -platformdirs==4.3.8 +platformdirs==4.4.0 # via virtualenv -pre-commit==4.2.0 +pre-commit==4.3.0 # via -r requirements.in publication==0.0.3 # via @@ -77,7 +77,7 @@ pyyaml==6.0.2 # via # -r requirements.in # pre-commit -requests==2.32.3 +requests==2.32.5 # via -r requirements.in six==1.17.0 # via python-dateutil @@ -91,9 +91,11 @@ typeguard==2.13.3 # cdk-monitoring-constructs # constructs # jsii -typing-extensions==4.14.0 - # via jsii -urllib3==2.4.0 +typing-extensions==4.15.0 + # via + # cattrs + # jsii +urllib3==2.5.0 # via requests -virtualenv==20.31.2 +virtualenv==20.34.0 # via pre-commit From fc3547556bb784c4b584ae2434b981ffd8c23aeb Mon Sep 17 00:00:00 2001 From: Peter Gorniak Date: Sat, 13 Sep 2025 12:53:35 -0700 Subject: [PATCH 5/5] bump precommit and run them --- .pre-commit-config.yaml | 4 ++-- app.py | 12 ++++++------ assets/lambda/chitbox_form.py | 9 +++++---- assets/lambda/domdiv_form.py | 2 +- assets/lambda/tuckbox_form.py | 11 ++++++----- 5 files changed, 20 insertions(+), 18 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0539568..5f67d4c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.4.8 + rev: v0.13.0 hooks: # Run the linter. - id: ruff @@ -14,7 +14,7 @@ repos: # Run the formatter. - id: ruff-format - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v6.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer diff --git a/app.py b/app.py index bc367b7..3c8e9a3 100644 --- a/app.py +++ b/app.py @@ -68,12 +68,12 @@ def __init__(self, scope, id_, **kwargs): # protect production stacks from accidental deletion kwargs["termination_protection"] = self.config.get("TERMINATION_PROTECTION") - assert ( - "SECRET_KEY" in self.config - ), "Need random SECRET_KEY specified in config.json" - assert ( - "CERTIFICATE_ARN" in self.config - ), "Need CERTIFICATE_ARN specified in config.json" + assert "SECRET_KEY" in self.config, ( + "Need random SECRET_KEY specified in config.json" + ) + assert "CERTIFICATE_ARN" in self.config, ( + "Need CERTIFICATE_ARN specified in config.json" + ) self.stage = self.config["STAGE"] self.stackname = f"{id_}-{self.stage}" diff --git a/assets/lambda/chitbox_form.py b/assets/lambda/chitbox_form.py index 9dbfb63..4e91785 100644 --- a/assets/lambda/chitbox_form.py +++ b/assets/lambda/chitbox_form.py @@ -1,11 +1,12 @@ from io import BytesIO import wtforms.fields as wtf_fields -from wtforms import validators -from flask_wtf import FlaskForm -from flask_wtf.file import FileField as FlaskFileField, FileAllowed -from flask_uploads import IMAGES from chitboxes.chitboxes import ChitBoxGenerator +from flask_uploads import IMAGES +from flask_wtf import FlaskForm +from flask_wtf.file import FileAllowed +from flask_wtf.file import FileField as FlaskFileField +from wtforms import validators class ChitboxForm(FlaskForm): diff --git a/assets/lambda/domdiv_form.py b/assets/lambda/domdiv_form.py index b8db69c..8dfa9a8 100644 --- a/assets/lambda/domdiv_form.py +++ b/assets/lambda/domdiv_form.py @@ -220,7 +220,7 @@ class DomDivForm(FlaskForm): default=False, ) expansion_dividers_long_name = wtf_fields.BooleanField( - label=("If Expansion Dividers, use edition " "on expansion dividers names"), + label=("If Expansion Dividers, use edition on expansion dividers names"), default=False, ) set_icon = wtf_fields.SelectField( diff --git a/assets/lambda/tuckbox_form.py b/assets/lambda/tuckbox_form.py index 7412439..aab6b38 100644 --- a/assets/lambda/tuckbox_form.py +++ b/assets/lambda/tuckbox_form.py @@ -1,13 +1,14 @@ -from io import BytesIO import re +from io import BytesIO -from loguru import logger import wtforms.fields as wtf_fields -from wtforms import validators -from flask_wtf import FlaskForm -from flask_wtf.file import FileField as FlaskFileField, FileAllowed from flask_uploads import IMAGES +from flask_wtf import FlaskForm +from flask_wtf.file import FileAllowed +from flask_wtf.file import FileField as FlaskFileField +from loguru import logger from tuckboxes.tuckboxes import TuckBoxGenerator +from wtforms import validators class TuckboxForm(FlaskForm):