From 53d5a6cc35b13919275ead7c6a4fea3df08b530d Mon Sep 17 00:00:00 2001 From: Mischback Date: Tue, 14 Jan 2020 21:44:47 +0100 Subject: [PATCH 001/110] Changed, how tox manages its requirements This is just applied for the util environment. It is changed to use the provided requirements/development.txt --- requirements/development.txt | 17 ++--------------- tox.ini | 2 +- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/requirements/development.txt b/requirements/development.txt index 79be0ae..f033fe7 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -1,15 +1,2 @@ -# coverage measures code coverage of Python programs -# See http://coverage.readthedocs.io/en/latest/ -coverage - -# **tox** automates testing -# See https://tox.readthedocs.io/en/latest/index.html -tox - -# **Sphinx** is used to create documentation -# See http://www.sphinx-doc.org/en/stable/tutorial.html -Sphinx - -# **flake8** is used to check for PEP8 compliance -# See http://flake8.pycqa.org/en/latest/index.html -flake8 +# currently used Django version +Django>=2.2, <3.0 diff --git a/tox.ini b/tox.ini index 7a15db0..a0a3efe 100644 --- a/tox.ini +++ b/tox.ini @@ -40,7 +40,7 @@ commands = basepython = python3 envdir = {toxworkdir}/current deps = - Django>=2.2, <3.0 + -r{toxinidir}/requirements/development.txt skip_install = true changedir = {envtmpdir} commands = From 781f84d38ed0f64194c3c688443673ae4335aefb Mon Sep 17 00:00:00 2001 From: Mischback Date: Fri, 17 Jan 2020 17:52:25 +0100 Subject: [PATCH 002/110] Building the context for Docker --- .dockerignore | 39 +++++++++++++++++++++++++++++++++++++++ Makefile | 19 ++++++++++++++++++- 2 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..8799979 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,39 @@ +# experimental .dockerignore + +# do NOT track Python compilation stuff +*.py[cod] +**/__python__ + +# ignore .git-related stuff +.git +**/.gitignore + +# ignore tox-related stuff +.tox +tox.ini + +# ignore any development related files +.dockerignore +.editorconfig +.travis.yml +Makefile + +# do NOT include ANY readme files +**/README* + +# do NOT include the documentation +# TODO: the original doc-folder of django-project-skeleton should be removed +# during initialising the project. +doc + +# do not include the Dockerfiles +configs/Docker + +# do not include any samples +**/*.sample + +# I started to put temporary notes into TODO files +**/*.todo + +# TODO: .gitignore excludes compiled language files. Is it safe to exclude the +# uncompiled language files for 'production'? diff --git a/Makefile b/Makefile index 6fe1924..c64f067 100644 --- a/Makefile +++ b/Makefile @@ -4,12 +4,20 @@ all: echo "" + echo "Development utilities" echo " clean Removes all temporary files" echo " current Runs the tox-environment for the current development" echo " doc Builds the documentation using 'Sphinx'" echo " doc-srv Serves the documentation on port 8082 (and automatically builds it)" echo " serve Runs the Django development server on port 8080" - echo " tox Runs complete tox test" + echo " tox Runs complete tox test environment" + echo "" + echo "Docker related commands" + echo " Please note, that if you are not 'root', you have to do 'sudo make ...'!" + echo "" + echo " docker/test-build-context" + echo " Builds a minimal docker container and shows the context" + echo " This is used to check the .dockerignore file" echo "" @@ -32,3 +40,12 @@ serve: tox: tox -q + +docker/test-build-context: + echo " \ + FROM busybox\n \ + COPY . /build-context\n \ + WORKDIR /build-context\n \ + CMD find ." \ + | docker build -t test-build-context -f- . \ + && docker container run --rm test-build-context From 06d2685a39ca4b80a92d70d1e6a00931c83a1acb Mon Sep 17 00:00:00 2001 From: Mischback Date: Fri, 17 Jan 2020 17:59:43 +0100 Subject: [PATCH 003/110] Minor Makefile improvements --- Makefile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c64f067..c92c0d3 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,14 @@ .SILENT: -.PHONY: all clean current doc doc-srv serve tox +.PHONY: clean current default doc doc-srv help serve tox -all: +default: help + echo "" + echo "You need to specify a subcommand." + exit 1 + + +help: echo "" echo "Development utilities" echo " clean Removes all temporary files" From 122608091abfc6f72aec48384ce003c9c2b69fcb Mon Sep 17 00:00:00 2001 From: Mischback Date: Mon, 20 Jan 2020 22:01:24 +0100 Subject: [PATCH 004/110] Working on Django Dockerfile --- configs/.gitignore | 12 ------ configs/Docker/django/Dockerfile | 69 ++++++++++++++++++++++++++++++++ requirements/production.txt | 3 ++ 3 files changed, 72 insertions(+), 12 deletions(-) delete mode 100644 configs/.gitignore create mode 100644 configs/Docker/django/Dockerfile create mode 100644 requirements/production.txt diff --git a/configs/.gitignore b/configs/.gitignore deleted file mode 100644 index 2762a76..0000000 --- a/configs/.gitignore +++ /dev/null @@ -1,12 +0,0 @@ -# specific .gitignore for this directory - -# ignore everything -* - -# except the explaining README -!README -# oh, and this file -!.gitignore - -# let's include samples of the config files -!*.sample diff --git a/configs/Docker/django/Dockerfile b/configs/Docker/django/Dockerfile new file mode 100644 index 0000000..1916ac3 --- /dev/null +++ b/configs/Docker/django/Dockerfile @@ -0,0 +1,69 @@ +############################################################################## +# STAGE 01 +# +# Fetching the base image and activate a virtual environment. +############################################################################## + +# base this on a slim Debian image running nothing more than Python +# Why Debian and not Alpine? Personal preference of Debian-based systems. +# Additionally, Alpine is not using glibc, which might introduce problems. If +# you want your image even smaller, you may switch to an Alpine based-image. +FROM python:3.8-slim-buster as stage01_activate_venv + +# create the virtual environment for our project +# TODO: the two environment variables are actually like "activating" the +# virtual environment. But they are based off a tutorial that uses +# "virtualenv" instead of "stdlib's venv". Verify, that this actually +# activates the environment. +ENV VIRTUAL_ENV=/opt/venv +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +RUN python3 -m venv $VIRTUAL_ENV + +# update pip inside of the virtual environment +RUN pip install --upgrade pip + + +############################################################################## +# STAGE 02 +# +# Using the requirements to build wheels for the final image. +############################################################################## +FROM stage01_activate_venv as stage02_building_wheels + +COPY ./requirements/production.txt ./requirements.txt + +# build wheels from requirements +RUN pip wheel --no-cache-dir --wheel-dir /wheels -r requirements.txt + + +############################################################################## +# STAGE 03 +# +# Actually install the packages in the virtual environment. +############################################################################## +FROM stage01_activate_venv as stage03_installing_packages + +# set environment variables +# TODO: Compare this to tox's settings! +# TODO: Check if these are still valid for "production"! +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 + +# install dependencies +COPY --from=stage02_building_wheels /wheels /wheels +RUN pip install --no-cache /wheels/* + + +############################################################################## +# STAGE 04 +# +# Prepare the Django environment, performing the following tasks: +# - preparing the database by running django-admin migrate +# - running django-admin collectstatic +############################################################################## +FROM stage03_installing_packages + +# create a user to be used (meaning: we won't run with root-permissions!) +#RUN useradd --create-home pythonuser +#USER pythonuser +#WORKDIR /home/pythonuser diff --git a/requirements/production.txt b/requirements/production.txt new file mode 100644 index 0000000..9d1dd96 --- /dev/null +++ b/requirements/production.txt @@ -0,0 +1,3 @@ +# currently used Django version +Django==2.2.9 +gunicorn From 7bae76b8b77b1b35a2ed139dc6aa453cb29d11fa Mon Sep 17 00:00:00 2001 From: Mischback Date: Mon, 27 Jan 2020 20:48:35 +0100 Subject: [PATCH 005/110] Switching to userspace --- configs/Docker/django/Dockerfile | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/configs/Docker/django/Dockerfile b/configs/Docker/django/Dockerfile index 1916ac3..f7e9264 100644 --- a/configs/Docker/django/Dockerfile +++ b/configs/Docker/django/Dockerfile @@ -30,6 +30,12 @@ RUN pip install --upgrade pip ############################################################################## FROM stage01_activate_venv as stage02_building_wheels +# set environment variables +# TODO: Compare this to tox's settings! +# TODO: Check if these are still valid for "production"! +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 + COPY ./requirements/production.txt ./requirements.txt # build wheels from requirements @@ -46,12 +52,13 @@ FROM stage01_activate_venv as stage03_installing_packages # set environment variables # TODO: Compare this to tox's settings! # TODO: Check if these are still valid for "production"! -ENV PYTHONDONTWRITEBYTECODE 1 -ENV PYTHONUNBUFFERED 1 +#ENV PYTHONDONTWRITEBYTECODE 1 +#ENV PYTHONUNBUFFERED 1 # install dependencies COPY --from=stage02_building_wheels /wheels /wheels -RUN pip install --no-cache /wheels/* +RUN pip install --no-cache /wheels/* \ + && rm -rf /wheels ############################################################################## @@ -64,6 +71,10 @@ RUN pip install --no-cache /wheels/* FROM stage03_installing_packages # create a user to be used (meaning: we won't run with root-permissions!) -#RUN useradd --create-home pythonuser -#USER pythonuser -#WORKDIR /home/pythonuser +RUN useradd -u 2342 -g 65534 -s /bin/false -M pythonuser \ + && mkdir /home/pythonuser +WORKDIR /home/pythonuser +COPY . . +RUN chown -R 2342:65534 * + +USER pythonuser From d3195c269ef764266d559a7fa5042bfa684326a8 Mon Sep 17 00:00:00 2001 From: Mischback Date: Mon, 27 Jan 2020 22:59:11 +0100 Subject: [PATCH 006/110] Started docker-compose.yml --- Makefile | 3 +++ configs/Docker/docker-compose.yml | 12 ++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 configs/Docker/docker-compose.yml diff --git a/Makefile b/Makefile index c92c0d3..9e15b09 100644 --- a/Makefile +++ b/Makefile @@ -55,3 +55,6 @@ docker/test-build-context: CMD find ." \ | docker build -t test-build-context -f- . \ && docker container run --rm test-build-context + +docker/build: + docker-compose -f configs/Docker/docker-compose.yml build diff --git a/configs/Docker/docker-compose.yml b/configs/Docker/docker-compose.yml new file mode 100644 index 0000000..dfaeec4 --- /dev/null +++ b/configs/Docker/docker-compose.yml @@ -0,0 +1,12 @@ +version: '3.5' + +services: + django: + build: + context: ../../ + dockerfile: configs/Docker/django/Dockerfile + # this should *magically* include a hash/datetime/unique identifier + image: dps-testing:653ab5f + command: python manage.py runserver 0.0.0.0:8000 + ports: + - 8000:8000 From e1ca71a32b461076011e2d9350455807b5ac20b0 Mon Sep 17 00:00:00 2001 From: Mischback Date: Tue, 28 Jan 2020 20:46:25 +0100 Subject: [PATCH 007/110] Testing another Dockerfile structure --- configs/Docker/django/Dockerfile | 74 +++++++------------------------ configs/Docker/docker-compose.yml | 2 +- 2 files changed, 18 insertions(+), 58 deletions(-) diff --git a/configs/Docker/django/Dockerfile b/configs/Docker/django/Dockerfile index f7e9264..28bfe71 100644 --- a/configs/Docker/django/Dockerfile +++ b/configs/Docker/django/Dockerfile @@ -1,80 +1,40 @@ -############################################################################## -# STAGE 01 -# -# Fetching the base image and activate a virtual environment. -############################################################################## - # base this on a slim Debian image running nothing more than Python # Why Debian and not Alpine? Personal preference of Debian-based systems. # Additionally, Alpine is not using glibc, which might introduce problems. If # you want your image even smaller, you may switch to an Alpine based-image. -FROM python:3.8-slim-buster as stage01_activate_venv +FROM python:3.8-slim-buster as base + +# set environment variables +# TODO: Check if these are still valid for "production"! +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 # create the virtual environment for our project -# TODO: the two environment variables are actually like "activating" the -# virtual environment. But they are based off a tutorial that uses -# "virtualenv" instead of "stdlib's venv". Verify, that this actually -# activates the environment. -ENV VIRTUAL_ENV=/opt/venv +ENV VIRTUAL_ENV=/venv ENV PATH="$VIRTUAL_ENV/bin:$PATH" RUN python3 -m venv $VIRTUAL_ENV # update pip inside of the virtual environment RUN pip install --upgrade pip - -############################################################################## -# STAGE 02 -# -# Using the requirements to build wheels for the final image. -############################################################################## -FROM stage01_activate_venv as stage02_building_wheels - -# set environment variables -# TODO: Compare this to tox's settings! -# TODO: Check if these are still valid for "production"! -ENV PYTHONDONTWRITEBYTECODE 1 -ENV PYTHONUNBUFFERED 1 - +# fetch the production requirements COPY ./requirements/production.txt ./requirements.txt -# build wheels from requirements -RUN pip wheel --no-cache-dir --wheel-dir /wheels -r requirements.txt - - -############################################################################## -# STAGE 03 -# -# Actually install the packages in the virtual environment. -############################################################################## -FROM stage01_activate_venv as stage03_installing_packages - -# set environment variables -# TODO: Compare this to tox's settings! -# TODO: Check if these are still valid for "production"! -#ENV PYTHONDONTWRITEBYTECODE 1 -#ENV PYTHONUNBUFFERED 1 - -# install dependencies -COPY --from=stage02_building_wheels /wheels /wheels -RUN pip install --no-cache /wheels/* \ - && rm -rf /wheels +# actually install the requirements into the virtual environment +RUN pip install --no-cache-dir -r requirements.txt -############################################################################## -# STAGE 04 -# -# Prepare the Django environment, performing the following tasks: -# - preparing the database by running django-admin migrate -# - running django-admin collectstatic -############################################################################## -FROM stage03_installing_packages +FROM python:3.8-slim-buster # create a user to be used (meaning: we won't run with root-permissions!) RUN useradd -u 2342 -g 65534 -s /bin/false -M pythonuser \ && mkdir /home/pythonuser WORKDIR /home/pythonuser -COPY . . -RUN chown -R 2342:65534 * + +# fetch the virtual environment from base +COPY --from=base --chown=2342:65534 /venv /venv + +COPY --chown=2342:65534 . . +# RUN chown -R 2342:65534 * USER pythonuser diff --git a/configs/Docker/docker-compose.yml b/configs/Docker/docker-compose.yml index dfaeec4..bdc0472 100644 --- a/configs/Docker/docker-compose.yml +++ b/configs/Docker/docker-compose.yml @@ -6,7 +6,7 @@ services: context: ../../ dockerfile: configs/Docker/django/Dockerfile # this should *magically* include a hash/datetime/unique identifier - image: dps-testing:653ab5f + image: mischback/dps:testing-${DPS} command: python manage.py runserver 0.0.0.0:8000 ports: - 8000:8000 From 050faa3d144e30905075bcfd610943af6dafc5e3 Mon Sep 17 00:00:00 2001 From: Mischback Date: Wed, 29 Jan 2020 22:13:32 +0100 Subject: [PATCH 008/110] Tag docker builds with git's commit hash --- Makefile | 6 +++++- configs/Docker/docker-compose.yml | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9e15b09..116f529 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,8 @@ .SILENT: -.PHONY: clean current default doc doc-srv help serve tox +.PHONY: clean current default doc doc-srv help serve tox \ + docker/build docker/test-build-context + +DPS_COMMIT_SHA1:=$(shell git rev-parse --short HEAD) default: help @@ -57,4 +60,5 @@ docker/test-build-context: && docker container run --rm test-build-context docker/build: + DPS_COMMIT_SHA1=$(DPS_COMMIT_SHA1) \ docker-compose -f configs/Docker/docker-compose.yml build diff --git a/configs/Docker/docker-compose.yml b/configs/Docker/docker-compose.yml index bdc0472..a4d45cb 100644 --- a/configs/Docker/docker-compose.yml +++ b/configs/Docker/docker-compose.yml @@ -6,7 +6,7 @@ services: context: ../../ dockerfile: configs/Docker/django/Dockerfile # this should *magically* include a hash/datetime/unique identifier - image: mischback/dps:testing-${DPS} + image: mischback/dps:testing-${DPS_COMMIT_SHA1} command: python manage.py runserver 0.0.0.0:8000 ports: - 8000:8000 From f4ad8d01adc9718c470c1ce0e63ce515c9ad929f Mon Sep 17 00:00:00 2001 From: Mischback Date: Wed, 29 Jan 2020 23:01:05 +0100 Subject: [PATCH 009/110] Update Dockerfile to automatically generate requirements --- configs/Docker/django/Dockerfile | 34 +++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/configs/Docker/django/Dockerfile b/configs/Docker/django/Dockerfile index 28bfe71..ac63ece 100644 --- a/configs/Docker/django/Dockerfile +++ b/configs/Docker/django/Dockerfile @@ -17,24 +17,48 @@ RUN python3 -m venv $VIRTUAL_ENV # update pip inside of the virtual environment RUN pip install --upgrade pip + +FROM base as requirements_compiler + +# TODO: install the requirements for Python dependency management +# RUN pip install pip-tools + +# TODO: fetch the requirements-directory +# COPY ./requirements/*.in /requirements + +# TODO: actually build the requirements for production environment +# TODO: if layered/included requirements are used, the files have to be compiled +# in the correct order. +# RUN pip-compile --generate-hashes \ +# --output-file /requirements/common.txt \ +# /requirements/common.in && \ +# pip-compile --generate-hashes \ +# --output-file /requirements/production.txt \ +# /requirements/production.in + + +FROM base as venv_builder + # fetch the production requirements COPY ./requirements/production.txt ./requirements.txt +# COPY --from=requirements_compiler /requirements/production.txt ./requirements.txt # actually install the requirements into the virtual environment RUN pip install --no-cache-dir -r requirements.txt -FROM python:3.8-slim-buster +FROM python:3.8-slim-buster as django_production # create a user to be used (meaning: we won't run with root-permissions!) -RUN useradd -u 2342 -g 65534 -s /bin/false -M pythonuser \ - && mkdir /home/pythonuser +RUN useradd -u 2342 -g 65534 -s /bin/false -M pythonuser && \ + mkdir /home/pythonuser WORKDIR /home/pythonuser # fetch the virtual environment from base -COPY --from=base --chown=2342:65534 /venv /venv +COPY --from=venv_builder --chown=2342:65534 /venv /venv +# fetch the actual project files COPY --chown=2342:65534 . . -# RUN chown -R 2342:65534 * +# drop user privileges USER pythonuser From 441f42ecf14bde45b4604e61d6069a8c48841961 Mon Sep 17 00:00:00 2001 From: Mischback Date: Thu, 30 Jan 2020 20:12:35 +0100 Subject: [PATCH 010/110] More and better tagging of images --- Makefile | 9 +++++++-- configs/Docker/docker-compose.yml | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 116f529..b49c8b6 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,11 @@ .PHONY: clean current default doc doc-srv help serve tox \ docker/build docker/test-build-context -DPS_COMMIT_SHA1:=$(shell git rev-parse --short HEAD) +DPS_DOCKER_REPO:="mischback/dps" +DPS_GIT_COMMIT_BRANCH:=$(shell git rev-parse --abbrev-ref HEAD) +DPS_GIT_COMMIT_SHA1:=$(shell git rev-parse --short HEAD) + +DPS_DOCKER_TAG="$(DPS_DOCKER_REPO):$(DPS_GIT_COMMIT_BRANCH)-$(DPS_GIT_COMMIT_SHA1)" default: help @@ -60,5 +64,6 @@ docker/test-build-context: && docker container run --rm test-build-context docker/build: - DPS_COMMIT_SHA1=$(DPS_COMMIT_SHA1) \ + DPS_DOCKER_TAG=$(DPS_DOCKER_TAG) \ docker-compose -f configs/Docker/docker-compose.yml build + docker tag $(DPS_DOCKER_TAG) "$(DPS_DOCKER_REPO):$(DPS_GIT_COMMIT_BRANCH)-current" diff --git a/configs/Docker/docker-compose.yml b/configs/Docker/docker-compose.yml index a4d45cb..fe00f6d 100644 --- a/configs/Docker/docker-compose.yml +++ b/configs/Docker/docker-compose.yml @@ -6,7 +6,7 @@ services: context: ../../ dockerfile: configs/Docker/django/Dockerfile # this should *magically* include a hash/datetime/unique identifier - image: mischback/dps:testing-${DPS_COMMIT_SHA1} + image: ${DPS_DOCKER_TAG} command: python manage.py runserver 0.0.0.0:8000 ports: - 8000:8000 From dee83ce953a0ea9327a281e6a488722f07a48bfa Mon Sep 17 00:00:00 2001 From: Mischback Date: Thu, 30 Jan 2020 21:33:25 +0100 Subject: [PATCH 011/110] Introducing pip-tools The project's requirements are automatically resolved during build step. --- configs/Docker/django/Dockerfile | 32 ++++++++++++---------- requirements/{production.txt => common.in} | 1 - requirements/development.txt | 2 -- requirements/production.in | 2 ++ 4 files changed, 20 insertions(+), 17 deletions(-) rename requirements/{production.txt => common.in} (83%) delete mode 100644 requirements/development.txt create mode 100644 requirements/production.in diff --git a/configs/Docker/django/Dockerfile b/configs/Docker/django/Dockerfile index ac63ece..e4d142f 100644 --- a/configs/Docker/django/Dockerfile +++ b/configs/Docker/django/Dockerfile @@ -20,31 +20,35 @@ RUN pip install --upgrade pip FROM base as requirements_compiler -# TODO: install the requirements for Python dependency management -# RUN pip install pip-tools +# install the requirements for Python dependency management +RUN pip install pip-tools -# TODO: fetch the requirements-directory -# COPY ./requirements/*.in /requirements +# fetch the requirements-directory +COPY ./requirements/*.in /requirements/ -# TODO: actually build the requirements for production environment +# actually build the requirements for production environment # TODO: if layered/included requirements are used, the files have to be compiled # in the correct order. -# RUN pip-compile --generate-hashes \ -# --output-file /requirements/common.txt \ -# /requirements/common.in && \ -# pip-compile --generate-hashes \ -# --output-file /requirements/production.txt \ -# /requirements/production.in +RUN pip-compile \ + --allow-unsafe \ + --generate-hashes \ + --output-file /requirements/common.txt \ + /requirements/common.in && \ + pip-compile \ + --allow-unsafe \ + --generate-hashes \ + --output-file /requirements/production.txt \ + /requirements/production.in FROM base as venv_builder # fetch the production requirements -COPY ./requirements/production.txt ./requirements.txt -# COPY --from=requirements_compiler /requirements/production.txt ./requirements.txt +# COPY ./requirements/production.txt ./requirements.txt +COPY --from=requirements_compiler /requirements/*.txt /requirements/ # actually install the requirements into the virtual environment -RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --no-cache-dir -r /requirements/common.txt -r /requirements/production.txt FROM python:3.8-slim-buster as django_production diff --git a/requirements/production.txt b/requirements/common.in similarity index 83% rename from requirements/production.txt rename to requirements/common.in index 9d1dd96..0703c09 100644 --- a/requirements/production.txt +++ b/requirements/common.in @@ -1,3 +1,2 @@ # currently used Django version Django==2.2.9 -gunicorn diff --git a/requirements/development.txt b/requirements/development.txt deleted file mode 100644 index f033fe7..0000000 --- a/requirements/development.txt +++ /dev/null @@ -1,2 +0,0 @@ -# currently used Django version -Django>=2.2, <3.0 diff --git a/requirements/production.in b/requirements/production.in new file mode 100644 index 0000000..a4a01ec --- /dev/null +++ b/requirements/production.in @@ -0,0 +1,2 @@ +-c common.txt +gunicorn From 8494a59e9dd925db1b6c1a80266501cc8f9007af Mon Sep 17 00:00:00 2001 From: Mischback Date: Fri, 31 Jan 2020 21:21:31 +0100 Subject: [PATCH 012/110] Generate requirements files with tox --- Makefile | 10 +++++++++- requirements/Makefile | 29 +++++++++++++++++++++++++++++ tox.ini | 12 ++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 requirements/Makefile diff --git a/Makefile b/Makefile index b49c8b6..7b9aaa0 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ .SILENT: -.PHONY: clean current default doc doc-srv help serve tox \ +.PHONY: clean current default doc doc-srv help requirements requirements-force \ + serve tox \ docker/build docker/test-build-context DPS_DOCKER_REPO:="mischback/dps" @@ -48,6 +49,13 @@ doc: doc-srv: doc tox -q -e doc-srv +requirements: + tox -q -e build_requirements + +requirements-force: + touch requirements/*.in + $(MAKE) requirements + serve: tox -q -e run diff --git a/requirements/Makefile b/requirements/Makefile new file mode 100644 index 0000000..6b625d9 --- /dev/null +++ b/requirements/Makefile @@ -0,0 +1,29 @@ +# Makefile to compile/update the actual requirements files using pip-tools' +# 'pip-compile' command. + +# get a list of all *.in files +OBJECTS = $(wildcard *.in) + +# construct the list of output files +OUTPUTS := $(OBJECTS:.in=.txt) + +# Make the command to actually compile the *.in files into *.txt configurable; +# the provided 'pip-compile' should be working within a 'tox' environment. +COMPILE_CMD = pip-compile + + +### RECEIPTS ### + +# by default update all requirements +all: $(OUTPUTS) + +# this is the actually compile step in generic form +%.txt: %.in + $(COMPILE_CMD) --allow-unsafe --generate-hashes --output-file $@ $< + + +# dependency chain +development.txt: common.txt +production.txt: common.txt + +.PHONY: all diff --git a/tox.ini b/tox.ini index a0a3efe..9ecf151 100644 --- a/tox.ini +++ b/tox.ini @@ -59,6 +59,18 @@ commands = {envbindir}/python foo/manage.py createsuperuser --settings=foo.settings.development {envbindir}/python foo/manage.py runserver 0:8080 --settings=foo.settings.development +[testenv:build_requirements] +basepython = {[testenv:util]basepython} +envdir = {toxworkdir}/build_requirements +deps = + pip-tools +skip_install = true +changedir = {toxinidir}/requirements +whitelist_externals = + make +commands = + {posargs:make} + [testenv:doc] basepython = {[testenv:util]basepython} envdir = {toxworkdir}/doc From dd3b911362d41790d23c4158fc1e580297b30530 Mon Sep 17 00:00:00 2001 From: Mischback Date: Tue, 4 Feb 2020 20:58:16 +0100 Subject: [PATCH 013/110] Actually use the built requirements in the Dockerfile The build process is further refined, using a single 'builder' (base) to set up the virtual environment, that is then copied over into the final image. --- Makefile | 22 +++++++++---- configs/Docker/django/Dockerfile | 53 ++++++++++++++------------------ 2 files changed, 39 insertions(+), 36 deletions(-) diff --git a/Makefile b/Makefile index 7b9aaa0..2412ec5 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,11 @@ .SILENT: .PHONY: clean current default doc doc-srv help requirements requirements-force \ serve tox \ - docker/build docker/test-build-context + docker/build docker/test-build-context \ + .docker/build .docker/test-build-context + +DOCKER_CMD:=docker +DOCKER_COMPOSE_CMD:=docker-compose DPS_DOCKER_REPO:="mischback/dps" DPS_GIT_COMMIT_BRANCH:=$(shell git rev-parse --abbrev-ref HEAD) @@ -62,7 +66,18 @@ serve: tox: tox -q +docker/build: requirements + sudo $(MAKE) .docker/build + +.docker/build: + DPS_DOCKER_TAG=$(DPS_DOCKER_TAG) \ + $(DOCKER_COMPOSE_CMD) -f configs/Docker/docker-compose.yml build + $(DOCKER_CMD) tag $(DPS_DOCKER_TAG) "$(DPS_DOCKER_REPO):$(DPS_GIT_COMMIT_BRANCH)-current" + docker/test-build-context: + sudo $(MAKE) .docker/test-build-context + +.docker/test-build-context: echo " \ FROM busybox\n \ COPY . /build-context\n \ @@ -70,8 +85,3 @@ docker/test-build-context: CMD find ." \ | docker build -t test-build-context -f- . \ && docker container run --rm test-build-context - -docker/build: - DPS_DOCKER_TAG=$(DPS_DOCKER_TAG) \ - docker-compose -f configs/Docker/docker-compose.yml build - docker tag $(DPS_DOCKER_TAG) "$(DPS_DOCKER_REPO):$(DPS_GIT_COMMIT_BRANCH)-current" diff --git a/configs/Docker/django/Dockerfile b/configs/Docker/django/Dockerfile index e4d142f..8c9a3f0 100644 --- a/configs/Docker/django/Dockerfile +++ b/configs/Docker/django/Dockerfile @@ -2,6 +2,16 @@ # Why Debian and not Alpine? Personal preference of Debian-based systems. # Additionally, Alpine is not using glibc, which might introduce problems. If # you want your image even smaller, you may switch to an Alpine based-image. + +### BASE +# The purpose of this image is to prepare the virtual environment, that will +# be used to run the Django stack in the final image. +# +# This image will be discarded and is merely a build artifact, leveraging +# Docker's multistage building feature to reduce the final image size. +# +# TODO: If there are any system dependencies, including build dependencies for +# the Python packages, they must be installed here aswell! FROM python:3.8-slim-buster as base # set environment variables @@ -17,40 +27,23 @@ RUN python3 -m venv $VIRTUAL_ENV # update pip inside of the virtual environment RUN pip install --upgrade pip - -FROM base as requirements_compiler - -# install the requirements for Python dependency management -RUN pip install pip-tools - -# fetch the requirements-directory -COPY ./requirements/*.in /requirements/ - -# actually build the requirements for production environment -# TODO: if layered/included requirements are used, the files have to be compiled -# in the correct order. -RUN pip-compile \ - --allow-unsafe \ - --generate-hashes \ - --output-file /requirements/common.txt \ - /requirements/common.in && \ - pip-compile \ - --allow-unsafe \ - --generate-hashes \ - --output-file /requirements/production.txt \ - /requirements/production.in - - -FROM base as venv_builder - # fetch the production requirements -# COPY ./requirements/production.txt ./requirements.txt -COPY --from=requirements_compiler /requirements/*.txt /requirements/ +# TODO: actually more than the required files will be copied. Should not be a +# problem, because this image is discarded after package installation. +COPY ./requirements/*.txt /requirements/ # actually install the requirements into the virtual environment -RUN pip install --no-cache-dir -r /requirements/common.txt -r /requirements/production.txt +RUN pip install --no-cache-dir \ + -r /requirements/common.txt \ + -r /requirements/production.txt +### DJANGO_PRODUCTION +# This image starts from a default debian based Python image. The virtual +# environment for running the Django stack is copied over from BASE. +# +# TODO: If there are any system dependencies, they have to be installed +# directly into this image. FROM python:3.8-slim-buster as django_production # create a user to be used (meaning: we won't run with root-permissions!) @@ -59,7 +52,7 @@ RUN useradd -u 2342 -g 65534 -s /bin/false -M pythonuser && \ WORKDIR /home/pythonuser # fetch the virtual environment from base -COPY --from=venv_builder --chown=2342:65534 /venv /venv +COPY --from=base --chown=2342:65534 /venv /venv # fetch the actual project files COPY --chown=2342:65534 . . From 4df5af9e813ffecce79ddd8220c40e48136393c0 Mon Sep 17 00:00:00 2001 From: Mischback Date: Tue, 4 Feb 2020 21:09:37 +0100 Subject: [PATCH 014/110] Activate the virtual environment --- Makefile | 2 ++ configs/Docker/django/Dockerfile | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/Makefile b/Makefile index 2412ec5..cfa6150 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,8 @@ help: clean: find . -iname "*.pyc" -delete find . -iname "__pycache__" -delete + rm requirements/common.txt + rm requirements/production.txt current: tox -q -e util diff --git a/configs/Docker/django/Dockerfile b/configs/Docker/django/Dockerfile index 8c9a3f0..dc56eed 100644 --- a/configs/Docker/django/Dockerfile +++ b/configs/Docker/django/Dockerfile @@ -54,6 +54,10 @@ WORKDIR /home/pythonuser # fetch the virtual environment from base COPY --from=base --chown=2342:65534 /venv /venv +# actually activate this virtual environment +ENV VIRTUAL_ENV=/venv +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + # fetch the actual project files COPY --chown=2342:65534 . . From 88bc9e9ece3fbdf0d7d310857dcef3cde405dea0 Mon Sep 17 00:00:00 2001 From: Mischback Date: Tue, 4 Feb 2020 21:58:39 +0100 Subject: [PATCH 015/110] Provide a default CMD --- configs/Docker/django/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configs/Docker/django/Dockerfile b/configs/Docker/django/Dockerfile index dc56eed..0b9bcef 100644 --- a/configs/Docker/django/Dockerfile +++ b/configs/Docker/django/Dockerfile @@ -63,3 +63,6 @@ COPY --chown=2342:65534 . . # drop user privileges USER pythonuser + +# start the Django development server by default +CMD ["python", "manage.py", "runserver"] From ce702964349f65ffe277d46f6b0c349e5c3d06e1 Mon Sep 17 00:00:00 2001 From: Mischback Date: Wed, 5 Feb 2020 22:22:55 +0100 Subject: [PATCH 016/110] Restart Makefile Actually, this project will have more than one Makefile, reflecting different stages of development and usage of django-project-skeleton. 1) The Makefile, that is included in the main directory when you download/use the repo as Django's startproject template: This Makefile is used during development and primarily includes receipts to support development. However, this will be the Makefile just after downloading the skeleton. So it has to include a receipt to prepare the project for further usage. 2) The Makefile, that is used during the lifespan of the project: This Makefile should support project-specific development, testing and deployment until production usage. --- Makefile | 81 ++++------------- .../{Makefile.sample => Makefile.1.4.sample} | 0 configs/Makefile.dev.sample | 89 +++++++++++++++++++ 3 files changed, 104 insertions(+), 66 deletions(-) rename configs/{Makefile.sample => Makefile.1.4.sample} (100%) create mode 100644 configs/Makefile.dev.sample diff --git a/Makefile b/Makefile index cfa6150..03b902a 100644 --- a/Makefile +++ b/Makefile @@ -1,80 +1,29 @@ .SILENT: -.PHONY: clean current default doc doc-srv help requirements requirements-force \ - serve tox \ - docker/build docker/test-build-context \ - .docker/build .docker/test-build-context +.PHONY: clean init \ + docker/test-build-context .docker/test-build-context DOCKER_CMD:=docker DOCKER_COMPOSE_CMD:=docker-compose DPS_DOCKER_REPO:="mischback/dps" -DPS_GIT_COMMIT_BRANCH:=$(shell git rev-parse --abbrev-ref HEAD) -DPS_GIT_COMMIT_SHA1:=$(shell git rev-parse --short HEAD) -DPS_DOCKER_TAG="$(DPS_DOCKER_REPO):$(DPS_GIT_COMMIT_BRANCH)-$(DPS_GIT_COMMIT_SHA1)" - -default: help - echo "" - echo "You need to specify a subcommand." - exit 1 - - -help: - echo "" - echo "Development utilities" - echo " clean Removes all temporary files" - echo " current Runs the tox-environment for the current development" - echo " doc Builds the documentation using 'Sphinx'" - echo " doc-srv Serves the documentation on port 8082 (and automatically builds it)" - echo " serve Runs the Django development server on port 8080" - echo " tox Runs complete tox test environment" - echo "" - echo "Docker related commands" - echo " Please note, that if you are not 'root', you have to do 'sudo make ...'!" - echo "" - echo " docker/test-build-context" - echo " Builds a minimal docker container and shows the context" - echo " This is used to check the .dockerignore file" - echo "" - - -# deletes all temporary files created by Django +# deletes all temporary and unwanted files clean: + # clean temporary Python files find . -iname "*.pyc" -delete find . -iname "__pycache__" -delete + # During development of django-project-skeleton, compiled requirement + # files are needed at some points, but the (compiled) requirements should + # not be included into version control of the skeleton. + # However, once the repository is fetched and used as a template for + # Django's startproject, the compiled requirements *should* be included + # into version control, thus, they can not simply included in a .gitignore. rm requirements/common.txt rm requirements/production.txt -current: - tox -q -e util - -doc: - tox -q -e doc - -doc-srv: doc - tox -q -e doc-srv - -requirements: - tox -q -e build_requirements - -requirements-force: - touch requirements/*.in - $(MAKE) requirements - -serve: - tox -q -e run - -tox: - tox -q - -docker/build: requirements - sudo $(MAKE) .docker/build - -.docker/build: - DPS_DOCKER_TAG=$(DPS_DOCKER_TAG) \ - $(DOCKER_COMPOSE_CMD) -f configs/Docker/docker-compose.yml build - $(DOCKER_CMD) tag $(DPS_DOCKER_TAG) "$(DPS_DOCKER_REPO):$(DPS_GIT_COMMIT_BRANCH)-current" +init: + echo "Initialising repository..." docker/test-build-context: sudo $(MAKE) .docker/test-build-context @@ -84,6 +33,6 @@ docker/test-build-context: FROM busybox\n \ COPY . /build-context\n \ WORKDIR /build-context\n \ - CMD find ." \ - | docker build -t test-build-context -f- . \ - && docker container run --rm test-build-context + CMD find ." | \ + $(DOCKER_CMD) build -t "$(DPS_DOCKER_REPO):test-build-context" -f- . && \ + $(DOCKER_CMD) container run --rm "$(DPS_DOCKER_REPO):test-build-context" diff --git a/configs/Makefile.sample b/configs/Makefile.1.4.sample similarity index 100% rename from configs/Makefile.sample rename to configs/Makefile.1.4.sample diff --git a/configs/Makefile.dev.sample b/configs/Makefile.dev.sample new file mode 100644 index 0000000..cfa6150 --- /dev/null +++ b/configs/Makefile.dev.sample @@ -0,0 +1,89 @@ +.SILENT: +.PHONY: clean current default doc doc-srv help requirements requirements-force \ + serve tox \ + docker/build docker/test-build-context \ + .docker/build .docker/test-build-context + +DOCKER_CMD:=docker +DOCKER_COMPOSE_CMD:=docker-compose + +DPS_DOCKER_REPO:="mischback/dps" +DPS_GIT_COMMIT_BRANCH:=$(shell git rev-parse --abbrev-ref HEAD) +DPS_GIT_COMMIT_SHA1:=$(shell git rev-parse --short HEAD) + +DPS_DOCKER_TAG="$(DPS_DOCKER_REPO):$(DPS_GIT_COMMIT_BRANCH)-$(DPS_GIT_COMMIT_SHA1)" + + +default: help + echo "" + echo "You need to specify a subcommand." + exit 1 + + +help: + echo "" + echo "Development utilities" + echo " clean Removes all temporary files" + echo " current Runs the tox-environment for the current development" + echo " doc Builds the documentation using 'Sphinx'" + echo " doc-srv Serves the documentation on port 8082 (and automatically builds it)" + echo " serve Runs the Django development server on port 8080" + echo " tox Runs complete tox test environment" + echo "" + echo "Docker related commands" + echo " Please note, that if you are not 'root', you have to do 'sudo make ...'!" + echo "" + echo " docker/test-build-context" + echo " Builds a minimal docker container and shows the context" + echo " This is used to check the .dockerignore file" + echo "" + + +# deletes all temporary files created by Django +clean: + find . -iname "*.pyc" -delete + find . -iname "__pycache__" -delete + rm requirements/common.txt + rm requirements/production.txt + +current: + tox -q -e util + +doc: + tox -q -e doc + +doc-srv: doc + tox -q -e doc-srv + +requirements: + tox -q -e build_requirements + +requirements-force: + touch requirements/*.in + $(MAKE) requirements + +serve: + tox -q -e run + +tox: + tox -q + +docker/build: requirements + sudo $(MAKE) .docker/build + +.docker/build: + DPS_DOCKER_TAG=$(DPS_DOCKER_TAG) \ + $(DOCKER_COMPOSE_CMD) -f configs/Docker/docker-compose.yml build + $(DOCKER_CMD) tag $(DPS_DOCKER_TAG) "$(DPS_DOCKER_REPO):$(DPS_GIT_COMMIT_BRANCH)-current" + +docker/test-build-context: + sudo $(MAKE) .docker/test-build-context + +.docker/test-build-context: + echo " \ + FROM busybox\n \ + COPY . /build-context\n \ + WORKDIR /build-context\n \ + CMD find ." \ + | docker build -t test-build-context -f- . \ + && docker container run --rm test-build-context From 1594758b4cce371e764d0c00e9b82871ea25488f Mon Sep 17 00:00:00 2001 From: Mischback Date: Thu, 6 Feb 2020 22:02:41 +0100 Subject: [PATCH 017/110] Modify tox.ini to allow Docker related testing during development --- tox.ini | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index 9ecf151..49d85eb 100644 --- a/tox.ini +++ b/tox.ini @@ -47,17 +47,17 @@ commands = django-admin startproject --template={toxinidir} foo {posargs:{envbindir}/python foo/manage.py check --settings=foo.settings.development} -[testenv:run] +[testenv:docker-testing] basepython = {[testenv:util]basepython} envdir = {[testenv:util]envdir} deps = {[testenv:util]deps} skip_install = {[testenv:util]skip_install} -changedir = {[testenv:util]changedir} +changedir = {envtmpdir}/foo +whitelist_externals = + make commands = - django-admin startproject --template={toxinidir} foo - {envbindir}/python foo/manage.py migrate --settings=foo.settings.development - {envbindir}/python foo/manage.py createsuperuser --settings=foo.settings.development - {envbindir}/python foo/manage.py runserver 0:8080 --settings=foo.settings.development + django-admin startproject --template={toxinidir} foo . + make init [testenv:build_requirements] basepython = {[testenv:util]basepython} From de410ef79f4391414dce6a1c47aaef6bb1a1b97d Mon Sep 17 00:00:00 2001 From: Mischback Date: Sun, 9 Feb 2020 20:43:33 +0100 Subject: [PATCH 018/110] Fetched Makefile.deployment --- configs/Makefile.deployment | 237 ++++++++++++++++++++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 configs/Makefile.deployment diff --git a/configs/Makefile.deployment b/configs/Makefile.deployment new file mode 100644 index 0000000..01699b9 --- /dev/null +++ b/configs/Makefile.deployment @@ -0,0 +1,237 @@ +# tox command (this assumes, that tox is available in your Python environment and your PATH is setup to allow direct calls) +TOX_CMD := tox -q -e + +# cmd to run django management commands +# if you rather use 'manage.py' instead of 'django-admin', switch the lines +# +# this assumes, that django is available in your Python environment and your PATH is setup to allow direct calls) +# DJANGOADMIN_CMD := django-admin +# this assumes, that your python executable is in your PATH and is indead accessible with python. Adjust according to your system's configuration +DJANGOADMIN_CMD := python {{ project_name }}/manage.py + +DJANGOADMIN_CMD_VERBOSITY := --verbosity=1 # {0,1,2,3}; 1 is default by Django + +### shortcut commands ### +run: dev/runserver + + +### development commands ### +dev/check: django/check +dev/migrate: django/migrate +dev/runserver: django/runserver-external4 +dev/static: django/staticfiles/find + + +### Django management commands ### +django/check: + $(MAKE) internal/django-mgmt django_cmd="check" + +django/compilemessages: + $(MAKE) internal/django-mgmt django_cmd="compilemessages" + +django/createcachetable: + $(MAKE) internal/django-mgmt django_cmd="createcachetable" + +django/dbshell: + # requires the command line client to be available in user's PATH + # dynamically determines the desired command line client from settings file + $(MAKE) internal/django-mgmt django_cmd="dbshell" + +django/diffsettings: + # compare with Django's default settings + # NOTE: compares the project's **development** settings with Django's defaults + $(MAKE) internal/django-mgmt django_cmd="diffsettings" + +django/diffsettings-all: + # show all settings, even if they correspond with Django's defaults + # NOTE: compares the project's **development** settings with Django's defaults + $(MAKE) internal/django-mgmt django_cmd="diffsettings --all" + +django/diffsettings-prod: + # compare **development** settings with **production** settings + # TODO: Has to be tested! + $(MAKE) internal/django-mgmt django_cmd="diffsettings --default={{ project_name }}.settings.production" + +django/dumpdata: + # apply indentation to increase readability + $(MAKE) internal/django-mgmt django_cmd="dumpdata --indent=4" + +django/flush: + # flushes the database + $(MAKE) internal/django-mgmt django_cmd="flush" + +djang/flush-plan: django/sqlflush + +django/flush-noinput: + # flushes the database without any prompts + $(MAKE) internal/django-mgmt django_cmd="flush --no-input" + +django/inspectdb: + # will not be wrapped, because this is a very edgy use case + # TODO: Can this output be colored, i.e. red? + echo "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments." + echo "I propose to run the command manually with your specific requirements, either using django-admin directly or by tox -e dev-django -- \"inspectdb [table [table ...]]\"" + +django/loaddata: + # TODO: Can this output be colored, i.e. red? + echo "This command is currently not implemented. If you are frequently loading fixtures into your database, you may modify the project's Makefile \"django/loaddata\" target" + +django/makemessages: + # TODO: Test this command! Is it sufficient or should '--all' be passed aswell? + $(MAKE) internal/django-mgmt django_cmd="makemessages" + +django/makemigrations: + # this command is provided for completeness, but should not be necessary for a Django project (relevance for apps!) + $(MAKE) internal/django-mgmt django_cmd="makemigrations" + +django/migrate: + # synchronize your models with your database + $(MAKE) internal/django-mgmt django_cmd="migrate" + +django/migrate-plan: + # show the migration operations, without actually executing them + $(MAKE) internal/django-mgmt django_cmd="migrate --plan" + +django/runserver: django/migrate + # runs the development server with default settings + # this will expose port 8000 on localhost, so it will not be accessible from another machine. + # see 'runserver-external' for an alternative + $(MAKE) internal/django-mgmt django_cmd="runserver" + +django/runserver-external4: django/migrate + # runs the development server with settings that allow access from another machine + # the server will be exposed on port 8000 + $(MAKE) internal/django-mgmt django_cmd="runserver 0.0.0.0:8000" + +django/runserver-external6: django/migrate + # runs the development server with settings that allow access from another machine + # the server will be exposed on port 8000 + $(MAKE) internal/django-mgmt django_cmd="runserver -6" + +django/sendtestemail: + # will not be wrapped, because this is a very edgy use case + # TODO: Can this output be colored, i.e. red? + echo "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments." + echo "I propose to run the command manually with your specific requirements, either using django-admin directly or by tox -e dev-django -- \"sendtestemail [email [email ...]]\"" + +django/shell: + # runs a python shell with the project's development settings + $(MAKE) internal/django-mgmt django_cmd="shell" + +django/showmigrations: + $(MAKE) internal/django-mgmt django_cmd="showmigrations" + +django/showmigrations-plan: + # show the migration operations, without actually executing them + $(MAKE) internal/django-mgmt django_cmd="showmigrations --plan" + +django/sqlflush: + # prints the SQL statements that would be executed for a 'flush' (see above) + $(MAKE) internal/django-mgmt django_cmd="sqlflush" + +django/sqlmigrate: + # will not be wrapped, because this is a very edgy use case + # TODO: Can this output be colored, i.e. red? + echo "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments." + echo "I propose to run the command manually with your specific requirements, either using django-admin directly or by tox -e dev-django -- \"sqlmigrate app_label migration_name\"" + +django/sqlsequencereset: + # will not be wrapped, because this is a very edgy use case + # TODO: Can this output be colored, i.e. red? + echo "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments." + echo "I propose to run the command manually with your specific requirements, either using django-admin directly or by tox -e dev-django -- \"sqlsequencereset app_label [app_label ...]\"" + +django/squashmigrations: + # will not be wrapped, because this is a very edgy use case + # this command is provided for completeness, but should not be necessary for a Django project (relevance for apps!) + # TODO: Can this output be colored, i.e. red? + echo "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments." + echo "I propose to run the command manually with your specific requirements, either using django-admin directly or by tox -e dev-django -- \"squashmigrations app_label [start_migration_name] migration_name\"" + +django/startapp: + # will not be wrapped, because this is a very edgy use case + # TODO: Can this output be colored, i.e. red? + echo "This command is beyond the scope of this project skeleton and too complex to be support by a Makefile" + echo "I propose to run the command manually with your specific requirements, either using django-admin directly or by tox -e dev-django -- \"startapp name [directory]\"" + echo "[directory] should be specified as \"apps/app_name\" if you wish to develop the app inside of this project." + # TODO: include a link to the definition of 'pluggable apps' for reference and to explain, why this might be a bad idea! + +django/startproject: + # will not be wrapped - obviously. This Makefile aims to manage an existing project + echo "This command is not provided. When you read this message, you should already have created a project." + +django/test: + # will not be wrapped. Testing is handled in other make targets. + echo "This command is not provided. Testing should be performed using the dedicated test related targets." + echo "Run \"make help\" for a list of commands and focus on the \"testing\" section." + +django/testserver: + # will not be wrapped. Testing is handled in other make targets. + echo "This command is not provided. Testing should be performed using the dedicated test related targets." + echo "Run \"make help\" for a list of commands and focus on the \"testing\" section." + +django/auth/changepassword: + # TODO: Can this output be colored, i.e. red? + # TODO: test this! with a matching user account and without... + echo "Trying to change the password of a Django user that matches your username..." + echo "If you want to change the password of another user, either using django-admin directly or by tox -e dev-django -- \"changepassword \"" + $(MAKE) internal/django-mgmt django_cmd="changepassword" + +django/auth/createsuperuser: + $(MAKE) internal/django-mgmt django_cmd="createsuperuser" + +django/auth/createsuperuser-noinput: + # TODO: This feature will be added, if Django>3.0 will be the main supported version! + echo "This feature will be implemented in a future release!" + $(MAKE) django/auth/createsuperuser + +django/contenttypes/remove_stale: + # only available if django.contrib.contenttypes is in INSTALLED_APPS + # TODO: check documentation; subcommands required? + $(MAKE) internal/django-mgmt django_cmd="remove_stale_contenttypes" + +django/sessions/clear: + # only available if django.contrib.sessions is in INSTALLED_APPS + # TODO: check documentation; subcommands required? + $(MAKE) internal/django-mgmt django_cmd="clearsessions" + +django/staticfiles/collect: + # only available if django.contrib.staticfiles is in INSTALLED_APPS + # TODO: check documentation; subcommands required? + $(MAKE) internal/django-mgmt django_cmd="collectstatic" + +django/staticfiles/find: + # only available if django.contrib.staticfiles is in INSTALLED_APPS + # TODO: check documentation; subcommands required? + $(MAKE) internal/django-mgmt django_cmd="findstatic" + +# main interface to run Django management commands +django_cmd ?= "" +internal/django-mgmt: internal/tox/dev-django +# internal/django-mgmt: internal/django/raw + +internal/django/raw: + $(DJANGOADMIN_CMD) $(django_cmd) $(DJANGOADMIN_CMD_VERBOSITY) --settings={{ project_name }}.settings.development + +internal/tox/dev-django: + $(TOX_CMD) dev-django -- $(django_cmd) $(DJANGOADMIN_CMD_VERBOSITY) + +.PHONY: run \ + \ + dev/check dev/migrate dev/runserver dev/static \ + \ + internal/django-mgmt internal/django/raw internal/tox/dev-django \ + \ + django/check django/compilemessages django/createcachetable \ + django/dbshell django/diffsettings django/diffsettings-all \ + django/diffsettings-prod django/dumpdata django/flush djang/flush-plan \ + django/flush-noinput django/inspectdb django/loaddata \ + django/makemessages django/makemigrations django/migrate \ + django/migrate-plan django/runserver django/runserver-external4 \ + django/runserver-external6 django/sendtestemail django/shell \ + django/showmigrations django/showmigrations-plan django/sqlflush \ + django/sqlmigrate django/sqlsequencereset django/squashmigrations \ + django/startapp django/startproject django/test django/testserver \ + django/auth/changepassword django/contenttypes/remove_stale \ + django/sessions/clear django/staticfiles/collect \ + django/staticfiles/find From 13a33d0a3e9574e990c4740472e3aab1715282d5 Mon Sep 17 00:00:00 2001 From: Mischback Date: Sun, 9 Feb 2020 21:46:55 +0100 Subject: [PATCH 019/110] Actually switching Makefile and tox.ini --- Makefile | 7 ++- configs/Makefile.1.4.sample | 108 ----------------------------------- configs/Makefile.deployment | 14 ++++- configs/tox.deployment | 109 ++++++++++++++++++++++++++++++++++++ requirements/development.in | 0 5 files changed, 126 insertions(+), 112 deletions(-) delete mode 100644 configs/Makefile.1.4.sample create mode 100644 configs/tox.deployment create mode 100644 requirements/development.in diff --git a/Makefile b/Makefile index 03b902a..f82874e 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .SILENT: -.PHONY: clean init \ +.PHONY: clean init tree \ docker/test-build-context .docker/test-build-context DOCKER_CMD:=docker @@ -24,6 +24,11 @@ clean: init: echo "Initialising repository..." + cp ./configs/tox.deployment ./tox.ini + cp ./configs/Makefile.deployment ./Makefile + +tree: + tree -a -I ".git|.tox|doc|run" --dirsfirst docker/test-build-context: sudo $(MAKE) .docker/test-build-context diff --git a/configs/Makefile.1.4.sample b/configs/Makefile.1.4.sample deleted file mode 100644 index c094928..0000000 --- a/configs/Makefile.1.4.sample +++ /dev/null @@ -1,108 +0,0 @@ -LOCALPATH := ./ -PYTHONPATH := $(LOCALPATH)/ -PYTHON_BIN := $(VIRTUAL_ENV)/bin - -DJANGO_TEST_SETTINGS_FILE := development -DJANGO_TEST_SETTINGS := {{ project_name }}.settings.$(DJANGO_TEST_SETTINGS_FILE) -DJANGO_TEST_POSTFIX := --settings=$(DJANGO_TEST_SETTINGS) --pythonpath=$(PYTHONPATH) - - -.PHONY: all clean coverage ensure_virtual_env flake8 flake lint \ - test test/dev test/prod \ - migrate setup/dev refresh refresh/dev refresh/prod update/dev - - -all: - @echo "Hello $(LOGNAME)! Welcome to django-project-skeleton" - @echo "" - @echo " clean Removes all temporary files" - @echo " coverage Runs the tests and shows code coverage" - @echo " flake8 Runs flake8 to check for PEP8 compliance" - @echo " = flake / lint" - @echo " migrate Applies all migrations" - @echo " refresh Refreshes the project by migrating the apps and" - @echo " collecting static assets" - @echo " refresh/dev Refreshes with development settings" - @echo " refresh/prod Refreshes with production settings" - @echo " setup/dev Sets up a development environment by installing" - @echo " necessary apps, running migrations and creating" - @echo " a superuser (django::django)" - @echo " test Runs the tests" - @echo " test/dev Runs the tests with development settings" - @echo " test/prod Runs the tests with production settings" - @echo " update/dev Shortcut for setup and refresh" - - -# performs the tests and measures code coverage -coverage: ensure_virtual_env test - $(PYTHON_BIN)/coverage html - $(PYTHON_BIN)/coverage report - - -# deletes all temporary files created by Django -clean: - @find . -iname "*.pyc" -delete - @find . -iname "__pycache__" -delete - @rm -rf .coverage coverage_html - - -# most of the commands can only be used inside of the virtual environment -ensure_virtual_env: - @if [ -z $$VIRTUAL_ENV ]; then \ - echo "You don't have a virtualenv enabled."; \ - echo "Please enable the virtualenv first!"; \ - exit 1; \ - fi - - -# runs flake8 to check for PEP8 compliance -flake8: ensure_virtual_env - $(PYTHON_BIN)/flake8 . - -flake: flake8 - -lint: flake8 - - -# runs the tests -# While we just have a bare project layout, this is more or less a dummy. -test: ensure_virtual_env - @echo "Using setting file '$(DJANGO_TEST_SETTINGS_FILE)'..." - @echo "" - @$(PYTHON_BIN)/coverage run $(PYTHON_BIN)/django-admin.py check $(DJANGO_TEST_POSTFIX) - -# runs the tests with development settings -test/dev: - $(MAKE) test DJANGO_TEST_SETTINGS_FILE=development - -# runs the tests with production settings -test/prod: - $(MAKE) test DJANGO_TEST_SETTINGS_FILE=production - - -# migrates the installed apps -migrate: ensure_virtual_env - $(PYTHON_BIN)/django-admin.py migrate $(DJANGO_TEST_POSTFIX) - -# sets up the development environment by installing required dependencies, -# migrates the apps and creates a dummy user (django::django) -setup/dev: ensure_virtual_env - @pip install -r requirements/development.txt - $(MAKE) migrate DJANGO_TEST_SETTINGS_FILE=development - @echo "from django.contrib.auth.models import User; User.objects.filter(email='admin@example.com').delete(); User.objects.create_superuser(username='django', email='admin@example.com', password='django')" | python manage.py shell - -# refreshes the project by migrating and collecting static assets -refresh: ensure_virtual_env - $(MAKE) clean - $(MAKE) migrate - $(PYTHON_BIN)/django-admin.py collectstatic --noinput $(DJANGO_TEST_POSTFIX) - -# refreshes with development settings -refresh/dev: - $(MAKE) refresh DJANGO_TEST_SETTINGS_FILE=development - -# refreshes with production settings -refresh/prod: - $(MAKE) refresh DJANGO_TEST_SETTINGS_FILE=production - -update/dev: setup/dev refresh/dev diff --git a/configs/Makefile.deployment b/configs/Makefile.deployment index 01699b9..75015d6 100644 --- a/configs/Makefile.deployment +++ b/configs/Makefile.deployment @@ -60,7 +60,7 @@ django/flush: # flushes the database $(MAKE) internal/django-mgmt django_cmd="flush" -djang/flush-plan: django/sqlflush +django/flush-plan: django/sqlflush django/flush-noinput: # flushes the database without any prompts @@ -213,18 +213,26 @@ internal/django-mgmt: internal/tox/dev-django internal/django/raw: $(DJANGOADMIN_CMD) $(django_cmd) $(DJANGOADMIN_CMD_VERBOSITY) --settings={{ project_name }}.settings.development -internal/tox/dev-django: +internal/tox/dev-django: util/requirements $(TOX_CMD) dev-django -- $(django_cmd) $(DJANGOADMIN_CMD_VERBOSITY) +util/requirements: + $(TOX_CMD) build_requirements + +util/requirements-force: + touch requirements/*.in + $(MAKE) util/requirements + .PHONY: run \ \ dev/check dev/migrate dev/runserver dev/static \ \ internal/django-mgmt internal/django/raw internal/tox/dev-django \ + util/requirements util/requirements-force \ \ django/check django/compilemessages django/createcachetable \ django/dbshell django/diffsettings django/diffsettings-all \ - django/diffsettings-prod django/dumpdata django/flush djang/flush-plan \ + django/diffsettings-prod django/dumpdata django/flush django/flush-plan \ django/flush-noinput django/inspectdb django/loaddata \ django/makemessages django/makemigrations django/migrate \ django/migrate-plan django/runserver django/runserver-external4 \ diff --git a/configs/tox.deployment b/configs/tox.deployment new file mode 100644 index 0000000..d422495 --- /dev/null +++ b/configs/tox.deployment @@ -0,0 +1,109 @@ +[tox] +# start with an empty list here +# TODO: verify that this actually **might** be empty! +envlist = + +# do not perform packaging operations +skipsdist = true + +# don't stop on missing interpreters +skip_missing_interpreters = true + +[testenv:base] +basepython = python3 +envdir = {toxworkdir}/base +deps = + -rrequirements/common.txt + -rrequirements/development.txt +skip_install = true +commands = + {posargs:{envbindir}/python {{ project_name }}/manage.py version --settings={{ project_name }}.settings.development} + +[testenv:dev-django] +basepython = {[testenv:base]basepython} +envdir = {[testenv:base]envdir} +deps = {[testenv:base]deps} +skip_install = {[testenv:base]skip_install} +commands = + {envbindir}/python {{ project_name }}/manage.py {posargs:version} --settings={{ project_name }}.settings.development + +[testenv:test-django] +basepython = {[testenv:base]basepython} +envdir = {[testenv:base]envdir} +deps = + {[testenv:base]deps} + coverage +skip_install = {[testenv:base]skip_install} +commands = + coverage run --parallel tests/runtests.py {posargs} + +[testenv:build_requirements] +basepython = {[testenv:base]basepython} +envdir = {toxworkdir}/build_requirements +deps = + pip-tools +skip_install = true +changedir = {toxinidir}/requirements +whitelist_externals = + make +commands = + {posargs:make} + + +################################################################################ +# The following sections actually provide settings for various tools +################################################################################ + +# This sections sets the options for coverage collecting +[coverage:run] +branch = True +omit = + */__init__.py + */migrations/* + */site-packages/* + +# This sections sets the options for coverage reporting +[coverage:report] +precision = 1 +show_missing = True +fail_under = 95 + + +# This section actually sets the options for flake8 +[flake8] +exclude = + .git, + .tox, + +# as per Django's Coding Style +# see https://docs.djangoproject.com/en/dev/internals/contributing/writing-code/coding-style/ +max-line-length = 119 + +ignore = + # as per Django's Coding Style + W601, + + +# This section actually sets the options for isort +[isort] +# these settings are taken from Django's isort configuration +# see https://github.com/django/django/blob/2.0.2/setup.cfg +combine_as_imports = True +default_section = THIRDPARTY +include_trailing_comma = True +line_length = 79 +multi_line_output = 5 +not_skip = __init__.py + +# project specific isort options +known_first_party = {{ project_name }} +known_django = django +sections = FUTURE, STDLIB, DJANGO, THIRDPARTY, FIRSTPARTY, LOCALFOLDER +import_heading_stdlib = Python imports +import_heading_django = Django imports +import_heading_firstparty = app imports +import_heading_localfolder = app imports +import_heading_thirdparty = external imports +skip_glob = + .tox, + */migrations/* diff --git a/requirements/development.in b/requirements/development.in new file mode 100644 index 0000000..e69de29 From d60adb38a0550d45a41815d05fb68f7fb651d594 Mon Sep 17 00:00:00 2001 From: Mischback Date: Mon, 10 Feb 2020 18:50:55 +0100 Subject: [PATCH 020/110] Prepare colourful output This requires 'tput' to be installed on your system (most probably it is!), but will gracefully provide monochrome text, if it is not. --- configs/Makefile.deployment | 74 ++++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 26 deletions(-) diff --git a/configs/Makefile.deployment b/configs/Makefile.deployment index 75015d6..55344e3 100644 --- a/configs/Makefile.deployment +++ b/configs/Makefile.deployment @@ -1,16 +1,21 @@ -# tox command (this assumes, that tox is available in your Python environment and your PATH is setup to allow direct calls) +# tox command (this assumes, that tox is available in your Python environment +# and your PATH is setup to allow direct calls) TOX_CMD := tox -q -e # cmd to run django management commands # if you rather use 'manage.py' instead of 'django-admin', switch the lines # -# this assumes, that django is available in your Python environment and your PATH is setup to allow direct calls) +# this assumes, that django is available in your Python environment and your +# PATH is setup to allow direct calls) # DJANGOADMIN_CMD := django-admin -# this assumes, that your python executable is in your PATH and is indead accessible with python. Adjust according to your system's configuration +# +# this assumes, that your Python executable is in your PATH and is indeed +# accessible with python. Adjust according to your system's configuration DJANGOADMIN_CMD := python {{ project_name }}/manage.py DJANGOADMIN_CMD_VERBOSITY := --verbosity=1 # {0,1,2,3}; 1 is default by Django + ### shortcut commands ### run: dev/runserver @@ -69,12 +74,12 @@ django/flush-noinput: django/inspectdb: # will not be wrapped, because this is a very edgy use case # TODO: Can this output be colored, i.e. red? - echo "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments." - echo "I propose to run the command manually with your specific requirements, either using django-admin directly or by tox -e dev-django -- \"inspectdb [table [table ...]]\"" + $(ECHO) "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments." + $(ECHO) "I propose to run the command manually with your specific requirements, either using django-admin directly or by tox -e dev-django -- \"inspectdb [table [table ...]]\"" django/loaddata: # TODO: Can this output be colored, i.e. red? - echo "This command is currently not implemented. If you are frequently loading fixtures into your database, you may modify the project's Makefile \"django/loaddata\" target" + $(ECHO) "This command is currently not implemented. If you are frequently loading fixtures into your database, you may modify the project's Makefile \"django/loaddata\" target" django/makemessages: # TODO: Test this command! Is it sufficient or should '--all' be passed aswell? @@ -111,8 +116,8 @@ django/runserver-external6: django/migrate django/sendtestemail: # will not be wrapped, because this is a very edgy use case # TODO: Can this output be colored, i.e. red? - echo "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments." - echo "I propose to run the command manually with your specific requirements, either using django-admin directly or by tox -e dev-django -- \"sendtestemail [email [email ...]]\"" + $(ECHO) "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments." + $(ECHO) "I propose to run the command manually with your specific requirements, either using django-admin directly or by tox -e dev-django -- \"sendtestemail [email [email ...]]\"" django/shell: # runs a python shell with the project's development settings @@ -132,49 +137,49 @@ django/sqlflush: django/sqlmigrate: # will not be wrapped, because this is a very edgy use case # TODO: Can this output be colored, i.e. red? - echo "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments." - echo "I propose to run the command manually with your specific requirements, either using django-admin directly or by tox -e dev-django -- \"sqlmigrate app_label migration_name\"" + $(ECHO) "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments." + $(ECHO) "I propose to run the command manually with your specific requirements, either using django-admin directly or by tox -e dev-django -- \"sqlmigrate app_label migration_name\"" django/sqlsequencereset: # will not be wrapped, because this is a very edgy use case # TODO: Can this output be colored, i.e. red? - echo "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments." - echo "I propose to run the command manually with your specific requirements, either using django-admin directly or by tox -e dev-django -- \"sqlsequencereset app_label [app_label ...]\"" + $(ECHO) "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments." + $(ECHO) "I propose to run the command manually with your specific requirements, either using django-admin directly or by tox -e dev-django -- \"sqlsequencereset app_label [app_label ...]\"" django/squashmigrations: # will not be wrapped, because this is a very edgy use case # this command is provided for completeness, but should not be necessary for a Django project (relevance for apps!) # TODO: Can this output be colored, i.e. red? - echo "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments." - echo "I propose to run the command manually with your specific requirements, either using django-admin directly or by tox -e dev-django -- \"squashmigrations app_label [start_migration_name] migration_name\"" + $(ECHO) "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments." + $(ECHO) "I propose to run the command manually with your specific requirements, either using django-admin directly or by tox -e dev-django -- \"squashmigrations app_label [start_migration_name] migration_name\"" django/startapp: # will not be wrapped, because this is a very edgy use case # TODO: Can this output be colored, i.e. red? - echo "This command is beyond the scope of this project skeleton and too complex to be support by a Makefile" - echo "I propose to run the command manually with your specific requirements, either using django-admin directly or by tox -e dev-django -- \"startapp name [directory]\"" - echo "[directory] should be specified as \"apps/app_name\" if you wish to develop the app inside of this project." + $(ECHO) "This command is beyond the scope of this project skeleton and too complex to be support by a Makefile" + $(ECHO) "I propose to run the command manually with your specific requirements, either using django-admin directly or by tox -e dev-django -- \"startapp name [directory]\"" + $(ECHO) "[directory] should be specified as \"apps/app_name\" if you wish to develop the app inside of this project." # TODO: include a link to the definition of 'pluggable apps' for reference and to explain, why this might be a bad idea! django/startproject: # will not be wrapped - obviously. This Makefile aims to manage an existing project - echo "This command is not provided. When you read this message, you should already have created a project." + $(ECHO) "This command is not provided. When you read this message, you should already have created a project." django/test: # will not be wrapped. Testing is handled in other make targets. - echo "This command is not provided. Testing should be performed using the dedicated test related targets." - echo "Run \"make help\" for a list of commands and focus on the \"testing\" section." + $(ECHO) "This command is not provided. Testing should be performed using the dedicated test related targets." + $(ECHO) "Run \"make help\" for a list of commands and focus on the \"testing\" section." django/testserver: # will not be wrapped. Testing is handled in other make targets. - echo "This command is not provided. Testing should be performed using the dedicated test related targets." - echo "Run \"make help\" for a list of commands and focus on the \"testing\" section." + $(ECHO) "This command is not provided. Testing should be performed using the dedicated test related targets." + $(ECHO) "Run \"make help\" for a list of commands and focus on the \"testing\" section." django/auth/changepassword: # TODO: Can this output be colored, i.e. red? # TODO: test this! with a matching user account and without... - echo "Trying to change the password of a Django user that matches your username..." - echo "If you want to change the password of another user, either using django-admin directly or by tox -e dev-django -- \"changepassword \"" + $(ECHO) "Trying to change the password of a Django user that matches your username..." + $(ECHO) "If you want to change the password of another user, either using django-admin directly or by tox -e dev-django -- \"changepassword \"" $(MAKE) internal/django-mgmt django_cmd="changepassword" django/auth/createsuperuser: @@ -182,7 +187,7 @@ django/auth/createsuperuser: django/auth/createsuperuser-noinput: # TODO: This feature will be added, if Django>3.0 will be the main supported version! - echo "This feature will be implemented in a future release!" + $(ECHO) "This feature will be implemented in a future release!" $(MAKE) django/auth/createsuperuser django/contenttypes/remove_stale: @@ -223,7 +228,24 @@ util/requirements-force: touch requirements/*.in $(MAKE) util/requirements -.PHONY: run \ + +current: + $(ECHO) "$(CRED)red$(CRES) / $(CGRE)green$(CRES) / $(CYEL)yellow$(CRES) / $(CCYA)blue$(CRES)" + + +# prepare coloring of echo outputs +CRED := $(shell tput setaf 1 2>/dev/null) +CGRE := $(shell tput setaf 2 2>/dev/null) +CYEL := $(shell tput setaf 3 2>/dev/null) +CCYA := $(shell tput setaf 6 2>/dev/null) +CRES := $(shell tput sgr0 2>/dev/null) +ECHO := echo + +# suppress unintended output +.SILENT: + +.PHONY: current \ + run \ \ dev/check dev/migrate dev/runserver dev/static \ \ From 0fb1ff950eec0abe7172505b12000230e68c7b16 Mon Sep 17 00:00:00 2001 From: Mischback Date: Mon, 10 Feb 2020 20:35:53 +0100 Subject: [PATCH 021/110] Add colours to the output --- configs/Makefile.deployment | 47 ++++++++++++++++++------------------- configs/tox.deployment | 4 ++-- 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/configs/Makefile.deployment b/configs/Makefile.deployment index 55344e3..b038f3c 100644 --- a/configs/Makefile.deployment +++ b/configs/Makefile.deployment @@ -1,3 +1,5 @@ +### Makefile configuration section ### +# # tox command (this assumes, that tox is available in your Python environment # and your PATH is setup to allow direct calls) TOX_CMD := tox -q -e @@ -11,8 +13,9 @@ TOX_CMD := tox -q -e # # this assumes, that your Python executable is in your PATH and is indeed # accessible with python. Adjust according to your system's configuration -DJANGOADMIN_CMD := python {{ project_name }}/manage.py +DJANGOADMIN_CMD := python manage.py +# easily control the verbosity of Django commands DJANGOADMIN_CMD_VERBOSITY := --verbosity=1 # {0,1,2,3}; 1 is default by Django @@ -73,13 +76,11 @@ django/flush-noinput: django/inspectdb: # will not be wrapped, because this is a very edgy use case - # TODO: Can this output be colored, i.e. red? $(ECHO) "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments." - $(ECHO) "I propose to run the command manually with your specific requirements, either using django-admin directly or by tox -e dev-django -- \"inspectdb [table [table ...]]\"" + $(ECHO) "I propose to run the command manually with your specific requirements, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"inspectdb [table [table ...]]\"$(CRES)." django/loaddata: - # TODO: Can this output be colored, i.e. red? - $(ECHO) "This command is currently not implemented. If you are frequently loading fixtures into your database, you may modify the project's Makefile \"django/loaddata\" target" + $(ECHO) "This command is currently not implemented. If you are frequently loading fixtures into your database, you may modify the project's Makefile $(CCYA)\"django/loaddata\"$(CRES) target." django/makemessages: # TODO: Test this command! Is it sufficient or should '--all' be passed aswell? @@ -100,7 +101,7 @@ django/migrate-plan: django/runserver: django/migrate # runs the development server with default settings # this will expose port 8000 on localhost, so it will not be accessible from another machine. - # see 'runserver-external' for an alternative + # see 'runserver-external[4|6]' for an alternative $(MAKE) internal/django-mgmt django_cmd="runserver" django/runserver-external4: django/migrate @@ -115,9 +116,8 @@ django/runserver-external6: django/migrate django/sendtestemail: # will not be wrapped, because this is a very edgy use case - # TODO: Can this output be colored, i.e. red? $(ECHO) "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments." - $(ECHO) "I propose to run the command manually with your specific requirements, either using django-admin directly or by tox -e dev-django -- \"sendtestemail [email [email ...]]\"" + $(ECHO) "I propose to run the command manually with your specific requirements, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"sendtestemail [email [email ...]]\"$(CRES)." django/shell: # runs a python shell with the project's development settings @@ -136,50 +136,48 @@ django/sqlflush: django/sqlmigrate: # will not be wrapped, because this is a very edgy use case - # TODO: Can this output be colored, i.e. red? $(ECHO) "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments." - $(ECHO) "I propose to run the command manually with your specific requirements, either using django-admin directly or by tox -e dev-django -- \"sqlmigrate app_label migration_name\"" + $(ECHO) "I propose to run the command manually with your specific requirements, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"sqlmigrate app_label migration_name\"$(CRES)." django/sqlsequencereset: # will not be wrapped, because this is a very edgy use case - # TODO: Can this output be colored, i.e. red? $(ECHO) "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments." - $(ECHO) "I propose to run the command manually with your specific requirements, either using django-admin directly or by tox -e dev-django -- \"sqlsequencereset app_label [app_label ...]\"" + $(ECHO) "I propose to run the command manually with your specific requirements, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"sqlsequencereset app_label [app_label ...]\"$(CRES)." django/squashmigrations: # will not be wrapped, because this is a very edgy use case # this command is provided for completeness, but should not be necessary for a Django project (relevance for apps!) - # TODO: Can this output be colored, i.e. red? $(ECHO) "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments." - $(ECHO) "I propose to run the command manually with your specific requirements, either using django-admin directly or by tox -e dev-django -- \"squashmigrations app_label [start_migration_name] migration_name\"" + $(ECHO) "I propose to run the command manually with your specific requirements, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"squashmigrations app_label [start_migration_name] migration_name\"$(CRES)." django/startapp: # will not be wrapped, because this is a very edgy use case - # TODO: Can this output be colored, i.e. red? $(ECHO) "This command is beyond the scope of this project skeleton and too complex to be support by a Makefile" - $(ECHO) "I propose to run the command manually with your specific requirements, either using django-admin directly or by tox -e dev-django -- \"startapp name [directory]\"" - $(ECHO) "[directory] should be specified as \"apps/app_name\" if you wish to develop the app inside of this project." + $(ECHO) "I propose to run the command manually with your specific requirements, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"startapp name [directory]\"$(CRES)." + $(ECHO) "[directory] should be specified as $(CCYA)\"apps/app_name\"$(CRES) if you wish to develop the app inside of this project." # TODO: include a link to the definition of 'pluggable apps' for reference and to explain, why this might be a bad idea! django/startproject: # will not be wrapped - obviously. This Makefile aims to manage an existing project - $(ECHO) "This command is not provided. When you read this message, you should already have created a project." + $(ECHO) "$(CRED)This command is not provided. When you read this message, you should already have created a project.$(CRES)" django/test: # will not be wrapped. Testing is handled in other make targets. - $(ECHO) "This command is not provided. Testing should be performed using the dedicated test related targets." - $(ECHO) "Run \"make help\" for a list of commands and focus on the \"testing\" section." + $(ECHO) "$(CRED)This command is not provided. Testing should be performed using the dedicated test related targets.$(CRES)" + $(ECHO) "Run $(CCYA)\"make help\"$(CRES) for a list of commands and focus on the $(CCYA)\"testing\"$(CRES) section." django/testserver: # will not be wrapped. Testing is handled in other make targets. - $(ECHO) "This command is not provided. Testing should be performed using the dedicated test related targets." - $(ECHO) "Run \"make help\" for a list of commands and focus on the \"testing\" section." + $(ECHO) "$(CRED)This command is not provided. Testing should be performed using the dedicated test related targets.$(CRES)" + $(ECHO) "Run $(CCYA)\"make help\"$(CRES) for a list of commands and focus on the $(CCYA)\"testing\"$(CRES) section." + +django/version: + $(MAKE) internal/django-mgmt django_cmd="version" django/auth/changepassword: - # TODO: Can this output be colored, i.e. red? # TODO: test this! with a matching user account and without... $(ECHO) "Trying to change the password of a Django user that matches your username..." - $(ECHO) "If you want to change the password of another user, either using django-admin directly or by tox -e dev-django -- \"changepassword \"" + $(ECHO) "If you want to change the password of another user, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"changepassword \"$(CRES)." $(MAKE) internal/django-mgmt django_cmd="changepassword" django/auth/createsuperuser: @@ -262,6 +260,7 @@ ECHO := echo django/showmigrations django/showmigrations-plan django/sqlflush \ django/sqlmigrate django/sqlsequencereset django/squashmigrations \ django/startapp django/startproject django/test django/testserver \ + django/version \ django/auth/changepassword django/contenttypes/remove_stale \ django/sessions/clear django/staticfiles/collect \ django/staticfiles/find diff --git a/configs/tox.deployment b/configs/tox.deployment index d422495..f3ace3b 100644 --- a/configs/tox.deployment +++ b/configs/tox.deployment @@ -17,7 +17,7 @@ deps = -rrequirements/development.txt skip_install = true commands = - {posargs:{envbindir}/python {{ project_name }}/manage.py version --settings={{ project_name }}.settings.development} + {posargs:{envbindir}/python manage.py version --settings={{ project_name }}.settings.development} [testenv:dev-django] basepython = {[testenv:base]basepython} @@ -25,7 +25,7 @@ envdir = {[testenv:base]envdir} deps = {[testenv:base]deps} skip_install = {[testenv:base]skip_install} commands = - {envbindir}/python {{ project_name }}/manage.py {posargs:version} --settings={{ project_name }}.settings.development + {envbindir}/python manage.py {posargs:version} --settings={{ project_name }}.settings.development [testenv:test-django] basepython = {[testenv:base]basepython} From 1833936de5236d0b3baf6af045599d7129e064cd Mon Sep 17 00:00:00 2001 From: Mischback Date: Tue, 11 Feb 2020 23:39:21 +0100 Subject: [PATCH 022/110] Makefile supports Docker build step The actual deployment Makefile received a lot of love, including - comments, comments, documentation - switched from 'echo' to '/usr/bin/printf' for max compatibility - some minor manual testing of Makefile commands/targets docker-compose.yml was updated to include the new naming scheme for images built through the Makefile interface. The tagging may be controlled by Makefile variables, that also are set up to support templating by Django's startproject. To tag every build with an unique identifier, a custom shell script was added in the new ./bin directory. This script needs documentation and comments. --- bin/get_build_id.sh | 12 ++++ configs/Docker/docker-compose.yml | 2 +- configs/Makefile.deployment | 111 +++++++++++++++++++++--------- 3 files changed, 90 insertions(+), 35 deletions(-) create mode 100755 bin/get_build_id.sh diff --git a/bin/get_build_id.sh b/bin/get_build_id.sh new file mode 100755 index 0000000..ff511dd --- /dev/null +++ b/bin/get_build_id.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +if git rev-parse --is-inside-work-tree > /dev/null 2>&1; then + sha=$(git rev-parse --short HEAD); + if test -n "$(git status --porcelain)"; then + /usr/bin/printf "%s-dirty\n" $sha; + else + /usr/bin/printf "%s\n" $sha; + fi +else + date +%Y%m%d-%H%M 2>/dev/null; +fi diff --git a/configs/Docker/docker-compose.yml b/configs/Docker/docker-compose.yml index fe00f6d..ce47131 100644 --- a/configs/Docker/docker-compose.yml +++ b/configs/Docker/docker-compose.yml @@ -6,7 +6,7 @@ services: context: ../../ dockerfile: configs/Docker/django/Dockerfile # this should *magically* include a hash/datetime/unique identifier - image: ${DPS_DOCKER_TAG} + image: ${DPS_BUILD_NAME_PREFIX}/django:${DPS_BUILD_ID} command: python manage.py runserver 0.0.0.0:8000 ports: - 8000:8000 diff --git a/configs/Makefile.deployment b/configs/Makefile.deployment index b038f3c..ae2209b 100644 --- a/configs/Makefile.deployment +++ b/configs/Makefile.deployment @@ -1,5 +1,35 @@ -### Makefile configuration section ### +##### Makefile configuration section ##### + + +### Docker related constants ### + +# Docker images will be tagged with the following naming scheme: +# {[registry hostname]/[project name]}/[service]:[DPS_BUILD_ID] +# - registry hostname: optionally specifies a registry hostname +# - project name: the project's name as defined during 'startproject' +# - service: defines the service represented by the image, e.g. Django, +# nginx, postgre, ... The service is set in 'docker-compose.yml' and +# is hardcoded there. +# - DPS_BUILD_ID: automatically determined ID. Will fetch the current +# git commit SHA1 hash or provide a timestamp. +# See bin/get_build_id.sh +# +# DPS_BUILD_NAME_PREFIX is used to provide the [registry hostname], [hostname] +# and [project name] (by default no [registry hostname] is provided). +DPS_BUILD_NAME_PREFIX := "{{ project_name }}" + + +### internal setup ### +# These variables/constants define, how the project is managed. + +# main interface to run Django management commands # +# internal/tox/dev-django will use tox environments to run Django managements +# commands. +# internal/django/raw will directly call the Django management commands. +internal/django-mgmt: internal/tox/dev-django +# internal/django-mgmt: internal/django/raw + # tox command (this assumes, that tox is available in your Python environment # and your PATH is setup to allow direct calls) TOX_CMD := tox -q -e @@ -18,6 +48,12 @@ DJANGOADMIN_CMD := python manage.py # easily control the verbosity of Django commands DJANGOADMIN_CMD_VERBOSITY := --verbosity=1 # {0,1,2,3}; 1 is default by Django +# Docker command +DOCKER_CMD := $(shell which docker) + +# Docker compose command +DOCKER_COMPOSE_CMD := $(shell which docker-compose) + ### shortcut commands ### run: dev/runserver @@ -30,7 +66,20 @@ dev/runserver: django/runserver-external4 dev/static: django/staticfiles/find +### Docker commands ### + +docker/build: util/requirements + sudo $(MAKE) internal/docker/build + +internal/docker/build: + DPS_BUILD_NAME_PREFIX=$(DPS_BUILD_NAME_PREFIX) DPS_BUILD_ID=$(DPS_BUILD_ID) \ + $(DOCKER_COMPOSE_CMD) -f configs/Docker/docker-compose.yml build + $(DOCKER_CMD) tag "$(DPS_BUILD_NAME_PREFIX)/django:$(DPS_BUILD_ID)" "$(DPS_BUILD_NAME_PREFIX)/django:latest" + + ### Django management commands ### +django_cmd ?= "" + django/check: $(MAKE) internal/django-mgmt django_cmd="check" @@ -57,7 +106,6 @@ django/diffsettings-all: django/diffsettings-prod: # compare **development** settings with **production** settings - # TODO: Has to be tested! $(MAKE) internal/django-mgmt django_cmd="diffsettings --default={{ project_name }}.settings.production" django/dumpdata: @@ -76,11 +124,11 @@ django/flush-noinput: django/inspectdb: # will not be wrapped, because this is a very edgy use case - $(ECHO) "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments." - $(ECHO) "I propose to run the command manually with your specific requirements, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"inspectdb [table [table ...]]\"$(CRES)." + $(ECHO) "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments.\n" + $(ECHO) "I propose to run the command manually with your specific requirements, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"inspectdb [table [table ...]]\"$(CRES).\n" django/loaddata: - $(ECHO) "This command is currently not implemented. If you are frequently loading fixtures into your database, you may modify the project's Makefile $(CCYA)\"django/loaddata\"$(CRES) target." + $(ECHO) "This command is currently not implemented. If you are frequently loading fixtures into your database, you may modify the project's Makefile $(CCYA)\"django/loaddata\"$(CRES) target.\n" django/makemessages: # TODO: Test this command! Is it sufficient or should '--all' be passed aswell? @@ -116,8 +164,8 @@ django/runserver-external6: django/migrate django/sendtestemail: # will not be wrapped, because this is a very edgy use case - $(ECHO) "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments." - $(ECHO) "I propose to run the command manually with your specific requirements, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"sendtestemail [email [email ...]]\"$(CRES)." + $(ECHO) "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments.\n" + $(ECHO) "I propose to run the command manually with your specific requirements, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"sendtestemail [email [email ...]]\"$(CRES).\n" django/shell: # runs a python shell with the project's development settings @@ -136,48 +184,48 @@ django/sqlflush: django/sqlmigrate: # will not be wrapped, because this is a very edgy use case - $(ECHO) "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments." - $(ECHO) "I propose to run the command manually with your specific requirements, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"sqlmigrate app_label migration_name\"$(CRES)." + $(ECHO) "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments.\n" + $(ECHO) "I propose to run the command manually with your specific requirements, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"sqlmigrate app_label migration_name\"$(CRES).\n" django/sqlsequencereset: # will not be wrapped, because this is a very edgy use case - $(ECHO) "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments." - $(ECHO) "I propose to run the command manually with your specific requirements, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"sqlsequencereset app_label [app_label ...]\"$(CRES)." + $(ECHO) "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments.\n" + $(ECHO) "I propose to run the command manually with your specific requirements, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"sqlsequencereset app_label [app_label ...]\"$(CRES).\n" django/squashmigrations: # will not be wrapped, because this is a very edgy use case # this command is provided for completeness, but should not be necessary for a Django project (relevance for apps!) - $(ECHO) "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments." - $(ECHO) "I propose to run the command manually with your specific requirements, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"squashmigrations app_label [start_migration_name] migration_name\"$(CRES)." + $(ECHO) "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments.\n" + $(ECHO) "I propose to run the command manually with your specific requirements, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"squashmigrations app_label [start_migration_name] migration_name\"$(CRES).\n" django/startapp: # will not be wrapped, because this is a very edgy use case - $(ECHO) "This command is beyond the scope of this project skeleton and too complex to be support by a Makefile" - $(ECHO) "I propose to run the command manually with your specific requirements, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"startapp name [directory]\"$(CRES)." - $(ECHO) "[directory] should be specified as $(CCYA)\"apps/app_name\"$(CRES) if you wish to develop the app inside of this project." + $(ECHO) "This command is beyond the scope of this project skeleton and too complex to be support by a Makefile.\n" + $(ECHO) "I propose to run the command manually with your specific requirements, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"startapp name [directory]\"$(CRES).\n" + $(ECHO) "[directory] should be specified as $(CCYA)\"apps/app_name\"$(CRES) if you wish to develop the app inside of this project.\n" # TODO: include a link to the definition of 'pluggable apps' for reference and to explain, why this might be a bad idea! django/startproject: # will not be wrapped - obviously. This Makefile aims to manage an existing project - $(ECHO) "$(CRED)This command is not provided. When you read this message, you should already have created a project.$(CRES)" + $(ECHO) "$(CRED)This command is not provided. When you read this message, you should already have created a project.$(CRES)\n" django/test: # will not be wrapped. Testing is handled in other make targets. - $(ECHO) "$(CRED)This command is not provided. Testing should be performed using the dedicated test related targets.$(CRES)" - $(ECHO) "Run $(CCYA)\"make help\"$(CRES) for a list of commands and focus on the $(CCYA)\"testing\"$(CRES) section." + $(ECHO) "$(CRED)This command is not provided. Testing should be performed using the dedicated test related targets.$(CRES)\n" + $(ECHO) "Run $(CCYA)\"make help\"$(CRES) for a list of commands and focus on the $(CCYA)\"testing\"$(CRES) section.\n" django/testserver: # will not be wrapped. Testing is handled in other make targets. - $(ECHO) "$(CRED)This command is not provided. Testing should be performed using the dedicated test related targets.$(CRES)" - $(ECHO) "Run $(CCYA)\"make help\"$(CRES) for a list of commands and focus on the $(CCYA)\"testing\"$(CRES) section." + $(ECHO) "$(CRED)This command is not provided. Testing should be performed using the dedicated test related targets.$(CRES)\n" + $(ECHO) "Run $(CCYA)\"make help\"$(CRES) for a list of commands and focus on the $(CCYA)\"testing\"$(CRES) section.\n" django/version: $(MAKE) internal/django-mgmt django_cmd="version" django/auth/changepassword: # TODO: test this! with a matching user account and without... - $(ECHO) "Trying to change the password of a Django user that matches your username..." - $(ECHO) "If you want to change the password of another user, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"changepassword \"$(CRES)." + $(ECHO) "Trying to change the password of a Django user that matches your username...\n" + $(ECHO) "If you want to change the password of another user, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"changepassword \"$(CRES).\n" $(MAKE) internal/django-mgmt django_cmd="changepassword" django/auth/createsuperuser: @@ -185,7 +233,7 @@ django/auth/createsuperuser: django/auth/createsuperuser-noinput: # TODO: This feature will be added, if Django>3.0 will be the main supported version! - $(ECHO) "This feature will be implemented in a future release!" + $(ECHO) "This feature will be implemented in a future release!\n" $(MAKE) django/auth/createsuperuser django/contenttypes/remove_stale: @@ -208,11 +256,6 @@ django/staticfiles/find: # TODO: check documentation; subcommands required? $(MAKE) internal/django-mgmt django_cmd="findstatic" -# main interface to run Django management commands -django_cmd ?= "" -internal/django-mgmt: internal/tox/dev-django -# internal/django-mgmt: internal/django/raw - internal/django/raw: $(DJANGOADMIN_CMD) $(django_cmd) $(DJANGOADMIN_CMD_VERBOSITY) --settings={{ project_name }}.settings.development @@ -227,17 +270,17 @@ util/requirements-force: $(MAKE) util/requirements -current: - $(ECHO) "$(CRED)red$(CRES) / $(CGRE)green$(CRES) / $(CYEL)yellow$(CRES) / $(CCYA)blue$(CRES)" - - # prepare coloring of echo outputs +ECHO := /usr/bin/printf + CRED := $(shell tput setaf 1 2>/dev/null) CGRE := $(shell tput setaf 2 2>/dev/null) CYEL := $(shell tput setaf 3 2>/dev/null) CCYA := $(shell tput setaf 6 2>/dev/null) CRES := $(shell tput sgr0 2>/dev/null) -ECHO := echo + +DPS_USERNAME := $(shell id -un) +DPS_BUILD_ID := $(shell ./bin/get_build_id.sh) # suppress unintended output .SILENT: From 713136de4d9c7bb83fa86a15e0405521eba5a632 Mon Sep 17 00:00:00 2001 From: Mischback Date: Tue, 11 Feb 2020 23:44:24 +0100 Subject: [PATCH 023/110] Removed outdated Makefile backup --- configs/Makefile.dev.sample | 89 ------------------------------------- 1 file changed, 89 deletions(-) delete mode 100644 configs/Makefile.dev.sample diff --git a/configs/Makefile.dev.sample b/configs/Makefile.dev.sample deleted file mode 100644 index cfa6150..0000000 --- a/configs/Makefile.dev.sample +++ /dev/null @@ -1,89 +0,0 @@ -.SILENT: -.PHONY: clean current default doc doc-srv help requirements requirements-force \ - serve tox \ - docker/build docker/test-build-context \ - .docker/build .docker/test-build-context - -DOCKER_CMD:=docker -DOCKER_COMPOSE_CMD:=docker-compose - -DPS_DOCKER_REPO:="mischback/dps" -DPS_GIT_COMMIT_BRANCH:=$(shell git rev-parse --abbrev-ref HEAD) -DPS_GIT_COMMIT_SHA1:=$(shell git rev-parse --short HEAD) - -DPS_DOCKER_TAG="$(DPS_DOCKER_REPO):$(DPS_GIT_COMMIT_BRANCH)-$(DPS_GIT_COMMIT_SHA1)" - - -default: help - echo "" - echo "You need to specify a subcommand." - exit 1 - - -help: - echo "" - echo "Development utilities" - echo " clean Removes all temporary files" - echo " current Runs the tox-environment for the current development" - echo " doc Builds the documentation using 'Sphinx'" - echo " doc-srv Serves the documentation on port 8082 (and automatically builds it)" - echo " serve Runs the Django development server on port 8080" - echo " tox Runs complete tox test environment" - echo "" - echo "Docker related commands" - echo " Please note, that if you are not 'root', you have to do 'sudo make ...'!" - echo "" - echo " docker/test-build-context" - echo " Builds a minimal docker container and shows the context" - echo " This is used to check the .dockerignore file" - echo "" - - -# deletes all temporary files created by Django -clean: - find . -iname "*.pyc" -delete - find . -iname "__pycache__" -delete - rm requirements/common.txt - rm requirements/production.txt - -current: - tox -q -e util - -doc: - tox -q -e doc - -doc-srv: doc - tox -q -e doc-srv - -requirements: - tox -q -e build_requirements - -requirements-force: - touch requirements/*.in - $(MAKE) requirements - -serve: - tox -q -e run - -tox: - tox -q - -docker/build: requirements - sudo $(MAKE) .docker/build - -.docker/build: - DPS_DOCKER_TAG=$(DPS_DOCKER_TAG) \ - $(DOCKER_COMPOSE_CMD) -f configs/Docker/docker-compose.yml build - $(DOCKER_CMD) tag $(DPS_DOCKER_TAG) "$(DPS_DOCKER_REPO):$(DPS_GIT_COMMIT_BRANCH)-current" - -docker/test-build-context: - sudo $(MAKE) .docker/test-build-context - -.docker/test-build-context: - echo " \ - FROM busybox\n \ - COPY . /build-context\n \ - WORKDIR /build-context\n \ - CMD find ." \ - | docker build -t test-build-context -f- . \ - && docker container run --rm test-build-context From ea118f20c9ea953cba96d0448d73d167bb46497b Mon Sep 17 00:00:00 2001 From: Mischback Date: Tue, 11 Feb 2020 23:47:00 +0100 Subject: [PATCH 024/110] Renamed the apache2 config to match new file naming scheme --- configs/{apache2_vhost.sample => apache2_vhost.deployment} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename configs/{apache2_vhost.sample => apache2_vhost.deployment} (100%) diff --git a/configs/apache2_vhost.sample b/configs/apache2_vhost.deployment similarity index 100% rename from configs/apache2_vhost.sample rename to configs/apache2_vhost.deployment From 993665f7813b2e847081d084f3e3810589de0e7b Mon Sep 17 00:00:00 2001 From: Mischback Date: Tue, 11 Feb 2020 23:49:17 +0100 Subject: [PATCH 025/110] Removed unnecessary line from Makefile --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index f82874e..47530d1 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,6 @@ docker/test-build-context .docker/test-build-context DOCKER_CMD:=docker -DOCKER_COMPOSE_CMD:=docker-compose DPS_DOCKER_REPO:="mischback/dps" From dd6cb67a0ee130b016f27a608bc3243a650cc8c1 Mon Sep 17 00:00:00 2001 From: Mischback Date: Wed, 12 Feb 2020 21:20:56 +0100 Subject: [PATCH 026/110] Comments on get_build_id.sh --- bin/get_build_id.sh | 45 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/bin/get_build_id.sh b/bin/get_build_id.sh index ff511dd..a585579 100755 --- a/bin/get_build_id.sh +++ b/bin/get_build_id.sh @@ -1,12 +1,53 @@ #!/bin/sh +# This script is part of django-project-skeleton and is used to determine an +# identifier to tag Docker images. +# +# The project's Makefile will call this script and then use its output, beside +# other values/constants to construct the image's tag. +# +# Preferably the project is version controlled by git. The script will +# determine the latest commit sha1 hash (referenced by HEAD) and will also +# determine, if there are unstaged/uncommitted changes in the repository. +# The sha1 hash will be used to tag the image (see Makefile), optionally with +# an indicator for an dirty repository (during testing/staging/deployment you +# will not want 'dirty' images, because they are not deterministically +# reproducible. Nonetheless, those images will be created and may be used). +# +# If the project is not version controlled, a timestamp will be used instead. +# This is *not* recommended, because it will not allow you to recreate a +# specific image, making debugging nearly impossible. +# Additionally, the timestamp depends on the build system's timezone, +# introducing further problems, especially if the image should be used by +# people based in other timezones. +# +# TL;DR: +# - use git to version control your project +# - use the provided Makefile to build your Docker images +# - profit + +# determine, if the project is under version control by git if git rev-parse --is-inside-work-tree > /dev/null 2>&1; then - sha=$(git rev-parse --short HEAD); - if test -n "$(git status --porcelain)"; then + + # get most recent git commit sha1 hash + sha=$(git rev-parse --short HEAD -- 2>/dev/null); + + # use the porcelain of git status to determine, if there a unstaged + # changes in the repository + # TODO: How portable is this? Should this be switched to a plumbing cmd? + if test -n "$(git status --porcelain -- 2>/dev/null)"; then /usr/bin/printf "%s-dirty\n" $sha; else /usr/bin/printf "%s\n" $sha; fi +# Oh noooo, git is not used... Provide a timestamp to identify the image! +# This should be the absolutely last resort to tag an image and is *not* +# recommended. You should really be using a VCS! else + # YYYYmmdd-HHMM -> 20200212-1953 + # Please note, that this depends on your system's timezone, which makes + # identifying the resulting image *really* hard. + # Additionally, this will *not* allow rebuilding images in a deterministic + # way. date +%Y%m%d-%H%M 2>/dev/null; fi From 0db25cb20c0ce9c0ba3bc042df67ef9abe9ffef6 Mon Sep 17 00:00:00 2001 From: Mischback Date: Wed, 12 Feb 2020 21:30:42 +0100 Subject: [PATCH 027/110] Adjust .dockerignore to exclude all deployment related utility files --- .dockerignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.dockerignore b/.dockerignore index 8799979..9ce16f1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -32,6 +32,9 @@ configs/Docker # do not include any samples **/*.sample +# do not include the deployment specific files +**/*.deployment + # I started to put temporary notes into TODO files **/*.todo From f90e1a590611c1dc35c7da31599373379151e297 Mon Sep 17 00:00:00 2001 From: Mischback Date: Wed, 12 Feb 2020 22:10:19 +0100 Subject: [PATCH 028/110] Building Docker images using the tox environment Success! The build process can now be launched directly from within this repo, no more need to use a seperate test bed. The current structure is deployed into a temporary tox environment, initialized and then the Docker build process is triggered. Images will be tagged with the sha1 tags of this repo. --- Makefile | 3 +++ configs/Makefile.deployment | 1 + tox.ini | 16 ++++++++-------- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 47530d1..5654e6a 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,9 @@ init: cp ./configs/tox.deployment ./tox.ini cp ./configs/Makefile.deployment ./Makefile +test/docker-build: + tox -q -e docker-testing + tree: tree -a -I ".git|.tox|doc|run" --dirsfirst diff --git a/configs/Makefile.deployment b/configs/Makefile.deployment index ae2209b..1d48070 100644 --- a/configs/Makefile.deployment +++ b/configs/Makefile.deployment @@ -57,6 +57,7 @@ DOCKER_COMPOSE_CMD := $(shell which docker-compose) ### shortcut commands ### run: dev/runserver +build: docker/build ### development commands ### diff --git a/tox.ini b/tox.ini index 49d85eb..737e9fa 100644 --- a/tox.ini +++ b/tox.ini @@ -31,10 +31,10 @@ deps = django21: Django>=2.1, <2.2 django22: Django>=2.2, <3.0 django30: Django>=3.0, <3.1 -changedir = {envtmpdir} +changedir = {envtmpdir}/foo commands = - django-admin startproject --template={toxinidir} foo - {envbindir}/python foo/manage.py check --settings=foo.settings.development + django-admin startproject --extension=py,deployment --template={toxinidir} foo . + {envbindir}/python manage.py check --settings=foo.settings.development [testenv:util] basepython = python3 @@ -42,10 +42,10 @@ envdir = {toxworkdir}/current deps = -r{toxinidir}/requirements/development.txt skip_install = true -changedir = {envtmpdir} +changedir = {envtmpdir}/foo commands = - django-admin startproject --template={toxinidir} foo - {posargs:{envbindir}/python foo/manage.py check --settings=foo.settings.development} + django-admin startproject --extension=py,deployment --template={toxinidir} foo . + {posargs:{envbindir}/python manage.py check --settings=foo.settings.development} [testenv:docker-testing] basepython = {[testenv:util]basepython} @@ -56,8 +56,9 @@ changedir = {envtmpdir}/foo whitelist_externals = make commands = - django-admin startproject --template={toxinidir} foo . + django-admin startproject --extension=py,deployment --template={toxinidir} foo . make init + {posargs:make docker/build} [testenv:build_requirements] basepython = {[testenv:util]basepython} @@ -94,7 +95,6 @@ commands = python -m http.server {posargs:8082} # Python3 command - ################################################################################ # The following sections actually provide settings for various tools ################################################################################ From 78d65b243504ffa3cb47d96fe7250bc2fe1f9162 Mon Sep 17 00:00:00 2001 From: Mischback Date: Sat, 15 Feb 2020 12:05:41 +0100 Subject: [PATCH 029/110] Fetched utility script to install packages --- .dockerignore | 3 ++- bin/apt-install | 29 +++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100755 bin/apt-install diff --git a/.dockerignore b/.dockerignore index 9ce16f1..abbbf3e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -16,7 +16,8 @@ tox.ini .dockerignore .editorconfig .travis.yml -Makefile +**/Makefile +requirements/*.in # do NOT include ANY readme files **/README* diff --git a/bin/apt-install b/bin/apt-install new file mode 100755 index 0000000..c86fc9c --- /dev/null +++ b/bin/apt-install @@ -0,0 +1,29 @@ +#!/bin/bash + +# This script is used to install packages into Debian-based images. +# +# It may be called with a list of Debian packages and will automatically +# perform the refresh of apt's list (apt-get upate) and then install the +# requested packages. +# To keep the Docker images small, apt's lists are removed. +# +# This sequence of action is found in many Debian-based images, usually +# implemented by a single RUN command with several '&&'-chained commands. +# This solution was discovered in Mozilla's bedrock repository[1]. It improves +# readability of the Dockerfile and is easy enough to provide. +# +# [1]: https://github.com/mozilla/bedrock + + +# stop the script, if any of the commands fails +set -e + +# update apt's sources lists +apt-get update + +# install the specified packages and their dependencies +# do NOT install 'recommended packages' +apt-get install -y --no-install-recommends "$@" + +# remove apt's sources lists +rm -rf /var/lib/apt/lists/* From 3831b6f63fde2b64783015ed6fc93c343e12fa24 Mon Sep 17 00:00:00 2001 From: Mischback Date: Sat, 15 Feb 2020 12:28:39 +0100 Subject: [PATCH 030/110] Prepare usage of this utility script --- configs/Docker/django/Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/configs/Docker/django/Dockerfile b/configs/Docker/django/Dockerfile index 0b9bcef..4709364 100644 --- a/configs/Docker/django/Dockerfile +++ b/configs/Docker/django/Dockerfile @@ -14,6 +14,9 @@ # the Python packages, they must be installed here aswell! FROM python:3.8-slim-buster as base +# fetch our custom install script +COPY bin/apt-install /usr/local/bin/ + # set environment variables # TODO: Check if these are still valid for "production"! ENV PYTHONDONTWRITEBYTECODE 1 @@ -46,6 +49,9 @@ RUN pip install --no-cache-dir \ # directly into this image. FROM python:3.8-slim-buster as django_production +# fetch our custom install script +COPY bin/apt-install /usr/local/bin/ + # create a user to be used (meaning: we won't run with root-permissions!) RUN useradd -u 2342 -g 65534 -s /bin/false -M pythonuser && \ mkdir /home/pythonuser From 8ea8c56f0ec3844155b92d92215f2fde641b1415 Mon Sep 17 00:00:00 2001 From: Mischback Date: Sat, 15 Feb 2020 21:53:59 +0100 Subject: [PATCH 031/110] Make the container runnable This commit is only related to the internal development/testing environment and the method must be transfered to the actual deployed project. --- Makefile | 46 ++++++++++++++++++++++++------- configs/Docker/docker-compose.yml | 7 +++-- tox.ini | 16 +++++------ 3 files changed, 49 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 5654e6a..1586688 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,29 @@ .SILENT: .PHONY: clean init tree \ - docker/test-build-context .docker/test-build-context + docker/build docker/build-context .docker/build-context -DOCKER_CMD:=docker +# Docker command +DOCKER_CMD := $(shell which docker) -DPS_DOCKER_REPO:="mischback/dps" +# Docker compose command +DOCKER_COMPOSE_CMD := $(shell which docker-compose) + +# Docker related testing is performed by using tox: +# - an environment is used to setup a testing project (docker-testing) +# - the project is setup in the temporary directory, thus this step is +# performed by every run +# - the Docker build is triggered using the corresponding Makefile +# Makefile.deployment +# - this Makefile will automatically tag the created image +# - to start the image, this tagging has to be reproduced here +# +# During project setup, this variable is set to the project's name. +# The described testing process will use 'dpstest' as the project's name +DPS_BUILD_NAME_PREFIX := "dpstest" +# While the image is build, the git commit's sha1 hash is used to tag the image +# Additionally, 'latest' is applied to the image aswell. +# In order to run the image, we rely on the 'latest' build. +DPS_BUILD_ID := "latest" # deletes all temporary and unwanted files @@ -26,20 +45,27 @@ init: cp ./configs/tox.deployment ./tox.ini cp ./configs/Makefile.deployment ./Makefile -test/docker-build: +docker/build: tox -q -e docker-testing +docker/build-context: + sudo $(MAKE) .docker/build-context + +docker/run: + sudo $(MAKE) .docker/run + tree: tree -a -I ".git|.tox|doc|run" --dirsfirst -docker/test-build-context: - sudo $(MAKE) .docker/test-build-context - -.docker/test-build-context: +.docker/build-context: echo " \ FROM busybox\n \ COPY . /build-context\n \ WORKDIR /build-context\n \ CMD find ." | \ - $(DOCKER_CMD) build -t "$(DPS_DOCKER_REPO):test-build-context" -f- . && \ - $(DOCKER_CMD) container run --rm "$(DPS_DOCKER_REPO):test-build-context" + $(DOCKER_CMD) build -t "$(DPS_BUILD_NAME_PREFIX)/build-context:latest" -f- . && \ + $(DOCKER_CMD) container run --rm "$(DPS_BUILD_NAME_PREFIX)/build-context:latest" + +.docker/run: + DPS_BUILD_NAME_PREFIX=$(DPS_BUILD_NAME_PREFIX) DPS_BUILD_ID=$(DPS_BUILD_ID) \ + $(DOCKER_COMPOSE_CMD) -f configs/Docker/docker-compose.yml up diff --git a/configs/Docker/docker-compose.yml b/configs/Docker/docker-compose.yml index ce47131..4eb5e1e 100644 --- a/configs/Docker/docker-compose.yml +++ b/configs/Docker/docker-compose.yml @@ -5,8 +5,11 @@ services: build: context: ../../ dockerfile: configs/Docker/django/Dockerfile - # this should *magically* include a hash/datetime/unique identifier - image: ${DPS_BUILD_NAME_PREFIX}/django:${DPS_BUILD_ID} command: python manage.py runserver 0.0.0.0:8000 + # providing the build and the image directive automatically tags the image + # automatic tagging is performed with environment variables provided + # through the Makefile and results in a tag with either a git commit hash + # or a datetime stamp (see Makefile.deployment) + image: ${DPS_BUILD_NAME_PREFIX}/django:${DPS_BUILD_ID} ports: - 8000:8000 diff --git a/tox.ini b/tox.ini index 737e9fa..8afa531 100644 --- a/tox.ini +++ b/tox.ini @@ -31,10 +31,10 @@ deps = django21: Django>=2.1, <2.2 django22: Django>=2.2, <3.0 django30: Django>=3.0, <3.1 -changedir = {envtmpdir}/foo +changedir = {envtmpdir}/dpstest commands = - django-admin startproject --extension=py,deployment --template={toxinidir} foo . - {envbindir}/python manage.py check --settings=foo.settings.development + django-admin startproject --extension=py,deployment --template={toxinidir} dpstest . + {envbindir}/python manage.py check --settings=dpstest.settings.development [testenv:util] basepython = python3 @@ -42,21 +42,21 @@ envdir = {toxworkdir}/current deps = -r{toxinidir}/requirements/development.txt skip_install = true -changedir = {envtmpdir}/foo +changedir = {envtmpdir}/dpstest commands = - django-admin startproject --extension=py,deployment --template={toxinidir} foo . - {posargs:{envbindir}/python manage.py check --settings=foo.settings.development} + django-admin startproject --extension=py,deployment --template={toxinidir} dpstest . + {posargs:{envbindir}/python manage.py check --settings=dpstest.settings.development} [testenv:docker-testing] basepython = {[testenv:util]basepython} envdir = {[testenv:util]envdir} deps = {[testenv:util]deps} skip_install = {[testenv:util]skip_install} -changedir = {envtmpdir}/foo +changedir = {envtmpdir}/dpstest whitelist_externals = make commands = - django-admin startproject --extension=py,deployment --template={toxinidir} foo . + django-admin startproject --extension=py,deployment --template={toxinidir} dpstest . make init {posargs:make docker/build} From 1c63abb368995a0fc36b337e4ce6bc5a90fb677c Mon Sep 17 00:00:00 2001 From: Mischback Date: Sat, 15 Feb 2020 22:10:55 +0100 Subject: [PATCH 032/110] Transfer docker command to Makefile.deployment --- Makefile | 2 +- configs/Makefile.deployment | 100 ++++++++++++++++++++---------------- 2 files changed, 57 insertions(+), 45 deletions(-) diff --git a/Makefile b/Makefile index 1586688..eb7e74c 100644 --- a/Makefile +++ b/Makefile @@ -68,4 +68,4 @@ tree: .docker/run: DPS_BUILD_NAME_PREFIX=$(DPS_BUILD_NAME_PREFIX) DPS_BUILD_ID=$(DPS_BUILD_ID) \ - $(DOCKER_COMPOSE_CMD) -f configs/Docker/docker-compose.yml up + $(DOCKER_COMPOSE_CMD) -f configs/Docker/docker-compose.yml up django diff --git a/configs/Makefile.deployment b/configs/Makefile.deployment index 1d48070..b99567e 100644 --- a/configs/Makefile.deployment +++ b/configs/Makefile.deployment @@ -67,61 +67,50 @@ dev/runserver: django/runserver-external4 dev/static: django/staticfiles/find -### Docker commands ### - -docker/build: util/requirements - sudo $(MAKE) internal/docker/build - -internal/docker/build: - DPS_BUILD_NAME_PREFIX=$(DPS_BUILD_NAME_PREFIX) DPS_BUILD_ID=$(DPS_BUILD_ID) \ - $(DOCKER_COMPOSE_CMD) -f configs/Docker/docker-compose.yml build - $(DOCKER_CMD) tag "$(DPS_BUILD_NAME_PREFIX)/django:$(DPS_BUILD_ID)" "$(DPS_BUILD_NAME_PREFIX)/django:latest" - - ### Django management commands ### django_cmd ?= "" django/check: - $(MAKE) internal/django-mgmt django_cmd="check" + $(MAKE) .internal/django-mgmt django_cmd="check" django/compilemessages: - $(MAKE) internal/django-mgmt django_cmd="compilemessages" + $(MAKE) .internal/django-mgmt django_cmd="compilemessages" django/createcachetable: - $(MAKE) internal/django-mgmt django_cmd="createcachetable" + $(MAKE) .internal/django-mgmt django_cmd="createcachetable" django/dbshell: # requires the command line client to be available in user's PATH # dynamically determines the desired command line client from settings file - $(MAKE) internal/django-mgmt django_cmd="dbshell" + $(MAKE) .internal/django-mgmt django_cmd="dbshell" django/diffsettings: # compare with Django's default settings # NOTE: compares the project's **development** settings with Django's defaults - $(MAKE) internal/django-mgmt django_cmd="diffsettings" + $(MAKE) .internal/django-mgmt django_cmd="diffsettings" django/diffsettings-all: # show all settings, even if they correspond with Django's defaults # NOTE: compares the project's **development** settings with Django's defaults - $(MAKE) internal/django-mgmt django_cmd="diffsettings --all" + $(MAKE) .internal/django-mgmt django_cmd="diffsettings --all" django/diffsettings-prod: # compare **development** settings with **production** settings - $(MAKE) internal/django-mgmt django_cmd="diffsettings --default={{ project_name }}.settings.production" + $(MAKE) .internal/django-mgmt django_cmd="diffsettings --default={{ project_name }}.settings.production" django/dumpdata: # apply indentation to increase readability - $(MAKE) internal/django-mgmt django_cmd="dumpdata --indent=4" + $(MAKE) .internal/django-mgmt django_cmd="dumpdata --indent=4" django/flush: # flushes the database - $(MAKE) internal/django-mgmt django_cmd="flush" + $(MAKE) .internal/django-mgmt django_cmd="flush" django/flush-plan: django/sqlflush django/flush-noinput: # flushes the database without any prompts - $(MAKE) internal/django-mgmt django_cmd="flush --no-input" + $(MAKE) .internal/django-mgmt django_cmd="flush --no-input" django/inspectdb: # will not be wrapped, because this is a very edgy use case @@ -133,35 +122,35 @@ django/loaddata: django/makemessages: # TODO: Test this command! Is it sufficient or should '--all' be passed aswell? - $(MAKE) internal/django-mgmt django_cmd="makemessages" + $(MAKE) .internal/django-mgmt django_cmd="makemessages" django/makemigrations: # this command is provided for completeness, but should not be necessary for a Django project (relevance for apps!) - $(MAKE) internal/django-mgmt django_cmd="makemigrations" + $(MAKE) .internal/django-mgmt django_cmd="makemigrations" django/migrate: # synchronize your models with your database - $(MAKE) internal/django-mgmt django_cmd="migrate" + $(MAKE) .internal/django-mgmt django_cmd="migrate" django/migrate-plan: # show the migration operations, without actually executing them - $(MAKE) internal/django-mgmt django_cmd="migrate --plan" + $(MAKE) .internal/django-mgmt django_cmd="migrate --plan" django/runserver: django/migrate # runs the development server with default settings # this will expose port 8000 on localhost, so it will not be accessible from another machine. # see 'runserver-external[4|6]' for an alternative - $(MAKE) internal/django-mgmt django_cmd="runserver" + $(MAKE) .internal/django-mgmt django_cmd="runserver" django/runserver-external4: django/migrate # runs the development server with settings that allow access from another machine # the server will be exposed on port 8000 - $(MAKE) internal/django-mgmt django_cmd="runserver 0.0.0.0:8000" + $(MAKE) .internal/django-mgmt django_cmd="runserver 0.0.0.0:8000" django/runserver-external6: django/migrate # runs the development server with settings that allow access from another machine # the server will be exposed on port 8000 - $(MAKE) internal/django-mgmt django_cmd="runserver -6" + $(MAKE) .internal/django-mgmt django_cmd="runserver -6" django/sendtestemail: # will not be wrapped, because this is a very edgy use case @@ -170,18 +159,18 @@ django/sendtestemail: django/shell: # runs a python shell with the project's development settings - $(MAKE) internal/django-mgmt django_cmd="shell" + $(MAKE) .internal/django-mgmt django_cmd="shell" django/showmigrations: - $(MAKE) internal/django-mgmt django_cmd="showmigrations" + $(MAKE) .internal/django-mgmt django_cmd="showmigrations" django/showmigrations-plan: # show the migration operations, without actually executing them - $(MAKE) internal/django-mgmt django_cmd="showmigrations --plan" + $(MAKE) .internal/django-mgmt django_cmd="showmigrations --plan" django/sqlflush: # prints the SQL statements that would be executed for a 'flush' (see above) - $(MAKE) internal/django-mgmt django_cmd="sqlflush" + $(MAKE) .internal/django-mgmt django_cmd="sqlflush" django/sqlmigrate: # will not be wrapped, because this is a very edgy use case @@ -221,16 +210,16 @@ django/testserver: $(ECHO) "Run $(CCYA)\"make help\"$(CRES) for a list of commands and focus on the $(CCYA)\"testing\"$(CRES) section.\n" django/version: - $(MAKE) internal/django-mgmt django_cmd="version" + $(MAKE) .internal/django-mgmt django_cmd="version" django/auth/changepassword: # TODO: test this! with a matching user account and without... $(ECHO) "Trying to change the password of a Django user that matches your username...\n" $(ECHO) "If you want to change the password of another user, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"changepassword \"$(CRES).\n" - $(MAKE) internal/django-mgmt django_cmd="changepassword" + $(MAKE) .internal/django-mgmt django_cmd="changepassword" django/auth/createsuperuser: - $(MAKE) internal/django-mgmt django_cmd="createsuperuser" + $(MAKE) .internal/django-mgmt django_cmd="createsuperuser" django/auth/createsuperuser-noinput: # TODO: This feature will be added, if Django>3.0 will be the main supported version! @@ -240,28 +229,35 @@ django/auth/createsuperuser-noinput: django/contenttypes/remove_stale: # only available if django.contrib.contenttypes is in INSTALLED_APPS # TODO: check documentation; subcommands required? - $(MAKE) internal/django-mgmt django_cmd="remove_stale_contenttypes" + $(MAKE) .internal/django-mgmt django_cmd="remove_stale_contenttypes" django/sessions/clear: # only available if django.contrib.sessions is in INSTALLED_APPS # TODO: check documentation; subcommands required? - $(MAKE) internal/django-mgmt django_cmd="clearsessions" + $(MAKE) .internal/django-mgmt django_cmd="clearsessions" django/staticfiles/collect: # only available if django.contrib.staticfiles is in INSTALLED_APPS # TODO: check documentation; subcommands required? - $(MAKE) internal/django-mgmt django_cmd="collectstatic" + $(MAKE) .internal/django-mgmt django_cmd="collectstatic" django/staticfiles/find: # only available if django.contrib.staticfiles is in INSTALLED_APPS # TODO: check documentation; subcommands required? - $(MAKE) internal/django-mgmt django_cmd="findstatic" + $(MAKE) .internal/django-mgmt django_cmd="findstatic" -internal/django/raw: - $(DJANGOADMIN_CMD) $(django_cmd) $(DJANGOADMIN_CMD_VERBOSITY) --settings={{ project_name }}.settings.development -internal/tox/dev-django: util/requirements - $(TOX_CMD) dev-django -- $(django_cmd) $(DJANGOADMIN_CMD_VERBOSITY) +### Docker commands ### + +docker/build: util/requirements + sudo $(MAKE) .internal/docker/build + +docker/run: docker/run/django + +docker/run/django: + sudo $(MAKE) .internal/docker/run/django + +### Utility commands ### util/requirements: $(TOX_CMD) build_requirements @@ -271,6 +267,22 @@ util/requirements-force: $(MAKE) util/requirements +.internal/django/raw: + $(DJANGOADMIN_CMD) $(django_cmd) $(DJANGOADMIN_CMD_VERBOSITY) --settings={{ project_name }}.settings.development + +.internal/docker/build: + DPS_BUILD_NAME_PREFIX=$(DPS_BUILD_NAME_PREFIX) DPS_BUILD_ID=$(DPS_BUILD_ID) \ + $(DOCKER_COMPOSE_CMD) -f configs/Docker/docker-compose.yml build + $(DOCKER_CMD) tag "$(DPS_BUILD_NAME_PREFIX)/django:$(DPS_BUILD_ID)" "$(DPS_BUILD_NAME_PREFIX)/django:latest" + +.internal/docker/run/django: + DPS_BUILD_NAME_PREFIX=$(DPS_BUILD_NAME_PREFIX) DPS_BUILD_ID=$(DPS_BUILD_ID) \ + $(DOCKER_COMPOSE_CMD) -f configs/Docker/docker-compose.yml up django + +.internal/tox/dev-django: util/requirements + $(TOX_CMD) dev-django -- $(django_cmd) $(DJANGOADMIN_CMD_VERBOSITY) + + # prepare coloring of echo outputs ECHO := /usr/bin/printf @@ -291,7 +303,7 @@ DPS_BUILD_ID := $(shell ./bin/get_build_id.sh) \ dev/check dev/migrate dev/runserver dev/static \ \ - internal/django-mgmt internal/django/raw internal/tox/dev-django \ + .internal/django-mgmt .internal/django/raw .internal/tox/dev-django \ util/requirements util/requirements-force \ \ django/check django/compilemessages django/createcachetable \ From 54e6ac7beddf97aecd5374e2f6c5c648888d13fd Mon Sep 17 00:00:00 2001 From: Mischback Date: Sat, 15 Feb 2020 23:18:46 +0100 Subject: [PATCH 033/110] Place Docker-related scripts in their own directory --- .dockerignore | 7 ++++++- {bin => configs/Docker/bin}/apt-install | 0 configs/Docker/django/Dockerfile | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) rename {bin => configs/Docker/bin}/apt-install (100%) diff --git a/.dockerignore b/.dockerignore index abbbf3e..4a29ff2 100644 --- a/.dockerignore +++ b/.dockerignore @@ -27,8 +27,13 @@ requirements/*.in # during initialising the project. doc -# do not include the Dockerfiles +# do NOT include the bin directory (these scripts are not used in production) +bin + +# do NOT include the Docker-related files configs/Docker +# but DO include the production-related scripts +!configs/Docker/bin/apt-install # do not include any samples **/*.sample diff --git a/bin/apt-install b/configs/Docker/bin/apt-install similarity index 100% rename from bin/apt-install rename to configs/Docker/bin/apt-install diff --git a/configs/Docker/django/Dockerfile b/configs/Docker/django/Dockerfile index 4709364..4fd8208 100644 --- a/configs/Docker/django/Dockerfile +++ b/configs/Docker/django/Dockerfile @@ -15,7 +15,7 @@ FROM python:3.8-slim-buster as base # fetch our custom install script -COPY bin/apt-install /usr/local/bin/ +COPY configs/Docker/bin/apt-install /usr/local/bin/ # set environment variables # TODO: Check if these are still valid for "production"! @@ -50,7 +50,7 @@ RUN pip install --no-cache-dir \ FROM python:3.8-slim-buster as django_production # fetch our custom install script -COPY bin/apt-install /usr/local/bin/ +COPY configs/Docker/bin/apt-install /usr/local/bin/ # create a user to be used (meaning: we won't run with root-permissions!) RUN useradd -u 2342 -g 65534 -s /bin/false -M pythonuser && \ From 4733a15dc48e8639e5003bb652521616b880f4d7 Mon Sep 17 00:00:00 2001 From: Mischback Date: Sun, 16 Feb 2020 00:08:02 +0100 Subject: [PATCH 034/110] Added gunicorn to the stack This needs more configuration, but will be done, when nginx is in the stack aswell. --- .dockerignore | 1 + Makefile | 1 + configs/Docker/bin/django-run.deployment | 4 ++++ configs/Docker/docker-compose.yml | 2 +- 4 files changed, 7 insertions(+), 1 deletion(-) create mode 100755 configs/Docker/bin/django-run.deployment diff --git a/.dockerignore b/.dockerignore index 4a29ff2..ba9f333 100644 --- a/.dockerignore +++ b/.dockerignore @@ -34,6 +34,7 @@ bin configs/Docker # but DO include the production-related scripts !configs/Docker/bin/apt-install +!configs/Docker/bin/django-run.sh # do not include any samples **/*.sample diff --git a/Makefile b/Makefile index eb7e74c..c2dea1a 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,7 @@ clean: init: echo "Initialising repository..." cp ./configs/tox.deployment ./tox.ini + mv ./configs/Docker/bin/django-run.deployment ./configs/Docker/bin/django-run.sh cp ./configs/Makefile.deployment ./Makefile docker/build: diff --git a/configs/Docker/bin/django-run.deployment b/configs/Docker/bin/django-run.deployment new file mode 100755 index 0000000..f7a17a6 --- /dev/null +++ b/configs/Docker/bin/django-run.deployment @@ -0,0 +1,4 @@ +#!/bin/sh + +exec gunicorn {{ project_name }}.wsgi:application \ + -b 0.0.0.0:8000 diff --git a/configs/Docker/docker-compose.yml b/configs/Docker/docker-compose.yml index 4eb5e1e..efa5525 100644 --- a/configs/Docker/docker-compose.yml +++ b/configs/Docker/docker-compose.yml @@ -5,7 +5,7 @@ services: build: context: ../../ dockerfile: configs/Docker/django/Dockerfile - command: python manage.py runserver 0.0.0.0:8000 + command: ./configs/Docker/bin/django-run.sh # providing the build and the image directive automatically tags the image # automatic tagging is performed with environment variables provided # through the Makefile and results in a tag with either a git commit hash From b1ca417d53b7538d611e67ad116cb3e08b57febc Mon Sep 17 00:00:00 2001 From: Mischback Date: Sun, 16 Feb 2020 19:06:59 +0100 Subject: [PATCH 035/110] API BREAK This commit breaks the documented inner working of the settings module. From now on, 'production' and 'development' are distinct configurations, both depending on 'common'. --- project_name/settings/common.py | 35 ++++++++++++++++------------ project_name/settings/development.py | 18 ++++---------- project_name/settings/production.py | 9 +++---- 3 files changed, 30 insertions(+), 32 deletions(-) diff --git a/project_name/settings/common.py b/project_name/settings/common.py index 6b835a6..fb6a630 100644 --- a/project_name/settings/common.py +++ b/project_name/settings/common.py @@ -78,9 +78,29 @@ }, ] +# the default WSGI application +WSGI_APPLICATION = '%s.wsgi.application' % SITE_NAME + +# the root URL configuration +ROOT_URLCONF = '%s.urls' % SITE_NAME + +# the URL for static files +STATIC_URL = '/static/' + +# the URL for media files +MEDIA_URL = '/media/' + +# adjust the minimal login +LOGIN_URL = 'core_login' +LOGIN_REDIRECT_URL = '/' +LOGOUT_REDIRECT_URL = 'core_login' + # Internationalization USE_I18N = False +# uncomment the following line to include i18n +# from .i18n import * + # ##### SECURITY CONFIGURATION ############################ @@ -95,21 +115,6 @@ MANAGERS = ADMINS -# ##### DJANGO RUNNING CONFIGURATION ###################### - -# the default WSGI application -WSGI_APPLICATION = '%s.wsgi.application' % SITE_NAME - -# the root URL configuration -ROOT_URLCONF = '%s.urls' % SITE_NAME - -# the URL for static files -STATIC_URL = '/static/' - -# the URL for media files -MEDIA_URL = '/media/' - - # ##### DEBUG CONFIGURATION ############################### DEBUG = False diff --git a/project_name/settings/development.py b/project_name/settings/development.py index c9236d8..628a466 100644 --- a/project_name/settings/development.py +++ b/project_name/settings/development.py @@ -1,23 +1,15 @@ # Python imports from os.path import join -# project imports +# fetch the common settings from .common import * -# uncomment the following line to include i18n -# from .i18n import * - - -# ##### DEBUG CONFIGURATION ############################### -DEBUG = True +# ##### APPLICATION CONFIGURATION ######################### # allow all hosts during development ALLOWED_HOSTS = ['*'] -# adjust the minimal login -LOGIN_URL = 'core_login' -LOGIN_REDIRECT_URL = '/' -LOGOUT_REDIRECT_URL = 'core_login' +INSTALLED_APPS = DEFAULT_APPS # ##### DATABASE CONFIGURATION ############################ @@ -28,6 +20,6 @@ } } -# ##### APPLICATION CONFIGURATION ######################### -INSTALLED_APPS = DEFAULT_APPS +# ##### DEBUG CONFIGURATION ############################### +DEBUG = True diff --git a/project_name/settings/production.py b/project_name/settings/production.py index 273968e..86e5aed 100644 --- a/project_name/settings/production.py +++ b/project_name/settings/production.py @@ -1,12 +1,13 @@ -# for now fetch the development settings only -from .development import * +# fetch the common settings +from .common import * -# turn off all debugging -DEBUG = False +# ##### APPLICATION CONFIGURATION ######################### # You will have to determine, which hostnames should be served by Django ALLOWED_HOSTS = [] +INSTALLED_APPS = DEFAULT_APPS + # ##### SECURITY CONFIGURATION ############################ # TODO: Make sure, that sensitive information uses https From 397858b394b7041e741b771f9e012f7298042b98 Mon Sep 17 00:00:00 2001 From: Mischback Date: Sun, 16 Feb 2020 22:13:51 +0100 Subject: [PATCH 036/110] Basic working nginx/gunicorn combination --- .dockerignore | 3 +- Makefile | 3 +- configs/Docker/bin/django-run.deployment | 2 ++ configs/Docker/docker-compose.yml | 11 ++++++ configs/Docker/nginx/Dockerfile | 3 ++ configs/Docker/nginx/nginx.conf.deployment | 39 ++++++++++++++++++++++ configs/Makefile.deployment | 1 + project_name/settings/docker-nginx.py | 7 ++++ 8 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 configs/Docker/nginx/Dockerfile create mode 100644 configs/Docker/nginx/nginx.conf.deployment create mode 100644 project_name/settings/docker-nginx.py diff --git a/.dockerignore b/.dockerignore index ba9f333..3f09b20 100644 --- a/.dockerignore +++ b/.dockerignore @@ -32,9 +32,10 @@ bin # do NOT include the Docker-related files configs/Docker -# but DO include the production-related scripts +# but DO include the production-related scripts/config files !configs/Docker/bin/apt-install !configs/Docker/bin/django-run.sh +!configs/Docker/nginx/nginx.conf # do not include any samples **/*.sample diff --git a/Makefile b/Makefile index c2dea1a..428eb97 100644 --- a/Makefile +++ b/Makefile @@ -44,6 +44,7 @@ init: echo "Initialising repository..." cp ./configs/tox.deployment ./tox.ini mv ./configs/Docker/bin/django-run.deployment ./configs/Docker/bin/django-run.sh + mv ./configs/Docker/nginx/nginx.conf.deployment ./configs/Docker/nginx/nginx.conf cp ./configs/Makefile.deployment ./Makefile docker/build: @@ -69,4 +70,4 @@ tree: .docker/run: DPS_BUILD_NAME_PREFIX=$(DPS_BUILD_NAME_PREFIX) DPS_BUILD_ID=$(DPS_BUILD_ID) \ - $(DOCKER_COMPOSE_CMD) -f configs/Docker/docker-compose.yml up django + $(DOCKER_COMPOSE_CMD) -f configs/Docker/docker-compose.yml up diff --git a/configs/Docker/bin/django-run.deployment b/configs/Docker/bin/django-run.deployment index f7a17a6..7eab30f 100755 --- a/configs/Docker/bin/django-run.deployment +++ b/configs/Docker/bin/django-run.deployment @@ -1,4 +1,6 @@ #!/bin/sh +export DJANGO_SETTINGS_MODULE="{{ project_name}}.settings.docker-nginx" + exec gunicorn {{ project_name }}.wsgi:application \ -b 0.0.0.0:8000 diff --git a/configs/Docker/docker-compose.yml b/configs/Docker/docker-compose.yml index efa5525..b241fea 100644 --- a/configs/Docker/docker-compose.yml +++ b/configs/Docker/docker-compose.yml @@ -13,3 +13,14 @@ services: image: ${DPS_BUILD_NAME_PREFIX}/django:${DPS_BUILD_ID} ports: - 8000:8000 + nginx: + build: + context: ./nginx/ + dockerfile: Dockerfile + # providing the build and the image directive automatically tags the image + # automatic tagging is performed with environment variables provided + # through the Makefile and results in a tag with either a git commit hash + # or a datetime stamp (see Makefile.deployment) + image: ${DPS_BUILD_NAME_PREFIX}/nginx:${DPS_BUILD_ID} + ports: + - 80:80 diff --git a/configs/Docker/nginx/Dockerfile b/configs/Docker/nginx/Dockerfile new file mode 100644 index 0000000..918c386 --- /dev/null +++ b/configs/Docker/nginx/Dockerfile @@ -0,0 +1,3 @@ +FROM nginx:1.17 + +COPY nginx.conf /etc/nginx/nginx.conf diff --git a/configs/Docker/nginx/nginx.conf.deployment b/configs/Docker/nginx/nginx.conf.deployment new file mode 100644 index 0000000..b3ed1c8 --- /dev/null +++ b/configs/Docker/nginx/nginx.conf.deployment @@ -0,0 +1,39 @@ + +worker_processes 1; + +events { + worker_connections 1024; + accept_mutex off; +} + +http { + + # define the Django server + upstream app_server { + server django:8000; + } + + # if no Host match, close the connection to prevent host spoofing + server { + listen 80 default_server; + return 444; + } + + server { + listen 80; + client_max_body_size 4G; + + server_name deb10.massive.home; + + keepalive_timeout 5; + + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host {{ project_name }}; + proxy_set_header Host $http_host; + proxy_redirect off; + proxy_pass http://app_server; + } + } +} diff --git a/configs/Makefile.deployment b/configs/Makefile.deployment index b99567e..3296db4 100644 --- a/configs/Makefile.deployment +++ b/configs/Makefile.deployment @@ -274,6 +274,7 @@ util/requirements-force: DPS_BUILD_NAME_PREFIX=$(DPS_BUILD_NAME_PREFIX) DPS_BUILD_ID=$(DPS_BUILD_ID) \ $(DOCKER_COMPOSE_CMD) -f configs/Docker/docker-compose.yml build $(DOCKER_CMD) tag "$(DPS_BUILD_NAME_PREFIX)/django:$(DPS_BUILD_ID)" "$(DPS_BUILD_NAME_PREFIX)/django:latest" + $(DOCKER_CMD) tag "$(DPS_BUILD_NAME_PREFIX)/nginx:$(DPS_BUILD_ID)" "$(DPS_BUILD_NAME_PREFIX)/nginx:latest" .internal/docker/run/django: DPS_BUILD_NAME_PREFIX=$(DPS_BUILD_NAME_PREFIX) DPS_BUILD_ID=$(DPS_BUILD_ID) \ diff --git a/project_name/settings/docker-nginx.py b/project_name/settings/docker-nginx.py new file mode 100644 index 0000000..4a31af5 --- /dev/null +++ b/project_name/settings/docker-nginx.py @@ -0,0 +1,7 @@ +# fetch the production settings +from .production import * + +# make Django work correctly behind the nginx proxy +# nginx **must** set the appropriate header! +ALLOWED_HOSTS = ['{{ project_name }}'] +USE_X_FORWARDED_HOST = True From 221f1d5cddb47f59a241fa47a2a3d88ac552dcd1 Mon Sep 17 00:00:00 2001 From: Mischback Date: Tue, 18 Feb 2020 22:04:29 +0100 Subject: [PATCH 037/110] Provide a gunicorn configuration by file --- .dockerignore | 1 + configs/Docker/bin/django-run.deployment | 3 +- configs/Docker/django/gunicorn_conf.py | 84 ++++++++++++++++++++++++ configs/Docker/env.sample | 8 +++ 4 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 configs/Docker/django/gunicorn_conf.py create mode 100644 configs/Docker/env.sample diff --git a/.dockerignore b/.dockerignore index 3f09b20..e419f80 100644 --- a/.dockerignore +++ b/.dockerignore @@ -35,6 +35,7 @@ configs/Docker # but DO include the production-related scripts/config files !configs/Docker/bin/apt-install !configs/Docker/bin/django-run.sh +!configs/Docker/django/gunicorn_conf.py !configs/Docker/nginx/nginx.conf # do not include any samples diff --git a/configs/Docker/bin/django-run.deployment b/configs/Docker/bin/django-run.deployment index 7eab30f..4862753 100755 --- a/configs/Docker/bin/django-run.deployment +++ b/configs/Docker/bin/django-run.deployment @@ -2,5 +2,4 @@ export DJANGO_SETTINGS_MODULE="{{ project_name}}.settings.docker-nginx" -exec gunicorn {{ project_name }}.wsgi:application \ - -b 0.0.0.0:8000 +exec gunicorn {{ project_name }}.wsgi:application --config='file:configs/Docker/django/gunicorn_conf.py' diff --git a/configs/Docker/django/gunicorn_conf.py b/configs/Docker/django/gunicorn_conf.py new file mode 100644 index 0000000..50e3bb5 --- /dev/null +++ b/configs/Docker/django/gunicorn_conf.py @@ -0,0 +1,84 @@ +# Python imports +import os + + +# bind Gunicorn to this HOST:PORT, may be a list! +# TODO: Is it really required to bind to 0.0.0.0 in Docker-based setup? +# TODO: Ensure, that :8000 ist not accessible from outside of the Docker-space +bind = '0.0.0.0:8000' +# bind = ['0.0.0.0:8000', '[::]:8000'] # provide binding for IPv4 and IPv6 + +# specify the number of workers +workers = os.environ.get('DPS_GUNICORN_NUM_WORKERS', 2) # Gunicorn default: 1 + +# specify the class of Gunicorn's workers +# make use of threaded workers +worker_class = os.environ.get('DPS_GUNICORN_WORKER_CLASS', 'gthread') # Gunicorn default: 'sync' + +# number of threads per worker (see above) +threads = os.environ.get('DPS_GUNICORN_NUM_THREADS', 4) # Gunicorn default: 1 + +# limit the lifespan of a worker by the number of handled requests +# this might prevent memory leaks +max_requests = 500 # Gunicorn default: 0 (unlimited) +max_requests_jitter = 25 # Gunicorn default: 0 (new in Gunicorn 19.2) + +# number of seconds to wait for requests on a Keep-Alive connection +# this setting is **not relevant** if using the 'sync' worker-class! +keepalive = os.environ.get('DPS_GUNICORN_KEEPALIVE', 10) # Gunicorn default: 2 + +# fix Gunicorn's heartbeat inside of containers +# http://docs.gunicorn.org/en/stable/faq.html#how-do-i-avoid-gunicorn-excessively-blocking-in-os-fchmod +# BLUF: have a tmpfs mount for worker_tmp_dir +# TODO: investigate the Docker image and search for a tmpfs mount (or create one!) +worker_tmp_dir = '/dev/shm' # Gunicorn default: None + +# disables the use of sendfile(). +# TODO: Check this setting and research its mechanics in a setup with NGINX/Gunicorn +# sendfile = None # Gunicorn default: None + +# specify a PID file +# TODO: Check if a PID file is relevant for a Docker-based setup +# pidfile = None # Gunicorn default: None + +# Control user/group of the worker process +# TODO: Should not be necessary for a Docker-based setup, where privileges are +# droppen in the image +# user = +# group = + + +### security related configuration ### +# limit_request_line = 4096 # Gunicorn default: 4096 +# limit_request_fields = 100 # Gunicorn default: 100 +# limit_request_field_size = 8190 # Gunicorn default: 8190 + +# these settings may be relevant if operating behind a proxy, that terminates HTTPS +# The DPS setup does not require this, because NGINX sets a header field that +# is evaluated by Django +# secure_scheme_headers = +# forwarded_allow_ips = + + +### logging related configuration ### +# Gunicorn uses Python's default logging library. Generally, the logging may be +# configured freely by passing a 'logconfig' file or 'logconfig_dict' Python +# dict. +# For this configuration, only relevant parts of the logging config are +# exposed/altered to match the Docker-based production environment. + +# While running behind a reverse proxy, Gunicorn's logs will only show the +# reverse proxy's IP. +# This project's setup uses an NGINX, which sets the 'X-Forwarded-For'-header. +access_log_format = os.environ.get( + 'DPS_GUNICORN_LOGFORMAT', + '%({x-forwarded-for}i)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"' +) + +# while running in a Docker container, logs should be put to stdout/stderr to be picked up +accesslog = os.environ.get('DPS_GUNICORN_ACCESSLOG_FILE', '-') +errorlog = os.environ.get('DPS_GUNICORN_ERRORLOG_FILE', '-') # this is actually the default value since Gunicorn 19.2 + +# make the log level configurable by an environment variable +# 'info' is provided as a default value, which is Gunicorn's default value aswell. +loglevel = os.environ.get('DPS_GUNICORN_LOGLEVEL', 'info') diff --git a/configs/Docker/env.sample b/configs/Docker/env.sample new file mode 100644 index 0000000..8f39b28 --- /dev/null +++ b/configs/Docker/env.sample @@ -0,0 +1,8 @@ +DPS_GUNICORN_NUM_WORKERS +DPS_GUNICORN_WORKER_CLASS +DPS_GUNICORN_NUM_THREADS +DPS_GUNICORN_KEEPALIVE +DPS_GUNICORN_ACCESSLOG_FILE +DPS_GUNICORN_ERRORLOG_FILE +DPS_GUNICORN_LOGFORMAT +DPS_GUNICORN_LOGLEVEL From 3e97e3803ac45c7b925cb266f142d143fded327e Mon Sep 17 00:00:00 2001 From: Mischback Date: Tue, 18 Feb 2020 22:09:23 +0100 Subject: [PATCH 038/110] Minor modification of Makefile --- Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile b/Makefile index 428eb97..86a5084 100644 --- a/Makefile +++ b/Makefile @@ -53,6 +53,9 @@ docker/build: docker/build-context: sudo $(MAKE) .docker/build-context +docker/images: + sudo $(MAKE) .docker/images + docker/run: sudo $(MAKE) .docker/run @@ -68,6 +71,9 @@ tree: $(DOCKER_CMD) build -t "$(DPS_BUILD_NAME_PREFIX)/build-context:latest" -f- . && \ $(DOCKER_CMD) container run --rm "$(DPS_BUILD_NAME_PREFIX)/build-context:latest" +.docker/images: + $(DOCKER_CMD) images + .docker/run: DPS_BUILD_NAME_PREFIX=$(DPS_BUILD_NAME_PREFIX) DPS_BUILD_ID=$(DPS_BUILD_ID) \ $(DOCKER_COMPOSE_CMD) -f configs/Docker/docker-compose.yml up From 81b71001ecca1920c86d03e18289be648ad53a47 Mon Sep 17 00:00:00 2001 From: Mischback Date: Tue, 18 Feb 2020 22:28:32 +0100 Subject: [PATCH 039/110] Do not expose Gunicorn publicly --- configs/Docker/django/Dockerfile | 3 +++ configs/Docker/docker-compose.yml | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/configs/Docker/django/Dockerfile b/configs/Docker/django/Dockerfile index 4fd8208..c1fccf1 100644 --- a/configs/Docker/django/Dockerfile +++ b/configs/Docker/django/Dockerfile @@ -67,6 +67,9 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" # fetch the actual project files COPY --chown=2342:65534 . . +# make Gunicorn's port 8000 available to other Docker images +EXPOSE 8000 + # drop user privileges USER pythonuser diff --git a/configs/Docker/docker-compose.yml b/configs/Docker/docker-compose.yml index b241fea..86c3aa6 100644 --- a/configs/Docker/docker-compose.yml +++ b/configs/Docker/docker-compose.yml @@ -11,8 +11,8 @@ services: # through the Makefile and results in a tag with either a git commit hash # or a datetime stamp (see Makefile.deployment) image: ${DPS_BUILD_NAME_PREFIX}/django:${DPS_BUILD_ID} - ports: - - 8000:8000 + expose: + - "8000" nginx: build: context: ./nginx/ @@ -23,4 +23,4 @@ services: # or a datetime stamp (see Makefile.deployment) image: ${DPS_BUILD_NAME_PREFIX}/nginx:${DPS_BUILD_ID} ports: - - 80:80 + - "80:80" From e4c3761b7ea82940c79e45eb7a9310e97b5f0e81 Mon Sep 17 00:00:00 2001 From: Mischback Date: Fri, 21 Feb 2020 21:21:03 +0100 Subject: [PATCH 040/110] Dropping support for Django < 1.11 --- tox.ini | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tox.ini b/tox.ini index 8afa531..2a0b042 100644 --- a/tox.ini +++ b/tox.ini @@ -2,10 +2,6 @@ [tox] envlist = - django17-py{27,32,33,34} - django18-py{27,33,34,35} - django19-py{27,34,35} - django110-py{27,34,35} django111-py{27,34,35,36,37} django20-py{34,35,36,37} django21-py{35,36,37} @@ -22,10 +18,6 @@ skip_missing_interpreters = true setenv = PYTHONDONTWRITEBYTECODE=1 deps = - django17: Django>=1.7, <1.8 - django18: Django>=1.8, <1.9 - django19: Django>=1.9, <1.10 - django110: Django>=1.10, <1.11 django111: Django>=1.11, <2.0 django20: Django>=2.0, <2.1 django21: Django>=2.1, <2.2 From ea824a05b1fc716e1322808962938eaa1581bc63 Mon Sep 17 00:00:00 2001 From: Mischback Date: Tue, 25 Feb 2020 21:02:47 +0100 Subject: [PATCH 041/110] Renamed project specific templates All project files, that need templating during `django-admin startproject` now have the suffix '.template'. The Makefile's *init* will take care of switching files and renaming them as required. --- .dockerignore | 2 +- Makefile | 16 ++++++++++------ ...ngo-run.deployment => run-django.sh.template} | 0 configs/Docker/docker-compose.yml | 2 +- ...nginx.conf.deployment => nginx.conf.template} | 0 ...e.deployment => Makefile.deployment.template} | 0 ....deployment => apache2_vhost.sample.template} | 0 .../{tox.deployment => tox.deployment.template} | 0 tox.ini | 6 +++--- 9 files changed, 15 insertions(+), 11 deletions(-) rename configs/Docker/bin/{django-run.deployment => run-django.sh.template} (100%) rename configs/Docker/nginx/{nginx.conf.deployment => nginx.conf.template} (100%) rename configs/{Makefile.deployment => Makefile.deployment.template} (100%) rename configs/{apache2_vhost.deployment => apache2_vhost.sample.template} (100%) rename configs/{tox.deployment => tox.deployment.template} (100%) diff --git a/.dockerignore b/.dockerignore index e419f80..ade2313 100644 --- a/.dockerignore +++ b/.dockerignore @@ -34,7 +34,7 @@ bin configs/Docker # but DO include the production-related scripts/config files !configs/Docker/bin/apt-install -!configs/Docker/bin/django-run.sh +!configs/Docker/bin/run-django.sh !configs/Docker/django/gunicorn_conf.py !configs/Docker/nginx/nginx.conf diff --git a/Makefile b/Makefile index 86a5084..9286780 100644 --- a/Makefile +++ b/Makefile @@ -41,11 +41,15 @@ clean: rm requirements/production.txt init: - echo "Initialising repository..." - cp ./configs/tox.deployment ./tox.ini - mv ./configs/Docker/bin/django-run.deployment ./configs/Docker/bin/django-run.sh - mv ./configs/Docker/nginx/nginx.conf.deployment ./configs/Docker/nginx/nginx.conf - cp ./configs/Makefile.deployment ./Makefile + # TODO: Include some output to this function + # create the final version of different files by removing the '.sample' suffix + mv ./configs/apache2_vhost.sample.template ./configs/apache2_vhost.sample + mv ./configs/Docker/bin/run-django.sh.template ./configs/Docker/bin/run-django.sh + mv ./configs/Docker/nginx/nginx.conf.template ./configs/Docker/nginx/nginx.conf + # switching the tox configuration file + mv ./configs/tox.deployment.template ./tox.ini + # switching the Makefile **should** be the last step + mv ./configs/Makefile.deployment.template ./Makefile docker/build: tox -q -e docker-testing @@ -60,7 +64,7 @@ docker/run: sudo $(MAKE) .docker/run tree: - tree -a -I ".git|.tox|doc|run" --dirsfirst + tree -a -I ".git|.tox|doc|run" --dirsfirst -C | less -r .docker/build-context: echo " \ diff --git a/configs/Docker/bin/django-run.deployment b/configs/Docker/bin/run-django.sh.template similarity index 100% rename from configs/Docker/bin/django-run.deployment rename to configs/Docker/bin/run-django.sh.template diff --git a/configs/Docker/docker-compose.yml b/configs/Docker/docker-compose.yml index 86c3aa6..3f393c7 100644 --- a/configs/Docker/docker-compose.yml +++ b/configs/Docker/docker-compose.yml @@ -5,7 +5,7 @@ services: build: context: ../../ dockerfile: configs/Docker/django/Dockerfile - command: ./configs/Docker/bin/django-run.sh + command: ./configs/Docker/bin/run-django.sh # providing the build and the image directive automatically tags the image # automatic tagging is performed with environment variables provided # through the Makefile and results in a tag with either a git commit hash diff --git a/configs/Docker/nginx/nginx.conf.deployment b/configs/Docker/nginx/nginx.conf.template similarity index 100% rename from configs/Docker/nginx/nginx.conf.deployment rename to configs/Docker/nginx/nginx.conf.template diff --git a/configs/Makefile.deployment b/configs/Makefile.deployment.template similarity index 100% rename from configs/Makefile.deployment rename to configs/Makefile.deployment.template diff --git a/configs/apache2_vhost.deployment b/configs/apache2_vhost.sample.template similarity index 100% rename from configs/apache2_vhost.deployment rename to configs/apache2_vhost.sample.template diff --git a/configs/tox.deployment b/configs/tox.deployment.template similarity index 100% rename from configs/tox.deployment rename to configs/tox.deployment.template diff --git a/tox.ini b/tox.ini index 2a0b042..de37044 100644 --- a/tox.ini +++ b/tox.ini @@ -25,7 +25,7 @@ deps = django30: Django>=3.0, <3.1 changedir = {envtmpdir}/dpstest commands = - django-admin startproject --extension=py,deployment --template={toxinidir} dpstest . + django-admin startproject --extension=py,template --template={toxinidir} dpstest . {envbindir}/python manage.py check --settings=dpstest.settings.development [testenv:util] @@ -36,7 +36,7 @@ deps = skip_install = true changedir = {envtmpdir}/dpstest commands = - django-admin startproject --extension=py,deployment --template={toxinidir} dpstest . + django-admin startproject --extension=py,template --template={toxinidir} dpstest . {posargs:{envbindir}/python manage.py check --settings=dpstest.settings.development} [testenv:docker-testing] @@ -48,7 +48,7 @@ changedir = {envtmpdir}/dpstest whitelist_externals = make commands = - django-admin startproject --extension=py,deployment --template={toxinidir} dpstest . + django-admin startproject --extension=py,template --template={toxinidir} dpstest . make init {posargs:make docker/build} From d8d66252749a31a0b552caf73b3b523f407669ed Mon Sep 17 00:00:00 2001 From: Mischback Date: Tue, 25 Feb 2020 22:16:56 +0100 Subject: [PATCH 042/110] Preparing nginx utility --- .dockerignore | 1 + configs/Docker/nginx/Dockerfile | 4 ++++ configs/Docker/nginx/entrypoint.sh | 3 +++ 3 files changed, 8 insertions(+) create mode 100755 configs/Docker/nginx/entrypoint.sh diff --git a/.dockerignore b/.dockerignore index ade2313..67ff30d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -37,6 +37,7 @@ configs/Docker !configs/Docker/bin/run-django.sh !configs/Docker/django/gunicorn_conf.py !configs/Docker/nginx/nginx.conf +!configs/Docker/nginx/entrypoint.sh # do not include any samples **/*.sample diff --git a/configs/Docker/nginx/Dockerfile b/configs/Docker/nginx/Dockerfile index 918c386..c95a427 100644 --- a/configs/Docker/nginx/Dockerfile +++ b/configs/Docker/nginx/Dockerfile @@ -1,3 +1,7 @@ FROM nginx:1.17 COPY nginx.conf /etc/nginx/nginx.conf +COPY entrypoint.sh /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] +CMD ["nginx", "-g", "daemon off;"] diff --git a/configs/Docker/nginx/entrypoint.sh b/configs/Docker/nginx/entrypoint.sh new file mode 100755 index 0000000..214eb4c --- /dev/null +++ b/configs/Docker/nginx/entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +exec "$@" From ffa8f6f791afb5ae905beff51884524bea624193 Mon Sep 17 00:00:00 2001 From: Mischback Date: Tue, 25 Feb 2020 22:31:42 +0100 Subject: [PATCH 043/110] Moving Django-related script into django directory --- .dockerignore | 2 +- Makefile | 2 +- .../run-django.sh.template => django/run-gunicorn.sh.template} | 0 configs/Docker/docker-compose.yml | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename configs/Docker/{bin/run-django.sh.template => django/run-gunicorn.sh.template} (100%) diff --git a/.dockerignore b/.dockerignore index 67ff30d..c5b1723 100644 --- a/.dockerignore +++ b/.dockerignore @@ -34,8 +34,8 @@ bin configs/Docker # but DO include the production-related scripts/config files !configs/Docker/bin/apt-install -!configs/Docker/bin/run-django.sh !configs/Docker/django/gunicorn_conf.py +!configs/Docker/django/run-gunicorn.sh !configs/Docker/nginx/nginx.conf !configs/Docker/nginx/entrypoint.sh diff --git a/Makefile b/Makefile index 9286780..704dd49 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ init: # TODO: Include some output to this function # create the final version of different files by removing the '.sample' suffix mv ./configs/apache2_vhost.sample.template ./configs/apache2_vhost.sample - mv ./configs/Docker/bin/run-django.sh.template ./configs/Docker/bin/run-django.sh + mv ./configs/Docker/django/run-gunicorn.sh.template ./configs/Docker/django/run-gunicorn.sh mv ./configs/Docker/nginx/nginx.conf.template ./configs/Docker/nginx/nginx.conf # switching the tox configuration file mv ./configs/tox.deployment.template ./tox.ini diff --git a/configs/Docker/bin/run-django.sh.template b/configs/Docker/django/run-gunicorn.sh.template similarity index 100% rename from configs/Docker/bin/run-django.sh.template rename to configs/Docker/django/run-gunicorn.sh.template diff --git a/configs/Docker/docker-compose.yml b/configs/Docker/docker-compose.yml index 3f393c7..7a1f4ee 100644 --- a/configs/Docker/docker-compose.yml +++ b/configs/Docker/docker-compose.yml @@ -5,7 +5,7 @@ services: build: context: ../../ dockerfile: configs/Docker/django/Dockerfile - command: ./configs/Docker/bin/run-django.sh + command: ./configs/Docker/django/run-gunicorn.sh # providing the build and the image directive automatically tags the image # automatic tagging is performed with environment variables provided # through the Makefile and results in a tag with either a git commit hash From 6d2604137bd20b1fe0be09be52396e3e723b8294 Mon Sep 17 00:00:00 2001 From: Mischback Date: Wed, 26 Feb 2020 21:55:57 +0100 Subject: [PATCH 044/110] Comments on the environment file --- .gitignore | 6 ++++-- configs/Docker/docker-compose.yml | 2 ++ configs/Docker/env.sample | 32 +++++++++++++++++++++++-------- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index bbb9bcd..c7837d1 100644 --- a/.gitignore +++ b/.gitignore @@ -13,5 +13,7 @@ coverage_html *.mo *.pot -# I started relying on putting temporary notes into TODO files -*.todo +# do NOT track environment files that are used for docker-compose, but do +# include the sample +configs/Docker/env.* +!configs/Docker/env.sample diff --git a/configs/Docker/docker-compose.yml b/configs/Docker/docker-compose.yml index 7a1f4ee..73f9fca 100644 --- a/configs/Docker/docker-compose.yml +++ b/configs/Docker/docker-compose.yml @@ -11,6 +11,8 @@ services: # through the Makefile and results in a tag with either a git commit hash # or a datetime stamp (see Makefile.deployment) image: ${DPS_BUILD_NAME_PREFIX}/django:${DPS_BUILD_ID} + env_file: + ./env.production expose: - "8000" nginx: diff --git a/configs/Docker/env.sample b/configs/Docker/env.sample index 8f39b28..f098e82 100644 --- a/configs/Docker/env.sample +++ b/configs/Docker/env.sample @@ -1,8 +1,24 @@ -DPS_GUNICORN_NUM_WORKERS -DPS_GUNICORN_WORKER_CLASS -DPS_GUNICORN_NUM_THREADS -DPS_GUNICORN_KEEPALIVE -DPS_GUNICORN_ACCESSLOG_FILE -DPS_GUNICORN_ERRORLOG_FILE -DPS_GUNICORN_LOGFORMAT -DPS_GUNICORN_LOGLEVEL +### GUNICORN related configuration + +# the number of Gunicorn worker processes +#DPS_GUNICORN_NUM_WORKERS=2 + +# the used worker class +#DPS_GUNICORN_WORKER_CLASS=gthread + +# the number of threads per worker process +#DPS_GUNICORN_NUM_THREADS=4 + +# how long should a Keep-Alive session be kept open +#DPS_GUNICORN_KEEPALIVE=10 + +# the file for access log / error log +# In Docker environments, logs should be written to stdout +#DPS_GUNICORN_ACCESSLOG_FILE=- +#DPS_GUNICORN_ERRORLOG_FILE=- + +# a custom log format +#DPS_GUNICORN_LOGFORMAT=%({x-forwarded-for}i)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s" + +# the log level +#DPS_GUNICORN_LOGLEVEL=info From c7a30ebd989a12f3c6cf6b8dfc4a4dbcc9ac5f9a Mon Sep 17 00:00:00 2001 From: Mischback Date: Wed, 26 Feb 2020 22:36:20 +0100 Subject: [PATCH 045/110] Update .dockerignore --- .dockerignore | 71 +++++++++++++++++++++++++-------------------------- 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/.dockerignore b/.dockerignore index c5b1723..b0dda4d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,52 +1,51 @@ # experimental .dockerignore -# do NOT track Python compilation stuff -*.py[cod] -**/__python__ - -# ignore .git-related stuff -.git -**/.gitignore - -# ignore tox-related stuff -.tox -tox.ini - -# ignore any development related files -.dockerignore -.editorconfig -.travis.yml -**/Makefile -requirements/*.in - -# do NOT include ANY readme files -**/README* - -# do NOT include the documentation -# TODO: the original doc-folder of django-project-skeleton should be removed -# during initialising the project. -doc - -# do NOT include the bin directory (these scripts are not used in production) +# do NOT include the bin directory +# these scripts are not used *inside* of Docker-based images. +# They may be used during the build/run process and provide utility functions, +# i.e. for tagging the images bin -# do NOT include the Docker-related files -configs/Docker -# but DO include the production-related scripts/config files +# do NOT include the provided configuration files +configs +# but DO include the production-related scripts/config files that are used +# inside of the images !configs/Docker/bin/apt-install !configs/Docker/django/gunicorn_conf.py !configs/Docker/django/run-gunicorn.sh !configs/Docker/nginx/nginx.conf !configs/Docker/nginx/entrypoint.sh +# do NOT include the documentation +# TODO: the original doc-folder of django-project-skeleton should be removed +# during initialisation of the project. On the other hand, there could be +# project specific documentation. However, this should be excluded from any +# image. +# TODO: 'doc' is already renamed to 'docs' in development! +doc + +# inside of the images, only the compiled requirements files with the '.txt' +# extension are used. No need to include the source files. +requirements/*.in + +# utility files/directories of the repository should not be included +**/.git +**/.tox +**/.dockerignore +**/.editorconfig +**/.gitignore +**/.travis.yml +**/Makefile +**/README* +**/tox.ini + # do not include any samples **/*.sample -# do not include the deployment specific files -**/*.deployment - -# I started to put temporary notes into TODO files -**/*.todo +# do NOT track Python compilation stuff +*.py[cod] +**/__python__ +**/__pycache__ # TODO: .gitignore excludes compiled language files. Is it safe to exclude the # uncompiled language files for 'production'? From 98282771b45602473f84366e95499a5d95e5b50a Mon Sep 17 00:00:00 2001 From: Mischback Date: Thu, 27 Feb 2020 19:10:38 +0100 Subject: [PATCH 046/110] Adjusting imports in settings files --- project_name/settings/common.py | 20 ++++++++++---------- project_name/settings/development.py | 4 ++-- project_name/settings/i18n.py | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/project_name/settings/common.py b/project_name/settings/common.py index fb6a630..3e88522 100644 --- a/project_name/settings/common.py +++ b/project_name/settings/common.py @@ -1,38 +1,38 @@ # Python imports -from os.path import abspath, basename, dirname, join, normpath +import os import sys # ##### PATH CONFIGURATION ################################ # fetch Django's project directory -DJANGO_ROOT = dirname(dirname(abspath(__file__))) +DJANGO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # fetch the project_root -PROJECT_ROOT = dirname(DJANGO_ROOT) +PROJECT_ROOT = os.path.dirname(DJANGO_ROOT) # the name of the whole site -SITE_NAME = basename(DJANGO_ROOT) +SITE_NAME = os.path.basename(DJANGO_ROOT) # collect static files here -STATIC_ROOT = join(PROJECT_ROOT, 'run', 'static') +STATIC_ROOT = os.path.join(PROJECT_ROOT, 'run', 'static') # collect media files here -MEDIA_ROOT = join(PROJECT_ROOT, 'run', 'media') +MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'run', 'media') # look for static assets here STATICFILES_DIRS = [ - join(PROJECT_ROOT, 'static'), + os.path.join(PROJECT_ROOT, 'static'), ] # look for templates here # This is an internal setting, used in the TEMPLATES directive PROJECT_TEMPLATES = [ - join(PROJECT_ROOT, 'templates'), + os.path.join(PROJECT_ROOT, 'templates'), ] # add apps/ to the Python path -sys.path.append(normpath(join(PROJECT_ROOT, 'apps'))) +sys.path.append(os.path.normpath(os.path.join(PROJECT_ROOT, 'apps'))) # ##### APPLICATION CONFIGURATION ######################### @@ -106,7 +106,7 @@ # We store the secret key here # The required SECRET_KEY is fetched at the end of this file -SECRET_FILE = normpath(join(PROJECT_ROOT, 'run', 'SECRET.key')) +SECRET_FILE = os.path.normpath(os.path.join(PROJECT_ROOT, 'run', 'SECRET.key')) # these persons receive error notification ADMINS = ( diff --git a/project_name/settings/development.py b/project_name/settings/development.py index 628a466..0f375ba 100644 --- a/project_name/settings/development.py +++ b/project_name/settings/development.py @@ -1,5 +1,5 @@ # Python imports -from os.path import join +import os # fetch the common settings from .common import * @@ -16,7 +16,7 @@ DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': join(PROJECT_ROOT, 'run', 'dev.sqlite3'), + 'NAME': os.path.join(PROJECT_ROOT, 'run', 'dev.sqlite3'), } } diff --git a/project_name/settings/i18n.py b/project_name/settings/i18n.py index a59e3ac..750630d 100644 --- a/project_name/settings/i18n.py +++ b/project_name/settings/i18n.py @@ -1,5 +1,5 @@ # Python imports -from os.path import join +import os # Django imports from django.utils.translation import ugettext_lazy as _ @@ -29,7 +29,7 @@ # Look for translations in these locations LOCALE_PATHS = ( - join(PROJECT_ROOT, 'locale'), + os.path.join(PROJECT_ROOT, 'locale'), ) # Inject the localization middleware into the right position From cd009e0921f9755a75f42cd35f2856149262f3ab Mon Sep 17 00:00:00 2001 From: Mischback Date: Thu, 27 Feb 2020 19:20:14 +0100 Subject: [PATCH 047/110] Fixed typo in Makefile (deployment) --- configs/Makefile.deployment.template | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/Makefile.deployment.template b/configs/Makefile.deployment.template index 3296db4..9a39d3c 100644 --- a/configs/Makefile.deployment.template +++ b/configs/Makefile.deployment.template @@ -27,8 +27,8 @@ DPS_BUILD_NAME_PREFIX := "{{ project_name }}" # internal/tox/dev-django will use tox environments to run Django managements # commands. # internal/django/raw will directly call the Django management commands. -internal/django-mgmt: internal/tox/dev-django -# internal/django-mgmt: internal/django/raw +.internal/django-mgmt: .internal/tox/dev-django +# .internal/django-mgmt: .internal/django/raw # tox command (this assumes, that tox is available in your Python environment # and your PATH is setup to allow direct calls) From 7080a01d35260280a1f53298cfc8b86b99f0ed63 Mon Sep 17 00:00:00 2001 From: Mischback Date: Thu, 27 Feb 2020 19:29:12 +0100 Subject: [PATCH 048/110] Fixed string fomratting in settings files Ok, let's pretend that this was fixed years ago... --- project_name/settings/common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/project_name/settings/common.py b/project_name/settings/common.py index 3e88522..ac2ff0d 100644 --- a/project_name/settings/common.py +++ b/project_name/settings/common.py @@ -79,10 +79,10 @@ ] # the default WSGI application -WSGI_APPLICATION = '%s.wsgi.application' % SITE_NAME +WSGI_APPLICATION = '{}.wsgi.application'.format(SITE_NAME) # the root URL configuration -ROOT_URLCONF = '%s.urls' % SITE_NAME +ROOT_URLCONF = '{}.urls'.format(SITE_NAME) # the URL for static files STATIC_URL = '/static/' @@ -130,4 +130,4 @@ with open(SECRET_FILE, 'w') as f: f.write(SECRET_KEY) except IOError: - raise Exception('Could not open %s for writing!' % SECRET_FILE) + raise Exception('Could not open {} for writing!'.format(SECRET_FILE)) From fc3fd3fb0e1c5fec2f9f2c6346ad2475c9b1880e Mon Sep 17 00:00:00 2001 From: Mischback Date: Thu, 27 Feb 2020 20:36:14 +0100 Subject: [PATCH 049/110] Make Django configurable by environment variables --- configs/Docker/env.sample | 34 +++++++++++++++++++++++++++ project_name/settings/common.py | 13 ++++++---- project_name/settings/docker-nginx.py | 5 +++- project_name/settings/production.py | 6 ++++- 4 files changed, 51 insertions(+), 7 deletions(-) diff --git a/configs/Docker/env.sample b/configs/Docker/env.sample index f098e82..a0e829a 100644 --- a/configs/Docker/env.sample +++ b/configs/Docker/env.sample @@ -1,3 +1,37 @@ +### DJANGO related configuration + +# Django's ALLOWED_HOSTS +# In *settings/production.py* this is by default an empty list. In the +# provided *settings/docker-nginx.py* this is set to the project's name while +# setting USE_X_FORWARDED_HOST = True. This means, that Django will not longer +# check, if it is responsible for the request but if the request comes from a +# trusted proxy (nginx in this case). +#DPS_DJANGO_ALLOWED_HOSTS= + +# Django's LOGIN_REDIRECT_URL +#DPS_DJANGO_LOGIN_REDIRECT_URL=/ + +# Django's LOGOUT_REDIRECT_URL +#DPS_DJANGO_LOGOUT_REDIRECT_URL=core_login + +# Django's MEDIA_URL +# the URL to serve media files from +# While this is part of Django's configuration, this environment variable is +# used by nginx's Docker image aswell +#DPS_DJANGO_MEDIA_URL=/media/ + +# Django's STATIC_URL +# the URL to serve static files from +# While this is part of Django's configuration, this environment variable is +# used by nginx's Docker image aswell +#DPS_DJANGO_STATIC_URL=/static/ + +# the (dotted) path to the WSGI app +# defaults to the app provided by Django +# see *settings/common.py* for the default value +#DPS_DJANGO_WSGI_APP= + + ### GUNICORN related configuration # the number of Gunicorn worker processes diff --git a/project_name/settings/common.py b/project_name/settings/common.py index ac2ff0d..55afad2 100644 --- a/project_name/settings/common.py +++ b/project_name/settings/common.py @@ -79,21 +79,24 @@ ] # the default WSGI application -WSGI_APPLICATION = '{}.wsgi.application'.format(SITE_NAME) +WSGI_APPLICATION = os.environ.get( + 'DPS_DJANGO_WSGI_APP', + '{}.wsgi.application'.format(SITE_NAME) +) # the root URL configuration ROOT_URLCONF = '{}.urls'.format(SITE_NAME) # the URL for static files -STATIC_URL = '/static/' +STATIC_URL = os.environ.get('DPS_DJANGO_STATIC_URL', '/static/') # the URL for media files -MEDIA_URL = '/media/' +MEDIA_URL = os.environ.get('DPS_DJANGO_MEDIA_URL', '/media/') # adjust the minimal login LOGIN_URL = 'core_login' -LOGIN_REDIRECT_URL = '/' -LOGOUT_REDIRECT_URL = 'core_login' +LOGIN_REDIRECT_URL = os.environ.get('DPS_DJANGO_LOGIN_REDIRECT_URL', '/') +LOGOUT_REDIRECT_URL = os.environ.get('DPS_DJANGO_LOGOUT_REDIRECT_URL', 'core_login') # Internationalization USE_I18N = False diff --git a/project_name/settings/docker-nginx.py b/project_name/settings/docker-nginx.py index 4a31af5..80247f1 100644 --- a/project_name/settings/docker-nginx.py +++ b/project_name/settings/docker-nginx.py @@ -1,7 +1,10 @@ +# Python imports +import os + # fetch the production settings from .production import * # make Django work correctly behind the nginx proxy # nginx **must** set the appropriate header! -ALLOWED_HOSTS = ['{{ project_name }}'] +ALLOWED_HOSTS = os.environ.get('DPS_DJANGO_ALLOWED_HOSTS', '{{ project_name }}').split() USE_X_FORWARDED_HOST = True diff --git a/project_name/settings/production.py b/project_name/settings/production.py index 86e5aed..f546427 100644 --- a/project_name/settings/production.py +++ b/project_name/settings/production.py @@ -1,10 +1,14 @@ +# Python imports +import os + # fetch the common settings from .common import * + # ##### APPLICATION CONFIGURATION ######################### # You will have to determine, which hostnames should be served by Django -ALLOWED_HOSTS = [] +ALLOWED_HOSTS = os.environ.get('DPS_DJANGO_ALLOWED_HOSTS', '').split() INSTALLED_APPS = DEFAULT_APPS From 4af865de193a59203a258c47f1a3e4f18fb72af9 Mon Sep 17 00:00:00 2001 From: Mischback Date: Thu, 27 Feb 2020 20:40:25 +0100 Subject: [PATCH 050/110] Minor documentation issue --- configs/Docker/django/gunicorn_conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/configs/Docker/django/gunicorn_conf.py b/configs/Docker/django/gunicorn_conf.py index 50e3bb5..c7df229 100644 --- a/configs/Docker/django/gunicorn_conf.py +++ b/configs/Docker/django/gunicorn_conf.py @@ -70,6 +70,7 @@ # While running behind a reverse proxy, Gunicorn's logs will only show the # reverse proxy's IP. # This project's setup uses an NGINX, which sets the 'X-Forwarded-For'-header. +# See: https://stackoverflow.com/questions/25737589/gunicorn-doesnt-log-real-ip-from-nginx access_log_format = os.environ.get( 'DPS_GUNICORN_LOGFORMAT', '%({x-forwarded-for}i)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"' From 39bed574da62c291692f87ea269bd5720b7d04ac Mon Sep 17 00:00:00 2001 From: Mischback Date: Thu, 27 Feb 2020 23:09:26 +0100 Subject: [PATCH 051/110] Got stuck with Django's logging inside of the container It seems like everything is working fine, but I can't see the logs in the Docker container, if I try to manually inject them from a second TTY using Django's shell and pure Python. TODO: Write a test function bound to a url to test if logging can be triggered from the outside. --- .../Docker/django/run-gunicorn.sh.template | 2 +- project_name/settings/docker-nginx.py | 10 --- project_name/settings/docker.py | 62 +++++++++++++++++++ 3 files changed, 63 insertions(+), 11 deletions(-) delete mode 100644 project_name/settings/docker-nginx.py create mode 100644 project_name/settings/docker.py diff --git a/configs/Docker/django/run-gunicorn.sh.template b/configs/Docker/django/run-gunicorn.sh.template index 4862753..c96de1f 100755 --- a/configs/Docker/django/run-gunicorn.sh.template +++ b/configs/Docker/django/run-gunicorn.sh.template @@ -1,5 +1,5 @@ #!/bin/sh -export DJANGO_SETTINGS_MODULE="{{ project_name}}.settings.docker-nginx" +export DJANGO_SETTINGS_MODULE="{{ project_name}}.settings.docker" exec gunicorn {{ project_name }}.wsgi:application --config='file:configs/Docker/django/gunicorn_conf.py' diff --git a/project_name/settings/docker-nginx.py b/project_name/settings/docker-nginx.py deleted file mode 100644 index 80247f1..0000000 --- a/project_name/settings/docker-nginx.py +++ /dev/null @@ -1,10 +0,0 @@ -# Python imports -import os - -# fetch the production settings -from .production import * - -# make Django work correctly behind the nginx proxy -# nginx **must** set the appropriate header! -ALLOWED_HOSTS = os.environ.get('DPS_DJANGO_ALLOWED_HOSTS', '{{ project_name }}').split() -USE_X_FORWARDED_HOST = True diff --git a/project_name/settings/docker.py b/project_name/settings/docker.py new file mode 100644 index 0000000..43dca95 --- /dev/null +++ b/project_name/settings/docker.py @@ -0,0 +1,62 @@ +# Python imports +import logging +import os + +# fetch the production settings +from .production import * + + +class MaxLevelFilter(logging.Filter): + """Only allow messages up to a given logging level. + + The idea is to direct any logs below a given level to stdout and above + that level to stderr. + + This is based on + https://stackoverflow.com/questions/1383254/logging-streamhandler-and-standard-streams + """ + + def __init__(self, max_level): + self.max_level = getattr(logging, max_level) + + def filter(self, record): + return record.levelno < self.max_level + + +# make Django work correctly behind the nginx proxy +# nginx **must** set the appropriate header! +ALLOWED_HOSTS = os.environ.get('DPS_DJANGO_ALLOWED_HOSTS', '{{ project_name }}').split() +USE_X_FORWARDED_HOST = True + +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'filters': { + 'max_level_filter': { + '()': '{{ project_name }}.settings.docker.MaxLevelFilter', + 'max_level': 'WARNING', + }, + }, + 'handlers': { + 'docker_stdout': { + 'level': 'DEBUG', + 'filters': ['max_level_filter'], + 'class': 'logging.StreamHandler', + 'stream': 'ext://sys.stdout' + }, + 'docker_stderr': { + 'level': 'WARNING', + 'class': 'logging.StreamHandler', + 'stream': 'ext://sys.stderr' + }, + }, + 'loggers': { + 'django': { + 'handlers': ['docker_stdout', 'docker_stderr'], + # TODO: Make this adjustable by environment as minimal logging level + 'level': 'INFO', + }, + }, +} + +logging.getLogger('django').info('Applied fixes for running Django in Docker behind nginx!') From c05ed23f611f1fc1f77a86381b7e5de71eb79df1 Mon Sep 17 00:00:00 2001 From: Mischback Date: Fri, 28 Feb 2020 19:30:18 +0100 Subject: [PATCH 052/110] Added formatter and root logger --- project_name/settings/docker.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/project_name/settings/docker.py b/project_name/settings/docker.py index 43dca95..395b81d 100644 --- a/project_name/settings/docker.py +++ b/project_name/settings/docker.py @@ -31,6 +31,12 @@ def filter(self, record): LOGGING = { 'version': 1, 'disable_existing_loggers': False, + 'formatters': { + 'docker_simple': { + 'format': '{levelname} {message}', + 'style': '{', + }, + }, 'filters': { 'max_level_filter': { '()': '{{ project_name }}.settings.docker.MaxLevelFilter', @@ -42,12 +48,14 @@ def filter(self, record): 'level': 'DEBUG', 'filters': ['max_level_filter'], 'class': 'logging.StreamHandler', - 'stream': 'ext://sys.stdout' + 'stream': 'ext://sys.stdout', + 'formatter': 'docker_simple', }, 'docker_stderr': { 'level': 'WARNING', 'class': 'logging.StreamHandler', - 'stream': 'ext://sys.stderr' + 'stream': 'ext://sys.stderr', + 'formatter': 'docker_simple', }, }, 'loggers': { @@ -57,6 +65,11 @@ def filter(self, record): 'level': 'INFO', }, }, + 'root': { + 'handlers': ['docker_stdout', 'docker_stderr'], + # TODO: Make this adjustable by environment as minimal logging level + 'level': 'INFO', + }, } logging.getLogger('django').info('Applied fixes for running Django in Docker behind nginx!') From f072c0d0ff2b3dfb281b86eb7d19ed850271b236 Mon Sep 17 00:00:00 2001 From: Mischback Date: Fri, 28 Feb 2020 20:31:47 +0100 Subject: [PATCH 053/110] Logging from Django actually works --- configs/Docker/env.sample | 3 ++ project_name/settings/docker.py | 52 +++++++-------------------------- 2 files changed, 14 insertions(+), 41 deletions(-) diff --git a/configs/Docker/env.sample b/configs/Docker/env.sample index a0e829a..20947ca 100644 --- a/configs/Docker/env.sample +++ b/configs/Docker/env.sample @@ -8,6 +8,9 @@ # trusted proxy (nginx in this case). #DPS_DJANGO_ALLOWED_HOSTS= +# The minimum level of log messages to show up for Django's logging +#DPS_DJANGO_LOGGING_LEVEL=INFO + # Django's LOGIN_REDIRECT_URL #DPS_DJANGO_LOGIN_REDIRECT_URL=/ diff --git a/project_name/settings/docker.py b/project_name/settings/docker.py index 395b81d..53def28 100644 --- a/project_name/settings/docker.py +++ b/project_name/settings/docker.py @@ -1,28 +1,10 @@ # Python imports -import logging import os # fetch the production settings from .production import * -class MaxLevelFilter(logging.Filter): - """Only allow messages up to a given logging level. - - The idea is to direct any logs below a given level to stdout and above - that level to stderr. - - This is based on - https://stackoverflow.com/questions/1383254/logging-streamhandler-and-standard-streams - """ - - def __init__(self, max_level): - self.max_level = getattr(logging, max_level) - - def filter(self, record): - return record.levelno < self.max_level - - # make Django work correctly behind the nginx proxy # nginx **must** set the appropriate header! ALLOWED_HOSTS = os.environ.get('DPS_DJANGO_ALLOWED_HOSTS', '{{ project_name }}').split() @@ -37,39 +19,27 @@ def filter(self, record): 'style': '{', }, }, - 'filters': { - 'max_level_filter': { - '()': '{{ project_name }}.settings.docker.MaxLevelFilter', - 'max_level': 'WARNING', - }, - }, 'handlers': { 'docker_stdout': { - 'level': 'DEBUG', - 'filters': ['max_level_filter'], - 'class': 'logging.StreamHandler', - 'stream': 'ext://sys.stdout', - 'formatter': 'docker_simple', - }, - 'docker_stderr': { - 'level': 'WARNING', 'class': 'logging.StreamHandler', - 'stream': 'ext://sys.stderr', 'formatter': 'docker_simple', + 'level': 'DEBUG', + 'stream': 'ext://sys.stdout', }, }, 'loggers': { + # all Django logs should end up here... 'django': { - 'handlers': ['docker_stdout', 'docker_stderr'], - # TODO: Make this adjustable by environment as minimal logging level - 'level': 'INFO', + # Django's 'mail_admins' handler is removed! + 'handlers': ['docker_stdout'], + 'level': os.environ.get('DPS_DJANGO_LOGGING_LEVEL', 'INFO').upper(), + # Whatever got catched here will not propagate to the root logger + 'propagate': False, }, }, + # the 'root' logger is just redefined to make it compatible with Docker 'root': { - 'handlers': ['docker_stdout', 'docker_stderr'], - # TODO: Make this adjustable by environment as minimal logging level - 'level': 'INFO', + 'handlers': ['docker_stdout'], + 'level': os.environ.get('DPS_DJANGO_LOGGING_LEVEL', 'INFO').upper(), }, } - -logging.getLogger('django').info('Applied fixes for running Django in Docker behind nginx!') From 49f7071681f862e9127dac2754c90450f64ae73d Mon Sep 17 00:00:00 2001 From: Mischback Date: Fri, 28 Feb 2020 21:43:22 +0100 Subject: [PATCH 054/110] Updated the formatter --- project_name/settings/docker.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/project_name/settings/docker.py b/project_name/settings/docker.py index 53def28..fa2c276 100644 --- a/project_name/settings/docker.py +++ b/project_name/settings/docker.py @@ -14,15 +14,15 @@ 'version': 1, 'disable_existing_loggers': False, 'formatters': { - 'docker_simple': { - 'format': '{levelname} {message}', - 'style': '{', + 'dps_docker_default': { + 'format': '%(asctime)-19s %(levelname)-8s [%(process)d] [%(name)s] %(message)s', + 'datefmt': '%Y-%m-%d %H:%M:%S' }, }, 'handlers': { 'docker_stdout': { 'class': 'logging.StreamHandler', - 'formatter': 'docker_simple', + 'formatter': 'dps_docker_default', 'level': 'DEBUG', 'stream': 'ext://sys.stdout', }, From f7b162faf5f479c6b701608fb68b27a9b754b6a6 Mon Sep 17 00:00:00 2001 From: Mischback Date: Fri, 28 Feb 2020 23:01:15 +0100 Subject: [PATCH 055/110] Fix Django to 2.2 --- requirements/common.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/common.in b/requirements/common.in index 0703c09..6473c7a 100644 --- a/requirements/common.in +++ b/requirements/common.in @@ -1,2 +1,2 @@ # currently used Django version -Django==2.2.9 +Django>=2.2,<3.0 From b8e3ebeb4a4088b4811a60ed18d7528967d73d1a Mon Sep 17 00:00:00 2001 From: Mischback Date: Fri, 28 Feb 2020 23:38:58 +0100 Subject: [PATCH 056/110] Completed logging Django container - logging is set up in Gunicorn's configuration - logging should be configured in one place for the whole container (DRY) - the applied logging conf is specific to Docker-based deployment in a production environment - Django's logging is completely replaced, but Django's logging calls will be propagated and thus logged by the root handler (to stdout) - bump Gunicorn requirement to 19.8 --- configs/Docker/django/gunicorn_conf.py | 48 ++++++++++++++++++++++---- configs/Docker/env.sample | 17 +++++---- project_name/settings/docker.py | 35 ++----------------- requirements/production.in | 2 +- 4 files changed, 55 insertions(+), 47 deletions(-) diff --git a/configs/Docker/django/gunicorn_conf.py b/configs/Docker/django/gunicorn_conf.py index c7df229..a557b5a 100644 --- a/configs/Docker/django/gunicorn_conf.py +++ b/configs/Docker/django/gunicorn_conf.py @@ -76,10 +76,46 @@ '%({x-forwarded-for}i)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"' ) -# while running in a Docker container, logs should be put to stdout/stderr to be picked up -accesslog = os.environ.get('DPS_GUNICORN_ACCESSLOG_FILE', '-') -errorlog = os.environ.get('DPS_GUNICORN_ERRORLOG_FILE', '-') # this is actually the default value since Gunicorn 19.2 -# make the log level configurable by an environment variable -# 'info' is provided as a default value, which is Gunicorn's default value aswell. -loglevel = os.environ.get('DPS_GUNICORN_LOGLEVEL', 'info') +logconfig_dict = { + 'version': 1, + 'disable_existing_loggers': True, + 'formatters': { + 'dps_docker_default': { + 'format': '%(asctime)-19s %(levelname)-8s [%(process)d] [%(name)s] %(message)s', + 'datefmt': '%Y-%m-%d %H:%M:%S' + }, + }, + 'handlers': { + 'docker_stdout': { + 'class': 'logging.StreamHandler', + 'formatter': 'dps_docker_default', + 'level': 'DEBUG', + 'stream': 'ext://sys.stdout', + }, + }, + 'loggers': { + # all Django logs should end up here... + 'django': { + # Django's 'mail_admins' handler is removed! + #'handlers': ['docker_stdout'], + 'level': os.environ.get('DPS_DJANGO_LOGLEVEL', 'INFO').upper(), + # Whatever got catched here will not propagate to the root logger + #'propagate': False, + }, + 'gunicorn': { + #'handlers': ['docker_stdout'], + 'level': os.environ.get('DPS_GUNICORN_LOGLEVEL', 'INFO').upper(), + # Whatever got catched here will not propagate to the root logger + #'propagate': False, + }, + 'gunicorn.access': { + 'propagate': os.environ.get('DPS_GUNICORN_SHOW_ACCESS_LOG', 'false').lower() == 'true', + }, + }, + # the 'root' logger is just redefined to make it compatible with Docker + 'root': { + 'handlers': ['docker_stdout'], + #'level': os.environ.get('DPS_DJANGO_LOGGING_LEVEL', 'INFO').upper(), + }, +} diff --git a/configs/Docker/env.sample b/configs/Docker/env.sample index 20947ca..0bf26bf 100644 --- a/configs/Docker/env.sample +++ b/configs/Docker/env.sample @@ -9,7 +9,7 @@ #DPS_DJANGO_ALLOWED_HOSTS= # The minimum level of log messages to show up for Django's logging -#DPS_DJANGO_LOGGING_LEVEL=INFO +#DPS_DJANGO_LOGLEVEL=INFO # Django's LOGIN_REDIRECT_URL #DPS_DJANGO_LOGIN_REDIRECT_URL=/ @@ -49,13 +49,16 @@ # how long should a Keep-Alive session be kept open #DPS_GUNICORN_KEEPALIVE=10 -# the file for access log / error log -# In Docker environments, logs should be written to stdout -#DPS_GUNICORN_ACCESSLOG_FILE=- -#DPS_GUNICORN_ERRORLOG_FILE=- - # a custom log format +# +# this is the provided default value, that basically mimics nginx's default access log #DPS_GUNICORN_LOGFORMAT=%({x-forwarded-for}i)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s" +# +# this is a shorter log that plays nicely with Python's logging setup from gunicorn_conf.py +#DPS_GUNICORN_LOGFORMAT=%({x-forwarded-for}i)s %(l)s %(s)s %(l)s "%(r)s" # the log level -#DPS_GUNICORN_LOGLEVEL=info +#DPS_GUNICORN_LOGLEVEL=INFO + +# determines if the accesslog should be included in the logs +#DPS_GUNICORN_SHOW_ACCESS_LOG=false diff --git a/project_name/settings/docker.py b/project_name/settings/docker.py index fa2c276..958b595 100644 --- a/project_name/settings/docker.py +++ b/project_name/settings/docker.py @@ -10,36 +10,5 @@ ALLOWED_HOSTS = os.environ.get('DPS_DJANGO_ALLOWED_HOSTS', '{{ project_name }}').split() USE_X_FORWARDED_HOST = True -LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'dps_docker_default': { - 'format': '%(asctime)-19s %(levelname)-8s [%(process)d] [%(name)s] %(message)s', - 'datefmt': '%Y-%m-%d %H:%M:%S' - }, - }, - 'handlers': { - 'docker_stdout': { - 'class': 'logging.StreamHandler', - 'formatter': 'dps_docker_default', - 'level': 'DEBUG', - 'stream': 'ext://sys.stdout', - }, - }, - 'loggers': { - # all Django logs should end up here... - 'django': { - # Django's 'mail_admins' handler is removed! - 'handlers': ['docker_stdout'], - 'level': os.environ.get('DPS_DJANGO_LOGGING_LEVEL', 'INFO').upper(), - # Whatever got catched here will not propagate to the root logger - 'propagate': False, - }, - }, - # the 'root' logger is just redefined to make it compatible with Docker - 'root': { - 'handlers': ['docker_stdout'], - 'level': os.environ.get('DPS_DJANGO_LOGGING_LEVEL', 'INFO').upper(), - }, -} +# Logging is set up in Gunicorn's configuration +LOGGING_CONFIG=None diff --git a/requirements/production.in b/requirements/production.in index a4a01ec..aff4b80 100644 --- a/requirements/production.in +++ b/requirements/production.in @@ -1,2 +1,2 @@ -c common.txt -gunicorn +gunicorn>=19.8 From 76b6c2aa19ec235c1ec012d70ae5dd4bb754a7d8 Mon Sep 17 00:00:00 2001 From: Mischback Date: Fri, 28 Feb 2020 23:53:46 +0100 Subject: [PATCH 057/110] Adjusted Gunicorn's default access log format --- configs/Docker/django/gunicorn_conf.py | 7 +------ configs/Docker/env.sample | 12 ++++++++---- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/configs/Docker/django/gunicorn_conf.py b/configs/Docker/django/gunicorn_conf.py index a557b5a..e0a3de0 100644 --- a/configs/Docker/django/gunicorn_conf.py +++ b/configs/Docker/django/gunicorn_conf.py @@ -73,7 +73,7 @@ # See: https://stackoverflow.com/questions/25737589/gunicorn-doesnt-log-real-ip-from-nginx access_log_format = os.environ.get( 'DPS_GUNICORN_LOGFORMAT', - '%({x-forwarded-for}i)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"' + '%({x-forwarded-for}i)s %(l)s %(s)s %(l)s "%(r)s"' ) @@ -100,14 +100,9 @@ # Django's 'mail_admins' handler is removed! #'handlers': ['docker_stdout'], 'level': os.environ.get('DPS_DJANGO_LOGLEVEL', 'INFO').upper(), - # Whatever got catched here will not propagate to the root logger - #'propagate': False, }, 'gunicorn': { - #'handlers': ['docker_stdout'], 'level': os.environ.get('DPS_GUNICORN_LOGLEVEL', 'INFO').upper(), - # Whatever got catched here will not propagate to the root logger - #'propagate': False, }, 'gunicorn.access': { 'propagate': os.environ.get('DPS_GUNICORN_SHOW_ACCESS_LOG', 'false').lower() == 'true', diff --git a/configs/Docker/env.sample b/configs/Docker/env.sample index 0bf26bf..dbec7b8 100644 --- a/configs/Docker/env.sample +++ b/configs/Docker/env.sample @@ -51,14 +51,18 @@ # a custom log format # -# this is the provided default value, that basically mimics nginx's default access log -#DPS_GUNICORN_LOGFORMAT=%({x-forwarded-for}i)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s" -# -# this is a shorter log that plays nicely with Python's logging setup from gunicorn_conf.py +# this is the default format that plays nicely with Python's logging setup +# from gunicorn_conf.py +# Please note, that by default, Gunicorn will **not** log access logs +# see DPS_GUNICORN_SHOW_ACCESS_LOG #DPS_GUNICORN_LOGFORMAT=%({x-forwarded-for}i)s %(l)s %(s)s %(l)s "%(r)s" +# +# this is a verbose format, that basically mimics nginx's default access log +#DPS_GUNICORN_LOGFORMAT=%({x-forwarded-for}i)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s" # the log level #DPS_GUNICORN_LOGLEVEL=INFO # determines if the accesslog should be included in the logs +# just set this to 'true' (without ') to enable access logs #DPS_GUNICORN_SHOW_ACCESS_LOG=false From 7d6cd6aa8b52fa3974142cf6a10b9b7ad2e0327d Mon Sep 17 00:00:00 2001 From: Mischback Date: Sat, 29 Feb 2020 00:02:04 +0100 Subject: [PATCH 058/110] Introduced overall log level --- configs/Docker/django/gunicorn_conf.py | 4 ++-- configs/Docker/env.sample | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/configs/Docker/django/gunicorn_conf.py b/configs/Docker/django/gunicorn_conf.py index e0a3de0..a26495b 100644 --- a/configs/Docker/django/gunicorn_conf.py +++ b/configs/Docker/django/gunicorn_conf.py @@ -90,7 +90,7 @@ 'docker_stdout': { 'class': 'logging.StreamHandler', 'formatter': 'dps_docker_default', - 'level': 'DEBUG', + 'level': os.environ.get('DPS_OVERALL_LOGLEVEL', 'INFO').upper(), 'stream': 'ext://sys.stdout', }, }, @@ -111,6 +111,6 @@ # the 'root' logger is just redefined to make it compatible with Docker 'root': { 'handlers': ['docker_stdout'], - #'level': os.environ.get('DPS_DJANGO_LOGGING_LEVEL', 'INFO').upper(), + 'level': os.environ.get('DPS_OVERALL_LOGLEVEL', 'INFO').upper(), }, } diff --git a/configs/Docker/env.sample b/configs/Docker/env.sample index dbec7b8..9b61b58 100644 --- a/configs/Docker/env.sample +++ b/configs/Docker/env.sample @@ -1,3 +1,8 @@ +### OVERALL configuration + +# set the overall (minimum) log level +#DPS_OVERALL_LOGLEVEL=INFO + ### DJANGO related configuration # Django's ALLOWED_HOSTS From aa1f343e1e2d89ea0c4685bf76c8abbc066ece1d Mon Sep 17 00:00:00 2001 From: Mischback Date: Sat, 29 Feb 2020 22:09:38 +0100 Subject: [PATCH 059/110] Fetching SECRET_KEY from the environment Optionally, the old method of generating and storing the SECRET_KEY in SECRET_FILE is provided as a fallback option. The settings module will log messages regarding this behaviour. --- configs/Docker/env.sample | 3 +++ project_name/settings/common.py | 36 ++++++++++++++++++++++++--------- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/configs/Docker/env.sample b/configs/Docker/env.sample index 9b61b58..27ecfcf 100644 --- a/configs/Docker/env.sample +++ b/configs/Docker/env.sample @@ -28,6 +28,9 @@ # used by nginx's Docker image aswell #DPS_DJANGO_MEDIA_URL=/media/ +# Django's SECRET_KEY +#DPS_DJANGO_SECRET_KEY= + # Django's STATIC_URL # the URL to serve static files from # While this is part of Django's configuration, this environment variable is diff --git a/project_name/settings/common.py b/project_name/settings/common.py index 55afad2..e1bc63a 100644 --- a/project_name/settings/common.py +++ b/project_name/settings/common.py @@ -1,7 +1,10 @@ # Python imports +import logging import os import sys +logger = logging.getLogger(__name__) + # ##### PATH CONFIGURATION ################################ @@ -122,15 +125,28 @@ DEBUG = False -# finally grab the SECRET KEY -try: - SECRET_KEY = open(SECRET_FILE).read().strip() -except IOError: +logger.debug('Trying to fetch SECRET_KEY from the environment...') +SECRET_KEY = os.environ.get('DPS_DJANGO_SECRET_KEY') +if SECRET_KEY is None: + logger.debug('Could not find key in the environment!') + + logger.debug('Trying to read SECRET_KEY from SECRET_FILE...') try: - from django.utils.crypto import get_random_string - chars = 'abcdefghijklmnopqrstuvwxyz0123456789!$%&()=+-_' - SECRET_KEY = get_random_string(50, chars) - with open(SECRET_FILE, 'w') as f: - f.write(SECRET_KEY) + SECRET_KEY = open(SECRET_FILE).read().strip() + logger.info('Read SECRET_KEY from SECRET_FILE.') except IOError: - raise Exception('Could not open {} for writing!'.format(SECRET_FILE)) + logger.debug('Could not open SECRET_FILE ({})!'.format(SECRET_FILE)) + + try: + from django.utils.crypto import get_random_string + chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!$%&()=+-_' + SECRET_KEY = get_random_string(50, chars) + with open(SECRET_FILE, 'w') as f: + f.write(SECRET_KEY) + + logger.info('Generated a new SECRET_KEY and stored it in SECRET_FILE ({})!'.format(SECRET_FILE)) + except IOError: + logger.exception('Could not open SECRET_FILE ({}) for writing!'.format(SECRET_FILE)) + raise Exception('Could not open {} for writing!'.format(SECRET_FILE)) +else: + logger.info('Fetched SECRET_KEY from environment.') From 4dcb49a93d91c3f157b0ed43b3d67425764b075b Mon Sep 17 00:00:00 2001 From: Mischback Date: Sat, 29 Feb 2020 22:35:10 +0100 Subject: [PATCH 060/110] Automatically create production environment file by Makefile --- Makefile | 6 +++++- configs/Makefile.deployment.template | 10 +++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 704dd49..1a7780b 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,11 @@ init: # switching the Makefile **should** be the last step mv ./configs/Makefile.deployment.template ./Makefile -docker/build: +configs/Docker/env.production: + $(ECHO) "Initializing environment file for production...\n" + cp configs/Docker/env.sample configs/Docker/env.production + +docker/build: configs/Docker/env.production tox -q -e docker-testing docker/build-context: diff --git a/configs/Makefile.deployment.template b/configs/Makefile.deployment.template index 9a39d3c..59d29b9 100644 --- a/configs/Makefile.deployment.template +++ b/configs/Makefile.deployment.template @@ -249,7 +249,7 @@ django/staticfiles/find: ### Docker commands ### -docker/build: util/requirements +docker/build: util/requirements configs/Docker/env.production sudo $(MAKE) .internal/docker/build docker/run: docker/run/django @@ -257,6 +257,14 @@ docker/run: docker/run/django docker/run/django: sudo $(MAKE) .internal/docker/run/django + +### Environment management ### + +configs/Docker/env.production: + $(ECHO) "Initializing environment file for production...\n" + cp configs/Docker/env.sample configs/Docker/env.production + + ### Utility commands ### util/requirements: From aeeddd724c2d5efef39364a465dad018afcdce2d Mon Sep 17 00:00:00 2001 From: Mischback Date: Sun, 1 Mar 2020 22:05:30 +0100 Subject: [PATCH 061/110] Automatically generate a SECRET_KEY in env.production --- Makefile | 3 ++- bin/generate_secret_key.sh | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100755 bin/generate_secret_key.sh diff --git a/Makefile b/Makefile index 1a7780b..7d4352a 100644 --- a/Makefile +++ b/Makefile @@ -52,8 +52,9 @@ init: mv ./configs/Makefile.deployment.template ./Makefile configs/Docker/env.production: - $(ECHO) "Initializing environment file for production...\n" + echo "Initializing environment file for production..." cp configs/Docker/env.sample configs/Docker/env.production + sed -i "s/#DPS_DJANGO_SECRET_KEY=/DPS_DJANGO_SECRET_KEY=$(shell ./bin/generate_secret_key.sh)/" configs/Docker/env.production docker/build: configs/Docker/env.production tox -q -e docker-testing diff --git a/bin/generate_secret_key.sh b/bin/generate_secret_key.sh new file mode 100755 index 0000000..9e2ba7e --- /dev/null +++ b/bin/generate_secret_key.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +# This script is part of django-project-skeleton and is used to generate an +# random string to be used as SECRET_KEY for Django's configuration. +# +# Ths script is - obviously - no reimplementation of the corresponding +# function provided by Django, but should provide a value, that may be +# considered "random enough". +# +# This script is used to generate a SECRET_KEY in the env.production file. +# That file is **NOT** under version control, for obvious reasons. + +cat /dev/urandom | tr -dc 'a-zA-Z0-9!@$%^()_-' | fold -w 50 | head -n 1 From e580fd6ec5a769f8727b3d737ab808d357603512 Mon Sep 17 00:00:00 2001 From: Mischback Date: Sun, 1 Mar 2020 22:17:09 +0100 Subject: [PATCH 062/110] env.production must be generated in both Makefiles --- configs/Makefile.deployment.template | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configs/Makefile.deployment.template b/configs/Makefile.deployment.template index 59d29b9..31248d6 100644 --- a/configs/Makefile.deployment.template +++ b/configs/Makefile.deployment.template @@ -261,8 +261,13 @@ docker/run/django: ### Environment management ### configs/Docker/env.production: + # this is explicitly **NOT** a phony target, because the production + # settings must only be generated **ONE TIME**. + # Some of these settings are initialised with random values, but hve to be + # constant for the lifetime of a project, e.g. the SECRET_KEY. $(ECHO) "Initializing environment file for production...\n" cp configs/Docker/env.sample configs/Docker/env.production + sed -i "s/#DPS_DJANGO_SECRET_KEY=/DPS_DJANGO_SECRET_KEY=$(shell ./bin/generate_secret_key.sh)/" configs/Docker/env.production ### Utility commands ### From 18c871af64fb2e8eddff4e517fe376b4aebc65ea Mon Sep 17 00:00:00 2001 From: Mischback Date: Sun, 1 Mar 2020 22:18:46 +0100 Subject: [PATCH 063/110] Remove env.production during development's clean --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 7d4352a..294bd7e 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,7 @@ clean: # into version control, thus, they can not simply included in a .gitignore. rm requirements/common.txt rm requirements/production.txt + rm configs/Docker/env.production init: # TODO: Include some output to this function From fb7e72ed4393c2e66f1256a606c47db62746c58d Mon Sep 17 00:00:00 2001 From: Mischback Date: Sun, 1 Mar 2020 22:27:47 +0100 Subject: [PATCH 064/110] Some more comments on generating SECRET_KEY --- bin/generate_secret_key.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/generate_secret_key.sh b/bin/generate_secret_key.sh index 9e2ba7e..a460028 100755 --- a/bin/generate_secret_key.sh +++ b/bin/generate_secret_key.sh @@ -6,6 +6,11 @@ # Ths script is - obviously - no reimplementation of the corresponding # function provided by Django, but should provide a value, that may be # considered "random enough". +# The code is taken from https://gist.github.com/earthgecko/3089509 , see +# https://github.com/django/django/blob/9386586f31b8a0bccf59a1bff647cd829d4e79aa/django/core/management/utils.py#L77 +# for Django's original implementation. +# +# '&' had to be removed from the available characters, because it breaks 'sed'. # # This script is used to generate a SECRET_KEY in the env.production file. # That file is **NOT** under version control, for obvious reasons. From c018180009f034a126dddfd5f13bad4473b387c0 Mon Sep 17 00:00:00 2001 From: Mischback Date: Mon, 2 Mar 2020 20:59:06 +0100 Subject: [PATCH 065/110] Added volume for static files --- configs/Docker/django/Dockerfile | 5 ++--- configs/Docker/docker-compose.yml | 7 +++++++ configs/Docker/nginx/Dockerfile | 3 +++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/configs/Docker/django/Dockerfile b/configs/Docker/django/Dockerfile index c1fccf1..36da730 100644 --- a/configs/Docker/django/Dockerfile +++ b/configs/Docker/django/Dockerfile @@ -53,9 +53,8 @@ FROM python:3.8-slim-buster as django_production COPY configs/Docker/bin/apt-install /usr/local/bin/ # create a user to be used (meaning: we won't run with root-permissions!) -RUN useradd -u 2342 -g 65534 -s /bin/false -M pythonuser && \ - mkdir /home/pythonuser -WORKDIR /home/pythonuser +RUN useradd -u 2342 -g 65534 -s /bin/false -M pythonuser +WORKDIR /webapp # fetch the virtual environment from base COPY --from=base --chown=2342:65534 /venv /venv diff --git a/configs/Docker/docker-compose.yml b/configs/Docker/docker-compose.yml index 73f9fca..3b3c6b0 100644 --- a/configs/Docker/docker-compose.yml +++ b/configs/Docker/docker-compose.yml @@ -15,6 +15,8 @@ services: ./env.production expose: - "8000" + volumes: + - static-files:/webapp/run/static nginx: build: context: ./nginx/ @@ -26,3 +28,8 @@ services: image: ${DPS_BUILD_NAME_PREFIX}/nginx:${DPS_BUILD_ID} ports: - "80:80" + volumes: + - static-files:/var/www/static + +volumes: + static-files: diff --git a/configs/Docker/nginx/Dockerfile b/configs/Docker/nginx/Dockerfile index c95a427..e47bbfc 100644 --- a/configs/Docker/nginx/Dockerfile +++ b/configs/Docker/nginx/Dockerfile @@ -3,5 +3,8 @@ FROM nginx:1.17 COPY nginx.conf /etc/nginx/nginx.conf COPY entrypoint.sh /entrypoint.sh +RUN mkdir /var/www && \ + mkdir /var/www/static + ENTRYPOINT ["/entrypoint.sh"] CMD ["nginx", "-g", "daemon off;"] From 764640e222af30abb902fc814d27e8d41fe1eace Mon Sep 17 00:00:00 2001 From: Mischback Date: Tue, 3 Mar 2020 21:28:05 +0100 Subject: [PATCH 066/110] Recreate nginx.conf --- Makefile | 4 +- configs/Docker/docker-compose.yml | 2 +- configs/Docker/nginx/Dockerfile | 3 -- configs/Docker/nginx/nginx.conf.template | 53 ++++++++++++++++++++---- 4 files changed, 49 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 294bd7e..75a7f23 100644 --- a/Makefile +++ b/Makefile @@ -66,9 +66,11 @@ docker/build-context: docker/images: sudo $(MAKE) .docker/images -docker/run: +docker/run: docker/build sudo $(MAKE) .docker/run +run: docker/run + tree: tree -a -I ".git|.tox|doc|run" --dirsfirst -C | less -r diff --git a/configs/Docker/docker-compose.yml b/configs/Docker/docker-compose.yml index 3b3c6b0..aea7021 100644 --- a/configs/Docker/docker-compose.yml +++ b/configs/Docker/docker-compose.yml @@ -29,7 +29,7 @@ services: ports: - "80:80" volumes: - - static-files:/var/www/static + - static-files:/var/www/django_files/static volumes: static-files: diff --git a/configs/Docker/nginx/Dockerfile b/configs/Docker/nginx/Dockerfile index e47bbfc..c95a427 100644 --- a/configs/Docker/nginx/Dockerfile +++ b/configs/Docker/nginx/Dockerfile @@ -3,8 +3,5 @@ FROM nginx:1.17 COPY nginx.conf /etc/nginx/nginx.conf COPY entrypoint.sh /entrypoint.sh -RUN mkdir /var/www && \ - mkdir /var/www/static - ENTRYPOINT ["/entrypoint.sh"] CMD ["nginx", "-g", "daemon off;"] diff --git a/configs/Docker/nginx/nginx.conf.template b/configs/Docker/nginx/nginx.conf.template index b3ed1c8..7efe466 100644 --- a/configs/Docker/nginx/nginx.conf.template +++ b/configs/Docker/nginx/nginx.conf.template @@ -6,26 +6,46 @@ events { accept_mutex off; } +# TODO: drop user privileges here! +# + http { # define the Django server upstream app_server { + # The upstream is identified by the service name provided in + # docker-compose.yml. Adjust accordingly, if you run your + # nginx outside of the scope that file / on bare metal. server django:8000; } - # if no Host match, close the connection to prevent host spoofing server { - listen 80 default_server; - return 444; - } + # TODO: This **must** be configurable by environment variables + server_name deb10.*; - server { + # TODO: Let's make this run with HTTP, but should finally be switched + # to HTTPS. listen 80; - client_max_body_size 4G; - server_name deb10.massive.home; + # TODO: This *should* be synchronized with Django's + # DATA_UPLOAD_MAX_MEMORY_SIZE, which defaults to 2621440 byte=2.5MB. + # This *could* be adjustable with environment variables, but on the + # other hand, this might be a very specific setting and adjustment + # only is necessary for specific applications that typically handle + # larger files. For now, it is just set to Django's default value. + client_max_body_size 2621440; + + # set the timeout for keep-alive connections for the whole server + keepalive_timeout 60; - keepalive_timeout 5; + # TODO: This *should* be adjustable with environment variables and + # has to be synchronized with Django's STATIC_URL, as defined by + # DPS_DJANGO_STATIC_URL in env.sample. + location /static/ { + # The Docker volume for static files is mounted to + # /var/www/django_files/static (see docker-compose.yml). + root /var/www/django_files; + } location / { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -34,6 +54,23 @@ http { proxy_set_header Host $http_host; proxy_redirect off; proxy_pass http://app_server; + + # TODO: This *should* be synchronized with Gunicorn's 'keepalive' + # setting (DPS_GUNICORN_KEEPALIVE) + keepalive_timeout 5; } } + + # if no server_name matches, close the connection to prevent host spoofing + server { + listen 80 default_server; + return 444; + } + + # automatically detect mime types + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # disable server tokens (on error pages / headers) + server_tokens off; } From 2866c49189df5bd8d6f89235cc3766478820216b Mon Sep 17 00:00:00 2001 From: Mischback Date: Tue, 3 Mar 2020 22:01:19 +0100 Subject: [PATCH 067/110] Synchronize time between host and containers This is not working as expected for the Django container. The first messages are logged with correct timestamps (these are issued by Gunicorn), but after Django successfully applied the provided configuration, the time zone for all Python-based logs is set to the value of Django's TIME_ZONE (which, if not specified, is set to 'America/Chicago'). TODO: Fix this issue; do **not** rely on Django's TIME_ZONE for the container's logging timestamps. Probably an own implementation may be necessary, to unify logging timestamps throughout all containers. However, this is highly necessary for any debugging, especially if any log aggregator is used. --- configs/Docker/docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/configs/Docker/docker-compose.yml b/configs/Docker/docker-compose.yml index aea7021..385f8c3 100644 --- a/configs/Docker/docker-compose.yml +++ b/configs/Docker/docker-compose.yml @@ -17,6 +17,8 @@ services: - "8000" volumes: - static-files:/webapp/run/static + # synchronize time with host machine + - /etc/localtime:/etc/localtime:ro nginx: build: context: ./nginx/ @@ -30,6 +32,8 @@ services: - "80:80" volumes: - static-files:/var/www/django_files/static + # synchronize time with host machine + - /etc/localtime:/etc/localtime:ro volumes: static-files: From 5c6810de7158cda31af7d561575eab553481a617 Mon Sep 17 00:00:00 2001 From: Mischback Date: Wed, 4 Mar 2020 18:49:43 +0100 Subject: [PATCH 068/110] Revoke last change The intention was, to prevent drift between the clocks of Docker containers and the host system. But that function is integrated in Docker. What the change actually did: Sync the timezones of containers and host. Obviously, that did not work out... --- configs/Docker/docker-compose.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/configs/Docker/docker-compose.yml b/configs/Docker/docker-compose.yml index 385f8c3..aea7021 100644 --- a/configs/Docker/docker-compose.yml +++ b/configs/Docker/docker-compose.yml @@ -17,8 +17,6 @@ services: - "8000" volumes: - static-files:/webapp/run/static - # synchronize time with host machine - - /etc/localtime:/etc/localtime:ro nginx: build: context: ./nginx/ @@ -32,8 +30,6 @@ services: - "80:80" volumes: - static-files:/var/www/django_files/static - # synchronize time with host machine - - /etc/localtime:/etc/localtime:ro volumes: static-files: From 191604a7428b1a5aebed8284af056cdad23369e1 Mon Sep 17 00:00:00 2001 From: Mischback Date: Wed, 4 Mar 2020 21:20:36 +0100 Subject: [PATCH 069/110] Introduce project specific time zone setting --- configs/Docker/env.sample | 12 ++++++++++++ project_name/settings/common.py | 3 +++ project_name/settings/i18n.py | 1 - 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/configs/Docker/env.sample b/configs/Docker/env.sample index 27ecfcf..05b7728 100644 --- a/configs/Docker/env.sample +++ b/configs/Docker/env.sample @@ -1,8 +1,20 @@ ### OVERALL configuration # set the overall (minimum) log level +# This setting is used by +# - [django] gunicorn_conf.py #DPS_OVERALL_LOGLEVEL=INFO +# Provide the timezone for all containers +# Must follow the list of time zones, see: +# https://en.wikipedia.org/wiki/List_of_tz_database_time_zones +# +# This setting is used by +# - [django] settings/common.py +#DPS_TIMEZONE=Etc/UTC +#DPS_TIMEZONE=Europe/Berlin + + ### DJANGO related configuration # Django's ALLOWED_HOSTS diff --git a/project_name/settings/common.py b/project_name/settings/common.py index e1bc63a..ae53cf2 100644 --- a/project_name/settings/common.py +++ b/project_name/settings/common.py @@ -40,6 +40,9 @@ # ##### APPLICATION CONFIGURATION ######################### +# set the project's default timezone +TIME_ZONE = os.environ.get('DPS_TIMEZONE', 'Etc/UTC') + # these are the apps DEFAULT_APPS = [ 'django.contrib.admin', diff --git a/project_name/settings/i18n.py b/project_name/settings/i18n.py index 750630d..95ed624 100644 --- a/project_name/settings/i18n.py +++ b/project_name/settings/i18n.py @@ -10,7 +10,6 @@ # ##### INTERNATIONALIZATION ############################## LANGUAGE_CODE = 'de' -TIME_ZONE = 'Europe/Berlin' # Internationalization USE_I18N = True From 4a05f131bedd7b60756d986f115feffaa932d143 Mon Sep 17 00:00:00 2001 From: Mischback Date: Wed, 4 Mar 2020 21:58:28 +0100 Subject: [PATCH 070/110] Use an entrypoint script to set container's time correctly For the used Debian-based images, setting the timezone requires root privileges, so 'gosu' is now used to drop privileges during container startup (in entrypoint-django.sh). --- .dockerignore | 1 + configs/Docker/django/Dockerfile | 11 ++++++++--- configs/Docker/django/entrypoint-django.sh | 13 +++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) create mode 100755 configs/Docker/django/entrypoint-django.sh diff --git a/.dockerignore b/.dockerignore index b0dda4d..79b60ba 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,6 +11,7 @@ configs # but DO include the production-related scripts/config files that are used # inside of the images !configs/Docker/bin/apt-install +!configs/Docker/django/entrypoint-django.sh !configs/Docker/django/gunicorn_conf.py !configs/Docker/django/run-gunicorn.sh !configs/Docker/nginx/nginx.conf diff --git a/configs/Docker/django/Dockerfile b/configs/Docker/django/Dockerfile index 36da730..09cfa93 100644 --- a/configs/Docker/django/Dockerfile +++ b/configs/Docker/django/Dockerfile @@ -52,6 +52,8 @@ FROM python:3.8-slim-buster as django_production # fetch our custom install script COPY configs/Docker/bin/apt-install /usr/local/bin/ +RUN apt-install gosu + # create a user to be used (meaning: we won't run with root-permissions!) RUN useradd -u 2342 -g 65534 -s /bin/false -M pythonuser WORKDIR /webapp @@ -63,6 +65,9 @@ COPY --from=base --chown=2342:65534 /venv /venv ENV VIRTUAL_ENV=/venv ENV PATH="$VIRTUAL_ENV/bin:$PATH" +# fetch the entrypoint script +COPY configs/Docker/django/entrypoint-django.sh /usr/local/bin/ + # fetch the actual project files COPY --chown=2342:65534 . . @@ -70,7 +75,7 @@ COPY --chown=2342:65534 . . EXPOSE 8000 # drop user privileges -USER pythonuser +#USER pythonuser -# start the Django development server by default -CMD ["python", "manage.py", "runserver"] +# set the entrypoint script +ENTRYPOINT ["entrypoint-django.sh"] diff --git a/configs/Docker/django/entrypoint-django.sh b/configs/Docker/django/entrypoint-django.sh new file mode 100755 index 0000000..eb2b753 --- /dev/null +++ b/configs/Docker/django/entrypoint-django.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +if [[ -v DPS_TIMEZONE ]]; then + + true \ + && echo ${DPS_TIMEZONE} > /etc/timezone \ + && ln -sf /usr/share/zoneinfo/${DPS_TIMEZONE} /etc/localtime \ + && dpkg-reconfigure -f noninteractive tzdata; + +fi + +# actually execute the provided command, but drop privileges using gosu +exec gosu pythonuser "$@" From 1752cca9b9f745ce4525a986e8eed5712a64992e Mon Sep 17 00:00:00 2001 From: Mischback Date: Wed, 4 Mar 2020 22:06:37 +0100 Subject: [PATCH 071/110] Clean Django Dockerfile --- configs/Docker/django/Dockerfile | 9 +++++---- configs/Docker/docker-compose.yml | 1 - 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/configs/Docker/django/Dockerfile b/configs/Docker/django/Dockerfile index 09cfa93..e71a8a5 100644 --- a/configs/Docker/django/Dockerfile +++ b/configs/Docker/django/Dockerfile @@ -66,16 +66,17 @@ ENV VIRTUAL_ENV=/venv ENV PATH="$VIRTUAL_ENV/bin:$PATH" # fetch the entrypoint script -COPY configs/Docker/django/entrypoint-django.sh /usr/local/bin/ +COPY configs/Docker/django/*.sh /usr/local/bin/ # fetch the actual project files +# TODO: copy in several steps with more/finer control COPY --chown=2342:65534 . . # make Gunicorn's port 8000 available to other Docker images EXPOSE 8000 -# drop user privileges -#USER pythonuser - # set the entrypoint script ENTRYPOINT ["entrypoint-django.sh"] + +# and the default command +CMD ["run-gunicorn.sh"] diff --git a/configs/Docker/docker-compose.yml b/configs/Docker/docker-compose.yml index aea7021..fc3b690 100644 --- a/configs/Docker/docker-compose.yml +++ b/configs/Docker/docker-compose.yml @@ -5,7 +5,6 @@ services: build: context: ../../ dockerfile: configs/Docker/django/Dockerfile - command: ./configs/Docker/django/run-gunicorn.sh # providing the build and the image directive automatically tags the image # automatic tagging is performed with environment variables provided # through the Makefile and results in a tag with either a git commit hash From f1883654693ed08c6d934b03a31dfe0fe70a0efd Mon Sep 17 00:00:00 2001 From: Mischback Date: Thu, 5 Mar 2020 21:12:01 +0100 Subject: [PATCH 072/110] Make setting the time zone DRY --- .dockerignore | 1 + configs/Docker/bin/set_timezone.sh | 24 ++++++++++++++++ configs/Docker/django/Dockerfile | 32 ++++++++++++++-------- configs/Docker/django/entrypoint-django.sh | 14 ++++------ 4 files changed, 51 insertions(+), 20 deletions(-) create mode 100755 configs/Docker/bin/set_timezone.sh diff --git a/.dockerignore b/.dockerignore index 79b60ba..d932bf5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,6 +11,7 @@ configs # but DO include the production-related scripts/config files that are used # inside of the images !configs/Docker/bin/apt-install +!configs/Docker/bin/set_timezone.sh !configs/Docker/django/entrypoint-django.sh !configs/Docker/django/gunicorn_conf.py !configs/Docker/django/run-gunicorn.sh diff --git a/configs/Docker/bin/set_timezone.sh b/configs/Docker/bin/set_timezone.sh new file mode 100755 index 0000000..f1cb7bd --- /dev/null +++ b/configs/Docker/bin/set_timezone.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Updates the container's timezone according to the provided environment +# variable DPS_TIMEZONE and sets UTC as default value. +# +# This script has to be run with root privileges in order to actually run +# `dpkg-reconfigure`. + +# check for the existence of DPS_TIMEZONE in environment +if [[ -v DPS_TIMEZONE ]]; then + true \ + && echo ${DPS_TIMEZONE} > /etc/timezone \ + && ln -sf /usr/share/zoneinfo/${DPS_TIMEZONE} /etc/localtime; + +# set UTC as default time zone +else + # TODO: Untestested code! Verify that this is working! Verify the existence of path /usr/share/zoneinfo/Etc/UTC! + true \ + && echo "Etc/UTC" > /etc/timezone \ + && ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime; +fi + +# actually apply the timezone systemwide +dpkg-reconfigure -f noninteractive tzdata; diff --git a/configs/Docker/django/Dockerfile b/configs/Docker/django/Dockerfile index e71a8a5..487328b 100644 --- a/configs/Docker/django/Dockerfile +++ b/configs/Docker/django/Dockerfile @@ -14,17 +14,17 @@ # the Python packages, they must be installed here aswell! FROM python:3.8-slim-buster as base -# fetch our custom install script -COPY configs/Docker/bin/apt-install /usr/local/bin/ - # set environment variables -# TODO: Check if these are still valid for "production"! ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 # create the virtual environment for our project ENV VIRTUAL_ENV=/venv -ENV PATH="$VIRTUAL_ENV/bin:$PATH" +ENV PATH="$VIRTUAL_ENV/bin:$PATH:/docker-bin" + +# fetch our custom install script +COPY configs/Docker/bin/apt-install /docker-bin/ + RUN python3 -m venv $VIRTUAL_ENV # update pip inside of the virtual environment @@ -49,8 +49,16 @@ RUN pip install --no-cache-dir \ # directly into this image. FROM python:3.8-slim-buster as django_production +# set environment variables +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 + +# actually activate this virtual environment +ENV VIRTUAL_ENV=/venv +ENV PATH="$VIRTUAL_ENV/bin:$PATH:/docker-bin" + # fetch our custom install script -COPY configs/Docker/bin/apt-install /usr/local/bin/ +COPY configs/Docker/bin/apt-install /docker-bin/ RUN apt-install gosu @@ -61,12 +69,12 @@ WORKDIR /webapp # fetch the virtual environment from base COPY --from=base --chown=2342:65534 /venv /venv -# actually activate this virtual environment -ENV VIRTUAL_ENV=/venv -ENV PATH="$VIRTUAL_ENV/bin:$PATH" +# fetch the entrypoint script (and includes) +COPY configs/Docker/django/entrypoint-django.sh /docker-bin/ +COPY configs/Docker/bin/set_timezone.sh /docker-bin/ -# fetch the entrypoint script -COPY configs/Docker/django/*.sh /usr/local/bin/ +# fetch the container's start script +COPY configs/Docker/django/run-gunicorn.sh /docker-bin/ # fetch the actual project files # TODO: copy in several steps with more/finer control @@ -75,7 +83,7 @@ COPY --chown=2342:65534 . . # make Gunicorn's port 8000 available to other Docker images EXPOSE 8000 -# set the entrypoint script +# set the entrypoint script... ENTRYPOINT ["entrypoint-django.sh"] # and the default command diff --git a/configs/Docker/django/entrypoint-django.sh b/configs/Docker/django/entrypoint-django.sh index eb2b753..23ee251 100755 --- a/configs/Docker/django/entrypoint-django.sh +++ b/configs/Docker/django/entrypoint-django.sh @@ -1,13 +1,11 @@ #!/bin/bash -if [[ -v DPS_TIMEZONE ]]; then +# determine the scripts current working directory (CWD) +CWD="${BASH_SOURCE%/*}"; +if [[ ! -d "$CWD" ]]; then CWD="$PWD"; fi - true \ - && echo ${DPS_TIMEZONE} > /etc/timezone \ - && ln -sf /usr/share/zoneinfo/${DPS_TIMEZONE} /etc/localtime \ - && dpkg-reconfigure -f noninteractive tzdata; +# include the script for setting the timezone +source "$CWD/set_timezone.sh" -fi - -# actually execute the provided command, but drop privileges using gosu +# execute the provided command, but drop privileges using gosu exec gosu pythonuser "$@" From 9f1c52e275805c24375f2e4e1a79ea1686346108 Mon Sep 17 00:00:00 2001 From: Mischback Date: Thu, 5 Mar 2020 21:30:36 +0100 Subject: [PATCH 073/110] Renamed Debian install script --- .dockerignore | 3 +-- configs/Docker/bin/{apt-install => apt-get_install.sh} | 0 configs/Docker/django/Dockerfile | 6 +++--- 3 files changed, 4 insertions(+), 5 deletions(-) rename configs/Docker/bin/{apt-install => apt-get_install.sh} (100%) diff --git a/.dockerignore b/.dockerignore index d932bf5..6897d9d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,8 +10,7 @@ bin configs # but DO include the production-related scripts/config files that are used # inside of the images -!configs/Docker/bin/apt-install -!configs/Docker/bin/set_timezone.sh +!configs/Docker/bin/* !configs/Docker/django/entrypoint-django.sh !configs/Docker/django/gunicorn_conf.py !configs/Docker/django/run-gunicorn.sh diff --git a/configs/Docker/bin/apt-install b/configs/Docker/bin/apt-get_install.sh similarity index 100% rename from configs/Docker/bin/apt-install rename to configs/Docker/bin/apt-get_install.sh diff --git a/configs/Docker/django/Dockerfile b/configs/Docker/django/Dockerfile index 487328b..8240b84 100644 --- a/configs/Docker/django/Dockerfile +++ b/configs/Docker/django/Dockerfile @@ -23,7 +23,7 @@ ENV VIRTUAL_ENV=/venv ENV PATH="$VIRTUAL_ENV/bin:$PATH:/docker-bin" # fetch our custom install script -COPY configs/Docker/bin/apt-install /docker-bin/ +COPY configs/Docker/bin/apt-get_install.sh /docker-bin/ RUN python3 -m venv $VIRTUAL_ENV @@ -58,9 +58,9 @@ ENV VIRTUAL_ENV=/venv ENV PATH="$VIRTUAL_ENV/bin:$PATH:/docker-bin" # fetch our custom install script -COPY configs/Docker/bin/apt-install /docker-bin/ +COPY configs/Docker/bin/apt-get_install.sh /docker-bin/ -RUN apt-install gosu +RUN apt-get_install.sh gosu # create a user to be used (meaning: we won't run with root-permissions!) RUN useradd -u 2342 -g 65534 -s /bin/false -M pythonuser From f8c2dabb82e6edf85d301f148693ff246f5d905d Mon Sep 17 00:00:00 2001 From: Mischback Date: Thu, 5 Mar 2020 21:43:26 +0100 Subject: [PATCH 074/110] Renamed some more scripts --- .dockerignore | 4 ++-- Makefile | 2 +- configs/Docker/django/Dockerfile | 8 ++++---- .../django/{entrypoint-django.sh => entrypoint_django.sh} | 0 ...{run-gunicorn.sh.template => run_gunicorn.sh.template} | 0 5 files changed, 7 insertions(+), 7 deletions(-) rename configs/Docker/django/{entrypoint-django.sh => entrypoint_django.sh} (100%) rename configs/Docker/django/{run-gunicorn.sh.template => run_gunicorn.sh.template} (100%) diff --git a/.dockerignore b/.dockerignore index 6897d9d..1300be2 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,9 +11,9 @@ configs # but DO include the production-related scripts/config files that are used # inside of the images !configs/Docker/bin/* -!configs/Docker/django/entrypoint-django.sh +!configs/Docker/django/entrypoint_django.sh !configs/Docker/django/gunicorn_conf.py -!configs/Docker/django/run-gunicorn.sh +!configs/Docker/django/run_gunicorn.sh !configs/Docker/nginx/nginx.conf !configs/Docker/nginx/entrypoint.sh diff --git a/Makefile b/Makefile index 75a7f23..1efef15 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ init: # TODO: Include some output to this function # create the final version of different files by removing the '.sample' suffix mv ./configs/apache2_vhost.sample.template ./configs/apache2_vhost.sample - mv ./configs/Docker/django/run-gunicorn.sh.template ./configs/Docker/django/run-gunicorn.sh + mv ./configs/Docker/django/run_gunicorn.sh.template ./configs/Docker/django/run_gunicorn.sh mv ./configs/Docker/nginx/nginx.conf.template ./configs/Docker/nginx/nginx.conf # switching the tox configuration file mv ./configs/tox.deployment.template ./tox.ini diff --git a/configs/Docker/django/Dockerfile b/configs/Docker/django/Dockerfile index 8240b84..acb6d15 100644 --- a/configs/Docker/django/Dockerfile +++ b/configs/Docker/django/Dockerfile @@ -70,11 +70,11 @@ WORKDIR /webapp COPY --from=base --chown=2342:65534 /venv /venv # fetch the entrypoint script (and includes) -COPY configs/Docker/django/entrypoint-django.sh /docker-bin/ +COPY configs/Docker/django/entrypoint_django.sh /docker-bin/ COPY configs/Docker/bin/set_timezone.sh /docker-bin/ # fetch the container's start script -COPY configs/Docker/django/run-gunicorn.sh /docker-bin/ +COPY configs/Docker/django/run_gunicorn.sh /docker-bin/ # fetch the actual project files # TODO: copy in several steps with more/finer control @@ -84,7 +84,7 @@ COPY --chown=2342:65534 . . EXPOSE 8000 # set the entrypoint script... -ENTRYPOINT ["entrypoint-django.sh"] +ENTRYPOINT ["entrypoint_django.sh"] # and the default command -CMD ["run-gunicorn.sh"] +CMD ["run_gunicorn.sh"] diff --git a/configs/Docker/django/entrypoint-django.sh b/configs/Docker/django/entrypoint_django.sh similarity index 100% rename from configs/Docker/django/entrypoint-django.sh rename to configs/Docker/django/entrypoint_django.sh diff --git a/configs/Docker/django/run-gunicorn.sh.template b/configs/Docker/django/run_gunicorn.sh.template similarity index 100% rename from configs/Docker/django/run-gunicorn.sh.template rename to configs/Docker/django/run_gunicorn.sh.template From 35d7fe5e7bdf3f990b20bae11860e932d1ef354c Mon Sep 17 00:00:00 2001 From: Mischback Date: Thu, 5 Mar 2020 22:00:50 +0100 Subject: [PATCH 075/110] Applied time zone setting to nginx container --- .dockerignore | 2 +- configs/Docker/docker-compose.yml | 6 ++++-- configs/Docker/nginx/Dockerfile | 7 ++++--- configs/Docker/nginx/entrypoint.sh | 3 --- configs/Docker/nginx/entrypoint_nginx.sh | 10 ++++++++++ 5 files changed, 19 insertions(+), 9 deletions(-) delete mode 100755 configs/Docker/nginx/entrypoint.sh create mode 100755 configs/Docker/nginx/entrypoint_nginx.sh diff --git a/.dockerignore b/.dockerignore index 1300be2..ff60762 100644 --- a/.dockerignore +++ b/.dockerignore @@ -15,7 +15,7 @@ configs !configs/Docker/django/gunicorn_conf.py !configs/Docker/django/run_gunicorn.sh !configs/Docker/nginx/nginx.conf -!configs/Docker/nginx/entrypoint.sh +!configs/Docker/nginx/entrypoint_nginx.sh # do NOT include the documentation # TODO: the original doc-folder of django-project-skeleton should be removed diff --git a/configs/Docker/docker-compose.yml b/configs/Docker/docker-compose.yml index fc3b690..7d63994 100644 --- a/configs/Docker/docker-compose.yml +++ b/configs/Docker/docker-compose.yml @@ -18,13 +18,15 @@ services: - static-files:/webapp/run/static nginx: build: - context: ./nginx/ - dockerfile: Dockerfile + context: ./ + dockerfile: nginx/Dockerfile # providing the build and the image directive automatically tags the image # automatic tagging is performed with environment variables provided # through the Makefile and results in a tag with either a git commit hash # or a datetime stamp (see Makefile.deployment) image: ${DPS_BUILD_NAME_PREFIX}/nginx:${DPS_BUILD_ID} + env_file: + ./env.production ports: - "80:80" volumes: diff --git a/configs/Docker/nginx/Dockerfile b/configs/Docker/nginx/Dockerfile index c95a427..3c9469c 100644 --- a/configs/Docker/nginx/Dockerfile +++ b/configs/Docker/nginx/Dockerfile @@ -1,7 +1,8 @@ FROM nginx:1.17 -COPY nginx.conf /etc/nginx/nginx.conf -COPY entrypoint.sh /entrypoint.sh +COPY bin/set_timezone.sh /docker-bin/ +COPY nginx/entrypoint_nginx.sh /docker-bin/ +COPY nginx/nginx.conf /etc/nginx/nginx.conf -ENTRYPOINT ["/entrypoint.sh"] +ENTRYPOINT ["/docker-bin/entrypoint_nginx.sh"] CMD ["nginx", "-g", "daemon off;"] diff --git a/configs/Docker/nginx/entrypoint.sh b/configs/Docker/nginx/entrypoint.sh deleted file mode 100755 index 214eb4c..0000000 --- a/configs/Docker/nginx/entrypoint.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -exec "$@" diff --git a/configs/Docker/nginx/entrypoint_nginx.sh b/configs/Docker/nginx/entrypoint_nginx.sh new file mode 100755 index 0000000..216498a --- /dev/null +++ b/configs/Docker/nginx/entrypoint_nginx.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# determine the scripts current working directory (CWD) +CWD="${BASH_SOURCE%/*}"; +if [[ ! -d "$CWD" ]]; then CWD="$PWD"; fi + +# include the script for setting the timezone +source "$CWD/set_timezone.sh" + +exec "$@" From 3d717f5a55198dc63b53c6324b03c65e6ac3db7d Mon Sep 17 00:00:00 2001 From: Mischback Date: Mon, 9 Mar 2020 20:46:52 +0100 Subject: [PATCH 076/110] Improved apt-get_install cleaning --- configs/Docker/bin/apt-get_install.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configs/Docker/bin/apt-get_install.sh b/configs/Docker/bin/apt-get_install.sh index c86fc9c..d8c4fb8 100755 --- a/configs/Docker/bin/apt-get_install.sh +++ b/configs/Docker/bin/apt-get_install.sh @@ -25,5 +25,8 @@ apt-get update # do NOT install 'recommended packages' apt-get install -y --no-install-recommends "$@" +apt-get autoremove +apt-get clean + # remove apt's sources lists rm -rf /var/lib/apt/lists/* From b834e3f55a0d6e4cf8f5340b330355323c684f72 Mon Sep 17 00:00:00 2001 From: Mischback Date: Mon, 9 Mar 2020 21:53:42 +0100 Subject: [PATCH 077/110] Applied -euo pipefail to all shell scripts --- bin/generate_secret_key.sh | 4 +++- bin/get_build_id.sh | 4 +++- configs/Docker/bin/apt-get_install.sh | 2 ++ configs/Docker/bin/set_timezone.sh | 2 ++ configs/Docker/django/entrypoint_django.sh | 2 ++ configs/Docker/django/run_gunicorn.sh.template | 2 ++ configs/Docker/nginx/entrypoint_nginx.sh | 2 ++ 7 files changed, 16 insertions(+), 2 deletions(-) diff --git a/bin/generate_secret_key.sh b/bin/generate_secret_key.sh index a460028..1f9abca 100755 --- a/bin/generate_secret_key.sh +++ b/bin/generate_secret_key.sh @@ -1,4 +1,6 @@ -#!/bin/sh +#!/bin/bash + +set -euo pipefail # This script is part of django-project-skeleton and is used to generate an # random string to be used as SECRET_KEY for Django's configuration. diff --git a/bin/get_build_id.sh b/bin/get_build_id.sh index a585579..2375a43 100755 --- a/bin/get_build_id.sh +++ b/bin/get_build_id.sh @@ -1,4 +1,6 @@ -#!/bin/sh +#!/bin/bash + +set -euo pipefail # This script is part of django-project-skeleton and is used to determine an # identifier to tag Docker images. diff --git a/configs/Docker/bin/apt-get_install.sh b/configs/Docker/bin/apt-get_install.sh index d8c4fb8..ec2c80c 100755 --- a/configs/Docker/bin/apt-get_install.sh +++ b/configs/Docker/bin/apt-get_install.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -euo pipefail + # This script is used to install packages into Debian-based images. # # It may be called with a list of Debian packages and will automatically diff --git a/configs/Docker/bin/set_timezone.sh b/configs/Docker/bin/set_timezone.sh index f1cb7bd..2c08552 100755 --- a/configs/Docker/bin/set_timezone.sh +++ b/configs/Docker/bin/set_timezone.sh @@ -6,6 +6,8 @@ # This script has to be run with root privileges in order to actually run # `dpkg-reconfigure`. +set -euo pipefail + # check for the existence of DPS_TIMEZONE in environment if [[ -v DPS_TIMEZONE ]]; then true \ diff --git a/configs/Docker/django/entrypoint_django.sh b/configs/Docker/django/entrypoint_django.sh index 23ee251..b495992 100755 --- a/configs/Docker/django/entrypoint_django.sh +++ b/configs/Docker/django/entrypoint_django.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -euo pipefail + # determine the scripts current working directory (CWD) CWD="${BASH_SOURCE%/*}"; if [[ ! -d "$CWD" ]]; then CWD="$PWD"; fi diff --git a/configs/Docker/django/run_gunicorn.sh.template b/configs/Docker/django/run_gunicorn.sh.template index c96de1f..8cf7595 100755 --- a/configs/Docker/django/run_gunicorn.sh.template +++ b/configs/Docker/django/run_gunicorn.sh.template @@ -1,5 +1,7 @@ #!/bin/sh +set -euo pipefail + export DJANGO_SETTINGS_MODULE="{{ project_name}}.settings.docker" exec gunicorn {{ project_name }}.wsgi:application --config='file:configs/Docker/django/gunicorn_conf.py' diff --git a/configs/Docker/nginx/entrypoint_nginx.sh b/configs/Docker/nginx/entrypoint_nginx.sh index 216498a..f9290d6 100755 --- a/configs/Docker/nginx/entrypoint_nginx.sh +++ b/configs/Docker/nginx/entrypoint_nginx.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -euo pipefail + # determine the scripts current working directory (CWD) CWD="${BASH_SOURCE%/*}"; if [[ ! -d "$CWD" ]]; then CWD="$PWD"; fi From a93732041e34ff251dc515085257687a43fad5b4 Mon Sep 17 00:00:00 2001 From: Mischback Date: Mon, 9 Mar 2020 22:14:09 +0100 Subject: [PATCH 078/110] Implement environment variable substitution in nginx.conf --- Makefile | 1 - configs/Docker/django/run_gunicorn.sh.template | 2 +- configs/Docker/env.sample | 7 +++++++ configs/Docker/nginx/Dockerfile | 2 +- configs/Docker/nginx/entrypoint_nginx.sh | 6 ++++++ configs/Docker/nginx/nginx.conf.template | 2 +- 6 files changed, 16 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 1efef15..bab7183 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,6 @@ init: # create the final version of different files by removing the '.sample' suffix mv ./configs/apache2_vhost.sample.template ./configs/apache2_vhost.sample mv ./configs/Docker/django/run_gunicorn.sh.template ./configs/Docker/django/run_gunicorn.sh - mv ./configs/Docker/nginx/nginx.conf.template ./configs/Docker/nginx/nginx.conf # switching the tox configuration file mv ./configs/tox.deployment.template ./tox.ini # switching the Makefile **should** be the last step diff --git a/configs/Docker/django/run_gunicorn.sh.template b/configs/Docker/django/run_gunicorn.sh.template index 8cf7595..52bc91e 100755 --- a/configs/Docker/django/run_gunicorn.sh.template +++ b/configs/Docker/django/run_gunicorn.sh.template @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -euo pipefail diff --git a/configs/Docker/env.sample b/configs/Docker/env.sample index 05b7728..69a2b4a 100644 --- a/configs/Docker/env.sample +++ b/configs/Docker/env.sample @@ -86,3 +86,10 @@ # determines if the accesslog should be included in the logs # just set this to 'true' (without ') to enable access logs #DPS_GUNICORN_SHOW_ACCESS_LOG=false + + +### NGINX related configuration + +# the DNS name of this project +# TODO: provide more documentation, at least a link to Nginx docs +#DPS_NGINX_SERVER_NAME=localhost diff --git a/configs/Docker/nginx/Dockerfile b/configs/Docker/nginx/Dockerfile index 3c9469c..722d86a 100644 --- a/configs/Docker/nginx/Dockerfile +++ b/configs/Docker/nginx/Dockerfile @@ -2,7 +2,7 @@ FROM nginx:1.17 COPY bin/set_timezone.sh /docker-bin/ COPY nginx/entrypoint_nginx.sh /docker-bin/ -COPY nginx/nginx.conf /etc/nginx/nginx.conf +COPY nginx/nginx.conf.template /docker-bin/ ENTRYPOINT ["/docker-bin/entrypoint_nginx.sh"] CMD ["nginx", "-g", "daemon off;"] diff --git a/configs/Docker/nginx/entrypoint_nginx.sh b/configs/Docker/nginx/entrypoint_nginx.sh index f9290d6..750f822 100755 --- a/configs/Docker/nginx/entrypoint_nginx.sh +++ b/configs/Docker/nginx/entrypoint_nginx.sh @@ -9,4 +9,10 @@ if [[ ! -d "$CWD" ]]; then CWD="$PWD"; fi # include the script for setting the timezone source "$CWD/set_timezone.sh" +envsubst ' \ + ${DPS_NGINX_SERVER_NAME} + ' \ + < /docker-bin/nginx.conf.template \ + > /etc/nginx/nginx.conf + exec "$@" diff --git a/configs/Docker/nginx/nginx.conf.template b/configs/Docker/nginx/nginx.conf.template index 7efe466..0c640dd 100644 --- a/configs/Docker/nginx/nginx.conf.template +++ b/configs/Docker/nginx/nginx.conf.template @@ -21,7 +21,7 @@ http { server { # TODO: This **must** be configurable by environment variables - server_name deb10.*; + server_name ${DPS_NGINX_SERVER_NAME}; # TODO: Let's make this run with HTTP, but should finally be switched # to HTTPS. From 95004b368a517ee5ae550ea44edf6a0dfc6723d3 Mon Sep 17 00:00:00 2001 From: Mischback Date: Tue, 10 Mar 2020 17:20:52 +0100 Subject: [PATCH 079/110] nginx.conf server_name --- configs/Docker/env.sample | 9 +++++++-- configs/Docker/nginx/Dockerfile | 3 +++ configs/Docker/nginx/nginx.conf.template | 4 +++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/configs/Docker/env.sample b/configs/Docker/env.sample index 69a2b4a..a8715cb 100644 --- a/configs/Docker/env.sample +++ b/configs/Docker/env.sample @@ -89,7 +89,12 @@ ### NGINX related configuration +# +# All default values are provided directly through the corresponding +# Dockerfile (configs/Docker/nginx/Dockerfile). -# the DNS name of this project -# TODO: provide more documentation, at least a link to Nginx docs +# Specifies the name that is used to identify the matching server block. Nginx +# supports wildcards aswell as regular expressions. +# See https://nginx.org/en/docs/http/server_names.html for further details. +# Default value: localhost #DPS_NGINX_SERVER_NAME=localhost diff --git a/configs/Docker/nginx/Dockerfile b/configs/Docker/nginx/Dockerfile index 722d86a..3027915 100644 --- a/configs/Docker/nginx/Dockerfile +++ b/configs/Docker/nginx/Dockerfile @@ -4,5 +4,8 @@ COPY bin/set_timezone.sh /docker-bin/ COPY nginx/entrypoint_nginx.sh /docker-bin/ COPY nginx/nginx.conf.template /docker-bin/ +ARG DPS_NGINX_SERVER_NAME=localhost +ENV DPS_NGINX_SERVER_NAME ${DPS_NGINX_SERVER_NAME} + ENTRYPOINT ["/docker-bin/entrypoint_nginx.sh"] CMD ["nginx", "-g", "daemon off;"] diff --git a/configs/Docker/nginx/nginx.conf.template b/configs/Docker/nginx/nginx.conf.template index 0c640dd..ae01286 100644 --- a/configs/Docker/nginx/nginx.conf.template +++ b/configs/Docker/nginx/nginx.conf.template @@ -20,7 +20,9 @@ http { } server { - # TODO: This **must** be configurable by environment variables + # Specifies the name that is used to identify the matching server + # block. Nginx supports wildcards aswell as regular expressions. See + # https://nginx.org/en/docs/http/server_names.html for further details. server_name ${DPS_NGINX_SERVER_NAME}; # TODO: Let's make this run with HTTP, but should finally be switched From a03b3690ce434853d3efe7daa798b09b9ce79aea Mon Sep 17 00:00:00 2001 From: Mischback Date: Tue, 10 Mar 2020 17:51:15 +0100 Subject: [PATCH 080/110] nginx.conf server_tokens --- configs/Docker/env.sample | 6 ++++++ configs/Docker/nginx/Dockerfile | 3 +++ configs/Docker/nginx/entrypoint_nginx.sh | 3 ++- configs/Docker/nginx/nginx.conf.template | 2 +- 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/configs/Docker/env.sample b/configs/Docker/env.sample index a8715cb..f059f6c 100644 --- a/configs/Docker/env.sample +++ b/configs/Docker/env.sample @@ -98,3 +98,9 @@ # See https://nginx.org/en/docs/http/server_names.html for further details. # Default value: localhost #DPS_NGINX_SERVER_NAME=localhost + +# Enables / disables emitting nginx version on error pages. +# See http://nginx.org/en/docs/http/ngx_http_core_module.html#server_tokens +# Supported values: on / build / off +# Default value: off +#DPS_NGINX_SERVER_TOKENS=off diff --git a/configs/Docker/nginx/Dockerfile b/configs/Docker/nginx/Dockerfile index 3027915..58fd374 100644 --- a/configs/Docker/nginx/Dockerfile +++ b/configs/Docker/nginx/Dockerfile @@ -7,5 +7,8 @@ COPY nginx/nginx.conf.template /docker-bin/ ARG DPS_NGINX_SERVER_NAME=localhost ENV DPS_NGINX_SERVER_NAME ${DPS_NGINX_SERVER_NAME} +ARG DPS_NGINX_SERVER_TOKENS=off +ENV DPS_NGINX_SERVER_TOKENS ${DPS_NGINX_SERVER_TOKENS} + ENTRYPOINT ["/docker-bin/entrypoint_nginx.sh"] CMD ["nginx", "-g", "daemon off;"] diff --git a/configs/Docker/nginx/entrypoint_nginx.sh b/configs/Docker/nginx/entrypoint_nginx.sh index 750f822..b536eb5 100755 --- a/configs/Docker/nginx/entrypoint_nginx.sh +++ b/configs/Docker/nginx/entrypoint_nginx.sh @@ -9,8 +9,9 @@ if [[ ! -d "$CWD" ]]; then CWD="$PWD"; fi # include the script for setting the timezone source "$CWD/set_timezone.sh" -envsubst ' \ +envsubst ' ${DPS_NGINX_SERVER_NAME} + ${DPS_NGINX_SERVER_TOKENS} ' \ < /docker-bin/nginx.conf.template \ > /etc/nginx/nginx.conf diff --git a/configs/Docker/nginx/nginx.conf.template b/configs/Docker/nginx/nginx.conf.template index ae01286..a6d35b2 100644 --- a/configs/Docker/nginx/nginx.conf.template +++ b/configs/Docker/nginx/nginx.conf.template @@ -74,5 +74,5 @@ http { default_type application/octet-stream; # disable server tokens (on error pages / headers) - server_tokens off; + server_tokens ${DPS_NGINX_SERVER_TOKENS}; } From 72246f057595f06f5add57c76a9f9130ed834144 Mon Sep 17 00:00:00 2001 From: Mischback Date: Fri, 13 Mar 2020 20:32:50 +0100 Subject: [PATCH 081/110] nginx.conf STATIC_URL --- configs/Docker/env.sample | 13 +++++++------ configs/Docker/nginx/Dockerfile | 3 +++ configs/Docker/nginx/entrypoint_nginx.sh | 1 + configs/Docker/nginx/nginx.conf.template | 11 ++++++----- project_name/settings/common.py | 2 +- 5 files changed, 18 insertions(+), 12 deletions(-) diff --git a/configs/Docker/env.sample b/configs/Docker/env.sample index f059f6c..080377b 100644 --- a/configs/Docker/env.sample +++ b/configs/Docker/env.sample @@ -14,6 +14,13 @@ #DPS_TIMEZONE=Etc/UTC #DPS_TIMEZONE=Europe/Berlin +# The url prefix for static files +# +# This setting is used by +# - [django] settings/common.py +# - [nginx] nginx.conf.template +#DPS_STATIC_URL=/static/ + ### DJANGO related configuration @@ -43,12 +50,6 @@ # Django's SECRET_KEY #DPS_DJANGO_SECRET_KEY= -# Django's STATIC_URL -# the URL to serve static files from -# While this is part of Django's configuration, this environment variable is -# used by nginx's Docker image aswell -#DPS_DJANGO_STATIC_URL=/static/ - # the (dotted) path to the WSGI app # defaults to the app provided by Django # see *settings/common.py* for the default value diff --git a/configs/Docker/nginx/Dockerfile b/configs/Docker/nginx/Dockerfile index 58fd374..b659279 100644 --- a/configs/Docker/nginx/Dockerfile +++ b/configs/Docker/nginx/Dockerfile @@ -4,6 +4,9 @@ COPY bin/set_timezone.sh /docker-bin/ COPY nginx/entrypoint_nginx.sh /docker-bin/ COPY nginx/nginx.conf.template /docker-bin/ +ARG DPS_STATIC_URL=/static/ +ENV DPS_STATIC_URL ${DPS_STATIC_URL} + ARG DPS_NGINX_SERVER_NAME=localhost ENV DPS_NGINX_SERVER_NAME ${DPS_NGINX_SERVER_NAME} diff --git a/configs/Docker/nginx/entrypoint_nginx.sh b/configs/Docker/nginx/entrypoint_nginx.sh index b536eb5..2576c81 100755 --- a/configs/Docker/nginx/entrypoint_nginx.sh +++ b/configs/Docker/nginx/entrypoint_nginx.sh @@ -10,6 +10,7 @@ if [[ ! -d "$CWD" ]]; then CWD="$PWD"; fi source "$CWD/set_timezone.sh" envsubst ' + ${DPS_STATIC_URL} ${DPS_NGINX_SERVER_NAME} ${DPS_NGINX_SERVER_TOKENS} ' \ diff --git a/configs/Docker/nginx/nginx.conf.template b/configs/Docker/nginx/nginx.conf.template index a6d35b2..d5b8413 100644 --- a/configs/Docker/nginx/nginx.conf.template +++ b/configs/Docker/nginx/nginx.conf.template @@ -40,13 +40,14 @@ http { # set the timeout for keep-alive connections for the whole server keepalive_timeout 60; - # TODO: This *should* be adjustable with environment variables and - # has to be synchronized with Django's STATIC_URL, as defined by - # DPS_DJANGO_STATIC_URL in env.sample. - location /static/ { + # serve the webapp's static files directly by Nginx + location ${DPS_STATIC_URL} { # The Docker volume for static files is mounted to # /var/www/django_files/static (see docker-compose.yml). - root /var/www/django_files; + alias /var/www/django_files/static/; + + # disable directory listings for this location + autoindex off; } location / { diff --git a/project_name/settings/common.py b/project_name/settings/common.py index ae53cf2..a5e27ba 100644 --- a/project_name/settings/common.py +++ b/project_name/settings/common.py @@ -94,7 +94,7 @@ ROOT_URLCONF = '{}.urls'.format(SITE_NAME) # the URL for static files -STATIC_URL = os.environ.get('DPS_DJANGO_STATIC_URL', '/static/') +STATIC_URL = os.environ.get('DPS_STATIC_URL', '/static/') # the URL for media files MEDIA_URL = os.environ.get('DPS_DJANGO_MEDIA_URL', '/media/') From a2857e602b703e779af4ded32f933b9ff564aae7 Mon Sep 17 00:00:00 2001 From: Mischback Date: Fri, 13 Mar 2020 21:21:20 +0100 Subject: [PATCH 082/110] nginx.conf keepalive_timeout --- configs/Docker/django/gunicorn_conf.py | 2 +- configs/Docker/env.sample | 11 ++- configs/Docker/nginx/Dockerfile | 3 + configs/Docker/nginx/entrypoint_nginx.sh | 1 + configs/Docker/nginx/nginx.conf.template | 95 ++++++++++++++++++++++-- 5 files changed, 100 insertions(+), 12 deletions(-) diff --git a/configs/Docker/django/gunicorn_conf.py b/configs/Docker/django/gunicorn_conf.py index a26495b..b068bf8 100644 --- a/configs/Docker/django/gunicorn_conf.py +++ b/configs/Docker/django/gunicorn_conf.py @@ -25,7 +25,7 @@ # number of seconds to wait for requests on a Keep-Alive connection # this setting is **not relevant** if using the 'sync' worker-class! -keepalive = os.environ.get('DPS_GUNICORN_KEEPALIVE', 10) # Gunicorn default: 2 +keepalive = os.environ.get('DPS_UPSTREAM_KEEPALIVE_TIMEOUT', 60) # Gunicorn default: 2 # fix Gunicorn's heartbeat inside of containers # http://docs.gunicorn.org/en/stable/faq.html#how-do-i-avoid-gunicorn-excessively-blocking-in-os-fchmod diff --git a/configs/Docker/env.sample b/configs/Docker/env.sample index 080377b..81225e0 100644 --- a/configs/Docker/env.sample +++ b/configs/Docker/env.sample @@ -21,6 +21,14 @@ # - [nginx] nginx.conf.template #DPS_STATIC_URL=/static/ +# How long should connections between Nginx and the upstream app server be +# kept open/alive. +# +# This settins is used by +# - [django] gunicorn_conf.py +# - [nginx] nginx.conf.template +#DPS_UPSTREAM_KEEPALIVE_TIMEOUT=60 + ### DJANGO related configuration @@ -67,9 +75,6 @@ # the number of threads per worker process #DPS_GUNICORN_NUM_THREADS=4 -# how long should a Keep-Alive session be kept open -#DPS_GUNICORN_KEEPALIVE=10 - # a custom log format # # this is the default format that plays nicely with Python's logging setup diff --git a/configs/Docker/nginx/Dockerfile b/configs/Docker/nginx/Dockerfile index b659279..9cf4c01 100644 --- a/configs/Docker/nginx/Dockerfile +++ b/configs/Docker/nginx/Dockerfile @@ -7,6 +7,9 @@ COPY nginx/nginx.conf.template /docker-bin/ ARG DPS_STATIC_URL=/static/ ENV DPS_STATIC_URL ${DPS_STATIC_URL} +ARG DPS_UPSTREAM_KEEPALIVE_TIMEOUT=60 +ENV DPS_UPSTREAM_KEEPALIVE_TIMEOUT ${DPS_UPSTREAM_KEEPALIVE_TIMEOUT} + ARG DPS_NGINX_SERVER_NAME=localhost ENV DPS_NGINX_SERVER_NAME ${DPS_NGINX_SERVER_NAME} diff --git a/configs/Docker/nginx/entrypoint_nginx.sh b/configs/Docker/nginx/entrypoint_nginx.sh index 2576c81..40dd6f3 100755 --- a/configs/Docker/nginx/entrypoint_nginx.sh +++ b/configs/Docker/nginx/entrypoint_nginx.sh @@ -11,6 +11,7 @@ source "$CWD/set_timezone.sh" envsubst ' ${DPS_STATIC_URL} + ${DPS_UPSTREAM_KEEPALIVE_TIMEOUT} ${DPS_NGINX_SERVER_NAME} ${DPS_NGINX_SERVER_TOKENS} ' \ diff --git a/configs/Docker/nginx/nginx.conf.template b/configs/Docker/nginx/nginx.conf.template index d5b8413..1a936f6 100644 --- a/configs/Docker/nginx/nginx.conf.template +++ b/configs/Docker/nginx/nginx.conf.template @@ -15,8 +15,40 @@ http { upstream app_server { # The upstream is identified by the service name provided in # docker-compose.yml. Adjust accordingly, if you run your - # nginx outside of the scope that file / on bare metal. + # nginx outside of the scope of that file / on bare metal. server django:8000; + + # Number of preserved keepalive connections + # This settings is highly dependent on the configuration of the + # upstream servers, particurlarly the configuration of Gunicorn: + # As stated in Nginx docs, "The connections parameter should be set to + # a number small enough to let upstream servers process new incoming + # connections as well." (http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive) + # The provided default configuration of Gunicorn uses 2 workers with 4 + # threads per worker. This enables 8 (logically) concurrent requests. + # Nginx's 'keepalive' setting should be lower than this maximum number + # of concurrent requests. + #keepalive ${DPS_NGINX_KEEPALIVE_NUM_CONN}; + keepalive 5; + + # Number of requests per keepalive connection + # Nginx default value: 100 + # As stated in Nginx docs, "Closing connections periodically is + # necessary to free per-connection memory allocations." + # (http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_requests) + # The provided default configuration of Gunicorn will handle 500 + # requests per process, before restarting the worker; this is not + # directly dependent on each other. + #keepalive_requests ${DPS_NGINX_KEEPALIVE_NUM_REQ}; + keepalive_requests 100; + + # Timeout for idle keepalive connections + # This setting should be synced with Gunicorn's 'keepalive' setting. + # Gunicorn's default value is 2, Nginx's one 60. + # In the provided setup, Nginx is the only "client" talking to + # Gunicorn, so a higher value for this setting should be acceptable. + # http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_timeout + keepalive_timeout ${DPS_UPSTREAM_KEEPALIVE_TIMEOUT}; } server { @@ -51,17 +83,64 @@ http { } location / { + # Proxy requests to the upstream servers defined above + proxy_pass http://app_server; + + # Disable caching of upstream completely + # Caching is only done by the upstream application server (Django). + # This might be subject to change, if microcaching is desirable to + # enhance the setups performance, but is highly dependent on the + # actual application. + proxy_cache off; + + # Enable HTTP/1.1 for the communication with upstream servers. + # This setting is required to actually make use of keepalive + # connections between Nginx and the upstream servers. + proxy_http_version 1.1; + + # Drop the clients connection header (could contain "close" to + # close a keepalive connection). + proxy_set_header Connection ""; + + # TODO: Check, if this is necessary to fix redirects from Django + # views or leave 'off' + proxy_redirect off; + + # Provide the requester's address proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # Provide the original request's protocol + # FIXME: When this setup is switched to https, probably the + # following Django setting is relevant: + # - SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Host {{ project_name }}; - proxy_set_header Host $http_host; - proxy_redirect off; - proxy_pass http://app_server; - # TODO: This *should* be synchronized with Gunicorn's 'keepalive' - # setting (DPS_GUNICORN_KEEPALIVE) - keepalive_timeout 5; + # TODO: develop a working combination of + # - setting the Host header + # - setting the X-Forwarded-Host header + # - Django's ALLOWED_HOSTS setting + # - Django's USE_X_FORWARDED_HOST setting + # that makes CSRF-protected requests work correctly. + # See: + # - https://stackoverflow.com/a/48646798 + # - https://ubuntu.com/blog/django-behind-a-proxy-fixing-absolute-urls + # - https://medium.com/@rui.jorge.rei/today-i-learned-nginx-reverse-proxying-for-django-projects-3ab17ad707f6 + # - https://code.djangoproject.com/ticket/9064 + # + # Proposed solution: + # - use "proxy_set_header Host $host;" to set a correct "Host" header when passing the request to upstream + # - "$host" will contain the value of the "Host" header of the request or the primary server name (see http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_set_header) + # - the primary server name is controllable by an environment variable; so it is usable in Django's ALLOWED_HOSTS aswell (needs renaming!) + # - do NOT set "X-Forwarded-Host" at all + # - do NOT use Django's USE_X_FORWARDED_HOST + # - probably the file [project_name]/settings/docker.py is not longer required + # + # TODO: Remove "{{ project_name }}" and rely on value that is set by environment variables, if possible + # Provide the originally requested host + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Host {{ project_name }}; } + } # if no server_name matches, close the connection to prevent host spoofing From 38f54a9f070959d9488c34c9e34ec4590e9db097 Mon Sep 17 00:00:00 2001 From: Mischback Date: Fri, 13 Mar 2020 21:32:55 +0100 Subject: [PATCH 083/110] nginx.conf keepalive --- configs/Docker/env.sample | 5 +++++ configs/Docker/nginx/Dockerfile | 3 +++ configs/Docker/nginx/entrypoint_nginx.sh | 1 + configs/Docker/nginx/nginx.conf.template | 3 +-- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/configs/Docker/env.sample b/configs/Docker/env.sample index 81225e0..a257196 100644 --- a/configs/Docker/env.sample +++ b/configs/Docker/env.sample @@ -99,6 +99,11 @@ # All default values are provided directly through the corresponding # Dockerfile (configs/Docker/nginx/Dockerfile). +# Specifies the number of preserved keepalive connections. +# See http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive +# Default value: 5 +#DPS_NGINX_KEEPALIVE_NUM_CONN=5 + # Specifies the name that is used to identify the matching server block. Nginx # supports wildcards aswell as regular expressions. # See https://nginx.org/en/docs/http/server_names.html for further details. diff --git a/configs/Docker/nginx/Dockerfile b/configs/Docker/nginx/Dockerfile index 9cf4c01..26c159f 100644 --- a/configs/Docker/nginx/Dockerfile +++ b/configs/Docker/nginx/Dockerfile @@ -10,6 +10,9 @@ ENV DPS_STATIC_URL ${DPS_STATIC_URL} ARG DPS_UPSTREAM_KEEPALIVE_TIMEOUT=60 ENV DPS_UPSTREAM_KEEPALIVE_TIMEOUT ${DPS_UPSTREAM_KEEPALIVE_TIMEOUT} +ARG DPS_NGINX_KEEPALIVE_NUM_CONN=5 +ENV DPS_NGINX_KEEPALIVE_NUM_CONN ${DPS_NGINX_KEEPALIVE_NUM_CONN} + ARG DPS_NGINX_SERVER_NAME=localhost ENV DPS_NGINX_SERVER_NAME ${DPS_NGINX_SERVER_NAME} diff --git a/configs/Docker/nginx/entrypoint_nginx.sh b/configs/Docker/nginx/entrypoint_nginx.sh index 40dd6f3..ec95acc 100755 --- a/configs/Docker/nginx/entrypoint_nginx.sh +++ b/configs/Docker/nginx/entrypoint_nginx.sh @@ -12,6 +12,7 @@ source "$CWD/set_timezone.sh" envsubst ' ${DPS_STATIC_URL} ${DPS_UPSTREAM_KEEPALIVE_TIMEOUT} + ${DPS_NGINX_KEEPALIVE_NUM_CONN} ${DPS_NGINX_SERVER_NAME} ${DPS_NGINX_SERVER_TOKENS} ' \ diff --git a/configs/Docker/nginx/nginx.conf.template b/configs/Docker/nginx/nginx.conf.template index 1a936f6..4f5f09f 100644 --- a/configs/Docker/nginx/nginx.conf.template +++ b/configs/Docker/nginx/nginx.conf.template @@ -28,8 +28,7 @@ http { # threads per worker. This enables 8 (logically) concurrent requests. # Nginx's 'keepalive' setting should be lower than this maximum number # of concurrent requests. - #keepalive ${DPS_NGINX_KEEPALIVE_NUM_CONN}; - keepalive 5; + keepalive ${DPS_NGINX_KEEPALIVE_NUM_CONN}; # Number of requests per keepalive connection # Nginx default value: 100 From 469fc2532030fd643cb09014ee8be704ec8c97c9 Mon Sep 17 00:00:00 2001 From: Mischback Date: Fri, 13 Mar 2020 21:42:22 +0100 Subject: [PATCH 084/110] nginx.conf keepalive_requests --- configs/Docker/env.sample | 5 +++++ configs/Docker/nginx/Dockerfile | 3 +++ configs/Docker/nginx/entrypoint_nginx.sh | 1 + configs/Docker/nginx/nginx.conf.template | 3 +-- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/configs/Docker/env.sample b/configs/Docker/env.sample index a257196..66b043e 100644 --- a/configs/Docker/env.sample +++ b/configs/Docker/env.sample @@ -104,6 +104,11 @@ # Default value: 5 #DPS_NGINX_KEEPALIVE_NUM_CONN=5 +# Maximum number of requests performed by a single keepalive connection +# See http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_requests +# Default value: 100 +#DPS_NGINX_KEEPALIVE_NUM_REQ=100 + # Specifies the name that is used to identify the matching server block. Nginx # supports wildcards aswell as regular expressions. # See https://nginx.org/en/docs/http/server_names.html for further details. diff --git a/configs/Docker/nginx/Dockerfile b/configs/Docker/nginx/Dockerfile index 26c159f..cacd19b 100644 --- a/configs/Docker/nginx/Dockerfile +++ b/configs/Docker/nginx/Dockerfile @@ -13,6 +13,9 @@ ENV DPS_UPSTREAM_KEEPALIVE_TIMEOUT ${DPS_UPSTREAM_KEEPALIVE_TIMEOUT} ARG DPS_NGINX_KEEPALIVE_NUM_CONN=5 ENV DPS_NGINX_KEEPALIVE_NUM_CONN ${DPS_NGINX_KEEPALIVE_NUM_CONN} +ARG DPS_NGINX_KEEPALIVE_NUM_REQ=100 +ENV DPS_NGINX_KEEPALIVE_NUM_REQ ${DPS_NGINX_KEEPALIVE_NUM_REQ} + ARG DPS_NGINX_SERVER_NAME=localhost ENV DPS_NGINX_SERVER_NAME ${DPS_NGINX_SERVER_NAME} diff --git a/configs/Docker/nginx/entrypoint_nginx.sh b/configs/Docker/nginx/entrypoint_nginx.sh index ec95acc..c604b73 100755 --- a/configs/Docker/nginx/entrypoint_nginx.sh +++ b/configs/Docker/nginx/entrypoint_nginx.sh @@ -13,6 +13,7 @@ envsubst ' ${DPS_STATIC_URL} ${DPS_UPSTREAM_KEEPALIVE_TIMEOUT} ${DPS_NGINX_KEEPALIVE_NUM_CONN} + ${DPS_NGINX_KEEPALIVE_NUM_REQ} ${DPS_NGINX_SERVER_NAME} ${DPS_NGINX_SERVER_TOKENS} ' \ diff --git a/configs/Docker/nginx/nginx.conf.template b/configs/Docker/nginx/nginx.conf.template index 4f5f09f..a80a0a2 100644 --- a/configs/Docker/nginx/nginx.conf.template +++ b/configs/Docker/nginx/nginx.conf.template @@ -38,8 +38,7 @@ http { # The provided default configuration of Gunicorn will handle 500 # requests per process, before restarting the worker; this is not # directly dependent on each other. - #keepalive_requests ${DPS_NGINX_KEEPALIVE_NUM_REQ}; - keepalive_requests 100; + keepalive_requests ${DPS_NGINX_KEEPALIVE_NUM_REQ}; # Timeout for idle keepalive connections # This setting should be synced with Gunicorn's 'keepalive' setting. From a7d0efc59d964a24d268c5e7fddef65d2d8c1574 Mon Sep 17 00:00:00 2001 From: Mischback Date: Fri, 13 Mar 2020 22:55:09 +0100 Subject: [PATCH 085/110] nginx.conf server_name (again) --- configs/Docker/env.sample | 33 +++++++++++++++++++----- configs/Docker/nginx/Dockerfile | 6 ++--- configs/Docker/nginx/entrypoint_nginx.sh | 2 +- configs/Docker/nginx/nginx.conf.template | 27 ++++++++++++------- project_name/settings/docker.py | 9 ++++--- 5 files changed, 52 insertions(+), 25 deletions(-) diff --git a/configs/Docker/env.sample b/configs/Docker/env.sample index 66b043e..115234f 100644 --- a/configs/Docker/env.sample +++ b/configs/Docker/env.sample @@ -1,8 +1,11 @@ ### OVERALL configuration -# set the overall (minimum) log level +# Sets the overall (minimum) log level +# # This setting is used by # - [django] gunicorn_conf.py +# +# Default value: INFO #DPS_OVERALL_LOGLEVEL=INFO # Provide the timezone for all containers @@ -11,14 +14,34 @@ # # This setting is used by # - [django] settings/common.py +# +# Default value: Etc/UTC #DPS_TIMEZONE=Etc/UTC #DPS_TIMEZONE=Europe/Berlin +# Specifies the name that is used to identify the matching server block. +# +# Nginx supports regular expressions and wildcards, however, this environment +# variable is also used in Django's ALLOWED_HOSTS setting and is therefore +# constrained to the given requirements. +# +# See https://nginx.org/en/docs/http/server_names.html +# See https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts +# +# This setting is used by +# - [django] settings/docker.py +# - [nginx] nginx.conf.template +# +# Default value: localhost +#DPS_SERVER_NAME=localhost + # The url prefix for static files # # This setting is used by # - [django] settings/common.py # - [nginx] nginx.conf.template +# +# Default value: /static/ #DPS_STATIC_URL=/static/ # How long should connections between Nginx and the upstream app server be @@ -27,6 +50,8 @@ # This settins is used by # - [django] gunicorn_conf.py # - [nginx] nginx.conf.template +# +# Default value: 60 #DPS_UPSTREAM_KEEPALIVE_TIMEOUT=60 @@ -109,12 +134,6 @@ # Default value: 100 #DPS_NGINX_KEEPALIVE_NUM_REQ=100 -# Specifies the name that is used to identify the matching server block. Nginx -# supports wildcards aswell as regular expressions. -# See https://nginx.org/en/docs/http/server_names.html for further details. -# Default value: localhost -#DPS_NGINX_SERVER_NAME=localhost - # Enables / disables emitting nginx version on error pages. # See http://nginx.org/en/docs/http/ngx_http_core_module.html#server_tokens # Supported values: on / build / off diff --git a/configs/Docker/nginx/Dockerfile b/configs/Docker/nginx/Dockerfile index cacd19b..6f60faa 100644 --- a/configs/Docker/nginx/Dockerfile +++ b/configs/Docker/nginx/Dockerfile @@ -4,6 +4,9 @@ COPY bin/set_timezone.sh /docker-bin/ COPY nginx/entrypoint_nginx.sh /docker-bin/ COPY nginx/nginx.conf.template /docker-bin/ +ARG DPS_SERVER_NAME=localhost +ENV DPS_SERVER_NAME ${DPS_SERVER_NAME} + ARG DPS_STATIC_URL=/static/ ENV DPS_STATIC_URL ${DPS_STATIC_URL} @@ -16,9 +19,6 @@ ENV DPS_NGINX_KEEPALIVE_NUM_CONN ${DPS_NGINX_KEEPALIVE_NUM_CONN} ARG DPS_NGINX_KEEPALIVE_NUM_REQ=100 ENV DPS_NGINX_KEEPALIVE_NUM_REQ ${DPS_NGINX_KEEPALIVE_NUM_REQ} -ARG DPS_NGINX_SERVER_NAME=localhost -ENV DPS_NGINX_SERVER_NAME ${DPS_NGINX_SERVER_NAME} - ARG DPS_NGINX_SERVER_TOKENS=off ENV DPS_NGINX_SERVER_TOKENS ${DPS_NGINX_SERVER_TOKENS} diff --git a/configs/Docker/nginx/entrypoint_nginx.sh b/configs/Docker/nginx/entrypoint_nginx.sh index c604b73..f7555d2 100755 --- a/configs/Docker/nginx/entrypoint_nginx.sh +++ b/configs/Docker/nginx/entrypoint_nginx.sh @@ -10,11 +10,11 @@ if [[ ! -d "$CWD" ]]; then CWD="$PWD"; fi source "$CWD/set_timezone.sh" envsubst ' + ${DPS_SERVER_NAME} ${DPS_STATIC_URL} ${DPS_UPSTREAM_KEEPALIVE_TIMEOUT} ${DPS_NGINX_KEEPALIVE_NUM_CONN} ${DPS_NGINX_KEEPALIVE_NUM_REQ} - ${DPS_NGINX_SERVER_NAME} ${DPS_NGINX_SERVER_TOKENS} ' \ < /docker-bin/nginx.conf.template \ diff --git a/configs/Docker/nginx/nginx.conf.template b/configs/Docker/nginx/nginx.conf.template index a80a0a2..bdcabae 100644 --- a/configs/Docker/nginx/nginx.conf.template +++ b/configs/Docker/nginx/nginx.conf.template @@ -53,7 +53,7 @@ http { # Specifies the name that is used to identify the matching server # block. Nginx supports wildcards aswell as regular expressions. See # https://nginx.org/en/docs/http/server_names.html for further details. - server_name ${DPS_NGINX_SERVER_NAME}; + server_name ${DPS_SERVER_NAME}; # TODO: Let's make this run with HTTP, but should finally be switched # to HTTPS. @@ -113,7 +113,9 @@ http { # - SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') proxy_set_header X-Forwarded-Proto $scheme; - # TODO: develop a working combination of + # Provide the (originally requested) hostname to the upstream + # + # DONE: develop a working combination of # - setting the Host header # - setting the X-Forwarded-Host header # - Django's ALLOWED_HOSTS setting @@ -125,18 +127,23 @@ http { # - https://medium.com/@rui.jorge.rei/today-i-learned-nginx-reverse-proxying-for-django-projects-3ab17ad707f6 # - https://code.djangoproject.com/ticket/9064 # - # Proposed solution: - # - use "proxy_set_header Host $host;" to set a correct "Host" header when passing the request to upstream - # - "$host" will contain the value of the "Host" header of the request or the primary server name (see http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_set_header) - # - the primary server name is controllable by an environment variable; so it is usable in Django's ALLOWED_HOSTS aswell (needs renaming!) + # Current solution: + # - use "proxy_set_header Host $host;" to set a correct + # "Host" header when passing the request to upstream + # - "$host" will contain the value of the "Host" header + # of the request or the primary server name + # (see http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_set_header) + # - the primary server name is controllable by an + # environment variable; so it is usable in Django's + # ALLOWED_HOSTS aswell + # - this introduces additional constraints on the value + # of the environment variable. It must be compatible + # with Nginx's and Django's requirements # - do NOT set "X-Forwarded-Host" at all # - do NOT use Django's USE_X_FORWARDED_HOST - # - probably the file [project_name]/settings/docker.py is not longer required # - # TODO: Remove "{{ project_name }}" and rely on value that is set by environment variables, if possible # Provide the originally requested host - proxy_set_header Host $http_host; - proxy_set_header X-Forwarded-Host {{ project_name }}; + proxy_set_header Host $host; } } diff --git a/project_name/settings/docker.py b/project_name/settings/docker.py index 958b595..9785590 100644 --- a/project_name/settings/docker.py +++ b/project_name/settings/docker.py @@ -5,10 +5,11 @@ from .production import * -# make Django work correctly behind the nginx proxy -# nginx **must** set the appropriate header! -ALLOWED_HOSTS = os.environ.get('DPS_DJANGO_ALLOWED_HOSTS', '{{ project_name }}').split() -USE_X_FORWARDED_HOST = True +# Make Django work correctly behind the Nginx proxy +# Nginx **must** set the appropriate header! +# In the current setup, Nginx sets the "Host" header to the "server_name". +ALLOWED_HOSTS = os.environ.get('DPS_SERVER_NAME', 'localhost').split() +#USE_X_FORWARDED_HOST = True # Logging is set up in Gunicorn's configuration LOGGING_CONFIG=None From 929c80b608030f648939e6889e3f9452bac528a6 Mon Sep 17 00:00:00 2001 From: Mischback Date: Tue, 17 Mar 2020 21:32:31 +0100 Subject: [PATCH 086/110] nginx.conf keepalive_timeout --- configs/Docker/env.sample | 4 ++++ configs/Docker/nginx/Dockerfile | 3 +++ configs/Docker/nginx/entrypoint_nginx.sh | 1 + configs/Docker/nginx/nginx.conf.template | 22 +++++++++++----------- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/configs/Docker/env.sample b/configs/Docker/env.sample index 115234f..cb1eb4c 100644 --- a/configs/Docker/env.sample +++ b/configs/Docker/env.sample @@ -134,6 +134,10 @@ # Default value: 100 #DPS_NGINX_KEEPALIVE_NUM_REQ=100 +# How long should client connections to Nginx should be kept open (in seconds) +# Default value: 60 / Nginx default: 75 +#DPS_NGINX_SERVER_KEEPALIVE=60 + # Enables / disables emitting nginx version on error pages. # See http://nginx.org/en/docs/http/ngx_http_core_module.html#server_tokens # Supported values: on / build / off diff --git a/configs/Docker/nginx/Dockerfile b/configs/Docker/nginx/Dockerfile index 6f60faa..b32eb7b 100644 --- a/configs/Docker/nginx/Dockerfile +++ b/configs/Docker/nginx/Dockerfile @@ -19,6 +19,9 @@ ENV DPS_NGINX_KEEPALIVE_NUM_CONN ${DPS_NGINX_KEEPALIVE_NUM_CONN} ARG DPS_NGINX_KEEPALIVE_NUM_REQ=100 ENV DPS_NGINX_KEEPALIVE_NUM_REQ ${DPS_NGINX_KEEPALIVE_NUM_REQ} +ARG DPS_NGINX_SERVER_KEEPALIVE=60 +ENV DPS_NGINX_SERVER_KEEPALIVE ${DPS_NGINX_SERVER_KEEPALIVE} + ARG DPS_NGINX_SERVER_TOKENS=off ENV DPS_NGINX_SERVER_TOKENS ${DPS_NGINX_SERVER_TOKENS} diff --git a/configs/Docker/nginx/entrypoint_nginx.sh b/configs/Docker/nginx/entrypoint_nginx.sh index f7555d2..2a21748 100755 --- a/configs/Docker/nginx/entrypoint_nginx.sh +++ b/configs/Docker/nginx/entrypoint_nginx.sh @@ -15,6 +15,7 @@ envsubst ' ${DPS_UPSTREAM_KEEPALIVE_TIMEOUT} ${DPS_NGINX_KEEPALIVE_NUM_CONN} ${DPS_NGINX_KEEPALIVE_NUM_REQ} + ${DPS_NGINX_SERVER_KEEPALIVE} ${DPS_NGINX_SERVER_TOKENS} ' \ < /docker-bin/nginx.conf.template \ diff --git a/configs/Docker/nginx/nginx.conf.template b/configs/Docker/nginx/nginx.conf.template index bdcabae..0bead71 100644 --- a/configs/Docker/nginx/nginx.conf.template +++ b/configs/Docker/nginx/nginx.conf.template @@ -59,17 +59,6 @@ http { # to HTTPS. listen 80; - # TODO: This *should* be synchronized with Django's - # DATA_UPLOAD_MAX_MEMORY_SIZE, which defaults to 2621440 byte=2.5MB. - # This *could* be adjustable with environment variables, but on the - # other hand, this might be a very specific setting and adjustment - # only is necessary for specific applications that typically handle - # larger files. For now, it is just set to Django's default value. - client_max_body_size 2621440; - - # set the timeout for keep-alive connections for the whole server - keepalive_timeout 60; - # serve the webapp's static files directly by Nginx location ${DPS_STATIC_URL} { # The Docker volume for static files is mounted to @@ -146,6 +135,17 @@ http { proxy_set_header Host $host; } + # TODO: This *should* be synchronized with Django's + # DATA_UPLOAD_MAX_MEMORY_SIZE, which defaults to 2621440 byte=2.5MB. + # This *could* be adjustable with environment variables, but on the + # other hand, this might be a very specific setting and adjustment + # only is necessary for specific applications that typically handle + # larger files. For now, it is just set to Django's default value. + client_max_body_size 2621440; + + # set the timeout for keep-alive connections for the whole server + keepalive_timeout ${DPS_NGINX_SERVER_KEEPALIVE}; + } # if no server_name matches, close the connection to prevent host spoofing From d0b605c998fd61fa59ae5db5d88144f0fde1924e Mon Sep 17 00:00:00 2001 From: Mischback Date: Tue, 17 Mar 2020 21:35:39 +0100 Subject: [PATCH 087/110] nginx.conf Added upstream keyword --- configs/Docker/env.sample | 4 ++-- configs/Docker/nginx/Dockerfile | 8 ++++---- configs/Docker/nginx/entrypoint_nginx.sh | 4 ++-- configs/Docker/nginx/nginx.conf.template | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/configs/Docker/env.sample b/configs/Docker/env.sample index cb1eb4c..c32621a 100644 --- a/configs/Docker/env.sample +++ b/configs/Docker/env.sample @@ -127,12 +127,12 @@ # Specifies the number of preserved keepalive connections. # See http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive # Default value: 5 -#DPS_NGINX_KEEPALIVE_NUM_CONN=5 +#DPS_NGINX_UPSTREAM_KEEPALIVE_NUM_CONN=5 # Maximum number of requests performed by a single keepalive connection # See http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_requests # Default value: 100 -#DPS_NGINX_KEEPALIVE_NUM_REQ=100 +#DPS_NGINX_UPSTREAM_KEEPALIVE_NUM_REQ=100 # How long should client connections to Nginx should be kept open (in seconds) # Default value: 60 / Nginx default: 75 diff --git a/configs/Docker/nginx/Dockerfile b/configs/Docker/nginx/Dockerfile index b32eb7b..29a0d8d 100644 --- a/configs/Docker/nginx/Dockerfile +++ b/configs/Docker/nginx/Dockerfile @@ -13,11 +13,11 @@ ENV DPS_STATIC_URL ${DPS_STATIC_URL} ARG DPS_UPSTREAM_KEEPALIVE_TIMEOUT=60 ENV DPS_UPSTREAM_KEEPALIVE_TIMEOUT ${DPS_UPSTREAM_KEEPALIVE_TIMEOUT} -ARG DPS_NGINX_KEEPALIVE_NUM_CONN=5 -ENV DPS_NGINX_KEEPALIVE_NUM_CONN ${DPS_NGINX_KEEPALIVE_NUM_CONN} +ARG DPS_NGINX_UPSTREAM_KEEPALIVE_NUM_CONN=5 +ENV DPS_NGINX_UPSTREAM_KEEPALIVE_NUM_CONN ${DPS_NGINX_UPSTREAM_KEEPALIVE_NUM_CONN} -ARG DPS_NGINX_KEEPALIVE_NUM_REQ=100 -ENV DPS_NGINX_KEEPALIVE_NUM_REQ ${DPS_NGINX_KEEPALIVE_NUM_REQ} +ARG DPS_NGINX_UPSTREAM_KEEPALIVE_NUM_REQ=100 +ENV DPS_NGINX_UPSTREAM_KEEPALIVE_NUM_REQ ${DPS_NGINX_UPSTREAM_KEEPALIVE_NUM_REQ} ARG DPS_NGINX_SERVER_KEEPALIVE=60 ENV DPS_NGINX_SERVER_KEEPALIVE ${DPS_NGINX_SERVER_KEEPALIVE} diff --git a/configs/Docker/nginx/entrypoint_nginx.sh b/configs/Docker/nginx/entrypoint_nginx.sh index 2a21748..3de2370 100755 --- a/configs/Docker/nginx/entrypoint_nginx.sh +++ b/configs/Docker/nginx/entrypoint_nginx.sh @@ -13,8 +13,8 @@ envsubst ' ${DPS_SERVER_NAME} ${DPS_STATIC_URL} ${DPS_UPSTREAM_KEEPALIVE_TIMEOUT} - ${DPS_NGINX_KEEPALIVE_NUM_CONN} - ${DPS_NGINX_KEEPALIVE_NUM_REQ} + ${DPS_NGINX_UPSTREAM_KEEPALIVE_NUM_CONN} + ${DPS_NGINX_UPSTREAM_KEEPALIVE_NUM_REQ} ${DPS_NGINX_SERVER_KEEPALIVE} ${DPS_NGINX_SERVER_TOKENS} ' \ diff --git a/configs/Docker/nginx/nginx.conf.template b/configs/Docker/nginx/nginx.conf.template index 0bead71..f38aeb8 100644 --- a/configs/Docker/nginx/nginx.conf.template +++ b/configs/Docker/nginx/nginx.conf.template @@ -28,7 +28,7 @@ http { # threads per worker. This enables 8 (logically) concurrent requests. # Nginx's 'keepalive' setting should be lower than this maximum number # of concurrent requests. - keepalive ${DPS_NGINX_KEEPALIVE_NUM_CONN}; + keepalive ${DPS_NGINX_UPSTREAM_KEEPALIVE_NUM_CONN}; # Number of requests per keepalive connection # Nginx default value: 100 @@ -38,7 +38,7 @@ http { # The provided default configuration of Gunicorn will handle 500 # requests per process, before restarting the worker; this is not # directly dependent on each other. - keepalive_requests ${DPS_NGINX_KEEPALIVE_NUM_REQ}; + keepalive_requests ${DPS_NGINX_UPSTREAM_KEEPALIVE_NUM_REQ}; # Timeout for idle keepalive connections # This setting should be synced with Gunicorn's 'keepalive' setting. From 740829b86b616bbd750156de04b0a44cc484f66e Mon Sep 17 00:00:00 2001 From: Mischback Date: Wed, 18 Mar 2020 21:33:33 +0100 Subject: [PATCH 088/110] nginx.conf Added gzip support --- configs/Docker/env.sample | 21 +++++++++++++++++++++ configs/Docker/nginx/Dockerfile | 12 ++++++++++++ configs/Docker/nginx/entrypoint_nginx.sh | 4 ++++ configs/Docker/nginx/nginx.conf.template | 14 ++++++++++++++ 4 files changed, 51 insertions(+) diff --git a/configs/Docker/env.sample b/configs/Docker/env.sample index c32621a..0f1dc64 100644 --- a/configs/Docker/env.sample +++ b/configs/Docker/env.sample @@ -143,3 +143,24 @@ # Supported values: on / build / off # Default value: off #DPS_NGINX_SERVER_TOKENS=off + +# Generally activates GZip compression +# Allowed value: on | off +# Default value: on / Nginx default: off +#DPS_NGINX_GZIP_ACTIVATION=on + +# GZip compression level for on the fly compression +# Allowed values: 1 - 9, where 1 is fastest and 9 is smallest +# Default value: 4 / Nginx default: 1 +#DPS_NGINX_GZIP_COMPRESSION=4 + +# Dynamic compression will only be done, if the content length is greater than +# this value. +# There is a tradeoff between the time required to perform the compression and +# the saved transfer size (and thus time). +# Default value: 500 / Nginx default: 20 +#DPS_NGINX_GZIP_MIN_LENGTH=500 + +# Activate compression for the following content types. +# Note: text/html is always compressed. +#DPS_NGINX_GZIP_TYPES=text/css font/woff diff --git a/configs/Docker/nginx/Dockerfile b/configs/Docker/nginx/Dockerfile index 29a0d8d..c9bdf94 100644 --- a/configs/Docker/nginx/Dockerfile +++ b/configs/Docker/nginx/Dockerfile @@ -25,5 +25,17 @@ ENV DPS_NGINX_SERVER_KEEPALIVE ${DPS_NGINX_SERVER_KEEPALIVE} ARG DPS_NGINX_SERVER_TOKENS=off ENV DPS_NGINX_SERVER_TOKENS ${DPS_NGINX_SERVER_TOKENS} +ARG DPS_NGINX_GZIP_ACTIVATION=on +ENV DPS_NGINX_GZIP_ACTIVATION ${DPS_NGINX_GZIP_ACTIVATION} + +ARG DPS_NGINX_GZIP_COMPRESSION=4 +ENV DPS_NGINX_GZIP_COMPRESSION ${DPS_NGINX_GZIP_COMPRESSION} + +ARG DPS_NGINX_GZIP_MIN_LENGTH=500 +ENV DPS_NGINX_GZIP_MIN_LENGTH ${DPS_NGINX_GZIP_MIN_LENGTH} + +ARG DPS_NGINX_GZIP_TYPES="text/css font/woff" +ENV DPS_NGINX_GZIP_TYPES ${DPS_NGINX_GZIP_TYPES} + ENTRYPOINT ["/docker-bin/entrypoint_nginx.sh"] CMD ["nginx", "-g", "daemon off;"] diff --git a/configs/Docker/nginx/entrypoint_nginx.sh b/configs/Docker/nginx/entrypoint_nginx.sh index 3de2370..767882f 100755 --- a/configs/Docker/nginx/entrypoint_nginx.sh +++ b/configs/Docker/nginx/entrypoint_nginx.sh @@ -17,6 +17,10 @@ envsubst ' ${DPS_NGINX_UPSTREAM_KEEPALIVE_NUM_REQ} ${DPS_NGINX_SERVER_KEEPALIVE} ${DPS_NGINX_SERVER_TOKENS} + ${DPS_NGINX_GZIP_ACTIVATION} + ${DPS_NGINX_GZIP_COMPRESSION} + ${DPS_NGINX_GZIP_MIN_LENGTH} + ${DPS_NGINX_GZIP_TYPES} ' \ < /docker-bin/nginx.conf.template \ > /etc/nginx/nginx.conf diff --git a/configs/Docker/nginx/nginx.conf.template b/configs/Docker/nginx/nginx.conf.template index f38aeb8..e986f96 100644 --- a/configs/Docker/nginx/nginx.conf.template +++ b/configs/Docker/nginx/nginx.conf.template @@ -146,6 +146,20 @@ http { # set the timeout for keep-alive connections for the whole server keepalive_timeout ${DPS_NGINX_SERVER_KEEPALIVE}; + # generally activate Gzip compression + gzip ${DPS_NGINX_GZIP_ACTIVATION}; + # set the compression level for on the fly compression + gzip_comp_level ${DPS_NGINX_GZIP_COMPRESSION}; + # disable gzip for some browsers + gzip_disable msie6; + # minimum content length (in byte) to be compressed + gzip_min_length ${DPS_NGINX_GZIP_MIN_LENGTH}; + # enable compression for all requests + gzip_proxied any; + # enable compression for the following content-types + # Note: "text/html" is always compressed! + gzip_types ${DPS_NGINX_GZIP_TYPES}; + } # if no server_name matches, close the connection to prevent host spoofing From b4eabd9d7a2c8c765a24024f6b2ca6903be81d41 Mon Sep 17 00:00:00 2001 From: Mischback Date: Thu, 19 Mar 2020 21:14:36 +0100 Subject: [PATCH 089/110] Switch to ManifestStaticFilesStorage --- project_name/settings/production.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/project_name/settings/production.py b/project_name/settings/production.py index f546427..40c56c0 100644 --- a/project_name/settings/production.py +++ b/project_name/settings/production.py @@ -12,6 +12,8 @@ INSTALLED_APPS = DEFAULT_APPS +STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.ManifestStaticFilesStorage' + # ##### SECURITY CONFIGURATION ############################ # TODO: Make sure, that sensitive information uses https From 521b1d5f87130e815cfb25a90886ef57c5b313b2 Mon Sep 17 00:00:00 2001 From: Mischback Date: Wed, 25 Mar 2020 20:09:49 +0100 Subject: [PATCH 090/110] Created init directory --- Makefile | 6 +++--- .../Makefile.deployment.template => init/Makefile.template | 0 configs/tox.deployment.template => init/tox.ini.template | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename configs/Makefile.deployment.template => init/Makefile.template (100%) rename configs/tox.deployment.template => init/tox.ini.template (100%) diff --git a/Makefile b/Makefile index bab7183..dfd4528 100644 --- a/Makefile +++ b/Makefile @@ -43,13 +43,13 @@ clean: init: # TODO: Include some output to this function - # create the final version of different files by removing the '.sample' suffix + # create the final version of different files by removing the '.template' suffix mv ./configs/apache2_vhost.sample.template ./configs/apache2_vhost.sample mv ./configs/Docker/django/run_gunicorn.sh.template ./configs/Docker/django/run_gunicorn.sh # switching the tox configuration file - mv ./configs/tox.deployment.template ./tox.ini + mv ./init/tox.ini.template ./tox.ini # switching the Makefile **should** be the last step - mv ./configs/Makefile.deployment.template ./Makefile + mv ./init/Makefile.template ./Makefile configs/Docker/env.production: echo "Initializing environment file for production..." diff --git a/configs/Makefile.deployment.template b/init/Makefile.template similarity index 100% rename from configs/Makefile.deployment.template rename to init/Makefile.template diff --git a/configs/tox.deployment.template b/init/tox.ini.template similarity index 100% rename from configs/tox.deployment.template rename to init/tox.ini.template From dbdc34daec91b8418a1174c9fd04437b2df735d6 Mon Sep 17 00:00:00 2001 From: Mischback Date: Wed, 25 Mar 2020 20:20:05 +0100 Subject: [PATCH 091/110] Created util directory --- Makefile | 4 ++-- {init => util/init}/Makefile.template | 0 {init => util/init}/tox.ini.template | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename {init => util/init}/Makefile.template (100%) rename {init => util/init}/tox.ini.template (100%) diff --git a/Makefile b/Makefile index dfd4528..50524f6 100644 --- a/Makefile +++ b/Makefile @@ -47,9 +47,9 @@ init: mv ./configs/apache2_vhost.sample.template ./configs/apache2_vhost.sample mv ./configs/Docker/django/run_gunicorn.sh.template ./configs/Docker/django/run_gunicorn.sh # switching the tox configuration file - mv ./init/tox.ini.template ./tox.ini + mv ./util/init/tox.ini.template ./tox.ini # switching the Makefile **should** be the last step - mv ./init/Makefile.template ./Makefile + mv ./util/init/Makefile.template ./Makefile configs/Docker/env.production: echo "Initializing environment file for production..." diff --git a/init/Makefile.template b/util/init/Makefile.template similarity index 100% rename from init/Makefile.template rename to util/init/Makefile.template diff --git a/init/tox.ini.template b/util/init/tox.ini.template similarity index 100% rename from init/tox.ini.template rename to util/init/tox.ini.template From 22fdc413604810e3d967e95224bd0764e2c617f9 Mon Sep 17 00:00:00 2001 From: Mischback Date: Wed, 25 Mar 2020 20:28:39 +0100 Subject: [PATCH 092/110] Moved project-specific bin to util --- Makefile | 2 +- {bin => util/bin}/generate_secret_key.sh | 0 {bin => util/bin}/get_build_id.sh | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename {bin => util/bin}/generate_secret_key.sh (100%) rename {bin => util/bin}/get_build_id.sh (100%) diff --git a/Makefile b/Makefile index 50524f6..7aa5214 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ init: configs/Docker/env.production: echo "Initializing environment file for production..." cp configs/Docker/env.sample configs/Docker/env.production - sed -i "s/#DPS_DJANGO_SECRET_KEY=/DPS_DJANGO_SECRET_KEY=$(shell ./bin/generate_secret_key.sh)/" configs/Docker/env.production + sed -i "s/#DPS_DJANGO_SECRET_KEY=/DPS_DJANGO_SECRET_KEY=$(shell ./util/bin/generate_secret_key.sh)/" configs/Docker/env.production docker/build: configs/Docker/env.production tox -q -e docker-testing diff --git a/bin/generate_secret_key.sh b/util/bin/generate_secret_key.sh similarity index 100% rename from bin/generate_secret_key.sh rename to util/bin/generate_secret_key.sh diff --git a/bin/get_build_id.sh b/util/bin/get_build_id.sh similarity index 100% rename from bin/get_build_id.sh rename to util/bin/get_build_id.sh From 5b63b081ffe4143b69522f799c308a2f3f8eb377 Mon Sep 17 00:00:00 2001 From: Mischback Date: Wed, 25 Mar 2020 20:44:38 +0100 Subject: [PATCH 093/110] Moved deployment-related files to util/deployment The files are not yet fixed! This means, all Docker-related scripts are currently not working and need manual adaptation. Beside the moved files, this is also required for .dockerignore and most probably the Makefiles. --- .gitignore | 4 ++-- {configs => util/deployment}/Docker/bin/apt-get_install.sh | 0 {configs => util/deployment}/Docker/bin/set_timezone.sh | 0 {configs => util/deployment}/Docker/django/Dockerfile | 0 .../deployment}/Docker/django/entrypoint_django.sh | 0 {configs => util/deployment}/Docker/django/gunicorn_conf.py | 0 .../deployment}/Docker/django/run_gunicorn.sh.template | 0 {configs => util/deployment}/Docker/docker-compose.yml | 0 {configs => util/deployment}/Docker/env.sample | 0 {configs => util/deployment}/Docker/nginx/Dockerfile | 0 {configs => util/deployment}/Docker/nginx/entrypoint_nginx.sh | 0 {configs => util/deployment}/Docker/nginx/nginx.conf.template | 0 {configs => util/deployment}/README | 0 {configs => util/deployment}/apache2_vhost.sample.template | 0 14 files changed, 2 insertions(+), 2 deletions(-) rename {configs => util/deployment}/Docker/bin/apt-get_install.sh (100%) rename {configs => util/deployment}/Docker/bin/set_timezone.sh (100%) rename {configs => util/deployment}/Docker/django/Dockerfile (100%) rename {configs => util/deployment}/Docker/django/entrypoint_django.sh (100%) rename {configs => util/deployment}/Docker/django/gunicorn_conf.py (100%) rename {configs => util/deployment}/Docker/django/run_gunicorn.sh.template (100%) rename {configs => util/deployment}/Docker/docker-compose.yml (100%) rename {configs => util/deployment}/Docker/env.sample (100%) rename {configs => util/deployment}/Docker/nginx/Dockerfile (100%) rename {configs => util/deployment}/Docker/nginx/entrypoint_nginx.sh (100%) rename {configs => util/deployment}/Docker/nginx/nginx.conf.template (100%) rename {configs => util/deployment}/README (100%) rename {configs => util/deployment}/apache2_vhost.sample.template (100%) diff --git a/.gitignore b/.gitignore index c7837d1..e38864c 100644 --- a/.gitignore +++ b/.gitignore @@ -15,5 +15,5 @@ coverage_html # do NOT track environment files that are used for docker-compose, but do # include the sample -configs/Docker/env.* -!configs/Docker/env.sample +util/deployment/Docker/env.* +!util/deployment/Docker/env.sample diff --git a/configs/Docker/bin/apt-get_install.sh b/util/deployment/Docker/bin/apt-get_install.sh similarity index 100% rename from configs/Docker/bin/apt-get_install.sh rename to util/deployment/Docker/bin/apt-get_install.sh diff --git a/configs/Docker/bin/set_timezone.sh b/util/deployment/Docker/bin/set_timezone.sh similarity index 100% rename from configs/Docker/bin/set_timezone.sh rename to util/deployment/Docker/bin/set_timezone.sh diff --git a/configs/Docker/django/Dockerfile b/util/deployment/Docker/django/Dockerfile similarity index 100% rename from configs/Docker/django/Dockerfile rename to util/deployment/Docker/django/Dockerfile diff --git a/configs/Docker/django/entrypoint_django.sh b/util/deployment/Docker/django/entrypoint_django.sh similarity index 100% rename from configs/Docker/django/entrypoint_django.sh rename to util/deployment/Docker/django/entrypoint_django.sh diff --git a/configs/Docker/django/gunicorn_conf.py b/util/deployment/Docker/django/gunicorn_conf.py similarity index 100% rename from configs/Docker/django/gunicorn_conf.py rename to util/deployment/Docker/django/gunicorn_conf.py diff --git a/configs/Docker/django/run_gunicorn.sh.template b/util/deployment/Docker/django/run_gunicorn.sh.template similarity index 100% rename from configs/Docker/django/run_gunicorn.sh.template rename to util/deployment/Docker/django/run_gunicorn.sh.template diff --git a/configs/Docker/docker-compose.yml b/util/deployment/Docker/docker-compose.yml similarity index 100% rename from configs/Docker/docker-compose.yml rename to util/deployment/Docker/docker-compose.yml diff --git a/configs/Docker/env.sample b/util/deployment/Docker/env.sample similarity index 100% rename from configs/Docker/env.sample rename to util/deployment/Docker/env.sample diff --git a/configs/Docker/nginx/Dockerfile b/util/deployment/Docker/nginx/Dockerfile similarity index 100% rename from configs/Docker/nginx/Dockerfile rename to util/deployment/Docker/nginx/Dockerfile diff --git a/configs/Docker/nginx/entrypoint_nginx.sh b/util/deployment/Docker/nginx/entrypoint_nginx.sh similarity index 100% rename from configs/Docker/nginx/entrypoint_nginx.sh rename to util/deployment/Docker/nginx/entrypoint_nginx.sh diff --git a/configs/Docker/nginx/nginx.conf.template b/util/deployment/Docker/nginx/nginx.conf.template similarity index 100% rename from configs/Docker/nginx/nginx.conf.template rename to util/deployment/Docker/nginx/nginx.conf.template diff --git a/configs/README b/util/deployment/README similarity index 100% rename from configs/README rename to util/deployment/README diff --git a/configs/apache2_vhost.sample.template b/util/deployment/apache2_vhost.sample.template similarity index 100% rename from configs/apache2_vhost.sample.template rename to util/deployment/apache2_vhost.sample.template From 76fa37c6676b928256d1e442900b4bc31d72e06e Mon Sep 17 00:00:00 2001 From: Mischback Date: Wed, 25 Mar 2020 20:47:49 +0100 Subject: [PATCH 094/110] Renamed doc to docs --- {doc => docs}/.gitignore | 0 {doc => docs}/Makefile | 0 {doc => docs}/source/apache2_vhost.rst | 0 {doc => docs}/source/conf.py | 0 {doc => docs}/source/index.rst | 0 {doc => docs}/source/quickstart.rst | 0 {doc => docs}/source/settings.rst | 0 {doc => docs}/source/structure.rst | 0 {doc => docs}/source/versions.rst | 0 9 files changed, 0 insertions(+), 0 deletions(-) rename {doc => docs}/.gitignore (100%) rename {doc => docs}/Makefile (100%) rename {doc => docs}/source/apache2_vhost.rst (100%) rename {doc => docs}/source/conf.py (100%) rename {doc => docs}/source/index.rst (100%) rename {doc => docs}/source/quickstart.rst (100%) rename {doc => docs}/source/settings.rst (100%) rename {doc => docs}/source/structure.rst (100%) rename {doc => docs}/source/versions.rst (100%) diff --git a/doc/.gitignore b/docs/.gitignore similarity index 100% rename from doc/.gitignore rename to docs/.gitignore diff --git a/doc/Makefile b/docs/Makefile similarity index 100% rename from doc/Makefile rename to docs/Makefile diff --git a/doc/source/apache2_vhost.rst b/docs/source/apache2_vhost.rst similarity index 100% rename from doc/source/apache2_vhost.rst rename to docs/source/apache2_vhost.rst diff --git a/doc/source/conf.py b/docs/source/conf.py similarity index 100% rename from doc/source/conf.py rename to docs/source/conf.py diff --git a/doc/source/index.rst b/docs/source/index.rst similarity index 100% rename from doc/source/index.rst rename to docs/source/index.rst diff --git a/doc/source/quickstart.rst b/docs/source/quickstart.rst similarity index 100% rename from doc/source/quickstart.rst rename to docs/source/quickstart.rst diff --git a/doc/source/settings.rst b/docs/source/settings.rst similarity index 100% rename from doc/source/settings.rst rename to docs/source/settings.rst diff --git a/doc/source/structure.rst b/docs/source/structure.rst similarity index 100% rename from doc/source/structure.rst rename to docs/source/structure.rst diff --git a/doc/source/versions.rst b/docs/source/versions.rst similarity index 100% rename from doc/source/versions.rst rename to docs/source/versions.rst From 3c0485072677e3b3c70be6b4b7ac1254582a29c8 Mon Sep 17 00:00:00 2001 From: Mischback Date: Wed, 25 Mar 2020 20:48:46 +0100 Subject: [PATCH 095/110] Fixed Makefile tree --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7aa5214..06de502 100644 --- a/Makefile +++ b/Makefile @@ -71,7 +71,7 @@ docker/run: docker/build run: docker/run tree: - tree -a -I ".git|.tox|doc|run" --dirsfirst -C | less -r + tree -a -I ".git|.tox|docs|run" --dirsfirst -C | less -r .docker/build-context: echo " \ From 9934acac534de6d343b1d174e4ddf3e8d3c95a15 Mon Sep 17 00:00:00 2001 From: Mischback Date: Wed, 25 Mar 2020 21:55:27 +0100 Subject: [PATCH 096/110] Provided hardcoded name for project-specific directory --- manage.py | 2 +- {project_name => project}/__init__.py | 0 {project_name => project}/settings/__init__.py | 0 {project_name => project}/settings/common.py | 0 {project_name => project}/settings/development.py | 0 {project_name => project}/settings/docker.py | 0 {project_name => project}/settings/i18n.py | 0 {project_name => project}/settings/production.py | 0 {project_name => project}/urls.py | 0 {project_name => project}/wsgi.py | 0 10 files changed, 1 insertion(+), 1 deletion(-) rename {project_name => project}/__init__.py (100%) rename {project_name => project}/settings/__init__.py (100%) rename {project_name => project}/settings/common.py (100%) rename {project_name => project}/settings/development.py (100%) rename {project_name => project}/settings/docker.py (100%) rename {project_name => project}/settings/i18n.py (100%) rename {project_name => project}/settings/production.py (100%) rename {project_name => project}/urls.py (100%) rename {project_name => project}/wsgi.py (100%) diff --git a/manage.py b/manage.py index c257e48..1b0cf6a 100755 --- a/manage.py +++ b/manage.py @@ -3,7 +3,7 @@ import sys if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "{{ project_name }}.settings.development") + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "project.settings.development") try: from django.core.management import execute_from_command_line except ImportError: diff --git a/project_name/__init__.py b/project/__init__.py similarity index 100% rename from project_name/__init__.py rename to project/__init__.py diff --git a/project_name/settings/__init__.py b/project/settings/__init__.py similarity index 100% rename from project_name/settings/__init__.py rename to project/settings/__init__.py diff --git a/project_name/settings/common.py b/project/settings/common.py similarity index 100% rename from project_name/settings/common.py rename to project/settings/common.py diff --git a/project_name/settings/development.py b/project/settings/development.py similarity index 100% rename from project_name/settings/development.py rename to project/settings/development.py diff --git a/project_name/settings/docker.py b/project/settings/docker.py similarity index 100% rename from project_name/settings/docker.py rename to project/settings/docker.py diff --git a/project_name/settings/i18n.py b/project/settings/i18n.py similarity index 100% rename from project_name/settings/i18n.py rename to project/settings/i18n.py diff --git a/project_name/settings/production.py b/project/settings/production.py similarity index 100% rename from project_name/settings/production.py rename to project/settings/production.py diff --git a/project_name/urls.py b/project/urls.py similarity index 100% rename from project_name/urls.py rename to project/urls.py diff --git a/project_name/wsgi.py b/project/wsgi.py similarity index 100% rename from project_name/wsgi.py rename to project/wsgi.py From 5eaadb490405dec5fb9b4ad5f4e5e0c6ab69f366 Mon Sep 17 00:00:00 2001 From: Mischback Date: Thu, 26 Mar 2020 18:26:30 +0100 Subject: [PATCH 097/110] Moved deployment-related files to a seperate directory --- .gitignore | 4 ++-- {util/deployment => deploy}/Docker/bin/apt-get_install.sh | 0 {util/deployment => deploy}/Docker/bin/set_timezone.sh | 0 {util/deployment => deploy}/Docker/django/Dockerfile | 0 .../deployment => deploy}/Docker/django/entrypoint_django.sh | 0 {util/deployment => deploy}/Docker/django/gunicorn_conf.py | 0 .../Docker/django/run_gunicorn.sh.template | 0 {util/deployment => deploy}/Docker/docker-compose.yml | 0 {util/deployment => deploy}/Docker/env.sample | 0 {util/deployment => deploy}/Docker/nginx/Dockerfile | 0 {util/deployment => deploy}/Docker/nginx/entrypoint_nginx.sh | 0 {util/deployment => deploy}/Docker/nginx/nginx.conf.template | 0 {util/deployment => deploy}/README | 0 {util/deployment => deploy}/apache2_vhost.sample.template | 0 14 files changed, 2 insertions(+), 2 deletions(-) rename {util/deployment => deploy}/Docker/bin/apt-get_install.sh (100%) rename {util/deployment => deploy}/Docker/bin/set_timezone.sh (100%) rename {util/deployment => deploy}/Docker/django/Dockerfile (100%) rename {util/deployment => deploy}/Docker/django/entrypoint_django.sh (100%) rename {util/deployment => deploy}/Docker/django/gunicorn_conf.py (100%) rename {util/deployment => deploy}/Docker/django/run_gunicorn.sh.template (100%) rename {util/deployment => deploy}/Docker/docker-compose.yml (100%) rename {util/deployment => deploy}/Docker/env.sample (100%) rename {util/deployment => deploy}/Docker/nginx/Dockerfile (100%) rename {util/deployment => deploy}/Docker/nginx/entrypoint_nginx.sh (100%) rename {util/deployment => deploy}/Docker/nginx/nginx.conf.template (100%) rename {util/deployment => deploy}/README (100%) rename {util/deployment => deploy}/apache2_vhost.sample.template (100%) diff --git a/.gitignore b/.gitignore index e38864c..26e9018 100644 --- a/.gitignore +++ b/.gitignore @@ -15,5 +15,5 @@ coverage_html # do NOT track environment files that are used for docker-compose, but do # include the sample -util/deployment/Docker/env.* -!util/deployment/Docker/env.sample +deploy/Docker/env.* +!deploy/Docker/env.sample diff --git a/util/deployment/Docker/bin/apt-get_install.sh b/deploy/Docker/bin/apt-get_install.sh similarity index 100% rename from util/deployment/Docker/bin/apt-get_install.sh rename to deploy/Docker/bin/apt-get_install.sh diff --git a/util/deployment/Docker/bin/set_timezone.sh b/deploy/Docker/bin/set_timezone.sh similarity index 100% rename from util/deployment/Docker/bin/set_timezone.sh rename to deploy/Docker/bin/set_timezone.sh diff --git a/util/deployment/Docker/django/Dockerfile b/deploy/Docker/django/Dockerfile similarity index 100% rename from util/deployment/Docker/django/Dockerfile rename to deploy/Docker/django/Dockerfile diff --git a/util/deployment/Docker/django/entrypoint_django.sh b/deploy/Docker/django/entrypoint_django.sh similarity index 100% rename from util/deployment/Docker/django/entrypoint_django.sh rename to deploy/Docker/django/entrypoint_django.sh diff --git a/util/deployment/Docker/django/gunicorn_conf.py b/deploy/Docker/django/gunicorn_conf.py similarity index 100% rename from util/deployment/Docker/django/gunicorn_conf.py rename to deploy/Docker/django/gunicorn_conf.py diff --git a/util/deployment/Docker/django/run_gunicorn.sh.template b/deploy/Docker/django/run_gunicorn.sh.template similarity index 100% rename from util/deployment/Docker/django/run_gunicorn.sh.template rename to deploy/Docker/django/run_gunicorn.sh.template diff --git a/util/deployment/Docker/docker-compose.yml b/deploy/Docker/docker-compose.yml similarity index 100% rename from util/deployment/Docker/docker-compose.yml rename to deploy/Docker/docker-compose.yml diff --git a/util/deployment/Docker/env.sample b/deploy/Docker/env.sample similarity index 100% rename from util/deployment/Docker/env.sample rename to deploy/Docker/env.sample diff --git a/util/deployment/Docker/nginx/Dockerfile b/deploy/Docker/nginx/Dockerfile similarity index 100% rename from util/deployment/Docker/nginx/Dockerfile rename to deploy/Docker/nginx/Dockerfile diff --git a/util/deployment/Docker/nginx/entrypoint_nginx.sh b/deploy/Docker/nginx/entrypoint_nginx.sh similarity index 100% rename from util/deployment/Docker/nginx/entrypoint_nginx.sh rename to deploy/Docker/nginx/entrypoint_nginx.sh diff --git a/util/deployment/Docker/nginx/nginx.conf.template b/deploy/Docker/nginx/nginx.conf.template similarity index 100% rename from util/deployment/Docker/nginx/nginx.conf.template rename to deploy/Docker/nginx/nginx.conf.template diff --git a/util/deployment/README b/deploy/README similarity index 100% rename from util/deployment/README rename to deploy/README diff --git a/util/deployment/apache2_vhost.sample.template b/deploy/apache2_vhost.sample.template similarity index 100% rename from util/deployment/apache2_vhost.sample.template rename to deploy/apache2_vhost.sample.template From a04ccb1fdaebad9caac508a1eea0b5d9aa77706c Mon Sep 17 00:00:00 2001 From: Mischback Date: Tue, 31 Mar 2020 20:53:10 +0200 Subject: [PATCH 098/110] Renamed project to config --- {project => config}/__init__.py | 0 {project => config}/settings/__init__.py | 0 {project => config}/settings/common.py | 0 {project => config}/settings/development.py | 0 {project => config}/settings/docker.py | 0 {project => config}/settings/i18n.py | 0 {project => config}/settings/production.py | 0 {project => config}/urls.py | 0 {project => config}/wsgi.py | 0 9 files changed, 0 insertions(+), 0 deletions(-) rename {project => config}/__init__.py (100%) rename {project => config}/settings/__init__.py (100%) rename {project => config}/settings/common.py (100%) rename {project => config}/settings/development.py (100%) rename {project => config}/settings/docker.py (100%) rename {project => config}/settings/i18n.py (100%) rename {project => config}/settings/production.py (100%) rename {project => config}/urls.py (100%) rename {project => config}/wsgi.py (100%) diff --git a/project/__init__.py b/config/__init__.py similarity index 100% rename from project/__init__.py rename to config/__init__.py diff --git a/project/settings/__init__.py b/config/settings/__init__.py similarity index 100% rename from project/settings/__init__.py rename to config/settings/__init__.py diff --git a/project/settings/common.py b/config/settings/common.py similarity index 100% rename from project/settings/common.py rename to config/settings/common.py diff --git a/project/settings/development.py b/config/settings/development.py similarity index 100% rename from project/settings/development.py rename to config/settings/development.py diff --git a/project/settings/docker.py b/config/settings/docker.py similarity index 100% rename from project/settings/docker.py rename to config/settings/docker.py diff --git a/project/settings/i18n.py b/config/settings/i18n.py similarity index 100% rename from project/settings/i18n.py rename to config/settings/i18n.py diff --git a/project/settings/production.py b/config/settings/production.py similarity index 100% rename from project/settings/production.py rename to config/settings/production.py diff --git a/project/urls.py b/config/urls.py similarity index 100% rename from project/urls.py rename to config/urls.py diff --git a/project/wsgi.py b/config/wsgi.py similarity index 100% rename from project/wsgi.py rename to config/wsgi.py From fdac3dac74e9a32a950ed88a1fd046a84e5aed57 Mon Sep 17 00:00:00 2001 From: Mischback Date: Tue, 31 Mar 2020 21:05:05 +0200 Subject: [PATCH 099/110] Moved the WSGI file to its own directory --- wsgi/__init__.py | 0 config/wsgi.py => wsgi/app.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 wsgi/__init__.py rename config/wsgi.py => wsgi/app.py (100%) diff --git a/wsgi/__init__.py b/wsgi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/config/wsgi.py b/wsgi/app.py similarity index 100% rename from config/wsgi.py rename to wsgi/app.py From cd5a336f5a71ca4a76d2682705a37965862b41be Mon Sep 17 00:00:00 2001 From: Mischback Date: Tue, 31 Mar 2020 21:30:15 +0200 Subject: [PATCH 100/110] Docker build process working Currently, there are still errors in the images, so this is NOT WORKING. --- Makefile | 14 +++++++------- deploy/Docker/django/Dockerfile | 10 +++++----- deploy/Docker/docker-compose.yml | 2 +- util/init/Makefile.template | 14 +++++++------- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 06de502..5305cac 100644 --- a/Makefile +++ b/Makefile @@ -44,19 +44,19 @@ clean: init: # TODO: Include some output to this function # create the final version of different files by removing the '.template' suffix - mv ./configs/apache2_vhost.sample.template ./configs/apache2_vhost.sample - mv ./configs/Docker/django/run_gunicorn.sh.template ./configs/Docker/django/run_gunicorn.sh + mv ./deploy/apache2_vhost.sample.template ./deploy/apache2_vhost.sample + mv ./deploy/Docker/django/run_gunicorn.sh.template ./deploy/Docker/django/run_gunicorn.sh # switching the tox configuration file mv ./util/init/tox.ini.template ./tox.ini # switching the Makefile **should** be the last step mv ./util/init/Makefile.template ./Makefile -configs/Docker/env.production: +deploy/Docker/env.production: echo "Initializing environment file for production..." - cp configs/Docker/env.sample configs/Docker/env.production - sed -i "s/#DPS_DJANGO_SECRET_KEY=/DPS_DJANGO_SECRET_KEY=$(shell ./util/bin/generate_secret_key.sh)/" configs/Docker/env.production + cp deploy/Docker/env.sample deploy/Docker/env.production + sed -i "s/#DPS_DJANGO_SECRET_KEY=/DPS_DJANGO_SECRET_KEY=$(shell ./util/bin/generate_secret_key.sh)/" deploy/Docker/env.production -docker/build: configs/Docker/env.production +docker/build: deploy/Docker/env.production tox -q -e docker-testing docker/build-context: @@ -87,4 +87,4 @@ tree: .docker/run: DPS_BUILD_NAME_PREFIX=$(DPS_BUILD_NAME_PREFIX) DPS_BUILD_ID=$(DPS_BUILD_ID) \ - $(DOCKER_COMPOSE_CMD) -f configs/Docker/docker-compose.yml up + $(DOCKER_COMPOSE_CMD) -f deploy/Docker/docker-compose.yml up diff --git a/deploy/Docker/django/Dockerfile b/deploy/Docker/django/Dockerfile index acb6d15..b6d029d 100644 --- a/deploy/Docker/django/Dockerfile +++ b/deploy/Docker/django/Dockerfile @@ -23,7 +23,7 @@ ENV VIRTUAL_ENV=/venv ENV PATH="$VIRTUAL_ENV/bin:$PATH:/docker-bin" # fetch our custom install script -COPY configs/Docker/bin/apt-get_install.sh /docker-bin/ +COPY deploy/Docker/bin/apt-get_install.sh /docker-bin/ RUN python3 -m venv $VIRTUAL_ENV @@ -58,7 +58,7 @@ ENV VIRTUAL_ENV=/venv ENV PATH="$VIRTUAL_ENV/bin:$PATH:/docker-bin" # fetch our custom install script -COPY configs/Docker/bin/apt-get_install.sh /docker-bin/ +COPY deploy/Docker/bin/apt-get_install.sh /docker-bin/ RUN apt-get_install.sh gosu @@ -70,11 +70,11 @@ WORKDIR /webapp COPY --from=base --chown=2342:65534 /venv /venv # fetch the entrypoint script (and includes) -COPY configs/Docker/django/entrypoint_django.sh /docker-bin/ -COPY configs/Docker/bin/set_timezone.sh /docker-bin/ +COPY deploy/Docker/django/entrypoint_django.sh /docker-bin/ +COPY deploy/Docker/bin/set_timezone.sh /docker-bin/ # fetch the container's start script -COPY configs/Docker/django/run_gunicorn.sh /docker-bin/ +COPY deploy/Docker/django/run_gunicorn.sh /docker-bin/ # fetch the actual project files # TODO: copy in several steps with more/finer control diff --git a/deploy/Docker/docker-compose.yml b/deploy/Docker/docker-compose.yml index 7d63994..18f01c8 100644 --- a/deploy/Docker/docker-compose.yml +++ b/deploy/Docker/docker-compose.yml @@ -4,7 +4,7 @@ services: django: build: context: ../../ - dockerfile: configs/Docker/django/Dockerfile + dockerfile: deploy/Docker/django/Dockerfile # providing the build and the image directive automatically tags the image # automatic tagging is performed with environment variables provided # through the Makefile and results in a tag with either a git commit hash diff --git a/util/init/Makefile.template b/util/init/Makefile.template index 31248d6..c1243ac 100644 --- a/util/init/Makefile.template +++ b/util/init/Makefile.template @@ -249,7 +249,7 @@ django/staticfiles/find: ### Docker commands ### -docker/build: util/requirements configs/Docker/env.production +docker/build: util/requirements deploy/Docker/env.production sudo $(MAKE) .internal/docker/build docker/run: docker/run/django @@ -260,14 +260,14 @@ docker/run/django: ### Environment management ### -configs/Docker/env.production: +deploy/Docker/env.production: # this is explicitly **NOT** a phony target, because the production # settings must only be generated **ONE TIME**. # Some of these settings are initialised with random values, but hve to be # constant for the lifetime of a project, e.g. the SECRET_KEY. $(ECHO) "Initializing environment file for production...\n" - cp configs/Docker/env.sample configs/Docker/env.production - sed -i "s/#DPS_DJANGO_SECRET_KEY=/DPS_DJANGO_SECRET_KEY=$(shell ./bin/generate_secret_key.sh)/" configs/Docker/env.production + cp deploy/Docker/env.sample deploy/Docker/env.production + sed -i "s/#DPS_DJANGO_SECRET_KEY=/DPS_DJANGO_SECRET_KEY=$(shell ./bin/generate_secret_key.sh)/" deploy/Docker/env.production ### Utility commands ### @@ -285,13 +285,13 @@ util/requirements-force: .internal/docker/build: DPS_BUILD_NAME_PREFIX=$(DPS_BUILD_NAME_PREFIX) DPS_BUILD_ID=$(DPS_BUILD_ID) \ - $(DOCKER_COMPOSE_CMD) -f configs/Docker/docker-compose.yml build + $(DOCKER_COMPOSE_CMD) -f deploy/Docker/docker-compose.yml build $(DOCKER_CMD) tag "$(DPS_BUILD_NAME_PREFIX)/django:$(DPS_BUILD_ID)" "$(DPS_BUILD_NAME_PREFIX)/django:latest" $(DOCKER_CMD) tag "$(DPS_BUILD_NAME_PREFIX)/nginx:$(DPS_BUILD_ID)" "$(DPS_BUILD_NAME_PREFIX)/nginx:latest" .internal/docker/run/django: DPS_BUILD_NAME_PREFIX=$(DPS_BUILD_NAME_PREFIX) DPS_BUILD_ID=$(DPS_BUILD_ID) \ - $(DOCKER_COMPOSE_CMD) -f configs/Docker/docker-compose.yml up django + $(DOCKER_COMPOSE_CMD) -f deploy/Docker/docker-compose.yml up django .internal/tox/dev-django: util/requirements $(TOX_CMD) dev-django -- $(django_cmd) $(DJANGOADMIN_CMD_VERBOSITY) @@ -307,7 +307,7 @@ CCYA := $(shell tput setaf 6 2>/dev/null) CRES := $(shell tput sgr0 2>/dev/null) DPS_USERNAME := $(shell id -un) -DPS_BUILD_ID := $(shell ./bin/get_build_id.sh) +DPS_BUILD_ID := $(shell ./util/bin/get_build_id.sh) # suppress unintended output .SILENT: From 31053b8c9a97d96d1d57922bff8dc9f4e3689347 Mon Sep 17 00:00:00 2001 From: Mischback Date: Tue, 31 Mar 2020 22:13:21 +0200 Subject: [PATCH 101/110] Make Docker-based images work again --- .dockerignore | 28 ++++++++----------- Makefile | 1 - config/settings/common.py | 2 +- deploy/Docker/django/Dockerfile | 8 +++++- deploy/Docker/django/run_gunicorn.sh | 7 +++++ deploy/Docker/django/run_gunicorn.sh.template | 7 ----- 6 files changed, 27 insertions(+), 26 deletions(-) create mode 100755 deploy/Docker/django/run_gunicorn.sh delete mode 100755 deploy/Docker/django/run_gunicorn.sh.template diff --git a/.dockerignore b/.dockerignore index ff60762..07eaa4a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,29 +1,25 @@ # experimental .dockerignore -# do NOT include the bin directory -# these scripts are not used *inside* of Docker-based images. -# They may be used during the build/run process and provide utility functions, -# i.e. for tagging the images -bin - -# do NOT include the provided configuration files -configs +# do NOT include the util directory +util + +# do NOT include the provided deployment-related files +deploy # but DO include the production-related scripts/config files that are used # inside of the images -!configs/Docker/bin/* -!configs/Docker/django/entrypoint_django.sh -!configs/Docker/django/gunicorn_conf.py -!configs/Docker/django/run_gunicorn.sh -!configs/Docker/nginx/nginx.conf -!configs/Docker/nginx/entrypoint_nginx.sh +!deploy/Docker/bin/* +!deploy/Docker/django/entrypoint_django.sh +!deploy/Docker/django/gunicorn_conf.py +!deploy/Docker/django/run_gunicorn.sh +!deploy/Docker/nginx/nginx.conf +!deploy/Docker/nginx/entrypoint_nginx.sh # do NOT include the documentation # TODO: the original doc-folder of django-project-skeleton should be removed # during initialisation of the project. On the other hand, there could be # project specific documentation. However, this should be excluded from any # image. -# TODO: 'doc' is already renamed to 'docs' in development! -doc +docs # inside of the images, only the compiled requirements files with the '.txt' # extension are used. No need to include the source files. diff --git a/Makefile b/Makefile index 5305cac..1c7387c 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,6 @@ init: # TODO: Include some output to this function # create the final version of different files by removing the '.template' suffix mv ./deploy/apache2_vhost.sample.template ./deploy/apache2_vhost.sample - mv ./deploy/Docker/django/run_gunicorn.sh.template ./deploy/Docker/django/run_gunicorn.sh # switching the tox configuration file mv ./util/init/tox.ini.template ./tox.ini # switching the Makefile **should** be the last step diff --git a/config/settings/common.py b/config/settings/common.py index a5e27ba..e093e93 100644 --- a/config/settings/common.py +++ b/config/settings/common.py @@ -87,7 +87,7 @@ # the default WSGI application WSGI_APPLICATION = os.environ.get( 'DPS_DJANGO_WSGI_APP', - '{}.wsgi.application'.format(SITE_NAME) + 'wsgi.app.application' ) # the root URL configuration diff --git a/deploy/Docker/django/Dockerfile b/deploy/Docker/django/Dockerfile index b6d029d..6943dae 100644 --- a/deploy/Docker/django/Dockerfile +++ b/deploy/Docker/django/Dockerfile @@ -69,16 +69,22 @@ WORKDIR /webapp # fetch the virtual environment from base COPY --from=base --chown=2342:65534 /venv /venv +# TODO: Can the following statements be combined into one? # fetch the entrypoint script (and includes) COPY deploy/Docker/django/entrypoint_django.sh /docker-bin/ COPY deploy/Docker/bin/set_timezone.sh /docker-bin/ # fetch the container's start script COPY deploy/Docker/django/run_gunicorn.sh /docker-bin/ +COPY deploy/Docker/django/gunicorn_conf.py /docker-bin/ # fetch the actual project files # TODO: copy in several steps with more/finer control -COPY --chown=2342:65534 . . +#COPY --chown=2342:65534 . . +COPY --chown=2342:65534 ./wsgi ./wsgi +COPY --chown=2342:65534 ./config ./config +COPY --chown=2342:65534 ./apps ./apps +COPY --chown=2342:65534 ./templates ./templates # make Gunicorn's port 8000 available to other Docker images EXPOSE 8000 diff --git a/deploy/Docker/django/run_gunicorn.sh b/deploy/Docker/django/run_gunicorn.sh new file mode 100755 index 0000000..a26b50f --- /dev/null +++ b/deploy/Docker/django/run_gunicorn.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -euo pipefail + +export DJANGO_SETTINGS_MODULE="config.settings.docker" + +exec gunicorn wsgi.app:application --config='file:/docker-bin/gunicorn_conf.py' diff --git a/deploy/Docker/django/run_gunicorn.sh.template b/deploy/Docker/django/run_gunicorn.sh.template deleted file mode 100755 index 52bc91e..0000000 --- a/deploy/Docker/django/run_gunicorn.sh.template +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -export DJANGO_SETTINGS_MODULE="{{ project_name}}.settings.docker" - -exec gunicorn {{ project_name }}.wsgi:application --config='file:configs/Docker/django/gunicorn_conf.py' From abb629b128b446e6919d4e80cd82ce3dfb6da097 Mon Sep 17 00:00:00 2001 From: Mischback Date: Fri, 3 Apr 2020 19:43:23 +0200 Subject: [PATCH 102/110] Update manage.py a) use the correct settings module b) default to use development settings, when executing 'manage.py' c) do not explicitly set the settings module in 'tox.ini', rely on the provided default value in 'manage.py' --- manage.py | 36 +++++++++++++++++++++--------------- tox.ini | 4 ++-- util/init/tox.ini.template | 4 ++-- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/manage.py b/manage.py index 1b0cf6a..d9581b4 100755 --- a/manage.py +++ b/manage.py @@ -1,22 +1,28 @@ #!/usr/bin/env python + +"""Django's command-line utility for administrative tasks. + +This is based on Django's default 'manage.py', but slightly modified to be +compatible with the custom project layout provided by +'django-project-skeleton'.""" + +# Python imports import os import sys -if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "project.settings.development") + +def main(): + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.development') try: from django.core.management import execute_from_command_line - except ImportError: - # The above import may fail for some other reason. Ensure that the - # issue is really that Django is missing to avoid masking other - # exceptions on Python 2. - try: - import django - except ImportError: - raise ImportError( - "Couldn't import Django. Are you sure it's installed and " - "available on your PYTHONPATH environment variable? Did you " - "forget to activate a virtual environment?" - ) - raise + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/tox.ini b/tox.ini index de37044..dad7bca 100644 --- a/tox.ini +++ b/tox.ini @@ -26,7 +26,7 @@ deps = changedir = {envtmpdir}/dpstest commands = django-admin startproject --extension=py,template --template={toxinidir} dpstest . - {envbindir}/python manage.py check --settings=dpstest.settings.development + {envbindir}/python manage.py check [testenv:util] basepython = python3 @@ -37,7 +37,7 @@ skip_install = true changedir = {envtmpdir}/dpstest commands = django-admin startproject --extension=py,template --template={toxinidir} dpstest . - {posargs:{envbindir}/python manage.py check --settings=dpstest.settings.development} + {posargs:{envbindir}/python manage.py check} [testenv:docker-testing] basepython = {[testenv:util]basepython} diff --git a/util/init/tox.ini.template b/util/init/tox.ini.template index f3ace3b..2e38c34 100644 --- a/util/init/tox.ini.template +++ b/util/init/tox.ini.template @@ -17,7 +17,7 @@ deps = -rrequirements/development.txt skip_install = true commands = - {posargs:{envbindir}/python manage.py version --settings={{ project_name }}.settings.development} + {posargs:{envbindir}/python manage.py version} [testenv:dev-django] basepython = {[testenv:base]basepython} @@ -25,7 +25,7 @@ envdir = {[testenv:base]envdir} deps = {[testenv:base]deps} skip_install = {[testenv:base]skip_install} commands = - {envbindir}/python manage.py {posargs:version} --settings={{ project_name }}.settings.development + {envbindir}/python manage.py {posargs:version} [testenv:test-django] basepython = {[testenv:base]basepython} From a1d809dfd6bd70f531a428528aea0b469b815af3 Mon Sep 17 00:00:00 2001 From: Mischback Date: Fri, 3 Apr 2020 19:50:15 +0200 Subject: [PATCH 103/110] Update Makefile to be consistent with tox.ini --- util/init/Makefile.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/init/Makefile.template b/util/init/Makefile.template index c1243ac..7cdaf57 100644 --- a/util/init/Makefile.template +++ b/util/init/Makefile.template @@ -281,7 +281,7 @@ util/requirements-force: .internal/django/raw: - $(DJANGOADMIN_CMD) $(django_cmd) $(DJANGOADMIN_CMD_VERBOSITY) --settings={{ project_name }}.settings.development + $(DJANGOADMIN_CMD) $(django_cmd) $(DJANGOADMIN_CMD_VERBOSITY) .internal/docker/build: DPS_BUILD_NAME_PREFIX=$(DPS_BUILD_NAME_PREFIX) DPS_BUILD_ID=$(DPS_BUILD_ID) \ From fe9432d0819ebc5e90cd7d3299ec3e0ab1db8499 Mon Sep 17 00:00:00 2001 From: Mischback Date: Fri, 3 Apr 2020 19:57:54 +0200 Subject: [PATCH 104/110] Moved/updated the wsgi interface --- {wsgi => sgi}/__init__.py | 0 sgi/wsgi.py | 25 +++++++++++++++++++++++++ wsgi/app.py | 14 -------------- 3 files changed, 25 insertions(+), 14 deletions(-) rename {wsgi => sgi}/__init__.py (100%) create mode 100644 sgi/wsgi.py delete mode 100644 wsgi/app.py diff --git a/wsgi/__init__.py b/sgi/__init__.py similarity index 100% rename from wsgi/__init__.py rename to sgi/__init__.py diff --git a/sgi/wsgi.py b/sgi/wsgi.py new file mode 100644 index 0000000..2b20624 --- /dev/null +++ b/sgi/wsgi.py @@ -0,0 +1,25 @@ +""" +WSGI config for {{ project_name }} project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/{{ docs_version }}/howto/deployment/wsgi/ + +This is based on Django's default 'wsgi.py', but slightly modified to be +compatible with the custom project layout provided by +'django-project-skeleton'.""" + +# Python imports +import os + +# Django imports +from django.core.wsgi import get_wsgi_application + +# Provide a default settings module +# WSGI is used to deploy the project, so the default value are the production +# settings. This may be overwritten in actual deployment setups. +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.production") + +# actually expose an application +application = get_wsgi_application() diff --git a/wsgi/app.py b/wsgi/app.py deleted file mode 100644 index 53a1acb..0000000 --- a/wsgi/app.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -WSGI config for {{ project_name }} project. - -It exposes the WSGI callable as a module-level variable named ``application``. - -For more information on this file, see -https://docs.djangoproject.com/en/1.8/howto/deployment/wsgi/ -""" - -import os -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "{{ project_name }}.settings.production") - -from django.core.wsgi import get_wsgi_application -application = get_wsgi_application() From 5ec544d02841ab385892073f350380aff2b4a0d0 Mon Sep 17 00:00:00 2001 From: Mischback Date: Fri, 3 Apr 2020 20:04:25 +0200 Subject: [PATCH 105/110] Updated Docker-based deployment files to use new wsgi location --- deploy/Docker/django/Dockerfile | 2 +- deploy/Docker/django/run_gunicorn.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/Docker/django/Dockerfile b/deploy/Docker/django/Dockerfile index 6943dae..b13c806 100644 --- a/deploy/Docker/django/Dockerfile +++ b/deploy/Docker/django/Dockerfile @@ -81,7 +81,7 @@ COPY deploy/Docker/django/gunicorn_conf.py /docker-bin/ # fetch the actual project files # TODO: copy in several steps with more/finer control #COPY --chown=2342:65534 . . -COPY --chown=2342:65534 ./wsgi ./wsgi +COPY --chown=2342:65534 ./sgi ./sgi COPY --chown=2342:65534 ./config ./config COPY --chown=2342:65534 ./apps ./apps COPY --chown=2342:65534 ./templates ./templates diff --git a/deploy/Docker/django/run_gunicorn.sh b/deploy/Docker/django/run_gunicorn.sh index a26b50f..f95e532 100755 --- a/deploy/Docker/django/run_gunicorn.sh +++ b/deploy/Docker/django/run_gunicorn.sh @@ -4,4 +4,4 @@ set -euo pipefail export DJANGO_SETTINGS_MODULE="config.settings.docker" -exec gunicorn wsgi.app:application --config='file:/docker-bin/gunicorn_conf.py' +exec gunicorn sgi.wsgi:application --config='file:/docker-bin/gunicorn_conf.py' From ecf16b6b45880459d17be0fec5025d51e63bbfed Mon Sep 17 00:00:00 2001 From: Mischback Date: Fri, 3 Apr 2020 20:09:15 +0200 Subject: [PATCH 106/110] Added asgi.py --- sgi/asgi.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 sgi/asgi.py diff --git a/sgi/asgi.py b/sgi/asgi.py new file mode 100644 index 0000000..6815ba6 --- /dev/null +++ b/sgi/asgi.py @@ -0,0 +1,25 @@ +""" +ASGI config for {{ project_name }} project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/{{ docs_version }}/howto/deployment/asgi/ + +This is based on Django's default 'asgi.py', but slightly modified to be +compatible with the custom project layout provided by +'django-project-skeleton'.""" + +# Python imports +import os + +# Django imports +from django.core.wsgi import get_asgi_application + +# Provide a default settings module +# WSGI is used to deploy the project, so the default value are the production +# settings. This may be overwritten in actual deployment setups. +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.production") + +# actually expose an application +application = get_asgi_application() From 2ab24b254b5d79e383ae19c8154baf22ce514259 Mon Sep 17 00:00:00 2001 From: Mischback Date: Sun, 5 Apr 2020 17:58:13 +0200 Subject: [PATCH 107/110] Minor fix in config.settings.common --- config/settings/common.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/config/settings/common.py b/config/settings/common.py index e093e93..d7a8cf5 100644 --- a/config/settings/common.py +++ b/config/settings/common.py @@ -84,11 +84,12 @@ }, ] -# the default WSGI application -WSGI_APPLICATION = os.environ.get( - 'DPS_DJANGO_WSGI_APP', - 'wsgi.app.application' -) +# The WSGI application to be used by Django's internal servers. +# Please note, Django's 'runserver' should **not** be used in production +# environments. +# If set to 'None', 'django.core.wsgi.get_wsgi_application()' will be used to +# determine the WSGI application. +WSGI_APPLICATION = os.environ.get('DPS_DJANGO_WSGI_APP', None) # the root URL configuration ROOT_URLCONF = '{}.urls'.format(SITE_NAME) From dd30df678e4386b09b6295d12208aaa119875270 Mon Sep 17 00:00:00 2001 From: Mischback Date: Sun, 5 Apr 2020 21:03:43 +0200 Subject: [PATCH 108/110] Get rid of docker-specific settings The docker-specific settings were included into the production settings, with some documentation, why and how Django's logging is deactivated. It also includes a fix for ALLOWED_HOSTS, to work nicely in our Docker-based deployment, but should be working fine in other setups aswell. --- config/settings/docker.py | 15 --------------- config/settings/production.py | 24 +++++++++++++++++++++++- deploy/Docker/django/run_gunicorn.sh | 2 +- deploy/Docker/env.sample | 15 ++++++++------- 4 files changed, 32 insertions(+), 24 deletions(-) delete mode 100644 config/settings/docker.py diff --git a/config/settings/docker.py b/config/settings/docker.py deleted file mode 100644 index 9785590..0000000 --- a/config/settings/docker.py +++ /dev/null @@ -1,15 +0,0 @@ -# Python imports -import os - -# fetch the production settings -from .production import * - - -# Make Django work correctly behind the Nginx proxy -# Nginx **must** set the appropriate header! -# In the current setup, Nginx sets the "Host" header to the "server_name". -ALLOWED_HOSTS = os.environ.get('DPS_SERVER_NAME', 'localhost').split() -#USE_X_FORWARDED_HOST = True - -# Logging is set up in Gunicorn's configuration -LOGGING_CONFIG=None diff --git a/config/settings/production.py b/config/settings/production.py index 40c56c0..6211645 100644 --- a/config/settings/production.py +++ b/config/settings/production.py @@ -8,12 +8,34 @@ # ##### APPLICATION CONFIGURATION ######################### # You will have to determine, which hostnames should be served by Django -ALLOWED_HOSTS = os.environ.get('DPS_DJANGO_ALLOWED_HOSTS', '').split() +# Determines, which hostnames should be served by Django. +# DPS_DJANGO_ALLOWED_HOSTS may contain a list of (acceptable) hostnames, while +# DPS_SERVER_NAME may only contain one hostname, as this value also determines +# the actual hostname, that is used by Nginx. +# +# If neither DPS_DJANGO_ALLOWED_HOSTS nor DPS_SERVER_NAME are provided, this +# will be an empty list and thus, no host will be served, rendering Django +# effectively useless. +ALLOWED_HOSTS = list( + set( + os.environ.get('DPS_DJANGO_ALLOWED_HOSTS', '').split() + + os.environ.get('DPS_SERVER_NAME', '').split() + ) +) INSTALLED_APPS = DEFAULT_APPS STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.ManifestStaticFilesStorage' +# Don't let Django configure logging in a production environment. +# Most probably, configuring logging should be done by WSGI Server, i.e. +# Gunicorn or uWSGI. +# For Docker-based deployments, logging is configured in 'gunicorn_conf.py'. +# +# Without setting LOGGING_CONFIG to 'None', Django will setup Python's logging +# module, as described https://docs.djangoproject.com/en/3.0/topics/logging/#default-logging-configuration +LOGGING_CONFIG=None + # ##### SECURITY CONFIGURATION ############################ # TODO: Make sure, that sensitive information uses https diff --git a/deploy/Docker/django/run_gunicorn.sh b/deploy/Docker/django/run_gunicorn.sh index f95e532..b7a1287 100755 --- a/deploy/Docker/django/run_gunicorn.sh +++ b/deploy/Docker/django/run_gunicorn.sh @@ -2,6 +2,6 @@ set -euo pipefail -export DJANGO_SETTINGS_MODULE="config.settings.docker" +export DJANGO_SETTINGS_MODULE="config.settings.production" exec gunicorn sgi.wsgi:application --config='file:/docker-bin/gunicorn_conf.py' diff --git a/deploy/Docker/env.sample b/deploy/Docker/env.sample index 0f1dc64..0ae160e 100644 --- a/deploy/Docker/env.sample +++ b/deploy/Docker/env.sample @@ -29,10 +29,14 @@ # See https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts # # This setting is used by -# - [django] settings/docker.py +# - [django] settings/production.py # - [nginx] nginx.conf.template # -# Default value: localhost +# This variable **must** be set, if django-project-skeleton's Docker-based +# deployment scripts are used, to keep Django and Nginx in sync. +# However, if you are using the webapp's Dockerfile on its own (probably with +# a custon reverse proxy), you should not use this settings and refer to +# DPS_DJANGO_ALLOWED_HOSTS instead. #DPS_SERVER_NAME=localhost # The url prefix for static files @@ -58,11 +62,8 @@ ### DJANGO related configuration # Django's ALLOWED_HOSTS -# In *settings/production.py* this is by default an empty list. In the -# provided *settings/docker-nginx.py* this is set to the project's name while -# setting USE_X_FORWARDED_HOST = True. This means, that Django will not longer -# check, if it is responsible for the request but if the request comes from a -# trusted proxy (nginx in this case). +# In *settings/production.py* this is by default an empty list but will be +# augmented by the DPS_SERVER_NAME setting. #DPS_DJANGO_ALLOWED_HOSTS= # The minimum level of log messages to show up for Django's logging From d070a090d93bc858c2d1a9c3f25fc3a87c38bff4 Mon Sep 17 00:00:00 2001 From: Mischback Date: Sun, 5 Apr 2020 22:00:33 +0200 Subject: [PATCH 109/110] Made DJANGO_SETTINGS_MODULE configurable --- deploy/Docker/django/Dockerfile | 7 +++++++ deploy/Docker/django/run_gunicorn.sh | 2 -- deploy/Docker/env.sample | 5 ++++- manage.py | 12 +++++++++++- sgi/asgi.py | 10 +++++++++- sgi/wsgi.py | 10 +++++++++- 6 files changed, 40 insertions(+), 6 deletions(-) diff --git a/deploy/Docker/django/Dockerfile b/deploy/Docker/django/Dockerfile index b13c806..b91429f 100644 --- a/deploy/Docker/django/Dockerfile +++ b/deploy/Docker/django/Dockerfile @@ -62,6 +62,10 @@ COPY deploy/Docker/bin/apt-get_install.sh /docker-bin/ RUN apt-get_install.sh gosu +# provide a default value for Django's settings module +ARG DPS_DJANGO_SETTINGS_MODULE=config.settings.production +ENV DPS_DJANGO_SETTINGS_MODULE ${DPS_DJANGO_SETTINGS_MODULE} + # create a user to be used (meaning: we won't run with root-permissions!) RUN useradd -u 2342 -g 65534 -s /bin/false -M pythonuser WORKDIR /webapp @@ -86,6 +90,9 @@ COPY --chown=2342:65534 ./config ./config COPY --chown=2342:65534 ./apps ./apps COPY --chown=2342:65534 ./templates ./templates +# TODO: This don't have to be in the image, but nice for debugging! +COPY --chown=2342:65534 ./manage.py ./manage.py + # make Gunicorn's port 8000 available to other Docker images EXPOSE 8000 diff --git a/deploy/Docker/django/run_gunicorn.sh b/deploy/Docker/django/run_gunicorn.sh index b7a1287..6069e29 100755 --- a/deploy/Docker/django/run_gunicorn.sh +++ b/deploy/Docker/django/run_gunicorn.sh @@ -2,6 +2,4 @@ set -euo pipefail -export DJANGO_SETTINGS_MODULE="config.settings.production" - exec gunicorn sgi.wsgi:application --config='file:/docker-bin/gunicorn_conf.py' diff --git a/deploy/Docker/env.sample b/deploy/Docker/env.sample index 0ae160e..b50b1a6 100644 --- a/deploy/Docker/env.sample +++ b/deploy/Docker/env.sample @@ -84,10 +84,13 @@ # Django's SECRET_KEY #DPS_DJANGO_SECRET_KEY= +# Django's setting module +#DPS_DJANGO_SETTINGS_MODULE=config.settings.production + # the (dotted) path to the WSGI app # defaults to the app provided by Django # see *settings/common.py* for the default value -#DPS_DJANGO_WSGI_APP= +#DPS_DJANGO_WSGI_APP=sgi.wsgi ### GUNICORN related configuration diff --git a/manage.py b/manage.py index d9581b4..fe1150d 100755 --- a/manage.py +++ b/manage.py @@ -12,7 +12,17 @@ def main(): - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.development') + # By default, 'manage.py' will use the development settings, provided + # in 'config.settings.development'. This might be overruled by explicitly + # setting DJANGO_SETTINGS_MODULE or using DPS_DJANGO_SETTINGS_MODULE. + os.environ.setdefault( + 'DJANGO_SETTINGS_MODULE', + os.environ.get( + 'DPS_DJANGO_SETTINGS_MODULE', + 'config.settings.development' + ) + ) + try: from django.core.management import execute_from_command_line except ImportError as exc: diff --git a/sgi/asgi.py b/sgi/asgi.py index 6815ba6..e26ba36 100644 --- a/sgi/asgi.py +++ b/sgi/asgi.py @@ -19,7 +19,15 @@ # Provide a default settings module # WSGI is used to deploy the project, so the default value are the production # settings. This may be overwritten in actual deployment setups. -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.production") +# This defaults to 'config.settings.production' but may be adjusted either by +# settings DJANGO_SETTINGS_MODULE directly or using DPS_DJANGO_SETTINGS_MODULE. +os.environ.setdefault( + 'DJANGO_SETTINGS_MODULE', + os.environ.get( + 'DPS_DJANGO_SETTINGS_MODULE', + 'config.settings.production' + ) +) # actually expose an application application = get_asgi_application() diff --git a/sgi/wsgi.py b/sgi/wsgi.py index 2b20624..dd383f6 100644 --- a/sgi/wsgi.py +++ b/sgi/wsgi.py @@ -19,7 +19,15 @@ # Provide a default settings module # WSGI is used to deploy the project, so the default value are the production # settings. This may be overwritten in actual deployment setups. -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.production") +# This defaults to 'config.settings.production' but may be adjusted either by +# settings DJANGO_SETTINGS_MODULE directly or using DPS_DJANGO_SETTINGS_MODULE. +os.environ.setdefault( + 'DJANGO_SETTINGS_MODULE', + os.environ.get( + 'DPS_DJANGO_SETTINGS_MODULE', + 'config.settings.production' + ) +) # actually expose an application application = get_wsgi_application() From 7755bcc44284e4ce949bc2bf34d411ccfe4f9cd4 Mon Sep 17 00:00:00 2001 From: Mischback Date: Sun, 5 Apr 2020 22:07:23 +0200 Subject: [PATCH 110/110] Drop support for Python2.7 --- .travis.yml | 1 - tox.ini | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 30c0ba6..41cfb60 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,6 @@ language: python # listing the Python versions to be tested python: - - "2.7" - "3.4" - "3.5" - "3.6" diff --git a/tox.ini b/tox.ini index dad7bca..a1d688d 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ [tox] envlist = - django111-py{27,34,35,36,37} + django111-py{34,35,36,37} django20-py{34,35,36,37} django21-py{35,36,37} django22-py{35,36,37,38}