diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..07eaa4a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,48 @@ +# experimental .dockerignore + +# do NOT include the util directory +util + +# do NOT include the provided deployment-related files +deploy +# but DO include the production-related scripts/config files that are used +# inside of the images +!deploy/Docker/bin/* +!deploy/Docker/django/entrypoint_django.sh +!deploy/Docker/django/gunicorn_conf.py +!deploy/Docker/django/run_gunicorn.sh +!deploy/Docker/nginx/nginx.conf +!deploy/Docker/nginx/entrypoint_nginx.sh + +# do NOT include the documentation +# TODO: the original doc-folder of django-project-skeleton should be removed +# during initialisation of the project. On the other hand, there could be +# project specific documentation. However, this should be excluded from any +# image. +docs + +# inside of the images, only the compiled requirements files with the '.txt' +# extension are used. No need to include the source files. +requirements/*.in + +# utility files/directories of the repository should not be included +**/.git +**/.tox +**/.dockerignore +**/.editorconfig +**/.gitignore +**/.travis.yml +**/Makefile +**/README* +**/tox.ini + +# do not include any samples +**/*.sample + +# do NOT track Python compilation stuff +*.py[cod] +**/__python__ +**/__pycache__ + +# TODO: .gitignore excludes compiled language files. Is it safe to exclude the +# uncompiled language files for 'production'? diff --git a/.gitignore b/.gitignore index bbb9bcd..26e9018 100644 --- a/.gitignore +++ b/.gitignore @@ -13,5 +13,7 @@ coverage_html *.mo *.pot -# I started relying on putting temporary notes into TODO files -*.todo +# do NOT track environment files that are used for docker-compose, but do +# include the sample +deploy/Docker/env.* +!deploy/Docker/env.sample diff --git a/.travis.yml b/.travis.yml index 30c0ba6..41cfb60 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,6 @@ language: python # listing the Python versions to be tested python: - - "2.7" - "3.4" - "3.5" - "3.6" diff --git a/Makefile b/Makefile index 6fe1924..1c7387c 100644 --- a/Makefile +++ b/Makefile @@ -1,34 +1,89 @@ .SILENT: -.PHONY: all clean current doc doc-srv serve tox +.PHONY: clean init tree \ + docker/build docker/build-context .docker/build-context +# Docker command +DOCKER_CMD := $(shell which docker) -all: - echo "" - echo " clean Removes all temporary files" - echo " current Runs the tox-environment for the current development" - echo " doc Builds the documentation using 'Sphinx'" - echo " doc-srv Serves the documentation on port 8082 (and automatically builds it)" - echo " serve Runs the Django development server on port 8080" - echo " tox Runs complete tox test" - echo "" +# Docker compose command +DOCKER_COMPOSE_CMD := $(shell which docker-compose) +# Docker related testing is performed by using tox: +# - an environment is used to setup a testing project (docker-testing) +# - the project is setup in the temporary directory, thus this step is +# performed by every run +# - the Docker build is triggered using the corresponding Makefile +# Makefile.deployment +# - this Makefile will automatically tag the created image +# - to start the image, this tagging has to be reproduced here +# +# During project setup, this variable is set to the project's name. +# The described testing process will use 'dpstest' as the project's name +DPS_BUILD_NAME_PREFIX := "dpstest" +# While the image is build, the git commit's sha1 hash is used to tag the image +# Additionally, 'latest' is applied to the image aswell. +# In order to run the image, we rely on the 'latest' build. +DPS_BUILD_ID := "latest" -# deletes all temporary files created by Django + +# deletes all temporary and unwanted files clean: + # clean temporary Python files find . -iname "*.pyc" -delete find . -iname "__pycache__" -delete + # During development of django-project-skeleton, compiled requirement + # files are needed at some points, but the (compiled) requirements should + # not be included into version control of the skeleton. + # However, once the repository is fetched and used as a template for + # Django's startproject, the compiled requirements *should* be included + # into version control, thus, they can not simply included in a .gitignore. + rm requirements/common.txt + rm requirements/production.txt + rm configs/Docker/env.production + +init: + # TODO: Include some output to this function + # create the final version of different files by removing the '.template' suffix + mv ./deploy/apache2_vhost.sample.template ./deploy/apache2_vhost.sample + # switching the tox configuration file + mv ./util/init/tox.ini.template ./tox.ini + # switching the Makefile **should** be the last step + mv ./util/init/Makefile.template ./Makefile + +deploy/Docker/env.production: + echo "Initializing environment file for production..." + cp deploy/Docker/env.sample deploy/Docker/env.production + sed -i "s/#DPS_DJANGO_SECRET_KEY=/DPS_DJANGO_SECRET_KEY=$(shell ./util/bin/generate_secret_key.sh)/" deploy/Docker/env.production + +docker/build: deploy/Docker/env.production + tox -q -e docker-testing + +docker/build-context: + sudo $(MAKE) .docker/build-context + +docker/images: + sudo $(MAKE) .docker/images + +docker/run: docker/build + sudo $(MAKE) .docker/run -current: - tox -q -e util +run: docker/run -doc: - tox -q -e doc +tree: + tree -a -I ".git|.tox|docs|run" --dirsfirst -C | less -r -doc-srv: doc - tox -q -e doc-srv +.docker/build-context: + echo " \ + FROM busybox\n \ + COPY . /build-context\n \ + WORKDIR /build-context\n \ + CMD find ." | \ + $(DOCKER_CMD) build -t "$(DPS_BUILD_NAME_PREFIX)/build-context:latest" -f- . && \ + $(DOCKER_CMD) container run --rm "$(DPS_BUILD_NAME_PREFIX)/build-context:latest" -serve: - tox -q -e run +.docker/images: + $(DOCKER_CMD) images -tox: - tox -q +.docker/run: + DPS_BUILD_NAME_PREFIX=$(DPS_BUILD_NAME_PREFIX) DPS_BUILD_ID=$(DPS_BUILD_ID) \ + $(DOCKER_COMPOSE_CMD) -f deploy/Docker/docker-compose.yml up diff --git a/project_name/__init__.py b/config/__init__.py similarity index 100% rename from project_name/__init__.py rename to config/__init__.py diff --git a/project_name/settings/__init__.py b/config/settings/__init__.py similarity index 100% rename from project_name/settings/__init__.py rename to config/settings/__init__.py diff --git a/config/settings/common.py b/config/settings/common.py new file mode 100644 index 0000000..d7a8cf5 --- /dev/null +++ b/config/settings/common.py @@ -0,0 +1,156 @@ +# Python imports +import logging +import os +import sys + +logger = logging.getLogger(__name__) + + +# ##### PATH CONFIGURATION ################################ + +# fetch Django's project directory +DJANGO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# fetch the project_root +PROJECT_ROOT = os.path.dirname(DJANGO_ROOT) + +# the name of the whole site +SITE_NAME = os.path.basename(DJANGO_ROOT) + +# collect static files here +STATIC_ROOT = os.path.join(PROJECT_ROOT, 'run', 'static') + +# collect media files here +MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'run', 'media') + +# look for static assets here +STATICFILES_DIRS = [ + os.path.join(PROJECT_ROOT, 'static'), +] + +# look for templates here +# This is an internal setting, used in the TEMPLATES directive +PROJECT_TEMPLATES = [ + os.path.join(PROJECT_ROOT, 'templates'), +] + +# add apps/ to the Python path +sys.path.append(os.path.normpath(os.path.join(PROJECT_ROOT, 'apps'))) + + +# ##### APPLICATION CONFIGURATION ######################### + +# set the project's default timezone +TIME_ZONE = os.environ.get('DPS_TIMEZONE', 'Etc/UTC') + +# these are the apps +DEFAULT_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', +] + +# Middlewares +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +# template stuff +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': PROJECT_TEMPLATES, + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.contrib.auth.context_processors.auth', + 'django.template.context_processors.debug', + 'django.template.context_processors.i18n', + 'django.template.context_processors.media', + 'django.template.context_processors.static', + 'django.template.context_processors.tz', + 'django.contrib.messages.context_processors.messages' + ], + }, + }, +] + +# The WSGI application to be used by Django's internal servers. +# Please note, Django's 'runserver' should **not** be used in production +# environments. +# If set to 'None', 'django.core.wsgi.get_wsgi_application()' will be used to +# determine the WSGI application. +WSGI_APPLICATION = os.environ.get('DPS_DJANGO_WSGI_APP', None) + +# the root URL configuration +ROOT_URLCONF = '{}.urls'.format(SITE_NAME) + +# the URL for static files +STATIC_URL = os.environ.get('DPS_STATIC_URL', '/static/') + +# the URL for media files +MEDIA_URL = os.environ.get('DPS_DJANGO_MEDIA_URL', '/media/') + +# adjust the minimal login +LOGIN_URL = 'core_login' +LOGIN_REDIRECT_URL = os.environ.get('DPS_DJANGO_LOGIN_REDIRECT_URL', '/') +LOGOUT_REDIRECT_URL = os.environ.get('DPS_DJANGO_LOGOUT_REDIRECT_URL', 'core_login') + +# Internationalization +USE_I18N = False + +# uncomment the following line to include i18n +# from .i18n import * + + +# ##### SECURITY CONFIGURATION ############################ + +# We store the secret key here +# The required SECRET_KEY is fetched at the end of this file +SECRET_FILE = os.path.normpath(os.path.join(PROJECT_ROOT, 'run', 'SECRET.key')) + +# these persons receive error notification +ADMINS = ( + ('your name', 'your_name@example.com'), +) +MANAGERS = ADMINS + + +# ##### DEBUG CONFIGURATION ############################### +DEBUG = False + + +logger.debug('Trying to fetch SECRET_KEY from the environment...') +SECRET_KEY = os.environ.get('DPS_DJANGO_SECRET_KEY') +if SECRET_KEY is None: + logger.debug('Could not find key in the environment!') + + logger.debug('Trying to read SECRET_KEY from SECRET_FILE...') + try: + SECRET_KEY = open(SECRET_FILE).read().strip() + logger.info('Read SECRET_KEY from SECRET_FILE.') + except IOError: + logger.debug('Could not open SECRET_FILE ({})!'.format(SECRET_FILE)) + + try: + from django.utils.crypto import get_random_string + chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!$%&()=+-_' + SECRET_KEY = get_random_string(50, chars) + with open(SECRET_FILE, 'w') as f: + f.write(SECRET_KEY) + + logger.info('Generated a new SECRET_KEY and stored it in SECRET_FILE ({})!'.format(SECRET_FILE)) + except IOError: + logger.exception('Could not open SECRET_FILE ({}) for writing!'.format(SECRET_FILE)) + raise Exception('Could not open {} for writing!'.format(SECRET_FILE)) +else: + logger.info('Fetched SECRET_KEY from environment.') diff --git a/project_name/settings/development.py b/config/settings/development.py similarity index 59% rename from project_name/settings/development.py rename to config/settings/development.py index c9236d8..0f375ba 100644 --- a/project_name/settings/development.py +++ b/config/settings/development.py @@ -1,33 +1,25 @@ # Python imports -from os.path import join +import os -# project imports +# fetch the common settings from .common import * -# uncomment the following line to include i18n -# from .i18n import * - - -# ##### DEBUG CONFIGURATION ############################### -DEBUG = True +# ##### APPLICATION CONFIGURATION ######################### # allow all hosts during development ALLOWED_HOSTS = ['*'] -# adjust the minimal login -LOGIN_URL = 'core_login' -LOGIN_REDIRECT_URL = '/' -LOGOUT_REDIRECT_URL = 'core_login' +INSTALLED_APPS = DEFAULT_APPS # ##### DATABASE CONFIGURATION ############################ DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': join(PROJECT_ROOT, 'run', 'dev.sqlite3'), + 'NAME': os.path.join(PROJECT_ROOT, 'run', 'dev.sqlite3'), } } -# ##### APPLICATION CONFIGURATION ######################### -INSTALLED_APPS = DEFAULT_APPS +# ##### DEBUG CONFIGURATION ############################### +DEBUG = True diff --git a/project_name/settings/i18n.py b/config/settings/i18n.py similarity index 90% rename from project_name/settings/i18n.py rename to config/settings/i18n.py index a59e3ac..95ed624 100644 --- a/project_name/settings/i18n.py +++ b/config/settings/i18n.py @@ -1,5 +1,5 @@ # Python imports -from os.path import join +import os # Django imports from django.utils.translation import ugettext_lazy as _ @@ -10,7 +10,6 @@ # ##### INTERNATIONALIZATION ############################## LANGUAGE_CODE = 'de' -TIME_ZONE = 'Europe/Berlin' # Internationalization USE_I18N = True @@ -29,7 +28,7 @@ # Look for translations in these locations LOCALE_PATHS = ( - join(PROJECT_ROOT, 'locale'), + os.path.join(PROJECT_ROOT, 'locale'), ) # Inject the localization middleware into the right position diff --git a/config/settings/production.py b/config/settings/production.py new file mode 100644 index 0000000..6211645 --- /dev/null +++ b/config/settings/production.py @@ -0,0 +1,62 @@ +# Python imports +import os + +# fetch the common settings +from .common import * + + +# ##### APPLICATION CONFIGURATION ######################### + +# You will have to determine, which hostnames should be served by Django +# Determines, which hostnames should be served by Django. +# DPS_DJANGO_ALLOWED_HOSTS may contain a list of (acceptable) hostnames, while +# DPS_SERVER_NAME may only contain one hostname, as this value also determines +# the actual hostname, that is used by Nginx. +# +# If neither DPS_DJANGO_ALLOWED_HOSTS nor DPS_SERVER_NAME are provided, this +# will be an empty list and thus, no host will be served, rendering Django +# effectively useless. +ALLOWED_HOSTS = list( + set( + os.environ.get('DPS_DJANGO_ALLOWED_HOSTS', '').split() + + os.environ.get('DPS_SERVER_NAME', '').split() + ) +) + +INSTALLED_APPS = DEFAULT_APPS + +STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.ManifestStaticFilesStorage' + +# Don't let Django configure logging in a production environment. +# Most probably, configuring logging should be done by WSGI Server, i.e. +# Gunicorn or uWSGI. +# For Docker-based deployments, logging is configured in 'gunicorn_conf.py'. +# +# Without setting LOGGING_CONFIG to 'None', Django will setup Python's logging +# module, as described https://docs.djangoproject.com/en/3.0/topics/logging/#default-logging-configuration +LOGGING_CONFIG=None + +# ##### SECURITY CONFIGURATION ############################ + +# TODO: Make sure, that sensitive information uses https +# TODO: Evaluate the following settings, before uncommenting them +# redirects all requests to https +# SECURE_SSL_REDIRECT = True +# session cookies will only be set, if https is used +# SESSION_COOKIE_SECURE = True +# how long is a session cookie valid? +# SESSION_COOKIE_AGE = 1209600 + +# validates passwords (very low security, but hey...) +# AUTH_PASSWORD_VALIDATORS = [ +# { 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', }, +# { 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', }, +# { 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', }, +# { 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', }, +# ] + +# the email address, these error notifications to admins come from +# SERVER_EMAIL = 'root@localhost' + +# how many days a password reset should work. I'd say even one day is too long +# PASSWORD_RESET_TIMEOUT_DAYS = 1 diff --git a/project_name/urls.py b/config/urls.py similarity index 100% rename from project_name/urls.py rename to config/urls.py diff --git a/configs/.gitignore b/configs/.gitignore deleted file mode 100644 index 2762a76..0000000 --- a/configs/.gitignore +++ /dev/null @@ -1,12 +0,0 @@ -# specific .gitignore for this directory - -# ignore everything -* - -# except the explaining README -!README -# oh, and this file -!.gitignore - -# let's include samples of the config files -!*.sample diff --git a/configs/Makefile.sample b/configs/Makefile.sample deleted file mode 100644 index c094928..0000000 --- a/configs/Makefile.sample +++ /dev/null @@ -1,108 +0,0 @@ -LOCALPATH := ./ -PYTHONPATH := $(LOCALPATH)/ -PYTHON_BIN := $(VIRTUAL_ENV)/bin - -DJANGO_TEST_SETTINGS_FILE := development -DJANGO_TEST_SETTINGS := {{ project_name }}.settings.$(DJANGO_TEST_SETTINGS_FILE) -DJANGO_TEST_POSTFIX := --settings=$(DJANGO_TEST_SETTINGS) --pythonpath=$(PYTHONPATH) - - -.PHONY: all clean coverage ensure_virtual_env flake8 flake lint \ - test test/dev test/prod \ - migrate setup/dev refresh refresh/dev refresh/prod update/dev - - -all: - @echo "Hello $(LOGNAME)! Welcome to django-project-skeleton" - @echo "" - @echo " clean Removes all temporary files" - @echo " coverage Runs the tests and shows code coverage" - @echo " flake8 Runs flake8 to check for PEP8 compliance" - @echo " = flake / lint" - @echo " migrate Applies all migrations" - @echo " refresh Refreshes the project by migrating the apps and" - @echo " collecting static assets" - @echo " refresh/dev Refreshes with development settings" - @echo " refresh/prod Refreshes with production settings" - @echo " setup/dev Sets up a development environment by installing" - @echo " necessary apps, running migrations and creating" - @echo " a superuser (django::django)" - @echo " test Runs the tests" - @echo " test/dev Runs the tests with development settings" - @echo " test/prod Runs the tests with production settings" - @echo " update/dev Shortcut for setup and refresh" - - -# performs the tests and measures code coverage -coverage: ensure_virtual_env test - $(PYTHON_BIN)/coverage html - $(PYTHON_BIN)/coverage report - - -# deletes all temporary files created by Django -clean: - @find . -iname "*.pyc" -delete - @find . -iname "__pycache__" -delete - @rm -rf .coverage coverage_html - - -# most of the commands can only be used inside of the virtual environment -ensure_virtual_env: - @if [ -z $$VIRTUAL_ENV ]; then \ - echo "You don't have a virtualenv enabled."; \ - echo "Please enable the virtualenv first!"; \ - exit 1; \ - fi - - -# runs flake8 to check for PEP8 compliance -flake8: ensure_virtual_env - $(PYTHON_BIN)/flake8 . - -flake: flake8 - -lint: flake8 - - -# runs the tests -# While we just have a bare project layout, this is more or less a dummy. -test: ensure_virtual_env - @echo "Using setting file '$(DJANGO_TEST_SETTINGS_FILE)'..." - @echo "" - @$(PYTHON_BIN)/coverage run $(PYTHON_BIN)/django-admin.py check $(DJANGO_TEST_POSTFIX) - -# runs the tests with development settings -test/dev: - $(MAKE) test DJANGO_TEST_SETTINGS_FILE=development - -# runs the tests with production settings -test/prod: - $(MAKE) test DJANGO_TEST_SETTINGS_FILE=production - - -# migrates the installed apps -migrate: ensure_virtual_env - $(PYTHON_BIN)/django-admin.py migrate $(DJANGO_TEST_POSTFIX) - -# sets up the development environment by installing required dependencies, -# migrates the apps and creates a dummy user (django::django) -setup/dev: ensure_virtual_env - @pip install -r requirements/development.txt - $(MAKE) migrate DJANGO_TEST_SETTINGS_FILE=development - @echo "from django.contrib.auth.models import User; User.objects.filter(email='admin@example.com').delete(); User.objects.create_superuser(username='django', email='admin@example.com', password='django')" | python manage.py shell - -# refreshes the project by migrating and collecting static assets -refresh: ensure_virtual_env - $(MAKE) clean - $(MAKE) migrate - $(PYTHON_BIN)/django-admin.py collectstatic --noinput $(DJANGO_TEST_POSTFIX) - -# refreshes with development settings -refresh/dev: - $(MAKE) refresh DJANGO_TEST_SETTINGS_FILE=development - -# refreshes with production settings -refresh/prod: - $(MAKE) refresh DJANGO_TEST_SETTINGS_FILE=production - -update/dev: setup/dev refresh/dev diff --git a/deploy/Docker/bin/apt-get_install.sh b/deploy/Docker/bin/apt-get_install.sh new file mode 100755 index 0000000..ec2c80c --- /dev/null +++ b/deploy/Docker/bin/apt-get_install.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -euo pipefail + +# This script is used to install packages into Debian-based images. +# +# It may be called with a list of Debian packages and will automatically +# perform the refresh of apt's list (apt-get upate) and then install the +# requested packages. +# To keep the Docker images small, apt's lists are removed. +# +# This sequence of action is found in many Debian-based images, usually +# implemented by a single RUN command with several '&&'-chained commands. +# This solution was discovered in Mozilla's bedrock repository[1]. It improves +# readability of the Dockerfile and is easy enough to provide. +# +# [1]: https://github.com/mozilla/bedrock + + +# stop the script, if any of the commands fails +set -e + +# update apt's sources lists +apt-get update + +# install the specified packages and their dependencies +# do NOT install 'recommended packages' +apt-get install -y --no-install-recommends "$@" + +apt-get autoremove +apt-get clean + +# remove apt's sources lists +rm -rf /var/lib/apt/lists/* diff --git a/deploy/Docker/bin/set_timezone.sh b/deploy/Docker/bin/set_timezone.sh new file mode 100755 index 0000000..2c08552 --- /dev/null +++ b/deploy/Docker/bin/set_timezone.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Updates the container's timezone according to the provided environment +# variable DPS_TIMEZONE and sets UTC as default value. +# +# This script has to be run with root privileges in order to actually run +# `dpkg-reconfigure`. + +set -euo pipefail + +# check for the existence of DPS_TIMEZONE in environment +if [[ -v DPS_TIMEZONE ]]; then + true \ + && echo ${DPS_TIMEZONE} > /etc/timezone \ + && ln -sf /usr/share/zoneinfo/${DPS_TIMEZONE} /etc/localtime; + +# set UTC as default time zone +else + # TODO: Untestested code! Verify that this is working! Verify the existence of path /usr/share/zoneinfo/Etc/UTC! + true \ + && echo "Etc/UTC" > /etc/timezone \ + && ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime; +fi + +# actually apply the timezone systemwide +dpkg-reconfigure -f noninteractive tzdata; diff --git a/deploy/Docker/django/Dockerfile b/deploy/Docker/django/Dockerfile new file mode 100644 index 0000000..b91429f --- /dev/null +++ b/deploy/Docker/django/Dockerfile @@ -0,0 +1,103 @@ +# base this on a slim Debian image running nothing more than Python +# Why Debian and not Alpine? Personal preference of Debian-based systems. +# Additionally, Alpine is not using glibc, which might introduce problems. If +# you want your image even smaller, you may switch to an Alpine based-image. + +### BASE +# The purpose of this image is to prepare the virtual environment, that will +# be used to run the Django stack in the final image. +# +# This image will be discarded and is merely a build artifact, leveraging +# Docker's multistage building feature to reduce the final image size. +# +# TODO: If there are any system dependencies, including build dependencies for +# the Python packages, they must be installed here aswell! +FROM python:3.8-slim-buster as base + +# set environment variables +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 + +# create the virtual environment for our project +ENV VIRTUAL_ENV=/venv +ENV PATH="$VIRTUAL_ENV/bin:$PATH:/docker-bin" + +# fetch our custom install script +COPY deploy/Docker/bin/apt-get_install.sh /docker-bin/ + +RUN python3 -m venv $VIRTUAL_ENV + +# update pip inside of the virtual environment +RUN pip install --upgrade pip + +# fetch the production requirements +# TODO: actually more than the required files will be copied. Should not be a +# problem, because this image is discarded after package installation. +COPY ./requirements/*.txt /requirements/ + +# actually install the requirements into the virtual environment +RUN pip install --no-cache-dir \ + -r /requirements/common.txt \ + -r /requirements/production.txt + + +### DJANGO_PRODUCTION +# This image starts from a default debian based Python image. The virtual +# environment for running the Django stack is copied over from BASE. +# +# TODO: If there are any system dependencies, they have to be installed +# directly into this image. +FROM python:3.8-slim-buster as django_production + +# set environment variables +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 + +# actually activate this virtual environment +ENV VIRTUAL_ENV=/venv +ENV PATH="$VIRTUAL_ENV/bin:$PATH:/docker-bin" + +# fetch our custom install script +COPY deploy/Docker/bin/apt-get_install.sh /docker-bin/ + +RUN apt-get_install.sh gosu + +# provide a default value for Django's settings module +ARG DPS_DJANGO_SETTINGS_MODULE=config.settings.production +ENV DPS_DJANGO_SETTINGS_MODULE ${DPS_DJANGO_SETTINGS_MODULE} + +# create a user to be used (meaning: we won't run with root-permissions!) +RUN useradd -u 2342 -g 65534 -s /bin/false -M pythonuser +WORKDIR /webapp + +# fetch the virtual environment from base +COPY --from=base --chown=2342:65534 /venv /venv + +# TODO: Can the following statements be combined into one? +# fetch the entrypoint script (and includes) +COPY deploy/Docker/django/entrypoint_django.sh /docker-bin/ +COPY deploy/Docker/bin/set_timezone.sh /docker-bin/ + +# fetch the container's start script +COPY deploy/Docker/django/run_gunicorn.sh /docker-bin/ +COPY deploy/Docker/django/gunicorn_conf.py /docker-bin/ + +# fetch the actual project files +# TODO: copy in several steps with more/finer control +#COPY --chown=2342:65534 . . +COPY --chown=2342:65534 ./sgi ./sgi +COPY --chown=2342:65534 ./config ./config +COPY --chown=2342:65534 ./apps ./apps +COPY --chown=2342:65534 ./templates ./templates + +# TODO: This don't have to be in the image, but nice for debugging! +COPY --chown=2342:65534 ./manage.py ./manage.py + +# make Gunicorn's port 8000 available to other Docker images +EXPOSE 8000 + +# set the entrypoint script... +ENTRYPOINT ["entrypoint_django.sh"] + +# and the default command +CMD ["run_gunicorn.sh"] diff --git a/deploy/Docker/django/entrypoint_django.sh b/deploy/Docker/django/entrypoint_django.sh new file mode 100755 index 0000000..b495992 --- /dev/null +++ b/deploy/Docker/django/entrypoint_django.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -euo pipefail + +# determine the scripts current working directory (CWD) +CWD="${BASH_SOURCE%/*}"; +if [[ ! -d "$CWD" ]]; then CWD="$PWD"; fi + +# include the script for setting the timezone +source "$CWD/set_timezone.sh" + +# execute the provided command, but drop privileges using gosu +exec gosu pythonuser "$@" diff --git a/deploy/Docker/django/gunicorn_conf.py b/deploy/Docker/django/gunicorn_conf.py new file mode 100644 index 0000000..b068bf8 --- /dev/null +++ b/deploy/Docker/django/gunicorn_conf.py @@ -0,0 +1,116 @@ +# Python imports +import os + + +# bind Gunicorn to this HOST:PORT, may be a list! +# TODO: Is it really required to bind to 0.0.0.0 in Docker-based setup? +# TODO: Ensure, that :8000 ist not accessible from outside of the Docker-space +bind = '0.0.0.0:8000' +# bind = ['0.0.0.0:8000', '[::]:8000'] # provide binding for IPv4 and IPv6 + +# specify the number of workers +workers = os.environ.get('DPS_GUNICORN_NUM_WORKERS', 2) # Gunicorn default: 1 + +# specify the class of Gunicorn's workers +# make use of threaded workers +worker_class = os.environ.get('DPS_GUNICORN_WORKER_CLASS', 'gthread') # Gunicorn default: 'sync' + +# number of threads per worker (see above) +threads = os.environ.get('DPS_GUNICORN_NUM_THREADS', 4) # Gunicorn default: 1 + +# limit the lifespan of a worker by the number of handled requests +# this might prevent memory leaks +max_requests = 500 # Gunicorn default: 0 (unlimited) +max_requests_jitter = 25 # Gunicorn default: 0 (new in Gunicorn 19.2) + +# number of seconds to wait for requests on a Keep-Alive connection +# this setting is **not relevant** if using the 'sync' worker-class! +keepalive = os.environ.get('DPS_UPSTREAM_KEEPALIVE_TIMEOUT', 60) # Gunicorn default: 2 + +# fix Gunicorn's heartbeat inside of containers +# http://docs.gunicorn.org/en/stable/faq.html#how-do-i-avoid-gunicorn-excessively-blocking-in-os-fchmod +# BLUF: have a tmpfs mount for worker_tmp_dir +# TODO: investigate the Docker image and search for a tmpfs mount (or create one!) +worker_tmp_dir = '/dev/shm' # Gunicorn default: None + +# disables the use of sendfile(). +# TODO: Check this setting and research its mechanics in a setup with NGINX/Gunicorn +# sendfile = None # Gunicorn default: None + +# specify a PID file +# TODO: Check if a PID file is relevant for a Docker-based setup +# pidfile = None # Gunicorn default: None + +# Control user/group of the worker process +# TODO: Should not be necessary for a Docker-based setup, where privileges are +# droppen in the image +# user = +# group = + + +### security related configuration ### +# limit_request_line = 4096 # Gunicorn default: 4096 +# limit_request_fields = 100 # Gunicorn default: 100 +# limit_request_field_size = 8190 # Gunicorn default: 8190 + +# these settings may be relevant if operating behind a proxy, that terminates HTTPS +# The DPS setup does not require this, because NGINX sets a header field that +# is evaluated by Django +# secure_scheme_headers = +# forwarded_allow_ips = + + +### logging related configuration ### +# Gunicorn uses Python's default logging library. Generally, the logging may be +# configured freely by passing a 'logconfig' file or 'logconfig_dict' Python +# dict. +# For this configuration, only relevant parts of the logging config are +# exposed/altered to match the Docker-based production environment. + +# While running behind a reverse proxy, Gunicorn's logs will only show the +# reverse proxy's IP. +# This project's setup uses an NGINX, which sets the 'X-Forwarded-For'-header. +# See: https://stackoverflow.com/questions/25737589/gunicorn-doesnt-log-real-ip-from-nginx +access_log_format = os.environ.get( + 'DPS_GUNICORN_LOGFORMAT', + '%({x-forwarded-for}i)s %(l)s %(s)s %(l)s "%(r)s"' +) + + +logconfig_dict = { + 'version': 1, + 'disable_existing_loggers': True, + 'formatters': { + 'dps_docker_default': { + 'format': '%(asctime)-19s %(levelname)-8s [%(process)d] [%(name)s] %(message)s', + 'datefmt': '%Y-%m-%d %H:%M:%S' + }, + }, + 'handlers': { + 'docker_stdout': { + 'class': 'logging.StreamHandler', + 'formatter': 'dps_docker_default', + 'level': os.environ.get('DPS_OVERALL_LOGLEVEL', 'INFO').upper(), + 'stream': 'ext://sys.stdout', + }, + }, + 'loggers': { + # all Django logs should end up here... + 'django': { + # Django's 'mail_admins' handler is removed! + #'handlers': ['docker_stdout'], + 'level': os.environ.get('DPS_DJANGO_LOGLEVEL', 'INFO').upper(), + }, + 'gunicorn': { + 'level': os.environ.get('DPS_GUNICORN_LOGLEVEL', 'INFO').upper(), + }, + 'gunicorn.access': { + 'propagate': os.environ.get('DPS_GUNICORN_SHOW_ACCESS_LOG', 'false').lower() == 'true', + }, + }, + # the 'root' logger is just redefined to make it compatible with Docker + 'root': { + 'handlers': ['docker_stdout'], + 'level': os.environ.get('DPS_OVERALL_LOGLEVEL', 'INFO').upper(), + }, +} diff --git a/deploy/Docker/django/run_gunicorn.sh b/deploy/Docker/django/run_gunicorn.sh new file mode 100755 index 0000000..6069e29 --- /dev/null +++ b/deploy/Docker/django/run_gunicorn.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -euo pipefail + +exec gunicorn sgi.wsgi:application --config='file:/docker-bin/gunicorn_conf.py' diff --git a/deploy/Docker/docker-compose.yml b/deploy/Docker/docker-compose.yml new file mode 100644 index 0000000..18f01c8 --- /dev/null +++ b/deploy/Docker/docker-compose.yml @@ -0,0 +1,36 @@ +version: '3.5' + +services: + django: + build: + context: ../../ + dockerfile: deploy/Docker/django/Dockerfile + # providing the build and the image directive automatically tags the image + # automatic tagging is performed with environment variables provided + # through the Makefile and results in a tag with either a git commit hash + # or a datetime stamp (see Makefile.deployment) + image: ${DPS_BUILD_NAME_PREFIX}/django:${DPS_BUILD_ID} + env_file: + ./env.production + expose: + - "8000" + volumes: + - static-files:/webapp/run/static + nginx: + build: + context: ./ + dockerfile: nginx/Dockerfile + # providing the build and the image directive automatically tags the image + # automatic tagging is performed with environment variables provided + # through the Makefile and results in a tag with either a git commit hash + # or a datetime stamp (see Makefile.deployment) + image: ${DPS_BUILD_NAME_PREFIX}/nginx:${DPS_BUILD_ID} + env_file: + ./env.production + ports: + - "80:80" + volumes: + - static-files:/var/www/django_files/static + +volumes: + static-files: diff --git a/deploy/Docker/env.sample b/deploy/Docker/env.sample new file mode 100644 index 0000000..b50b1a6 --- /dev/null +++ b/deploy/Docker/env.sample @@ -0,0 +1,170 @@ +### OVERALL configuration + +# Sets the overall (minimum) log level +# +# This setting is used by +# - [django] gunicorn_conf.py +# +# Default value: INFO +#DPS_OVERALL_LOGLEVEL=INFO + +# Provide the timezone for all containers +# Must follow the list of time zones, see: +# https://en.wikipedia.org/wiki/List_of_tz_database_time_zones +# +# This setting is used by +# - [django] settings/common.py +# +# Default value: Etc/UTC +#DPS_TIMEZONE=Etc/UTC +#DPS_TIMEZONE=Europe/Berlin + +# Specifies the name that is used to identify the matching server block. +# +# Nginx supports regular expressions and wildcards, however, this environment +# variable is also used in Django's ALLOWED_HOSTS setting and is therefore +# constrained to the given requirements. +# +# See https://nginx.org/en/docs/http/server_names.html +# See https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts +# +# This setting is used by +# - [django] settings/production.py +# - [nginx] nginx.conf.template +# +# This variable **must** be set, if django-project-skeleton's Docker-based +# deployment scripts are used, to keep Django and Nginx in sync. +# However, if you are using the webapp's Dockerfile on its own (probably with +# a custon reverse proxy), you should not use this settings and refer to +# DPS_DJANGO_ALLOWED_HOSTS instead. +#DPS_SERVER_NAME=localhost + +# The url prefix for static files +# +# This setting is used by +# - [django] settings/common.py +# - [nginx] nginx.conf.template +# +# Default value: /static/ +#DPS_STATIC_URL=/static/ + +# How long should connections between Nginx and the upstream app server be +# kept open/alive. +# +# This settins is used by +# - [django] gunicorn_conf.py +# - [nginx] nginx.conf.template +# +# Default value: 60 +#DPS_UPSTREAM_KEEPALIVE_TIMEOUT=60 + + +### DJANGO related configuration + +# Django's ALLOWED_HOSTS +# In *settings/production.py* this is by default an empty list but will be +# augmented by the DPS_SERVER_NAME setting. +#DPS_DJANGO_ALLOWED_HOSTS= + +# The minimum level of log messages to show up for Django's logging +#DPS_DJANGO_LOGLEVEL=INFO + +# Django's LOGIN_REDIRECT_URL +#DPS_DJANGO_LOGIN_REDIRECT_URL=/ + +# Django's LOGOUT_REDIRECT_URL +#DPS_DJANGO_LOGOUT_REDIRECT_URL=core_login + +# Django's MEDIA_URL +# the URL to serve media files from +# While this is part of Django's configuration, this environment variable is +# used by nginx's Docker image aswell +#DPS_DJANGO_MEDIA_URL=/media/ + +# Django's SECRET_KEY +#DPS_DJANGO_SECRET_KEY= + +# Django's setting module +#DPS_DJANGO_SETTINGS_MODULE=config.settings.production + +# the (dotted) path to the WSGI app +# defaults to the app provided by Django +# see *settings/common.py* for the default value +#DPS_DJANGO_WSGI_APP=sgi.wsgi + + +### GUNICORN related configuration + +# the number of Gunicorn worker processes +#DPS_GUNICORN_NUM_WORKERS=2 + +# the used worker class +#DPS_GUNICORN_WORKER_CLASS=gthread + +# the number of threads per worker process +#DPS_GUNICORN_NUM_THREADS=4 + +# a custom log format +# +# this is the default format that plays nicely with Python's logging setup +# from gunicorn_conf.py +# Please note, that by default, Gunicorn will **not** log access logs +# see DPS_GUNICORN_SHOW_ACCESS_LOG +#DPS_GUNICORN_LOGFORMAT=%({x-forwarded-for}i)s %(l)s %(s)s %(l)s "%(r)s" +# +# this is a verbose format, that basically mimics nginx's default access log +#DPS_GUNICORN_LOGFORMAT=%({x-forwarded-for}i)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s" + +# the log level +#DPS_GUNICORN_LOGLEVEL=INFO + +# determines if the accesslog should be included in the logs +# just set this to 'true' (without ') to enable access logs +#DPS_GUNICORN_SHOW_ACCESS_LOG=false + + +### NGINX related configuration +# +# All default values are provided directly through the corresponding +# Dockerfile (configs/Docker/nginx/Dockerfile). + +# Specifies the number of preserved keepalive connections. +# See http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive +# Default value: 5 +#DPS_NGINX_UPSTREAM_KEEPALIVE_NUM_CONN=5 + +# Maximum number of requests performed by a single keepalive connection +# See http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_requests +# Default value: 100 +#DPS_NGINX_UPSTREAM_KEEPALIVE_NUM_REQ=100 + +# How long should client connections to Nginx should be kept open (in seconds) +# Default value: 60 / Nginx default: 75 +#DPS_NGINX_SERVER_KEEPALIVE=60 + +# Enables / disables emitting nginx version on error pages. +# See http://nginx.org/en/docs/http/ngx_http_core_module.html#server_tokens +# Supported values: on / build / off +# Default value: off +#DPS_NGINX_SERVER_TOKENS=off + +# Generally activates GZip compression +# Allowed value: on | off +# Default value: on / Nginx default: off +#DPS_NGINX_GZIP_ACTIVATION=on + +# GZip compression level for on the fly compression +# Allowed values: 1 - 9, where 1 is fastest and 9 is smallest +# Default value: 4 / Nginx default: 1 +#DPS_NGINX_GZIP_COMPRESSION=4 + +# Dynamic compression will only be done, if the content length is greater than +# this value. +# There is a tradeoff between the time required to perform the compression and +# the saved transfer size (and thus time). +# Default value: 500 / Nginx default: 20 +#DPS_NGINX_GZIP_MIN_LENGTH=500 + +# Activate compression for the following content types. +# Note: text/html is always compressed. +#DPS_NGINX_GZIP_TYPES=text/css font/woff diff --git a/deploy/Docker/nginx/Dockerfile b/deploy/Docker/nginx/Dockerfile new file mode 100644 index 0000000..c9bdf94 --- /dev/null +++ b/deploy/Docker/nginx/Dockerfile @@ -0,0 +1,41 @@ +FROM nginx:1.17 + +COPY bin/set_timezone.sh /docker-bin/ +COPY nginx/entrypoint_nginx.sh /docker-bin/ +COPY nginx/nginx.conf.template /docker-bin/ + +ARG DPS_SERVER_NAME=localhost +ENV DPS_SERVER_NAME ${DPS_SERVER_NAME} + +ARG DPS_STATIC_URL=/static/ +ENV DPS_STATIC_URL ${DPS_STATIC_URL} + +ARG DPS_UPSTREAM_KEEPALIVE_TIMEOUT=60 +ENV DPS_UPSTREAM_KEEPALIVE_TIMEOUT ${DPS_UPSTREAM_KEEPALIVE_TIMEOUT} + +ARG DPS_NGINX_UPSTREAM_KEEPALIVE_NUM_CONN=5 +ENV DPS_NGINX_UPSTREAM_KEEPALIVE_NUM_CONN ${DPS_NGINX_UPSTREAM_KEEPALIVE_NUM_CONN} + +ARG DPS_NGINX_UPSTREAM_KEEPALIVE_NUM_REQ=100 +ENV DPS_NGINX_UPSTREAM_KEEPALIVE_NUM_REQ ${DPS_NGINX_UPSTREAM_KEEPALIVE_NUM_REQ} + +ARG DPS_NGINX_SERVER_KEEPALIVE=60 +ENV DPS_NGINX_SERVER_KEEPALIVE ${DPS_NGINX_SERVER_KEEPALIVE} + +ARG DPS_NGINX_SERVER_TOKENS=off +ENV DPS_NGINX_SERVER_TOKENS ${DPS_NGINX_SERVER_TOKENS} + +ARG DPS_NGINX_GZIP_ACTIVATION=on +ENV DPS_NGINX_GZIP_ACTIVATION ${DPS_NGINX_GZIP_ACTIVATION} + +ARG DPS_NGINX_GZIP_COMPRESSION=4 +ENV DPS_NGINX_GZIP_COMPRESSION ${DPS_NGINX_GZIP_COMPRESSION} + +ARG DPS_NGINX_GZIP_MIN_LENGTH=500 +ENV DPS_NGINX_GZIP_MIN_LENGTH ${DPS_NGINX_GZIP_MIN_LENGTH} + +ARG DPS_NGINX_GZIP_TYPES="text/css font/woff" +ENV DPS_NGINX_GZIP_TYPES ${DPS_NGINX_GZIP_TYPES} + +ENTRYPOINT ["/docker-bin/entrypoint_nginx.sh"] +CMD ["nginx", "-g", "daemon off;"] diff --git a/deploy/Docker/nginx/entrypoint_nginx.sh b/deploy/Docker/nginx/entrypoint_nginx.sh new file mode 100755 index 0000000..767882f --- /dev/null +++ b/deploy/Docker/nginx/entrypoint_nginx.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -euo pipefail + +# determine the scripts current working directory (CWD) +CWD="${BASH_SOURCE%/*}"; +if [[ ! -d "$CWD" ]]; then CWD="$PWD"; fi + +# include the script for setting the timezone +source "$CWD/set_timezone.sh" + +envsubst ' + ${DPS_SERVER_NAME} + ${DPS_STATIC_URL} + ${DPS_UPSTREAM_KEEPALIVE_TIMEOUT} + ${DPS_NGINX_UPSTREAM_KEEPALIVE_NUM_CONN} + ${DPS_NGINX_UPSTREAM_KEEPALIVE_NUM_REQ} + ${DPS_NGINX_SERVER_KEEPALIVE} + ${DPS_NGINX_SERVER_TOKENS} + ${DPS_NGINX_GZIP_ACTIVATION} + ${DPS_NGINX_GZIP_COMPRESSION} + ${DPS_NGINX_GZIP_MIN_LENGTH} + ${DPS_NGINX_GZIP_TYPES} + ' \ + < /docker-bin/nginx.conf.template \ + > /etc/nginx/nginx.conf + +exec "$@" diff --git a/deploy/Docker/nginx/nginx.conf.template b/deploy/Docker/nginx/nginx.conf.template new file mode 100644 index 0000000..e986f96 --- /dev/null +++ b/deploy/Docker/nginx/nginx.conf.template @@ -0,0 +1,177 @@ + +worker_processes 1; + +events { + worker_connections 1024; + accept_mutex off; +} + +# TODO: drop user privileges here! +# + +http { + + # define the Django server + upstream app_server { + # The upstream is identified by the service name provided in + # docker-compose.yml. Adjust accordingly, if you run your + # nginx outside of the scope of that file / on bare metal. + server django:8000; + + # Number of preserved keepalive connections + # This settings is highly dependent on the configuration of the + # upstream servers, particurlarly the configuration of Gunicorn: + # As stated in Nginx docs, "The connections parameter should be set to + # a number small enough to let upstream servers process new incoming + # connections as well." (http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive) + # The provided default configuration of Gunicorn uses 2 workers with 4 + # threads per worker. This enables 8 (logically) concurrent requests. + # Nginx's 'keepalive' setting should be lower than this maximum number + # of concurrent requests. + keepalive ${DPS_NGINX_UPSTREAM_KEEPALIVE_NUM_CONN}; + + # Number of requests per keepalive connection + # Nginx default value: 100 + # As stated in Nginx docs, "Closing connections periodically is + # necessary to free per-connection memory allocations." + # (http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_requests) + # The provided default configuration of Gunicorn will handle 500 + # requests per process, before restarting the worker; this is not + # directly dependent on each other. + keepalive_requests ${DPS_NGINX_UPSTREAM_KEEPALIVE_NUM_REQ}; + + # Timeout for idle keepalive connections + # This setting should be synced with Gunicorn's 'keepalive' setting. + # Gunicorn's default value is 2, Nginx's one 60. + # In the provided setup, Nginx is the only "client" talking to + # Gunicorn, so a higher value for this setting should be acceptable. + # http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_timeout + keepalive_timeout ${DPS_UPSTREAM_KEEPALIVE_TIMEOUT}; + } + + server { + # Specifies the name that is used to identify the matching server + # block. Nginx supports wildcards aswell as regular expressions. See + # https://nginx.org/en/docs/http/server_names.html for further details. + server_name ${DPS_SERVER_NAME}; + + # TODO: Let's make this run with HTTP, but should finally be switched + # to HTTPS. + listen 80; + + # serve the webapp's static files directly by Nginx + location ${DPS_STATIC_URL} { + # The Docker volume for static files is mounted to + # /var/www/django_files/static (see docker-compose.yml). + alias /var/www/django_files/static/; + + # disable directory listings for this location + autoindex off; + } + + location / { + # Proxy requests to the upstream servers defined above + proxy_pass http://app_server; + + # Disable caching of upstream completely + # Caching is only done by the upstream application server (Django). + # This might be subject to change, if microcaching is desirable to + # enhance the setups performance, but is highly dependent on the + # actual application. + proxy_cache off; + + # Enable HTTP/1.1 for the communication with upstream servers. + # This setting is required to actually make use of keepalive + # connections between Nginx and the upstream servers. + proxy_http_version 1.1; + + # Drop the clients connection header (could contain "close" to + # close a keepalive connection). + proxy_set_header Connection ""; + + # TODO: Check, if this is necessary to fix redirects from Django + # views or leave 'off' + proxy_redirect off; + + # Provide the requester's address + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # Provide the original request's protocol + # FIXME: When this setup is switched to https, probably the + # following Django setting is relevant: + # - SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') + proxy_set_header X-Forwarded-Proto $scheme; + + # Provide the (originally requested) hostname to the upstream + # + # DONE: develop a working combination of + # - setting the Host header + # - setting the X-Forwarded-Host header + # - Django's ALLOWED_HOSTS setting + # - Django's USE_X_FORWARDED_HOST setting + # that makes CSRF-protected requests work correctly. + # See: + # - https://stackoverflow.com/a/48646798 + # - https://ubuntu.com/blog/django-behind-a-proxy-fixing-absolute-urls + # - https://medium.com/@rui.jorge.rei/today-i-learned-nginx-reverse-proxying-for-django-projects-3ab17ad707f6 + # - https://code.djangoproject.com/ticket/9064 + # + # Current solution: + # - use "proxy_set_header Host $host;" to set a correct + # "Host" header when passing the request to upstream + # - "$host" will contain the value of the "Host" header + # of the request or the primary server name + # (see http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_set_header) + # - the primary server name is controllable by an + # environment variable; so it is usable in Django's + # ALLOWED_HOSTS aswell + # - this introduces additional constraints on the value + # of the environment variable. It must be compatible + # with Nginx's and Django's requirements + # - do NOT set "X-Forwarded-Host" at all + # - do NOT use Django's USE_X_FORWARDED_HOST + # + # Provide the originally requested host + proxy_set_header Host $host; + } + + # TODO: This *should* be synchronized with Django's + # DATA_UPLOAD_MAX_MEMORY_SIZE, which defaults to 2621440 byte=2.5MB. + # This *could* be adjustable with environment variables, but on the + # other hand, this might be a very specific setting and adjustment + # only is necessary for specific applications that typically handle + # larger files. For now, it is just set to Django's default value. + client_max_body_size 2621440; + + # set the timeout for keep-alive connections for the whole server + keepalive_timeout ${DPS_NGINX_SERVER_KEEPALIVE}; + + # generally activate Gzip compression + gzip ${DPS_NGINX_GZIP_ACTIVATION}; + # set the compression level for on the fly compression + gzip_comp_level ${DPS_NGINX_GZIP_COMPRESSION}; + # disable gzip for some browsers + gzip_disable msie6; + # minimum content length (in byte) to be compressed + gzip_min_length ${DPS_NGINX_GZIP_MIN_LENGTH}; + # enable compression for all requests + gzip_proxied any; + # enable compression for the following content-types + # Note: "text/html" is always compressed! + gzip_types ${DPS_NGINX_GZIP_TYPES}; + + } + + # if no server_name matches, close the connection to prevent host spoofing + server { + listen 80 default_server; + return 444; + } + + # automatically detect mime types + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # disable server tokens (on error pages / headers) + server_tokens ${DPS_NGINX_SERVER_TOKENS}; +} diff --git a/configs/README b/deploy/README similarity index 100% rename from configs/README rename to deploy/README diff --git a/configs/apache2_vhost.sample b/deploy/apache2_vhost.sample.template similarity index 100% rename from configs/apache2_vhost.sample rename to deploy/apache2_vhost.sample.template diff --git a/doc/.gitignore b/docs/.gitignore similarity index 100% rename from doc/.gitignore rename to docs/.gitignore diff --git a/doc/Makefile b/docs/Makefile similarity index 100% rename from doc/Makefile rename to docs/Makefile diff --git a/doc/source/apache2_vhost.rst b/docs/source/apache2_vhost.rst similarity index 100% rename from doc/source/apache2_vhost.rst rename to docs/source/apache2_vhost.rst diff --git a/doc/source/conf.py b/docs/source/conf.py similarity index 100% rename from doc/source/conf.py rename to docs/source/conf.py diff --git a/doc/source/index.rst b/docs/source/index.rst similarity index 100% rename from doc/source/index.rst rename to docs/source/index.rst diff --git a/doc/source/quickstart.rst b/docs/source/quickstart.rst similarity index 100% rename from doc/source/quickstart.rst rename to docs/source/quickstart.rst diff --git a/doc/source/settings.rst b/docs/source/settings.rst similarity index 100% rename from doc/source/settings.rst rename to docs/source/settings.rst diff --git a/doc/source/structure.rst b/docs/source/structure.rst similarity index 100% rename from doc/source/structure.rst rename to docs/source/structure.rst diff --git a/doc/source/versions.rst b/docs/source/versions.rst similarity index 100% rename from doc/source/versions.rst rename to docs/source/versions.rst diff --git a/manage.py b/manage.py index c257e48..fe1150d 100755 --- a/manage.py +++ b/manage.py @@ -1,22 +1,38 @@ #!/usr/bin/env python + +"""Django's command-line utility for administrative tasks. + +This is based on Django's default 'manage.py', but slightly modified to be +compatible with the custom project layout provided by +'django-project-skeleton'.""" + +# Python imports import os import sys -if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "{{ project_name }}.settings.development") + +def main(): + # By default, 'manage.py' will use the development settings, provided + # in 'config.settings.development'. This might be overruled by explicitly + # setting DJANGO_SETTINGS_MODULE or using DPS_DJANGO_SETTINGS_MODULE. + os.environ.setdefault( + 'DJANGO_SETTINGS_MODULE', + os.environ.get( + 'DPS_DJANGO_SETTINGS_MODULE', + 'config.settings.development' + ) + ) + try: from django.core.management import execute_from_command_line - except ImportError: - # The above import may fail for some other reason. Ensure that the - # issue is really that Django is missing to avoid masking other - # exceptions on Python 2. - try: - import django - except ImportError: - raise ImportError( - "Couldn't import Django. Are you sure it's installed and " - "available on your PYTHONPATH environment variable? Did you " - "forget to activate a virtual environment?" - ) - raise + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/project_name/settings/common.py b/project_name/settings/common.py deleted file mode 100644 index 6b835a6..0000000 --- a/project_name/settings/common.py +++ /dev/null @@ -1,128 +0,0 @@ -# Python imports -from os.path import abspath, basename, dirname, join, normpath -import sys - - -# ##### PATH CONFIGURATION ################################ - -# fetch Django's project directory -DJANGO_ROOT = dirname(dirname(abspath(__file__))) - -# fetch the project_root -PROJECT_ROOT = dirname(DJANGO_ROOT) - -# the name of the whole site -SITE_NAME = basename(DJANGO_ROOT) - -# collect static files here -STATIC_ROOT = join(PROJECT_ROOT, 'run', 'static') - -# collect media files here -MEDIA_ROOT = join(PROJECT_ROOT, 'run', 'media') - -# look for static assets here -STATICFILES_DIRS = [ - join(PROJECT_ROOT, 'static'), -] - -# look for templates here -# This is an internal setting, used in the TEMPLATES directive -PROJECT_TEMPLATES = [ - join(PROJECT_ROOT, 'templates'), -] - -# add apps/ to the Python path -sys.path.append(normpath(join(PROJECT_ROOT, 'apps'))) - - -# ##### APPLICATION CONFIGURATION ######################### - -# these are the apps -DEFAULT_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', -] - -# Middlewares -MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', -] - -# template stuff -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': PROJECT_TEMPLATES, - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.contrib.auth.context_processors.auth', - 'django.template.context_processors.debug', - 'django.template.context_processors.i18n', - 'django.template.context_processors.media', - 'django.template.context_processors.static', - 'django.template.context_processors.tz', - 'django.contrib.messages.context_processors.messages' - ], - }, - }, -] - -# Internationalization -USE_I18N = False - - -# ##### SECURITY CONFIGURATION ############################ - -# We store the secret key here -# The required SECRET_KEY is fetched at the end of this file -SECRET_FILE = normpath(join(PROJECT_ROOT, 'run', 'SECRET.key')) - -# these persons receive error notification -ADMINS = ( - ('your name', 'your_name@example.com'), -) -MANAGERS = ADMINS - - -# ##### DJANGO RUNNING CONFIGURATION ###################### - -# the default WSGI application -WSGI_APPLICATION = '%s.wsgi.application' % SITE_NAME - -# the root URL configuration -ROOT_URLCONF = '%s.urls' % SITE_NAME - -# the URL for static files -STATIC_URL = '/static/' - -# the URL for media files -MEDIA_URL = '/media/' - - -# ##### DEBUG CONFIGURATION ############################### -DEBUG = False - - -# finally grab the SECRET KEY -try: - SECRET_KEY = open(SECRET_FILE).read().strip() -except IOError: - try: - from django.utils.crypto import get_random_string - chars = 'abcdefghijklmnopqrstuvwxyz0123456789!$%&()=+-_' - SECRET_KEY = get_random_string(50, chars) - with open(SECRET_FILE, 'w') as f: - f.write(SECRET_KEY) - except IOError: - raise Exception('Could not open %s for writing!' % SECRET_FILE) diff --git a/project_name/settings/production.py b/project_name/settings/production.py deleted file mode 100644 index 273968e..0000000 --- a/project_name/settings/production.py +++ /dev/null @@ -1,33 +0,0 @@ -# for now fetch the development settings only -from .development import * - -# turn off all debugging -DEBUG = False - -# You will have to determine, which hostnames should be served by Django -ALLOWED_HOSTS = [] - -# ##### SECURITY CONFIGURATION ############################ - -# TODO: Make sure, that sensitive information uses https -# TODO: Evaluate the following settings, before uncommenting them -# redirects all requests to https -# SECURE_SSL_REDIRECT = True -# session cookies will only be set, if https is used -# SESSION_COOKIE_SECURE = True -# how long is a session cookie valid? -# SESSION_COOKIE_AGE = 1209600 - -# validates passwords (very low security, but hey...) -# AUTH_PASSWORD_VALIDATORS = [ -# { 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', }, -# { 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', }, -# { 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', }, -# { 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', }, -# ] - -# the email address, these error notifications to admins come from -# SERVER_EMAIL = 'root@localhost' - -# how many days a password reset should work. I'd say even one day is too long -# PASSWORD_RESET_TIMEOUT_DAYS = 1 diff --git a/project_name/wsgi.py b/project_name/wsgi.py deleted file mode 100644 index 53a1acb..0000000 --- a/project_name/wsgi.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -WSGI config for {{ project_name }} project. - -It exposes the WSGI callable as a module-level variable named ``application``. - -For more information on this file, see -https://docs.djangoproject.com/en/1.8/howto/deployment/wsgi/ -""" - -import os -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "{{ project_name }}.settings.production") - -from django.core.wsgi import get_wsgi_application -application = get_wsgi_application() diff --git a/requirements/Makefile b/requirements/Makefile new file mode 100644 index 0000000..6b625d9 --- /dev/null +++ b/requirements/Makefile @@ -0,0 +1,29 @@ +# Makefile to compile/update the actual requirements files using pip-tools' +# 'pip-compile' command. + +# get a list of all *.in files +OBJECTS = $(wildcard *.in) + +# construct the list of output files +OUTPUTS := $(OBJECTS:.in=.txt) + +# Make the command to actually compile the *.in files into *.txt configurable; +# the provided 'pip-compile' should be working within a 'tox' environment. +COMPILE_CMD = pip-compile + + +### RECEIPTS ### + +# by default update all requirements +all: $(OUTPUTS) + +# this is the actually compile step in generic form +%.txt: %.in + $(COMPILE_CMD) --allow-unsafe --generate-hashes --output-file $@ $< + + +# dependency chain +development.txt: common.txt +production.txt: common.txt + +.PHONY: all diff --git a/requirements/common.in b/requirements/common.in new file mode 100644 index 0000000..6473c7a --- /dev/null +++ b/requirements/common.in @@ -0,0 +1,2 @@ +# currently used Django version +Django>=2.2,<3.0 diff --git a/requirements/development.in b/requirements/development.in new file mode 100644 index 0000000..e69de29 diff --git a/requirements/development.txt b/requirements/development.txt deleted file mode 100644 index 79be0ae..0000000 --- a/requirements/development.txt +++ /dev/null @@ -1,15 +0,0 @@ -# coverage measures code coverage of Python programs -# See http://coverage.readthedocs.io/en/latest/ -coverage - -# **tox** automates testing -# See https://tox.readthedocs.io/en/latest/index.html -tox - -# **Sphinx** is used to create documentation -# See http://www.sphinx-doc.org/en/stable/tutorial.html -Sphinx - -# **flake8** is used to check for PEP8 compliance -# See http://flake8.pycqa.org/en/latest/index.html -flake8 diff --git a/requirements/production.in b/requirements/production.in new file mode 100644 index 0000000..aff4b80 --- /dev/null +++ b/requirements/production.in @@ -0,0 +1,2 @@ +-c common.txt +gunicorn>=19.8 diff --git a/sgi/__init__.py b/sgi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sgi/asgi.py b/sgi/asgi.py new file mode 100644 index 0000000..e26ba36 --- /dev/null +++ b/sgi/asgi.py @@ -0,0 +1,33 @@ +""" +ASGI config for {{ project_name }} project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/{{ docs_version }}/howto/deployment/asgi/ + +This is based on Django's default 'asgi.py', but slightly modified to be +compatible with the custom project layout provided by +'django-project-skeleton'.""" + +# Python imports +import os + +# Django imports +from django.core.wsgi import get_asgi_application + +# Provide a default settings module +# WSGI is used to deploy the project, so the default value are the production +# settings. This may be overwritten in actual deployment setups. +# This defaults to 'config.settings.production' but may be adjusted either by +# settings DJANGO_SETTINGS_MODULE directly or using DPS_DJANGO_SETTINGS_MODULE. +os.environ.setdefault( + 'DJANGO_SETTINGS_MODULE', + os.environ.get( + 'DPS_DJANGO_SETTINGS_MODULE', + 'config.settings.production' + ) +) + +# actually expose an application +application = get_asgi_application() diff --git a/sgi/wsgi.py b/sgi/wsgi.py new file mode 100644 index 0000000..dd383f6 --- /dev/null +++ b/sgi/wsgi.py @@ -0,0 +1,33 @@ +""" +WSGI config for {{ project_name }} project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/{{ docs_version }}/howto/deployment/wsgi/ + +This is based on Django's default 'wsgi.py', but slightly modified to be +compatible with the custom project layout provided by +'django-project-skeleton'.""" + +# Python imports +import os + +# Django imports +from django.core.wsgi import get_wsgi_application + +# Provide a default settings module +# WSGI is used to deploy the project, so the default value are the production +# settings. This may be overwritten in actual deployment setups. +# This defaults to 'config.settings.production' but may be adjusted either by +# settings DJANGO_SETTINGS_MODULE directly or using DPS_DJANGO_SETTINGS_MODULE. +os.environ.setdefault( + 'DJANGO_SETTINGS_MODULE', + os.environ.get( + 'DPS_DJANGO_SETTINGS_MODULE', + 'config.settings.production' + ) +) + +# actually expose an application +application = get_wsgi_application() diff --git a/tox.ini b/tox.ini index 7a15db0..a1d688d 100644 --- a/tox.ini +++ b/tox.ini @@ -2,11 +2,7 @@ [tox] envlist = - django17-py{27,32,33,34} - django18-py{27,33,34,35} - django19-py{27,34,35} - django110-py{27,34,35} - django111-py{27,34,35,36,37} + django111-py{34,35,36,37} django20-py{34,35,36,37} django21-py{35,36,37} django22-py{35,36,37,38} @@ -22,42 +18,51 @@ skip_missing_interpreters = true setenv = PYTHONDONTWRITEBYTECODE=1 deps = - django17: Django>=1.7, <1.8 - django18: Django>=1.8, <1.9 - django19: Django>=1.9, <1.10 - django110: Django>=1.10, <1.11 django111: Django>=1.11, <2.0 django20: Django>=2.0, <2.1 django21: Django>=2.1, <2.2 django22: Django>=2.2, <3.0 django30: Django>=3.0, <3.1 -changedir = {envtmpdir} +changedir = {envtmpdir}/dpstest commands = - django-admin startproject --template={toxinidir} foo - {envbindir}/python foo/manage.py check --settings=foo.settings.development + django-admin startproject --extension=py,template --template={toxinidir} dpstest . + {envbindir}/python manage.py check [testenv:util] basepython = python3 envdir = {toxworkdir}/current deps = - Django>=2.2, <3.0 + -r{toxinidir}/requirements/development.txt skip_install = true -changedir = {envtmpdir} +changedir = {envtmpdir}/dpstest commands = - django-admin startproject --template={toxinidir} foo - {posargs:{envbindir}/python foo/manage.py check --settings=foo.settings.development} + django-admin startproject --extension=py,template --template={toxinidir} dpstest . + {posargs:{envbindir}/python manage.py check} -[testenv:run] +[testenv:docker-testing] basepython = {[testenv:util]basepython} envdir = {[testenv:util]envdir} deps = {[testenv:util]deps} skip_install = {[testenv:util]skip_install} -changedir = {[testenv:util]changedir} +changedir = {envtmpdir}/dpstest +whitelist_externals = + make +commands = + django-admin startproject --extension=py,template --template={toxinidir} dpstest . + make init + {posargs:make docker/build} + +[testenv:build_requirements] +basepython = {[testenv:util]basepython} +envdir = {toxworkdir}/build_requirements +deps = + pip-tools +skip_install = true +changedir = {toxinidir}/requirements +whitelist_externals = + make commands = - django-admin startproject --template={toxinidir} foo - {envbindir}/python foo/manage.py migrate --settings=foo.settings.development - {envbindir}/python foo/manage.py createsuperuser --settings=foo.settings.development - {envbindir}/python foo/manage.py runserver 0:8080 --settings=foo.settings.development + {posargs:make} [testenv:doc] basepython = {[testenv:util]basepython} @@ -82,7 +87,6 @@ commands = python -m http.server {posargs:8082} # Python3 command - ################################################################################ # The following sections actually provide settings for various tools ################################################################################ diff --git a/util/bin/generate_secret_key.sh b/util/bin/generate_secret_key.sh new file mode 100755 index 0000000..1f9abca --- /dev/null +++ b/util/bin/generate_secret_key.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -euo pipefail + +# This script is part of django-project-skeleton and is used to generate an +# random string to be used as SECRET_KEY for Django's configuration. +# +# Ths script is - obviously - no reimplementation of the corresponding +# function provided by Django, but should provide a value, that may be +# considered "random enough". +# The code is taken from https://gist.github.com/earthgecko/3089509 , see +# https://github.com/django/django/blob/9386586f31b8a0bccf59a1bff647cd829d4e79aa/django/core/management/utils.py#L77 +# for Django's original implementation. +# +# '&' had to be removed from the available characters, because it breaks 'sed'. +# +# This script is used to generate a SECRET_KEY in the env.production file. +# That file is **NOT** under version control, for obvious reasons. + +cat /dev/urandom | tr -dc 'a-zA-Z0-9!@$%^()_-' | fold -w 50 | head -n 1 diff --git a/util/bin/get_build_id.sh b/util/bin/get_build_id.sh new file mode 100755 index 0000000..2375a43 --- /dev/null +++ b/util/bin/get_build_id.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +set -euo pipefail + +# This script is part of django-project-skeleton and is used to determine an +# identifier to tag Docker images. +# +# The project's Makefile will call this script and then use its output, beside +# other values/constants to construct the image's tag. +# +# Preferably the project is version controlled by git. The script will +# determine the latest commit sha1 hash (referenced by HEAD) and will also +# determine, if there are unstaged/uncommitted changes in the repository. +# The sha1 hash will be used to tag the image (see Makefile), optionally with +# an indicator for an dirty repository (during testing/staging/deployment you +# will not want 'dirty' images, because they are not deterministically +# reproducible. Nonetheless, those images will be created and may be used). +# +# If the project is not version controlled, a timestamp will be used instead. +# This is *not* recommended, because it will not allow you to recreate a +# specific image, making debugging nearly impossible. +# Additionally, the timestamp depends on the build system's timezone, +# introducing further problems, especially if the image should be used by +# people based in other timezones. +# +# TL;DR: +# - use git to version control your project +# - use the provided Makefile to build your Docker images +# - profit + +# determine, if the project is under version control by git +if git rev-parse --is-inside-work-tree > /dev/null 2>&1; then + + # get most recent git commit sha1 hash + sha=$(git rev-parse --short HEAD -- 2>/dev/null); + + # use the porcelain of git status to determine, if there a unstaged + # changes in the repository + # TODO: How portable is this? Should this be switched to a plumbing cmd? + if test -n "$(git status --porcelain -- 2>/dev/null)"; then + /usr/bin/printf "%s-dirty\n" $sha; + else + /usr/bin/printf "%s\n" $sha; + fi +# Oh noooo, git is not used... Provide a timestamp to identify the image! +# This should be the absolutely last resort to tag an image and is *not* +# recommended. You should really be using a VCS! +else + # YYYYmmdd-HHMM -> 20200212-1953 + # Please note, that this depends on your system's timezone, which makes + # identifying the resulting image *really* hard. + # Additionally, this will *not* allow rebuilding images in a deterministic + # way. + date +%Y%m%d-%H%M 2>/dev/null; +fi diff --git a/util/init/Makefile.template b/util/init/Makefile.template new file mode 100644 index 0000000..7cdaf57 --- /dev/null +++ b/util/init/Makefile.template @@ -0,0 +1,336 @@ +##### Makefile configuration section ##### + + +### Docker related constants ### + +# Docker images will be tagged with the following naming scheme: +# {[registry hostname]/[project name]}/[service]:[DPS_BUILD_ID] +# - registry hostname: optionally specifies a registry hostname +# - project name: the project's name as defined during 'startproject' +# - service: defines the service represented by the image, e.g. Django, +# nginx, postgre, ... The service is set in 'docker-compose.yml' and +# is hardcoded there. +# - DPS_BUILD_ID: automatically determined ID. Will fetch the current +# git commit SHA1 hash or provide a timestamp. +# See bin/get_build_id.sh +# +# DPS_BUILD_NAME_PREFIX is used to provide the [registry hostname], [hostname] +# and [project name] (by default no [registry hostname] is provided). +DPS_BUILD_NAME_PREFIX := "{{ project_name }}" + + +### internal setup ### +# These variables/constants define, how the project is managed. + +# main interface to run Django management commands +# +# internal/tox/dev-django will use tox environments to run Django managements +# commands. +# internal/django/raw will directly call the Django management commands. +.internal/django-mgmt: .internal/tox/dev-django +# .internal/django-mgmt: .internal/django/raw + +# tox command (this assumes, that tox is available in your Python environment +# and your PATH is setup to allow direct calls) +TOX_CMD := tox -q -e + +# cmd to run django management commands +# if you rather use 'manage.py' instead of 'django-admin', switch the lines +# +# this assumes, that django is available in your Python environment and your +# PATH is setup to allow direct calls) +# DJANGOADMIN_CMD := django-admin +# +# this assumes, that your Python executable is in your PATH and is indeed +# accessible with python. Adjust according to your system's configuration +DJANGOADMIN_CMD := python manage.py + +# easily control the verbosity of Django commands +DJANGOADMIN_CMD_VERBOSITY := --verbosity=1 # {0,1,2,3}; 1 is default by Django + +# Docker command +DOCKER_CMD := $(shell which docker) + +# Docker compose command +DOCKER_COMPOSE_CMD := $(shell which docker-compose) + + +### shortcut commands ### +run: dev/runserver +build: docker/build + + +### development commands ### +dev/check: django/check +dev/migrate: django/migrate +dev/runserver: django/runserver-external4 +dev/static: django/staticfiles/find + + +### Django management commands ### +django_cmd ?= "" + +django/check: + $(MAKE) .internal/django-mgmt django_cmd="check" + +django/compilemessages: + $(MAKE) .internal/django-mgmt django_cmd="compilemessages" + +django/createcachetable: + $(MAKE) .internal/django-mgmt django_cmd="createcachetable" + +django/dbshell: + # requires the command line client to be available in user's PATH + # dynamically determines the desired command line client from settings file + $(MAKE) .internal/django-mgmt django_cmd="dbshell" + +django/diffsettings: + # compare with Django's default settings + # NOTE: compares the project's **development** settings with Django's defaults + $(MAKE) .internal/django-mgmt django_cmd="diffsettings" + +django/diffsettings-all: + # show all settings, even if they correspond with Django's defaults + # NOTE: compares the project's **development** settings with Django's defaults + $(MAKE) .internal/django-mgmt django_cmd="diffsettings --all" + +django/diffsettings-prod: + # compare **development** settings with **production** settings + $(MAKE) .internal/django-mgmt django_cmd="diffsettings --default={{ project_name }}.settings.production" + +django/dumpdata: + # apply indentation to increase readability + $(MAKE) .internal/django-mgmt django_cmd="dumpdata --indent=4" + +django/flush: + # flushes the database + $(MAKE) .internal/django-mgmt django_cmd="flush" + +django/flush-plan: django/sqlflush + +django/flush-noinput: + # flushes the database without any prompts + $(MAKE) .internal/django-mgmt django_cmd="flush --no-input" + +django/inspectdb: + # will not be wrapped, because this is a very edgy use case + $(ECHO) "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments.\n" + $(ECHO) "I propose to run the command manually with your specific requirements, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"inspectdb [table [table ...]]\"$(CRES).\n" + +django/loaddata: + $(ECHO) "This command is currently not implemented. If you are frequently loading fixtures into your database, you may modify the project's Makefile $(CCYA)\"django/loaddata\"$(CRES) target.\n" + +django/makemessages: + # TODO: Test this command! Is it sufficient or should '--all' be passed aswell? + $(MAKE) .internal/django-mgmt django_cmd="makemessages" + +django/makemigrations: + # this command is provided for completeness, but should not be necessary for a Django project (relevance for apps!) + $(MAKE) .internal/django-mgmt django_cmd="makemigrations" + +django/migrate: + # synchronize your models with your database + $(MAKE) .internal/django-mgmt django_cmd="migrate" + +django/migrate-plan: + # show the migration operations, without actually executing them + $(MAKE) .internal/django-mgmt django_cmd="migrate --plan" + +django/runserver: django/migrate + # runs the development server with default settings + # this will expose port 8000 on localhost, so it will not be accessible from another machine. + # see 'runserver-external[4|6]' for an alternative + $(MAKE) .internal/django-mgmt django_cmd="runserver" + +django/runserver-external4: django/migrate + # runs the development server with settings that allow access from another machine + # the server will be exposed on port 8000 + $(MAKE) .internal/django-mgmt django_cmd="runserver 0.0.0.0:8000" + +django/runserver-external6: django/migrate + # runs the development server with settings that allow access from another machine + # the server will be exposed on port 8000 + $(MAKE) .internal/django-mgmt django_cmd="runserver -6" + +django/sendtestemail: + # will not be wrapped, because this is a very edgy use case + $(ECHO) "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments.\n" + $(ECHO) "I propose to run the command manually with your specific requirements, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"sendtestemail [email [email ...]]\"$(CRES).\n" + +django/shell: + # runs a python shell with the project's development settings + $(MAKE) .internal/django-mgmt django_cmd="shell" + +django/showmigrations: + $(MAKE) .internal/django-mgmt django_cmd="showmigrations" + +django/showmigrations-plan: + # show the migration operations, without actually executing them + $(MAKE) .internal/django-mgmt django_cmd="showmigrations --plan" + +django/sqlflush: + # prints the SQL statements that would be executed for a 'flush' (see above) + $(MAKE) .internal/django-mgmt django_cmd="sqlflush" + +django/sqlmigrate: + # will not be wrapped, because this is a very edgy use case + $(ECHO) "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments.\n" + $(ECHO) "I propose to run the command manually with your specific requirements, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"sqlmigrate app_label migration_name\"$(CRES).\n" + +django/sqlsequencereset: + # will not be wrapped, because this is a very edgy use case + $(ECHO) "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments.\n" + $(ECHO) "I propose to run the command manually with your specific requirements, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"sqlsequencereset app_label [app_label ...]\"$(CRES).\n" + +django/squashmigrations: + # will not be wrapped, because this is a very edgy use case + # this command is provided for completeness, but should not be necessary for a Django project (relevance for apps!) + $(ECHO) "This command is beyond the scope of this project skeleton and requires manual, project-specific adjustments.\n" + $(ECHO) "I propose to run the command manually with your specific requirements, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"squashmigrations app_label [start_migration_name] migration_name\"$(CRES).\n" + +django/startapp: + # will not be wrapped, because this is a very edgy use case + $(ECHO) "This command is beyond the scope of this project skeleton and too complex to be support by a Makefile.\n" + $(ECHO) "I propose to run the command manually with your specific requirements, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"startapp name [directory]\"$(CRES).\n" + $(ECHO) "[directory] should be specified as $(CCYA)\"apps/app_name\"$(CRES) if you wish to develop the app inside of this project.\n" + # TODO: include a link to the definition of 'pluggable apps' for reference and to explain, why this might be a bad idea! + +django/startproject: + # will not be wrapped - obviously. This Makefile aims to manage an existing project + $(ECHO) "$(CRED)This command is not provided. When you read this message, you should already have created a project.$(CRES)\n" + +django/test: + # will not be wrapped. Testing is handled in other make targets. + $(ECHO) "$(CRED)This command is not provided. Testing should be performed using the dedicated test related targets.$(CRES)\n" + $(ECHO) "Run $(CCYA)\"make help\"$(CRES) for a list of commands and focus on the $(CCYA)\"testing\"$(CRES) section.\n" + +django/testserver: + # will not be wrapped. Testing is handled in other make targets. + $(ECHO) "$(CRED)This command is not provided. Testing should be performed using the dedicated test related targets.$(CRES)\n" + $(ECHO) "Run $(CCYA)\"make help\"$(CRES) for a list of commands and focus on the $(CCYA)\"testing\"$(CRES) section.\n" + +django/version: + $(MAKE) .internal/django-mgmt django_cmd="version" + +django/auth/changepassword: + # TODO: test this! with a matching user account and without... + $(ECHO) "Trying to change the password of a Django user that matches your username...\n" + $(ECHO) "If you want to change the password of another user, either using $(CCYA)django-admin$(CRES) directly or by $(CCYA)tox -e dev-django -- \"changepassword \"$(CRES).\n" + $(MAKE) .internal/django-mgmt django_cmd="changepassword" + +django/auth/createsuperuser: + $(MAKE) .internal/django-mgmt django_cmd="createsuperuser" + +django/auth/createsuperuser-noinput: + # TODO: This feature will be added, if Django>3.0 will be the main supported version! + $(ECHO) "This feature will be implemented in a future release!\n" + $(MAKE) django/auth/createsuperuser + +django/contenttypes/remove_stale: + # only available if django.contrib.contenttypes is in INSTALLED_APPS + # TODO: check documentation; subcommands required? + $(MAKE) .internal/django-mgmt django_cmd="remove_stale_contenttypes" + +django/sessions/clear: + # only available if django.contrib.sessions is in INSTALLED_APPS + # TODO: check documentation; subcommands required? + $(MAKE) .internal/django-mgmt django_cmd="clearsessions" + +django/staticfiles/collect: + # only available if django.contrib.staticfiles is in INSTALLED_APPS + # TODO: check documentation; subcommands required? + $(MAKE) .internal/django-mgmt django_cmd="collectstatic" + +django/staticfiles/find: + # only available if django.contrib.staticfiles is in INSTALLED_APPS + # TODO: check documentation; subcommands required? + $(MAKE) .internal/django-mgmt django_cmd="findstatic" + + +### Docker commands ### + +docker/build: util/requirements deploy/Docker/env.production + sudo $(MAKE) .internal/docker/build + +docker/run: docker/run/django + +docker/run/django: + sudo $(MAKE) .internal/docker/run/django + + +### Environment management ### + +deploy/Docker/env.production: + # this is explicitly **NOT** a phony target, because the production + # settings must only be generated **ONE TIME**. + # Some of these settings are initialised with random values, but hve to be + # constant for the lifetime of a project, e.g. the SECRET_KEY. + $(ECHO) "Initializing environment file for production...\n" + cp deploy/Docker/env.sample deploy/Docker/env.production + sed -i "s/#DPS_DJANGO_SECRET_KEY=/DPS_DJANGO_SECRET_KEY=$(shell ./bin/generate_secret_key.sh)/" deploy/Docker/env.production + + +### Utility commands ### + +util/requirements: + $(TOX_CMD) build_requirements + +util/requirements-force: + touch requirements/*.in + $(MAKE) util/requirements + + +.internal/django/raw: + $(DJANGOADMIN_CMD) $(django_cmd) $(DJANGOADMIN_CMD_VERBOSITY) + +.internal/docker/build: + DPS_BUILD_NAME_PREFIX=$(DPS_BUILD_NAME_PREFIX) DPS_BUILD_ID=$(DPS_BUILD_ID) \ + $(DOCKER_COMPOSE_CMD) -f deploy/Docker/docker-compose.yml build + $(DOCKER_CMD) tag "$(DPS_BUILD_NAME_PREFIX)/django:$(DPS_BUILD_ID)" "$(DPS_BUILD_NAME_PREFIX)/django:latest" + $(DOCKER_CMD) tag "$(DPS_BUILD_NAME_PREFIX)/nginx:$(DPS_BUILD_ID)" "$(DPS_BUILD_NAME_PREFIX)/nginx:latest" + +.internal/docker/run/django: + DPS_BUILD_NAME_PREFIX=$(DPS_BUILD_NAME_PREFIX) DPS_BUILD_ID=$(DPS_BUILD_ID) \ + $(DOCKER_COMPOSE_CMD) -f deploy/Docker/docker-compose.yml up django + +.internal/tox/dev-django: util/requirements + $(TOX_CMD) dev-django -- $(django_cmd) $(DJANGOADMIN_CMD_VERBOSITY) + + +# prepare coloring of echo outputs +ECHO := /usr/bin/printf + +CRED := $(shell tput setaf 1 2>/dev/null) +CGRE := $(shell tput setaf 2 2>/dev/null) +CYEL := $(shell tput setaf 3 2>/dev/null) +CCYA := $(shell tput setaf 6 2>/dev/null) +CRES := $(shell tput sgr0 2>/dev/null) + +DPS_USERNAME := $(shell id -un) +DPS_BUILD_ID := $(shell ./util/bin/get_build_id.sh) + +# suppress unintended output +.SILENT: + +.PHONY: current \ + run \ + \ + dev/check dev/migrate dev/runserver dev/static \ + \ + .internal/django-mgmt .internal/django/raw .internal/tox/dev-django \ + util/requirements util/requirements-force \ + \ + django/check django/compilemessages django/createcachetable \ + django/dbshell django/diffsettings django/diffsettings-all \ + django/diffsettings-prod django/dumpdata django/flush django/flush-plan \ + django/flush-noinput django/inspectdb django/loaddata \ + django/makemessages django/makemigrations django/migrate \ + django/migrate-plan django/runserver django/runserver-external4 \ + django/runserver-external6 django/sendtestemail django/shell \ + django/showmigrations django/showmigrations-plan django/sqlflush \ + django/sqlmigrate django/sqlsequencereset django/squashmigrations \ + django/startapp django/startproject django/test django/testserver \ + django/version \ + django/auth/changepassword django/contenttypes/remove_stale \ + django/sessions/clear django/staticfiles/collect \ + django/staticfiles/find diff --git a/util/init/tox.ini.template b/util/init/tox.ini.template new file mode 100644 index 0000000..2e38c34 --- /dev/null +++ b/util/init/tox.ini.template @@ -0,0 +1,109 @@ +[tox] +# start with an empty list here +# TODO: verify that this actually **might** be empty! +envlist = + +# do not perform packaging operations +skipsdist = true + +# don't stop on missing interpreters +skip_missing_interpreters = true + +[testenv:base] +basepython = python3 +envdir = {toxworkdir}/base +deps = + -rrequirements/common.txt + -rrequirements/development.txt +skip_install = true +commands = + {posargs:{envbindir}/python manage.py version} + +[testenv:dev-django] +basepython = {[testenv:base]basepython} +envdir = {[testenv:base]envdir} +deps = {[testenv:base]deps} +skip_install = {[testenv:base]skip_install} +commands = + {envbindir}/python manage.py {posargs:version} + +[testenv:test-django] +basepython = {[testenv:base]basepython} +envdir = {[testenv:base]envdir} +deps = + {[testenv:base]deps} + coverage +skip_install = {[testenv:base]skip_install} +commands = + coverage run --parallel tests/runtests.py {posargs} + +[testenv:build_requirements] +basepython = {[testenv:base]basepython} +envdir = {toxworkdir}/build_requirements +deps = + pip-tools +skip_install = true +changedir = {toxinidir}/requirements +whitelist_externals = + make +commands = + {posargs:make} + + +################################################################################ +# The following sections actually provide settings for various tools +################################################################################ + +# This sections sets the options for coverage collecting +[coverage:run] +branch = True +omit = + */__init__.py + */migrations/* + */site-packages/* + +# This sections sets the options for coverage reporting +[coverage:report] +precision = 1 +show_missing = True +fail_under = 95 + + +# This section actually sets the options for flake8 +[flake8] +exclude = + .git, + .tox, + +# as per Django's Coding Style +# see https://docs.djangoproject.com/en/dev/internals/contributing/writing-code/coding-style/ +max-line-length = 119 + +ignore = + # as per Django's Coding Style + W601, + + +# This section actually sets the options for isort +[isort] +# these settings are taken from Django's isort configuration +# see https://github.com/django/django/blob/2.0.2/setup.cfg +combine_as_imports = True +default_section = THIRDPARTY +include_trailing_comma = True +line_length = 79 +multi_line_output = 5 +not_skip = __init__.py + +# project specific isort options +known_first_party = {{ project_name }} +known_django = django +sections = FUTURE, STDLIB, DJANGO, THIRDPARTY, FIRSTPARTY, LOCALFOLDER +import_heading_stdlib = Python imports +import_heading_django = Django imports +import_heading_firstparty = app imports +import_heading_localfolder = app imports +import_heading_thirdparty = external imports +skip_glob = + .tox, + */migrations/*