From 5cbef7a3082c1e1072f23a940d38222d90d6abc3 Mon Sep 17 00:00:00 2001 From: crccheck Date: Sun, 31 Aug 2014 12:53:35 -0500 Subject: [PATCH 1/4] add basic dockerfile --- .dockerignore | 6 ++++++ .gitignore | 1 + Dockerfile | 7 +++++++ Makefile | 8 ++++++++ 4 files changed, 22 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..4ec1aed --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +*.dump +*.sql +*_cache +.git +data/* +site/* diff --git a/.gitignore b/.gitignore index 7000ab1..00eb3fe 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ GoogleV3_cache # Private files .env +env-* example_project/local_settings.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6efd42f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,7 @@ +FROM texastribune/base +MAINTAINER cchang@texastribune.org + +ADD . /app +WORKDIR /app +ENV PYTHONPATH /app +RUN pip install -r requirements.txt diff --git a/Makefile b/Makefile index 339c050..d634d4b 100644 --- a/Makefile +++ b/Makefile @@ -68,3 +68,11 @@ serve: .PHONY: help test resetdb scrape pushdb site upload serve + +# DOCKER # +build: + docker build -t texastribune/elevators . + +shell: + docker run --rm --name debug -i -t --link pgplus:postgis \ + --env-file env-docker texastribune/elevators /bin/bash From d23c9253d095300147d3a4f0fc1d254d5b6cfdae Mon Sep 17 00:00:00 2001 From: crccheck Date: Sun, 31 Aug 2014 13:13:30 -0500 Subject: [PATCH 2/4] get a gunicorn container working --- Dockerfile | 5 ++++- Makefile | 7 ++++++- requirements.txt | 10 +++++----- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6efd42f..ae64426 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,4 +4,7 @@ MAINTAINER cchang@texastribune.org ADD . /app WORKDIR /app ENV PYTHONPATH /app -RUN pip install -r requirements.txt +RUN pip install --quiet -r requirements.txt +# needed to keep manage.py from trying to import too much +ENV DEBUG 0 +RUN python example_project/manage.py collectstatic --noinput diff --git a/Makefile b/Makefile index d634d4b..23eea03 100644 --- a/Makefile +++ b/Makefile @@ -74,5 +74,10 @@ build: docker build -t texastribune/elevators . shell: - docker run --rm --name debug -i -t --link pgplus:postgis \ + docker run --rm --name elevators -i -t --link pgplus:postgis \ --env-file env-docker texastribune/elevators /bin/bash + +gunicorn: + docker run --rm --name elevators --link pgplus:postgis \ + --env-file env-docker -p 8000:8000 texastribune/elevators \ + gunicorn example_project.wsgi --bind 0.0.0.0:8000 --log-file - diff --git a/requirements.txt b/requirements.txt index 47baeb2..dfa4c3e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -Django==1.6.5 +Django==1.6.6 -dj-database-url>=0.2.1 -project_runpy -psycopg2>=2.4.5 -gunicorn==0.17.2 +dj-database-url==0.3.0 +project_runpy==0.3.1 +psycopg2==2.5.4 +gunicorn==19.1.1 From d4b2f509f92792a87c124cddccfacebf4325e64d Mon Sep 17 00:00:00 2001 From: crccheck Date: Sun, 31 Aug 2014 15:35:03 -0500 Subject: [PATCH 3/4] simplify spider script using docker --- Makefile | 9 ++++++++- bin/download_site.sh | 26 +++++++++----------------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 23eea03..3005c85 100644 --- a/Makefile +++ b/Makefile @@ -78,6 +78,13 @@ shell: --env-file env-docker texastribune/elevators /bin/bash gunicorn: - docker run --rm --name elevators --link pgplus:postgis \ + docker run --detach --name elevators-wsgi --link pgplus:postgis \ --env-file env-docker -p 8000:8000 texastribune/elevators \ gunicorn example_project.wsgi --bind 0.0.0.0:8000 --log-file - + +# download script doesn't need concurrency so only use -c 1 +benchmark: gunicorn + docker run --rm --link elevators-wsgi:wsgi -t \ + zz ab -n 10 http://wsgi:8000/ + docker logs elevators-wsgi + docker rm -f elevators-wsgi diff --git a/bin/download_site.sh b/bin/download_site.sh index b32efb5..fa56423 100755 --- a/bin/download_site.sh +++ b/bin/download_site.sh @@ -1,28 +1,20 @@ +#!/usr/bin/env bash + # Instructions: # # run from project root directory -set +e -MANAGE="python ./example_project/manage.py" -PORT=8008 - - -DEBUG=0 $MANAGE runserver --nothreading --noreload $PORT & -pid=$! -echo "runserver pid: $pid" +# Make sure the docker image is up to date +make build -# make sure to kill the server if terminated early -trap "kill $pid; echo bye $pid" EXIT +docker run --detach --name elevators-wsgi --link pgplus:postgis \ + --env-file env-docker -p 8000 texastribune/elevators \ + gunicorn example_project.wsgi --bind 0.0.0.0:8000 --log-file - +MANAGE="python ./example_project/manage.py" +PORT=$(docker port elevators-site 8000 | cut -d : -f 2) # give time for the servers to get up sleep 1 -$MANAGE collectstatic --noinput - mkdir -p site cd site && wget -r localhost:$PORT --force-html -e robots=off -nH -nv --max-redirect 0 - -# kill server, run in a subprocess so we can suppress "Terminated" message -(kill $pid 2>&1) > /dev/null - -echo "bye" From c76860237e1e17c7cf8cbd5b4c0526ca599c1256 Mon Sep 17 00:00:00 2001 From: crccheck Date: Sun, 31 Aug 2014 17:41:09 -0500 Subject: [PATCH 4/4] add nginx to docker to try and make it faster --- .dockerignore | 1 + Dockerfile | 5 +++-- Makefile | 7 ++++--- bin/download_site.sh | 9 ++------- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/.dockerignore b/.dockerignore index 4ec1aed..a46dab1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,3 +4,4 @@ .git data/* site/* +static_root diff --git a/Dockerfile b/Dockerfile index ae64426..c55cf5c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,9 @@ -FROM texastribune/base +FROM texastribune/gunicorn MAINTAINER cchang@texastribune.org +RUN sed -i "s/wsgi:application/example_project.wsgi:application/" /etc/supervisor/conf.d/gunicorn.supervisor.conf + ADD . /app -WORKDIR /app ENV PYTHONPATH /app RUN pip install --quiet -r requirements.txt # needed to keep manage.py from trying to import too much diff --git a/Makefile b/Makefile index 3005c85..7154592 100644 --- a/Makefile +++ b/Makefile @@ -75,15 +75,16 @@ build: shell: docker run --rm --name elevators -i -t --link pgplus:postgis \ - --env-file env-docker texastribune/elevators /bin/bash + --volumes-from elevators-wsgi \ + --env-file env-docker --entrypoint /bin/bash texastribune/elevators gunicorn: docker run --detach --name elevators-wsgi --link pgplus:postgis \ - --env-file env-docker -p 8000:8000 texastribune/elevators \ - gunicorn example_project.wsgi --bind 0.0.0.0:8000 --log-file - + --env-file env-docker -p 8000:8000 texastribune/elevators # download script doesn't need concurrency so only use -c 1 benchmark: gunicorn + sleep 5 docker run --rm --link elevators-wsgi:wsgi -t \ zz ab -n 10 http://wsgi:8000/ docker logs elevators-wsgi diff --git a/bin/download_site.sh b/bin/download_site.sh index fa56423..7eab998 100755 --- a/bin/download_site.sh +++ b/bin/download_site.sh @@ -5,16 +5,11 @@ # run from project root directory # Make sure the docker image is up to date -make build - -docker run --detach --name elevators-wsgi --link pgplus:postgis \ - --env-file env-docker -p 8000 texastribune/elevators \ - gunicorn example_project.wsgi --bind 0.0.0.0:8000 --log-file - -MANAGE="python ./example_project/manage.py" +make build gunicorn PORT=$(docker port elevators-site 8000 | cut -d : -f 2) # give time for the servers to get up -sleep 1 +sleep 3 mkdir -p site cd site && wget -r localhost:$PORT --force-html -e robots=off -nH -nv --max-redirect 0