From 2dec81a01493a2efbaa23512510cd3faa438535e Mon Sep 17 00:00:00 2001
From: crccheck <c@crccheck.com>
Date: Sat, 31 Jan 2015 09:37:48 -0600
Subject: [PATCH 01/23] fix how 'next_inspection' could be null

---
 Makefile               | 2 +-
 tx_elevators/models.py | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 94d7019..9ce9df7 100644
--- a/Makefile
+++ b/Makefile
@@ -48,7 +48,7 @@ scrape:
 
 
 import:
-	python tx_elevators/scripts/scrape.py data/elevator_data_file.csv
+	time python tx_elevators/scripts/scrape.py data/elevator_data_file.csv
 
 
 dbpush:
diff --git a/tx_elevators/models.py b/tx_elevators/models.py
index 0f49e57..575d9dd 100644
--- a/tx_elevators/models.py
+++ b/tx_elevators/models.py
@@ -130,6 +130,8 @@ class Elevator(models.Model):
     # DT_EXPIRY
     next_inspection = models.DateField(
         u'Date of Next Inspection',
+        # may be null for time traveling future elevators
+        null=True,
     )
     # ELV_5YEAR
     last_5year = models.DateField(

From 4b421c34c8470e0e4959a50cafd8822fe1585a8c Mon Sep 17 00:00:00 2001
From: crccheck <c@crccheck.com>
Date: Sat, 31 Jan 2015 09:47:17 -0600
Subject: [PATCH 02/23] document timing info for a data import

---
 Makefile | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 9ce9df7..09acf52 100644
--- a/Makefile
+++ b/Makefile
@@ -47,8 +47,12 @@ scrape:
 	@echo "should geocode the top 1000 too: $(MANAGE) geocode"
 
 
+# timing for a trivial import
+# real	1m51.994s
+# user	1m24.935s
+# sys	0m4.226s
 import:
-	time python tx_elevators/scripts/scrape.py data/elevator_data_file.csv
+	python tx_elevators/scripts/scrape.py data/elevator_data_file.csv
 
 
 dbpush:

From 813925c4346215ca7e9c424bb8e82b555d981288 Mon Sep 17 00:00:00 2001
From: crccheck <c@crccheck.com>
Date: Sat, 31 Jan 2015 10:11:19 -0600
Subject: [PATCH 03/23] upgrade to django 1.7

---
 Makefile                    |  1 +
 example_project/settings.py |  1 -
 requirements-dev.txt        | 10 +++-------
 requirements.txt            |  6 +++---
 4 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/Makefile b/Makefile
index 09acf52..f0e5f01 100644
--- a/Makefile
+++ b/Makefile
@@ -14,6 +14,7 @@ help:
 	@echo "  make serve   - serve the spided pages locally (on port 8088)"
 
 
+# TODO actually write some tests
 test:
 	$(MANAGE) test tx_elevators
 
diff --git a/example_project/settings.py b/example_project/settings.py
index 86357ae..99f9404 100644
--- a/example_project/settings.py
+++ b/example_project/settings.py
@@ -180,7 +180,6 @@ def project_dir(*paths):
     # extra apps used for development
     INSTALLED_APPS += [
         'django_extensions',
-        'debug_toolbar',
 
         'django.contrib.sessions',
         'django.contrib.auth',
diff --git a/requirements-dev.txt b/requirements-dev.txt
index cb5e4a9..b522bf3 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,11 +1,7 @@
 -r requirements.txt
 
-boto==2.23.0
+boto==2.36.0
 # geopy==0.97
-https://github.com/crccheck/geopy/tarball/add-goog-components
-django-debug-toolbar>=0.9
-django-extensions>=0.9
+# https://github.com/crccheck/geopy/tarball/add-goog-components
+django-extensions==1.5.0
 factory-boy
-ipdb
-ipython
-Werkzeug>=0.8.3
diff --git a/requirements.txt b/requirements.txt
index 47baeb2..d7b40cd 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
-Django==1.6.5
+Django==1.7.4
 
-dj-database-url>=0.2.1
+dj-database-url==0.3.0
 project_runpy
 psycopg2>=2.4.5
-gunicorn==0.17.2
+gunicorn==19.2.0

From e18111ede2e54962f47ee54981ff58cc922c2f5e Mon Sep 17 00:00:00 2001
From: crccheck <c@crccheck.com>
Date: Sat, 31 Jan 2015 10:12:14 -0600
Subject: [PATCH 04/23] make parallel s3 upload the default

---
 Makefile                                    |  12 +-
 tx_elevators/management/commands/sync_s3.py | 345 --------------------
 2 files changed, 1 insertion(+), 356 deletions(-)
 delete mode 100644 tx_elevators/management/commands/sync_s3.py

diff --git a/Makefile b/Makefile
index f0e5f01..be72cde 100644
--- a/Makefile
+++ b/Makefile
@@ -78,21 +78,11 @@ site:
 serve:
 	cd site && python -m SimpleHTTPServer 8088
 
-# 24340 files uploaded.
-# 3 files skipped.
-# real	200m23.933s
-
-# 25611 files uploaded.
-# 2662 files skipped.
-# real	122m28.098s
-upload:
-	LOGGING=WARN DEBUG=0 $(MANAGE) sync_s3 --dir site --gzip
-
 # requires installing https://github.com/twpayne/s3-parallel-put
 # uses 8 threads by default
 #
 # INFO:s3-parallel-put[statter-12800]:put 137686194 bytes in 28270 files in 697.4 seconds (197436 bytes/s, 40.5 files/s)
-upload2:
+upload:
 	cd site && s3-parallel-put --bucket=${AWS_BUCKET_NAME} \
 	  --grant public-read --header "Cache-Control:max-age=2592000" --gzip  .
 
diff --git a/tx_elevators/management/commands/sync_s3.py b/tx_elevators/management/commands/sync_s3.py
deleted file mode 100644
index 55b4288..0000000
--- a/tx_elevators/management/commands/sync_s3.py
+++ /dev/null
@@ -1,345 +0,0 @@
-"""
-Sync Media to S3
-================
-
-Django command that scans all files in your settings.MEDIA_ROOT folder and
-uploads them to S3 with the same directory structure.
-
-This command can optionally do the following but it is off by default:
-* gzip compress any CSS and Javascript files it finds and adds the appropriate
-  'Content-Encoding' header.
-* set a far future 'Expires' header for optimal caching.
-
-Note: This script requires the Python boto library and valid Amazon Web
-Services API keys.
-
-Required settings.py variables:
-AWS_ACCESS_KEY_ID = ''
-AWS_SECRET_ACCESS_KEY = ''
-AWS_BUCKET_NAME = ''
-
-When you call this command with the `--renamegzip` param, it will add
-the '.gz' extension to the file name. But Safari just doesn't recognize
-'.gz' files and your site won't work on it! To fix this problem, you can
-set any other extension (like .jgz) in the `SYNC_S3_RENAME_GZIP_EXT`
-variable.
-
-Command options are:
-  -p PREFIX, --prefix=PREFIX
-                        The prefix to prepend to the path on S3.
-  --gzip                Enables gzipping CSS and Javascript files.
-  --expires             Enables setting a far future expires header.
-  --force               Skip the file mtime check to force upload of all
-                        files.
-  --filter-list         Override default directory and file exclusion
-                        filters. (enter as comma seperated line)
-  --renamegzip          Enables renaming of gzipped files by appending '.gz'.
-                        to the original file name. This way your original
-                        assets will not be replaced by the gzipped ones.
-                        You can change the extension setting the
-                        `SYNC_S3_RENAME_GZIP_EXT` var in your settings.py
-                        file.
-  --invalidate          Invalidates the objects in CloudFront after uploaading
-                        stuff to s3.
-
-
-TODO:
- * Use fnmatch (or regex) to allow more complex FILTER_LIST rules.
-
-"""
-import datetime
-import email
-import mimetypes
-from optparse import make_option
-import os
-import time
-import gzip
-try:
-    from cStringIO import StringIO
-    assert StringIO
-except ImportError:
-    from StringIO import StringIO
-
-
-from django.conf import settings
-from django.core.management.base import BaseCommand, CommandError
-
-# Make sure boto is available
-try:
-    import boto
-    import boto.exception
-except ImportError:
-    raise ImportError("The boto Python library is not installed.")
-
-
-class Command(BaseCommand):
-    # Extra variables to avoid passing these around
-    AWS_ACCESS_KEY_ID = ''
-    AWS_SECRET_ACCESS_KEY = ''
-    AWS_BUCKET_NAME = ''
-    AWS_CLOUDFRONT_DISTRIBUTION = ''
-    SYNC_S3_RENAME_GZIP_EXT = ''
-
-    DIRECTORY = ''
-    FILTER_LIST = ['.DS_Store', '.svn', '.hg', '.git', 'Thumbs.db']
-    GZIP_CONTENT_TYPES = (
-        'text/css',
-        'application/json',
-        'application/javascript',
-        'application/x-javascript',
-        'text/javascript',
-        'text/html',
-    )
-
-    uploaded_files = []
-    upload_count = 0
-    skip_count = 0
-
-    option_list = BaseCommand.option_list + (
-        make_option('-p', '--prefix',
-                    dest='prefix',
-                    default=getattr(settings, 'SYNC_MEDIA_S3_PREFIX', ''),
-                    help="The prefix to prepend to the path on S3."),
-        make_option('-d', '--dir',
-                    dest='dir', default=settings.MEDIA_ROOT,
-                    help="The root directory to use instead of your MEDIA_ROOT"),
-        make_option('--gzip',
-                    action='store_true', dest='gzip', default=False,
-                    help="Enables gzipping CSS and Javascript files."),
-        make_option('--renamegzip',
-                    action='store_true', dest='renamegzip', default=False,
-                    help="Enables renaming of gzipped assets to have '.gz' appended to the filename."),
-        make_option('--expires',
-                    action='store_true', dest='expires', default=False,
-                    help="Enables setting a far future expires header."),
-        make_option('--force',
-                    action='store_true', dest='force', default=False,
-                    help="Skip the file mtime check to force upload of all files."),
-        make_option('--filter-list', dest='filter_list',
-                    action='store', default='',
-                    help="Override default directory and file exclusion filters. (enter as comma seperated line)"),
-        make_option('--invalidate', dest='invalidate', default=False,
-                    action='store_true',
-                    help='Invalidates the associated objects in CloudFront')
-    )
-
-    help = 'Syncs the complete MEDIA_ROOT structure and files to S3 into the given bucket name.'
-    args = 'bucket_name'
-
-    can_import_settings = True
-
-    def handle(self, *args, **options):
-
-        # Check for AWS keys in settings
-        if not hasattr(settings, 'AWS_ACCESS_KEY_ID') or not hasattr(settings, 'AWS_SECRET_ACCESS_KEY'):
-            raise CommandError('Missing AWS keys from settings file.  Please supply both AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY.')
-        else:
-            self.AWS_ACCESS_KEY_ID = settings.AWS_ACCESS_KEY_ID
-            self.AWS_SECRET_ACCESS_KEY = settings.AWS_SECRET_ACCESS_KEY
-
-        if not hasattr(settings, 'AWS_BUCKET_NAME'):
-            raise CommandError('Missing bucket name from settings file. Please add the AWS_BUCKET_NAME to your settings file.')
-        else:
-            if not settings.AWS_BUCKET_NAME:
-                raise CommandError('AWS_BUCKET_NAME cannot be empty.')
-        self.AWS_BUCKET_NAME = settings.AWS_BUCKET_NAME
-
-        if not hasattr(settings, 'MEDIA_ROOT'):
-            raise CommandError('MEDIA_ROOT must be set in your settings.')
-        else:
-            if not settings.MEDIA_ROOT:
-                raise CommandError('MEDIA_ROOT must be set in your settings.')
-
-        self.AWS_CLOUDFRONT_DISTRIBUTION = getattr(settings, 'AWS_CLOUDFRONT_DISTRIBUTION', '')
-
-        self.SYNC_S3_RENAME_GZIP_EXT = \
-            getattr(settings, 'SYNC_S3_RENAME_GZIP_EXT', '.gz')
-
-        self.verbosity = int(options.get('verbosity'))
-        self.prefix = options.get('prefix')
-        self.do_gzip = options.get('gzip')
-        self.rename_gzip = options.get('renamegzip')
-        self.do_expires = options.get('expires')
-        self.do_force = options.get('force')
-        self.invalidate = options.get('invalidate')
-        self.DIRECTORY = options.get('dir')
-        self.FILTER_LIST = getattr(settings, 'FILTER_LIST', self.FILTER_LIST)
-        filter_list = options.get('filter_list')
-        if filter_list:
-            # command line option overrides default filter_list and
-            # settings.filter_list
-            self.FILTER_LIST = filter_list.split(',')
-
-        # Now call the syncing method to walk the MEDIA_ROOT directory and
-        # upload all files found.
-        self.sync_s3()
-
-        # Sending the invalidation request to CloudFront if the user
-        # requested this action
-        if self.invalidate:
-            self.invalidate_objects_cf()
-
-        print("")
-        print("%d files uploaded." % self.upload_count)
-        print("%d files skipped." % self.skip_count)
-
-    def open_cf(self):
-        """
-        Returns an open connection to CloudFront
-        """
-        return boto.connect_cloudfront(
-            self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY)
-
-    def invalidate_objects_cf(self):
-        """
-        Split the invalidation request in groups of 1000 objects
-        """
-        if not self.AWS_CLOUDFRONT_DISTRIBUTION:
-            raise CommandError(
-                'An object invalidation was requested but the variable '
-                'AWS_CLOUDFRONT_DISTRIBUTION is not present in your settings.')
-
-        # We can't send more than 1000 objects in the same invalidation
-        # request.
-        chunk = 1000
-
-        # Connecting to CloudFront
-        conn = self.open_cf()
-
-        # Splitting the object list
-        objs = self.uploaded_files
-        chunks = [objs[i:i + chunk] for i in range(0, len(objs), chunk)]
-
-        # Invalidation requests
-        for paths in chunks:
-            conn.create_invalidation_request(
-                self.AWS_CLOUDFRONT_DISTRIBUTION, paths)
-
-    def sync_s3(self):
-        """
-        Walks the media directory and syncs files to S3
-        """
-        bucket, key = self.open_s3()
-        os.path.walk(self.DIRECTORY, self.upload_s3, (bucket, key, self.AWS_BUCKET_NAME, self.DIRECTORY))
-
-    def compress_string(self, s):
-        """Gzip a given string."""
-        zbuf = StringIO()
-        zfile = gzip.GzipFile(mode='wb', compresslevel=6, fileobj=zbuf)
-        zfile.write(s)
-        zfile.close()
-        return zbuf.getvalue()
-
-    def open_s3(self):
-        """
-        Opens connection to S3 returning bucket and key
-        """
-        conn = boto.connect_s3(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY)
-        try:
-            bucket = conn.get_bucket(self.AWS_BUCKET_NAME)
-        except boto.exception.S3ResponseError:
-            bucket = conn.create_bucket(self.AWS_BUCKET_NAME)
-        return bucket, boto.s3.key.Key(bucket)
-
-    def upload_s3(self, arg, dirname, names):
-        """
-        This is the callback to os.path.walk and where much of the work happens
-        """
-        bucket, key, bucket_name, root_dir = arg
-
-        # Skip directories we don't want to sync
-        if os.path.basename(dirname) in self.FILTER_LIST:
-            # prevent walk from processing subfiles/subdirs below the ignored one
-            del names[:]
-            return
-
-        # Later we assume the MEDIA_ROOT ends with a trailing slash
-        if not root_dir.endswith(os.path.sep):
-            root_dir = root_dir + os.path.sep
-
-        for file in names:
-            headers = {}
-
-            if file in self.FILTER_LIST:
-                continue  # Skip files we don't want to sync
-
-            filename = os.path.join(dirname, file)
-            if os.path.isdir(filename):
-                continue  # Don't try to upload directories
-
-            file_key = filename[len(root_dir):]
-            if self.prefix:
-                file_key = '%s/%s' % (self.prefix, file_key)
-
-            # Check if file on S3 is older than local file, if so, upload
-            if not self.do_force:
-                s3_key = bucket.get_key(file_key)
-                if s3_key:
-                    s3_datetime = datetime.datetime(*time.strptime(
-                        s3_key.last_modified, '%a, %d %b %Y %H:%M:%S %Z')[0:6])
-                    local_datetime = datetime.datetime.utcfromtimestamp(
-                        os.stat(filename).st_mtime)
-                    if local_datetime < s3_datetime:
-                        self.skip_count += 1
-                        if self.verbosity > 1:
-                            print("File %s hasn't been modified since last being uploaded" % file_key)
-                        continue
-
-            # File is newer, let's process and upload
-            if self.verbosity > 0:
-                print("Uploading %s..." % file_key)
-
-            content_type = mimetypes.guess_type(filename)[0]
-            if content_type:
-                headers['Content-Type'] = content_type
-            file_obj = open(filename, 'rb')
-            file_size = os.fstat(file_obj.fileno()).st_size
-            filedata = file_obj.read()
-            if self.do_gzip:
-                # Gzipping only if file is large enough (>1K is recommended)
-                # and only if file is a common text type (not a binary file)
-                if file_size > 1024 and content_type in self.GZIP_CONTENT_TYPES:
-                    filedata = self.compress_string(filedata)
-                    if self.rename_gzip:
-                        # If rename_gzip is True, then rename the file
-                        # by appending an extension (like '.gz)' to
-                        # original filename.
-                        file_key = '%s.%s' % (
-                            file_key, self.SYNC_S3_RENAME_GZIP_EXT)
-                    headers['Content-Encoding'] = 'gzip'
-                    if self.verbosity > 1:
-                        print("\tgzipped: %dk to %dk" % (file_size / 1024, len(filedata) / 1024))
-
-            # XXX always set 1 month cache control
-            headers['Cache-Control'] = 'max-age %d' % (3600 * 24 * 30)
-            if self.do_expires:
-                # HTTP/1.0
-                headers['Expires'] = '%s GMT' % (email.Utils.formatdate(time.mktime((datetime.datetime.now() + datetime.timedelta(days=365 * 2)).timetuple())))
-                # HTTP/1.1
-                headers['Cache-Control'] = 'max-age %d' % (3600 * 24 * 365 * 2)
-                if self.verbosity > 1:
-                    print("\texpires: %s" % headers['Expires'])
-                    print("\tcache-control: %s" % headers['Cache-Control'])
-
-            try:
-                key.name = file_key
-                key.set_contents_from_string(filedata, headers, replace=True)
-                key.set_acl('public-read')
-            except boto.exception.S3CreateError as e:
-                print("Failed: %s" % e)
-            except Exception as e:
-                print(e)
-                raise
-            else:
-                self.upload_count += 1
-                self.uploaded_files.append(file_key)
-
-            file_obj.close()
-
-# Backwards compatibility for Django r9110
-if not [opt for opt in Command.option_list if opt.dest == 'verbosity']:
-    Command.option_list += (
-        make_option('-v', '--verbosity',
-                    dest='verbosity', default=1, action='count',
-                    help="Verbose mode. Multiple -v options increase the verbosity."),
-    )

From 0be456dd6f9000586d12e6f7028898b8e40b010a Mon Sep 17 00:00:00 2001
From: crccheck <c@crccheck.com>
Date: Sat, 31 Jan 2015 10:16:56 -0600
Subject: [PATCH 05/23] shutup, django

---
 example_project/settings.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/example_project/settings.py b/example_project/settings.py
index 99f9404..9d75828 100644
--- a/example_project/settings.py
+++ b/example_project/settings.py
@@ -192,6 +192,8 @@ def project_dir(*paths):
         'django.contrib.messages.middleware.MessageMiddleware',
     ]
 
+# STFU DJANGO, STOP COMPLAINING
+TEST_RUNNER = 'django.test.runner.DiscoverRunner'
 
 try:
     from .local_settings import *

From 624ae6c2910ba1df977ea5a1a1c4108bf66aa75e Mon Sep 17 00:00:00 2001
From: crccheck <c@crccheck.com>
Date: Sat, 31 Jan 2015 10:23:46 -0600
Subject: [PATCH 06/23] update scraper for django 1.7 syntax

---
 tx_elevators/scripts/scrape.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tx_elevators/scripts/scrape.py b/tx_elevators/scripts/scrape.py
index 7ddcdcc..3c25b12 100644
--- a/tx_elevators/scripts/scrape.py
+++ b/tx_elevators/scripts/scrape.py
@@ -111,7 +111,8 @@ def post_process():
     # Elevator.objects.filter(year_installed__gt=2013).update(year_installed=None)
 
 if __name__ == "__main__":
-    logger = logging.getLogger(__name__)
+    import django; django.setup()  # NOQA
+    logger = logging.getLogger('tx_elevators.scrape')
     path = sys.argv[1]
     process(path)
     post_process()

From e60227ca0fb42749c819a0d6b97190d2b6f7d6e9 Mon Sep 17 00:00:00 2001
From: crccheck <c@crccheck.com>
Date: Fri, 20 Mar 2015 01:03:11 -0500
Subject: [PATCH 07/23] silence s3 put so overall output is easier to follow

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index be72cde..cc4116f 100644
--- a/Makefile
+++ b/Makefile
@@ -83,8 +83,8 @@ serve:
 #
 # INFO:s3-parallel-put[statter-12800]:put 137686194 bytes in 28270 files in 697.4 seconds (197436 bytes/s, 40.5 files/s)
 upload:
-	cd site && s3-parallel-put --bucket=${AWS_BUCKET_NAME} \
-	  --grant public-read --header "Cache-Control:max-age=2592000" --gzip  .
+	cd site && s3-parallel-put --quiet --bucket=${AWS_BUCKET_NAME} \
+	  --grant public-read --header "Cache-Control:max-age=2592000" --gzip .
 
 
 .PHONY: help test resetdb scrape pushdb site upload serve

From d517ae4fa00ef5e6a034c8e824cda3f4a26dff13 Mon Sep 17 00:00:00 2001
From: crccheck <c@crccheck.com>
Date: Thu, 11 Jun 2015 23:10:36 -0500
Subject: [PATCH 08/23] tweak postgres dump/restore docs

---
 Makefile | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index cc4116f..e948f45 100644
--- a/Makefile
+++ b/Makefile
@@ -27,12 +27,12 @@ resetdb:
 # Backup the local database
 #
 # To restore
-#   cat tx_elevators-2014-08-31.dump | \
-#   docker run --rm --link postgis:postgis -t crccheck/postgis \
-#   pg_restore -U docker -h postgis --dbname elevators
+#   cat tx_elevators-2014-11-01.dump | \
+#     docker run --rm --link elevators_1:db -t crccheck/postgis \
+#     pg_restore -U docker -h db --dbname elevators
 dumpdb:
-	docker run --rm --link postgis:postgis -t crccheck/postgis \
-	  pg_dump -U docker -h postgis -p 5432 -Fc elevators > tx_elevators-$$(date +"%Y-%m-%d").dump
+	docker run --rm --link elevators_1:db -t crccheck/postgis \
+	  pg_dump -U docker -h db -Fc elevators > tx_elevators-$$(date +"%Y-%m-%d").dump
 
 # Dump building geocodes
 #

From ab921858fb3e5ab031410b2bd4de5b13aedf6e39 Mon Sep 17 00:00:00 2001
From: crccheck <c@crccheck.com>
Date: Thu, 11 Jun 2015 23:25:18 -0500
Subject: [PATCH 09/23] switch to migrate instead of syncdb

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index e948f45..35746fb 100644
--- a/Makefile
+++ b/Makefile
@@ -21,7 +21,7 @@ test:
 
 resetdb:
 	$(MANAGE) reset_db --router=default --noinput
-	$(MANAGE) syncdb --noinput
+	$(MANAGE) migrate --noinput
 
 
 # Backup the local database

From c9b4d5cb0bbde1cf33ff4abdd469485f78ab0c59 Mon Sep 17 00:00:00 2001
From: crccheck <c@crccheck.com>
Date: Thu, 11 Jun 2015 23:27:37 -0500
Subject: [PATCH 10/23] tweak readme, delete heroku deploy instructions

---
 README.md  | 34 ++++++++++++++++++++++++++++++
 README.rst | 62 ------------------------------------------------------
 2 files changed, 34 insertions(+), 62 deletions(-)
 create mode 100644 README.md
 delete mode 100644 README.rst

diff --git a/README.md b/README.md
new file mode 100644
index 0000000..af6f467
--- /dev/null
+++ b/README.md
@@ -0,0 +1,34 @@
+TX Elevators
+============
+
+Dev Setup
+---------
+
+Installing Requirements:
+
+    pip install -r requirements-dev.txt
+
+
+Using Postgresql instead of Sqlite as your database:
+
+    export DATABASE_URL='postgres:///tx_elevators'
+
+
+Getting Data
+------------
+
+If you don't have a database set up, `DEBUG=1 make resetdb` will create one for
+you. Running `make scrape` will download a fresh copy of the CSV and import the
+data. Afterwards, you can run `manage.py geocode` to geocode the data.
+
+
+Deploying to S3
+---------------
+
+Partial instructions for deploying to a [hosted site on S3]:
+
+1. Make sure you're not in debug mode.
+2. Make sure this project is running locally on `http://localhost:8000`.
+3. Run `make site upload`
+
+  [hosted site on S3]: http://docs.aws.amazon.com/AmazonS3/latest/dev/WebsiteHosting.html
diff --git a/README.rst b/README.rst
deleted file mode 100644
index 91badad..0000000
--- a/README.rst
+++ /dev/null
@@ -1,62 +0,0 @@
-============
-Tx Elevators
-============
-Dev Setup
----------
-
-Installing Requirements::
-
-    pip install -r requirements-dev.txt
-
-
-Using Postgresql instead of Sqlite as your database::
-
-    export DATABASE_URL='postgres:///tx_elevators'
-
-
-Getting Data
-------------
-
-If you don't have a database set up, ``DEBUG=1 make resetdb`` will create one
-for you. Running ``make scrape`` will download a fresh copy of the CSV and
-import the data. Afterwards, you can run ``manage.py geocode`` to geocode the
-data.
-
-
-Deploying to Heroku
--------------------
-
-Create a new app and give it a database::
-
-    $ heroku apps:create
-    $ heroku addons:add heroku-postgresql:dev
-
-Promote the database to ``DATABASE_URL``::
-
-    $ heroku config | grep HEROKU_POSTGRESQL
-    HEROKU_POSTGRESQL_RED_URL: postgres://user3123:passkja83kd8@ec2-117-21-174-214.compute-1.amazonaws.com:6212/db982398
-    $ heroku pg:promote RED
-
-Install the pgbackups addon::
-
-    $ heroku addons:add pgbackups
-
-Migrate data from your local Postgresql to Heroku (https://devcenter.heroku.com/articles/heroku-postgres-import-export)::
-
-    $ pg_dump -Fc --no-acl --no-owner tx_elevators > tx_elevators.dump
-
-Upload ``tx_elevators.dump`` someplace on the Internets and pull it into Heroku::
-
-    $ heroku pgbackups:restore DATABASE http://example.com/tx_elevators.dump
-
-
-Deploying to S3
----------------
-
-Partial instructions for deploying to a `hosted site on S3`_):
-
-1. Make sure you're not in debug mode.
-2. Make sure this project is running locally on ``http://localhost:8000``.
-3. Run ``make site upload``
-
-.. _hosted site on S3: http://docs.aws.amazon.com/AmazonS3/latest/dev/WebsiteHosting.html

From 1a97a6bec6071e0d64c980c9048e36ff9e619618 Mon Sep 17 00:00:00 2001
From: crccheck <c@crccheck.com>
Date: Thu, 11 Jun 2015 23:28:35 -0500
Subject: [PATCH 11/23] combine to one requirements.txt for simplicity

---
 README.md            | 2 +-
 requirements-dev.txt | 7 -------
 requirements.txt     | 6 ++++++
 3 files changed, 7 insertions(+), 8 deletions(-)
 delete mode 100644 requirements-dev.txt

diff --git a/README.md b/README.md
index af6f467..cd992e0 100644
--- a/README.md
+++ b/README.md
@@ -6,7 +6,7 @@ Dev Setup
 
 Installing Requirements:
 
-    pip install -r requirements-dev.txt
+    pip install -r requirements.txt
 
 
 Using Postgresql instead of Sqlite as your database:
diff --git a/requirements-dev.txt b/requirements-dev.txt
deleted file mode 100644
index b522bf3..0000000
--- a/requirements-dev.txt
+++ /dev/null
@@ -1,7 +0,0 @@
--r requirements.txt
-
-boto==2.36.0
-# geopy==0.97
-# https://github.com/crccheck/geopy/tarball/add-goog-components
-django-extensions==1.5.0
-factory-boy
diff --git a/requirements.txt b/requirements.txt
index d7b40cd..ecfe74c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,3 +4,9 @@ dj-database-url==0.3.0
 project_runpy
 psycopg2>=2.4.5
 gunicorn==19.2.0
+
+boto==2.36.0
+# geopy==0.97
+# https://github.com/crccheck/geopy/tarball/add-goog-components
+django-extensions==1.5.0
+factory-boy

From aaa25a221bd8f9461db3947fe896ba94b3121ddd Mon Sep 17 00:00:00 2001
From: crccheck <chris.chang@tabbedout.com>
Date: Thu, 11 Jun 2015 23:32:46 -0500
Subject: [PATCH 12/23] Document import timing

---
 Makefile | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 35746fb..de1cd06 100644
--- a/Makefile
+++ b/Makefile
@@ -48,10 +48,8 @@ scrape:
 	@echo "should geocode the top 1000 too: $(MANAGE) geocode"
 
 
-# timing for a trivial import
-# real	1m51.994s
-# user	1m24.935s
-# sys	0m4.226s
+# timing for trivial import real	1m51.994s
+# timing for a fresh import real	4m15.279s
 import:
 	python tx_elevators/scripts/scrape.py data/elevator_data_file.csv
 

From d406ecacf950721f5e98866d035e5148af6bb6ce Mon Sep 17 00:00:00 2001
From: crccheck <c@crccheck.com>
Date: Thu, 11 Jun 2015 23:35:20 -0500
Subject: [PATCH 13/23] add a progress bar to the import

---
 requirements.txt               | 1 +
 tx_elevators/scripts/scrape.py | 9 ++++++---
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index ecfe74c..f86320f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,3 +10,4 @@ boto==2.36.0
 # https://github.com/crccheck/geopy/tarball/add-goog-components
 django-extensions==1.5.0
 factory-boy
+tqdm==1.0
diff --git a/tx_elevators/scripts/scrape.py b/tx_elevators/scripts/scrape.py
index 3c25b12..92d19b8 100644
--- a/tx_elevators/scripts/scrape.py
+++ b/tx_elevators/scripts/scrape.py
@@ -10,6 +10,7 @@
 import sys
 
 from tx_elevators.models import Building, Elevator
+from tqdm import tqdm
 
 
 def setfield(obj, fieldname, value):
@@ -92,9 +93,10 @@ def process_row(row):
 def process(path):
     with open(path, 'rU') as f:
         reader = csv.DictReader(f)
-        for i, row in enumerate(reader):
-            if not i % 1000:
-                logger.info("Processing Row %s" % i)
+        for total, row in enumerate(f):  # subtract 1 for header row
+            pass
+        f.seek(0)
+        for row in tqdm(reader, total=total, leave=True):
             try:
                 process_row(format_row(row))
             except Exception as e:
@@ -102,6 +104,7 @@ def process(path):
                 import ipdb
                 ipdb.set_trace()
                 raise
+        print('')
 
 
 def post_process():

From b0ed76c4db7f8fdeeb529676067c931fc613a71a Mon Sep 17 00:00:00 2001
From: crccheck <c@crccheck.com>
Date: Thu, 11 Jun 2015 23:56:50 -0500
Subject: [PATCH 14/23] bump django yet again to 1.8

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index f86320f..49ded00 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-Django==1.7.4
+Django==1.8.2
 
 dj-database-url==0.3.0
 project_runpy

From 4c50544359e8269ed1a43f3f4057882e8e55c554 Mon Sep 17 00:00:00 2001
From: crccheck <c@crccheck.com>
Date: Thu, 11 Jun 2015 23:58:07 -0500
Subject: [PATCH 15/23] switch to dj-obj-update package to handle model
 update_or_create

---
 requirements.txt               |  1 +
 tx_elevators/scripts/scrape.py | 44 +++++-----------------------------
 2 files changed, 7 insertions(+), 38 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 49ded00..3153eef 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -11,3 +11,4 @@ boto==2.36.0
 django-extensions==1.5.0
 factory-boy
 tqdm==1.0
+dj-obj-update==0.2.0
diff --git a/tx_elevators/scripts/scrape.py b/tx_elevators/scripts/scrape.py
index 92d19b8..4100c17 100644
--- a/tx_elevators/scripts/scrape.py
+++ b/tx_elevators/scripts/scrape.py
@@ -9,35 +9,11 @@
 import logging
 import sys
 
+from obj_update import obj_update_or_create
 from tx_elevators.models import Building, Elevator
 from tqdm import tqdm
 
 
-def setfield(obj, fieldname, value):
-    """Fancy setattr with debugging."""
-    old = getattr(obj, fieldname)
-    if str(old) != str(value):
-        setattr(obj, fieldname, value)
-        if not hasattr(obj, '_is_dirty'):
-            obj._is_dirty = []
-        obj._is_dirty.append("%s %s->%s" % (fieldname, old, value))
-
-
-def update(obj, data):
-    """
-    Fancy way to update `obj` with `data` dict.
-
-    Returns True if data changed and  was saved.
-    """
-    for key, value in data.items():
-        setfield(obj, key, value)
-    if getattr(obj, '_is_dirty', None):
-        logger.debug(obj._is_dirty)
-        obj.save()
-        del obj._is_dirty
-        return True
-
-
 def format_row(row):
     # trim white space
     for key, value in row.items():
@@ -64,12 +40,8 @@ def process_row(row):
         owner=row['ONAME1'],
         contact=row['CNAME1'],
     )
-    building, created = Building.objects.get_or_create(
-        elbi=row['LICNO'],
-        defaults=default_data,
-    )
-    if not created:
-        update(building, default_data)
+    building, __ = obj_update_or_create(
+        Building, elbi=row['LICNO'], defaults=default_data)
 
     default_data = dict(
         tdlr_id=row['IDNO'],
@@ -82,12 +54,8 @@ def process_row(row):
         year_installed=row['YR_INSTALL'],
         building=building
     )
-    elevator, created = Elevator.objects.get_or_create(
-        decal=row['SUB_NO'],
-        defaults=default_data,
-    )
-    if not created:
-        update(elevator, default_data)
+    elevator, __ = obj_update_or_create(
+        Elevator, decal=row['SUB_NO'], defaults=default_data)
 
 
 def process(path):
@@ -104,7 +72,7 @@ def process(path):
                 import ipdb
                 ipdb.set_trace()
                 raise
-        print('')
+        print('')  # Fix for tqdm leave=True does not print a newline
 
 
 def post_process():

From 9d322b361b2aa14115a456f52a4a1973b88f922a Mon Sep 17 00:00:00 2001
From: crccheck <c@crccheck.com>
Date: Sat, 13 Jun 2015 20:03:18 -0500
Subject: [PATCH 16/23] add more progress bar indicators to data loaders

---
 example_project/settings.py                 | 3 +--
 tx_elevators/management/commands/loadgeo.py | 6 +++++-
 tx_elevators/scripts/scrape.py              | 4 +++-
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/example_project/settings.py b/example_project/settings.py
index 9d75828..1365a54 100644
--- a/example_project/settings.py
+++ b/example_project/settings.py
@@ -146,9 +146,8 @@ def project_dir(*paths):
 LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,
-    # Log everything
     'root': {
-        'level': env.get('LOGGING', 'DEBUG'),
+        'level': env.get('LOGGING', 'WARNING'),
         'handlers': ['console'],
     },
     'filters': {
diff --git a/tx_elevators/management/commands/loadgeo.py b/tx_elevators/management/commands/loadgeo.py
index c08d597..edc4da3 100644
--- a/tx_elevators/management/commands/loadgeo.py
+++ b/tx_elevators/management/commands/loadgeo.py
@@ -5,6 +5,7 @@
 import logging
 
 from django.core.management.base import BaseCommand
+from tqdm import tqdm
 
 
 class Command(BaseCommand):
@@ -17,8 +18,11 @@ def handle(self, path, *args, **options):
         logger = logging.getLogger(__name__)
 
         with open(path) as csvfile:
+            for total, row in enumerate(csvfile, start=1):
+                pass
+            csvfile.seek(0)
             reader = csv.reader(csvfile)
-            for row in reader:
+            for row in tqdm(reader, total=total, leave=True):
                 elbi, latitude, longitude = row
                 building = Building.objects.filter(elbi=elbi).update(
                     latitude=latitude,
diff --git a/tx_elevators/scripts/scrape.py b/tx_elevators/scripts/scrape.py
index 4100c17..3ae7c97 100644
--- a/tx_elevators/scripts/scrape.py
+++ b/tx_elevators/scripts/scrape.py
@@ -14,6 +14,9 @@
 from tqdm import tqdm
 
 
+logger = logging.getLogger('tx_elevators.scrape')
+
+
 def format_row(row):
     # trim white space
     for key, value in row.items():
@@ -83,7 +86,6 @@ def post_process():
 
 if __name__ == "__main__":
     import django; django.setup()  # NOQA
-    logger = logging.getLogger('tx_elevators.scrape')
     path = sys.argv[1]
     process(path)
     post_process()

From 9b07f9c515703d9502c3be558afcde9208618fd4 Mon Sep 17 00:00:00 2001
From: crccheck <c@crccheck.com>
Date: Sat, 13 Jun 2015 20:11:55 -0500
Subject: [PATCH 17/23] reenable geopy

---
 requirements.txt | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 3153eef..96746ee 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,8 +6,7 @@ psycopg2>=2.4.5
 gunicorn==19.2.0
 
 boto==2.36.0
-# geopy==0.97
-# https://github.com/crccheck/geopy/tarball/add-goog-components
+geopy==0.97.1
 django-extensions==1.5.0
 factory-boy
 tqdm==1.0

From 929b7bafe24f0c14a136f0ff37a2ff6305b7a1d2 Mon Sep 17 00:00:00 2001
From: crccheck <c@crccheck.com>
Date: Sat, 13 Jun 2015 20:37:51 -0500
Subject: [PATCH 18/23] delete deprecated url templatetag

---
 tx_elevators/templates/tx_elevators/building_detail.html | 1 -
 tx_elevators/templates/tx_elevators/layouts/base.html    | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/tx_elevators/templates/tx_elevators/building_detail.html b/tx_elevators/templates/tx_elevators/building_detail.html
index 33c2cf6..893c95b 100644
--- a/tx_elevators/templates/tx_elevators/building_detail.html
+++ b/tx_elevators/templates/tx_elevators/building_detail.html
@@ -1,5 +1,4 @@
 {% extends "tx_elevators/layouts/base.html" %}
-{% load url from future %}
 
 {% block head_title %}{{ object }}{% endblock %}
 {% block page_title %}{{ object }}{% endblock %}
diff --git a/tx_elevators/templates/tx_elevators/layouts/base.html b/tx_elevators/templates/tx_elevators/layouts/base.html
index 230758f..68ff4b0 100644
--- a/tx_elevators/templates/tx_elevators/layouts/base.html
+++ b/tx_elevators/templates/tx_elevators/layouts/base.html
@@ -1,4 +1,4 @@
-{% load url from future %}<!DOCTYPE html>
+<!DOCTYPE html>
 <html>
   <head>
     <meta charset=utf-8>

From 2d330ff6ad6838b02819958f5754633b59f0edd1 Mon Sep 17 00:00:00 2001
From: crccheck <c@crccheck.com>
Date: Sat, 13 Jun 2015 20:45:18 -0500
Subject: [PATCH 19/23] remove procfile since this ain't goin' on heroku

---
 Procfile         | 1 -
 requirements.txt | 3 +--
 2 files changed, 1 insertion(+), 3 deletions(-)
 delete mode 100644 Procfile

diff --git a/Procfile b/Procfile
deleted file mode 100644
index c38c1c5..0000000
--- a/Procfile
+++ /dev/null
@@ -1 +0,0 @@
-web: python example_project/manage.py collectstatic --noinput && gunicorn example_project.wsgi --workers 2 --bind 0.0.0.0:$PORT
diff --git a/requirements.txt b/requirements.txt
index 96746ee..a475df8 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,11 +3,10 @@ Django==1.8.2
 dj-database-url==0.3.0
 project_runpy
 psycopg2>=2.4.5
-gunicorn==19.2.0
 
 boto==2.36.0
 geopy==0.97.1
-django-extensions==1.5.0
+django-extensions==1.5.5
 factory-boy
 tqdm==1.0
 dj-obj-update==0.2.0

From 974f703ddecafe9e5d96a7e847b489a787c7ee41 Mon Sep 17 00:00:00 2001
From: crccheck <c@crccheck.com>
Date: Sat, 13 Jun 2015 21:06:29 -0500
Subject: [PATCH 20/23] fix get_queryset/get_query_set

---
 tx_elevators/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tx_elevators/models.py b/tx_elevators/models.py
index 575d9dd..45629c7 100644
--- a/tx_elevators/models.py
+++ b/tx_elevators/models.py
@@ -102,7 +102,7 @@ def for_table(self):
         """
         The queryset that should be used inside the building_detail template.
         """
-        return self.get_query_set().order_by(
+        return self.get_queryset().order_by(
             '-floors',
             '-equipment_type',
             'drive_type',

From 3ae9c197df78d074e2f44b59060c9251ff7e60c6 Mon Sep 17 00:00:00 2001
From: crccheck <c@crccheck.com>
Date: Fri, 1 Apr 2016 04:19:55 +0100
Subject: [PATCH 21/23] workflow tweaks

---
 Makefile              | 43 +++++--------------------------------------
 data/Makefile         |  7 +++----
 tx_elevators/views.py |  2 +-
 3 files changed, 9 insertions(+), 43 deletions(-)

diff --git a/Makefile b/Makefile
index de1cd06..41eac3f 100644
--- a/Makefile
+++ b/Makefile
@@ -2,16 +2,8 @@ PROJECT=./example_project
 MANAGE=python $(PROJECT)/manage.py
 SITE_URL=localhost:8000
 
-help:
-	@echo "make commands:"
-	@echo "  make help    - this help"
-	@echo "  make test    - run test suite"
-	@echo "  make resetdb - drop and recreate the database"
-	@echo "  make scrape  - get data and import"
-	@echo "  make import  - import data"
-	@echo "  make site    - spider $(SITE_URL) and save pages locally"
-	@echo "  make upload  - sync spidered pages to S3"
-	@echo "  make serve   - serve the spided pages locally (on port 8088)"
+help: ## Shows this help
+	@echo "$$(grep -h '#\{2\}' $(MAKEFILE_LIST) | sed 's/: #\{2\} /	/' | column -t -s '	')"
 
 
 # TODO actually write some tests
@@ -19,30 +11,17 @@ test:
 	$(MANAGE) test tx_elevators
 
 
-resetdb:
+resetdb: ## Reset the dev database
 	$(MANAGE) reset_db --router=default --noinput
 	$(MANAGE) migrate --noinput
 
-
-# Backup the local database
-#
-# To restore
-#   cat tx_elevators-2014-11-01.dump | \
-#     docker run --rm --link elevators_1:db -t crccheck/postgis \
-#     pg_restore -U docker -h db --dbname elevators
-dumpdb:
-	docker run --rm --link elevators_1:db -t crccheck/postgis \
-	  pg_dump -U docker -h db -Fc elevators > tx_elevators-$$(date +"%Y-%m-%d").dump
-
-# Dump building geocodes
-#
 # Note that `geocode` will still re-lookup bad addresses
 #
 # To restore: `django loadgeo data/geocoding.csv`
-dumpgeo:
+dumpgeo: ## Dump building geo data
 	$(MANAGE) dumpgeo > data/geocoding.csv
 
-scrape:
+scrape: ## Scrape new data
 	cd data && $(MAKE) $(MFLAGS) clean elevator_data_file.csv
 	python tx_elevators/scripts/scrape.py data/elevator_data_file.csv
 	@echo "should geocode the top 1000 too: $(MANAGE) geocode"
@@ -54,15 +33,6 @@ import:
 	python tx_elevators/scripts/scrape.py data/elevator_data_file.csv
 
 
-dbpush:
-	test $(SCP_DUMP)
-	test $(SCP_URL)
-	pg_dump -Fc --no-acl --no-owner tx_elevators > tx_elevators.dump
-	scp tx_elevators.dump $(SCP_DUMP)
-	heroku pgbackups:restore DATABASE $(SCP_URL)
-	rm tx_elevators.dump
-
-
 # FINISHED --2013-04-01 00:10:54--
 # Total wall clock time: 43m 29s
 # Downloaded: 24343 files, 92M in 5.3s (17.3 MB/s)
@@ -83,6 +53,3 @@ serve:
 upload:
 	cd site && s3-parallel-put --quiet --bucket=${AWS_BUCKET_NAME} \
 	  --grant public-read --header "Cache-Control:max-age=2592000" --gzip .
-
-
-.PHONY: help test resetdb scrape pushdb site upload serve
diff --git a/data/Makefile b/data/Makefile
index 1a802ca..855d546 100644
--- a/data/Makefile
+++ b/data/Makefile
@@ -1,9 +1,9 @@
+CSV = https://www.license.state.tx.us/DownLoadableContent/Elevator/elevator_data_file.csv
+
 # Source Elevator Data CSV:
 # https://www.license.state.tx.us/ElevatorSearch/HelpPage.asp#data
 elevator_data_file.csv:
-	wget https://www.license.state.tx.us/DownLoadableContent/Elevator/elevator_data_file.csv
-	mv $@ $@.orig
-	cat $@.orig | csvsort > $@
+	curl $(CSV) | csvsort > $@
 
 
 sample_elevator_data_file.csv: elevator_data_file.csv
@@ -12,7 +12,6 @@ sample_elevator_data_file.csv: elevator_data_file.csv
 
 clean:
 	rm -f elevator_data_file.csv
-	rm -f elevator_data_file.csv.orig
 
 
 .PHONY: clean
diff --git a/tx_elevators/views.py b/tx_elevators/views.py
index 723d0dc..1f4a690 100644
--- a/tx_elevators/views.py
+++ b/tx_elevators/views.py
@@ -73,7 +73,7 @@ def get_context_data(self, **kwargs):
         context['moving_sidewalks'] = elevators.filter(
             equipment_type='MOVING SIDEWALK')
         context['escalators'] = elevators.filter(equipment_type='ESCALATOR')
-        context['future'] = elevators.filter(year_installed__gt=2015).\
+        context['future'] = elevators.filter(year_installed__gt=2016).\
             select_related('building').order_by('year_installed')
         context['past'] = elevators.filter(year_installed__lt=1000).\
             select_related('building').order_by('year_installed')

From 4e1962695f74c91426477414da534a69eb2eed63 Mon Sep 17 00:00:00 2001
From: crccheck <c@crccheck.com>
Date: Fri, 1 Apr 2016 04:50:43 +0100
Subject: [PATCH 22/23] Refactor site download to be all Makefile

---
 .gitignore           |  3 ++-
 Makefile             | 27 ++++++++++++++++++++++-----
 bin/download_site.sh | 28 ----------------------------
 3 files changed, 24 insertions(+), 34 deletions(-)
 delete mode 100755 bin/download_site.sh

diff --git a/.gitignore b/.gitignore
index 00eb3fe..836278c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,11 +23,12 @@ example_project/local_settings.py
 
 # Randomly generated files
 *.log
+*.pid
 *.pot
 *.pyc
 .DS_Store
 .sass-cache
-site/
+._*
 
 
 # Files generated by Heroku
diff --git a/Makefile b/Makefile
index 41eac3f..784d14f 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
 PROJECT=./example_project
 MANAGE=python $(PROJECT)/manage.py
-SITE_URL=localhost:8000
+PORT=14327
 
 help: ## Shows this help
 	@echo "$$(grep -h '#\{2\}' $(MAKEFILE_LIST) | sed 's/: #\{2\} /	/' | column -t -s '	')"
@@ -33,6 +33,17 @@ import:
 	python tx_elevators/scripts/scrape.py data/elevator_data_file.csv
 
 
+web/start:
+	$(MANAGE) collectstatic --noinput
+	DEBUG=0 $(MANAGE) runserver $(PORT) --nothreading --noreload & echo $$! > web.pid
+	sleep 1
+
+web/stop: web.pid
+	# pkill -P $$(cat web.pid)
+	kill $$(cat web.pid)
+	rm web.pid
+
+
 # FINISHED --2013-04-01 00:10:54--
 # Total wall clock time: 43m 29s
 # Downloaded: 24343 files, 92M in 5.3s (17.3 MB/s)
@@ -40,16 +51,22 @@ import:
 # FINISHED --2014-11-01 16:38:55--
 # Total wall clock time: 9m 4s
 # Downloaded: 25615 files, 120M in 0.8s (150 MB/s)
-site:
-	bin/download_site.sh
+#
+# FINISHED --2016-04-01 04:48:54--
+# Total wall clock time: 4m 59s
+# Downloaded: 26757 files, 126M in 0.1s (971 MB/s)
+site: web/start
+	mkdir -p ._site
+	cd ._site && wget -r localhost:$(PORT) --force-html -e robots=off -nH -nv --max-redirect 0 || true
+	@$(MAKE) web/stop
 
 serve:
-	cd site && python -m SimpleHTTPServer 8088
+	cd ._site && python -m SimpleHTTPServer 8088
 
 # requires installing https://github.com/twpayne/s3-parallel-put
 # uses 8 threads by default
 #
 # INFO:s3-parallel-put[statter-12800]:put 137686194 bytes in 28270 files in 697.4 seconds (197436 bytes/s, 40.5 files/s)
 upload:
-	cd site && s3-parallel-put --quiet --bucket=${AWS_BUCKET_NAME} \
+	cd ._site && s3-parallel-put --quiet --bucket=${AWS_BUCKET_NAME} \
 	  --grant public-read --header "Cache-Control:max-age=2592000" --gzip .
diff --git a/bin/download_site.sh b/bin/download_site.sh
deleted file mode 100755
index b32efb5..0000000
--- a/bin/download_site.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-# Instructions:
-#
-# run from project root directory
-
-set +e
-MANAGE="python ./example_project/manage.py"
-PORT=8008
-
-
-DEBUG=0 $MANAGE runserver --nothreading --noreload $PORT &
-pid=$!
-echo "runserver pid: $pid"
-
-# make sure to kill the server if terminated early
-trap "kill $pid; echo bye $pid" EXIT
-
-# give time for the servers to get up
-sleep 1
-
-$MANAGE collectstatic --noinput
-
-mkdir -p site
-cd site && wget -r localhost:$PORT --force-html -e robots=off -nH -nv --max-redirect 0
-
-# kill server, run in a subprocess so we can suppress "Terminated" message
-(kill $pid 2>&1) > /dev/null
-
-echo "bye"

From 3f2d1743d819ca71db2c5d7be07ade72bf75f595 Mon Sep 17 00:00:00 2001
From: crccheck <c@crccheck.com>
Date: Fri, 28 Oct 2016 03:58:35 +0000
Subject: [PATCH 23/23] switch to regular aws-cli instead of s3-parallel-put
 since it's broken

---
 Makefile | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/Makefile b/Makefile
index 784d14f..f04e488 100644
--- a/Makefile
+++ b/Makefile
@@ -63,10 +63,7 @@ site: web/start
 serve:
 	cd ._site && python -m SimpleHTTPServer 8088
 
-# requires installing https://github.com/twpayne/s3-parallel-put
-# uses 8 threads by default
-#
-# INFO:s3-parallel-put[statter-12800]:put 137686194 bytes in 28270 files in 697.4 seconds (197436 bytes/s, 40.5 files/s)
 upload:
-	cd ._site && s3-parallel-put --quiet --bucket=${AWS_BUCKET_NAME} \
-	  --grant public-read --header "Cache-Control:max-age=2592000" --gzip .
+	aws s3 sync ._site s3://$(AWS_BUCKET_NAME)/ \
+	  --cache-control "max-age=2592000" \
+	  --acl "public-read"