From 39f591f992a21881318f5656f2861f8a4a239597 Mon Sep 17 00:00:00 2001 From: Darren Weber Date: Tue, 6 Aug 2019 11:02:36 -0700 Subject: [PATCH 1/4] Install gdal 2.4.2 on lambci/lambda:build-python3.7 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6a87915..d830918 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM lambci/lambda:build-python3.6 +FROM lambci/lambda:build-python3.7 ENV \ LANG=en_US.UTF-8 \ From 3fa8e1fee9385cd96842c7424addcc2c40966649 Mon Sep 17 00:00:00 2001 From: Darren Weber Date: Tue, 6 Aug 2019 11:19:42 -0700 Subject: [PATCH 2/4] Use GDAL and python versions in the image tag - set BUILD as IMAGE:TAG - use BUILD for most make rules --- Makefile | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index eeb7f27..da32b9f 100644 --- a/Makefile +++ b/Makefile @@ -1,31 +1,34 @@ SHELL = /bin/bash -TAG = 2.4.2 -IMAGE := ${DOCKER_USERNAME}/amazonlinux-gdal:${TAG} +PY_VERSION = 3.7 +GDAL_VERSION = 2.4.2 +TAG = gdal${GDAL_VERSION}-py${PY_VERSION} +IMAGE := amazonlinux-gdal +BUILD := ${IMAGE}:${TAG} all: build push build: - docker build -f Dockerfile -t amazonlinux-gdal:${TAG} . + docker build -f Dockerfile -t ${BUILD} . -shell: - docker run --name amazonlinux --volume $(shell pwd)/:/data --rm -it ${IMAGE} /bin/bash +shell: build + docker run --name amazonlinux --volume $(shell pwd)/:/data --rm -it ${BUILD} /bin/bash debug-gdal: build - docker run \ - --name amazonlinux \ - -itd amazonlinux-gdal:${TAG} /bin/bash + docker run --name amazonlinux \ + -itd ${BUILD} /bin/bash docker exec -it amazonlinux bash -c 'ldd /var/task/lib/libgdal.so' docker exec -it amazonlinux bash -c 'readelf -d /var/task/lib/libgdal.so' docker stop amazonlinux docker rm amazonlinux test: - docker run amazonlinux-gdal:${TAG} bash -c "gdalinfo --version | grep '${TAG}'" - docker run amazonlinux-gdal:${TAG} bash -c "gdalinfo --formats | grep 'JP2OpenJPEG'" + docker run ${BUILD} bash -c "gdalinfo --version | grep '${GDAL_VERSION}'" + docker run ${BUILD} bash -c "gdalinfo --formats | grep 'JP2OpenJPEG'" + docker run ${BUILD} bash -c "python --version | grep '${PY_VERSION}'" push: - docker tag amazonlinux-gdal:${TAG} ${IMAGE} - docker push ${IMAGE} + #docker tag ${IMAGE} ${BUILD} + docker push ${DOCKER_USERNAME}/${BUILD} clean: docker stop amazonlinux From d8276fcd9bc959fcd9d455ef3fe8ac8834960c18 Mon Sep 17 00:00:00 2001 From: Darren Weber Date: Tue, 6 Aug 2019 14:24:54 -0700 Subject: [PATCH 3/4] Image bundled with cython,numpy,gdal,rasterio python bindings - numpy >= 1.17 needs a c99 compiler setting --- Dockerfile | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index d830918..751e6af 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,8 @@ RUN yum install -y automake16 libpng-devel nasm ENV PREFIX /var/task -# versions of packages +# versions of packages, see also rasterio test matrix at: +# - https://github.com/mapbox/rasterio/blob/master/.travis.yml ENV \ PKGCONFIG_VERSION=0.29.2 \ PROJ_VERSION=5.2.0 \ @@ -179,5 +180,9 @@ ENV \ ENV PATH=$PREFIX/bin:$PATH -RUN pip3 install pip -U -RUN pip3 install cython numpy --no-binary numpy +# numpy 1.17 requires an explicit c99 compiler option +# - https://github.com/numpy/numpy/pull/12783/files +ENV CFLAGS='-std=c99' +RUN pip3 install pip -U && \ + pip3 install cython numpy "gdal==${GDAL_VERSION}" rasterio --no-binary :all: + From 4e6349a86e2bea033d91c6f3e29b78922ea5cf56 Mon Sep 17 00:00:00 2001 From: Darren Weber Date: Tue, 6 Aug 2019 19:05:01 -0700 Subject: [PATCH 4/4] make rules to build and package a lambda layer in /opt --- .gitignore | 1 + Dockerfile | 5 ++++- Makefile | 32 ++++++++++++++++++++++++++++---- 3 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c4c4ffc --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.zip diff --git a/Dockerfile b/Dockerfile index 751e6af..aead3c4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,10 @@ ENV \ RUN yum makecache fast RUN yum install -y automake16 libpng-devel nasm -ENV PREFIX /var/task +# labda runtime: /var/task +# labda layer: /opt +ARG prefix=/var/task +ENV PREFIX=${prefix} # versions of packages, see also rasterio test matrix at: # - https://github.com/mapbox/rasterio/blob/master/.travis.yml diff --git a/Makefile b/Makefile index da32b9f..2f0eeeb 100644 --- a/Makefile +++ b/Makefile @@ -27,9 +27,33 @@ test: docker run ${BUILD} bash -c "python --version | grep '${PY_VERSION}'" push: - #docker tag ${IMAGE} ${BUILD} docker push ${DOCKER_USERNAME}/${BUILD} -clean: - docker stop amazonlinux - docker rm amazonlinux +container-clean: + docker stop amazonlinux > /dev/null 2>&1 || true + docker rm amazonlinux > /dev/null 2>&1 || true + +# --- +# lambda layer build and package using /opt + +LAYER_BUILD = ${BUILD}-layer +LAYER_PACKAGE := amazonlinux-${TAG}-layer.zip + +lambda-layer-build: + docker build -f Dockerfile -t ${LAYER_BUILD} --build-arg prefix=/opt . + +lambda-layer-shell: lambda-layer-build container-clean + docker run --name amazonlinux --volume $(shell pwd)/:/data --rm -it ${LAYER_BUILD} /bin/bash + +lambda-layer-package: lambda-layer-build container-clean + docker run --name amazonlinux -itd ${LAYER_BUILD} /bin/bash + docker exec -it amazonlinux bash -c 'mkdir -p $${PREFIX}/python/lib/python${PY_VERSION}/site-packages' + docker exec -it amazonlinux bash -c 'rsync -a /var/lang/lib/python${PY_VERSION}/site-packages/ $${PREFIX}/python/lib/python${PY_VERSION}/site-packages/' + docker exec -it amazonlinux bash -c 'cd $${PREFIX} && zip -r9 --symlinks /tmp/package.zip python' + docker exec -it amazonlinux bash -c 'cd $${PREFIX} && zip -r9 --symlinks /tmp/package.zip lib/*.so*' + docker exec -it amazonlinux bash -c 'cd $${PREFIX} && zip -r9 --symlinks /tmp/package.zip lib64/*.so*' + docker exec -it amazonlinux bash -c 'cd $${PREFIX} && zip -r9 --symlinks /tmp/package.zip bin' + docker exec -it amazonlinux bash -c 'cd $${PREFIX} && zip -r9 /tmp/package.zip share' + docker cp amazonlinux:/tmp/package.zip ${LAYER_PACKAGE} + docker stop amazonlinux && docker rm amazonlinux +