diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a4f851c..abbe4133 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,18 +1,21 @@ -name: CI +name: Docker Image CI on: push: branches: - - master + - '*' pull_request: branches: - - master + - master jobs: - ci: + + build: + + runs-on: ubuntu-latest + steps: - - uses: actions/checkout@v1 - - run: docker build -t "${PWD##*/}" . - - run: docker run "${PWD##*/}" python -V - - run: docker run "${PWD##*/}" version + - uses: actions/checkout@v2 + - name: Build the Docker image + run: docker build . --file Dockerfile --tag dataopssre/docker-airflow2:$(date +%s) \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..3f69b484 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,38 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: CI + +on: + release: + types: [published] + +jobs: + push_to_registry: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Log in to Docker Hub + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: dataopssre/docker-airflow2 + + - name: Build and push Docker image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/.github/workflows/publish_helm_chart.yml b/.github/workflows/publish_helm_chart.yml new file mode 100644 index 00000000..9bc77aa4 --- /dev/null +++ b/.github/workflows/publish_helm_chart.yml @@ -0,0 +1,36 @@ +name: Release Charts + +on: + push: + branches: + - master + +jobs: + release: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Configure Git + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + + - name: Install Helm + uses: azure/setup-helm@v1 + with: + version: v3.4.0 + + - name: Add Helm repo dependency + run: helm repo add bitnami https://charts.bitnami.com/bitnami + + - name: Run chart-releaser + uses: helm/chart-releaser-action@v1.2.1 + with: + charts_dir: charts + env: + CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + CR_RELEASE_NAME_TEMPLATE: "helm-chart/{{ .Name }}-{{ .Version }}" diff --git a/.gitignore b/.gitignore index 991a0fb0..bfdfaa42 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ Session.vim *.tmlanguage.cache *.tmPreferences.cache *.stTheme.cache +.idea/ # workspace files are user-specific *.sublime-workspace diff --git a/Dockerfile b/Dockerfile index 02782d0c..9c632f81 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,86 +1,23 @@ -# VERSION 1.10.9 -# AUTHOR: Matthieu "Puckel_" Roisil -# DESCRIPTION: Basic Airflow container -# BUILD: docker build --rm -t puckel/docker-airflow . -# SOURCE: https://github.com/puckel/docker-airflow +FROM apache/airflow:2.2.3-python3.9 -FROM python:3.7-slim-buster -LABEL maintainer="Puckel_" +LABEL maintainer="dataops-sre" -# Never prompt the user for choices on installation/configuration of packages -ENV DEBIAN_FRONTEND noninteractive -ENV TERM linux +ARG AIRFLOW_VERSION=2.2.3 +ARG PYTHON_VERSION=3.9 -# Airflow -ARG AIRFLOW_VERSION=1.10.9 -ARG AIRFLOW_USER_HOME=/usr/local/airflow ARG AIRFLOW_DEPS="" ARG PYTHON_DEPS="" -ENV AIRFLOW_HOME=${AIRFLOW_USER_HOME} -# Define en_US. -ENV LANGUAGE en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LC_ALL en_US.UTF-8 -ENV LC_CTYPE en_US.UTF-8 -ENV LC_MESSAGES en_US.UTF-8 +RUN pip install apache-airflow[crypto,celery,postgres,hive,jdbc,mysql,ssh,kubernetes,snowflake${AIRFLOW_DEPS:+,}${AIRFLOW_DEPS}]==${AIRFLOW_VERSION} \ + # plyvel Required by airflow, unknown why not installed as a dependency + && pip install plyvel==1.3.0 \ + # pyarrow 5.0 required by snowflake but possibly incompatible by airflow + && pip install pyarrow==5.0.0 \ + && if [ -n "${PYTHON_DEPS}" ]; then pip install ${PYTHON_DEPS}; fi -# Disable noisy "Handling signal" log messages: -# ENV GUNICORN_CMD_ARGS --log-level WARNING - -RUN set -ex \ - && buildDeps=' \ - freetds-dev \ - libkrb5-dev \ - libsasl2-dev \ - libssl-dev \ - libffi-dev \ - libpq-dev \ - git \ - ' \ - && apt-get update -yqq \ - && apt-get upgrade -yqq \ - && apt-get install -yqq --no-install-recommends \ - $buildDeps \ - freetds-bin \ - build-essential \ - default-libmysqlclient-dev \ - apt-utils \ - curl \ - rsync \ - netcat \ - locales \ - && sed -i 's/^# en_US.UTF-8 UTF-8$/en_US.UTF-8 UTF-8/g' /etc/locale.gen \ - && locale-gen \ - && update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 \ - && useradd -ms /bin/bash -d ${AIRFLOW_USER_HOME} airflow \ - && pip install -U pip setuptools wheel \ - && pip install pytz \ - && pip install pyOpenSSL \ - && pip install ndg-httpsclient \ - && pip install pyasn1 \ - && pip install apache-airflow[crypto,celery,postgres,hive,jdbc,mysql,ssh${AIRFLOW_DEPS:+,}${AIRFLOW_DEPS}]==${AIRFLOW_VERSION} \ - && pip install 'redis==3.2' \ - && if [ -n "${PYTHON_DEPS}" ]; then pip install ${PYTHON_DEPS}; fi \ - && apt-get purge --auto-remove -yqq $buildDeps \ - && apt-get autoremove -yqq --purge \ - && apt-get clean \ - && rm -rf \ - /var/lib/apt/lists/* \ - /tmp/* \ - /var/tmp/* \ - /usr/share/man \ - /usr/share/doc \ - /usr/share/doc-base COPY script/entrypoint.sh /entrypoint.sh -COPY config/airflow.cfg ${AIRFLOW_USER_HOME}/airflow.cfg - -RUN chown -R airflow: ${AIRFLOW_USER_HOME} - -EXPOSE 8080 5555 8793 +COPY config/webserver_config.py $AIRFLOW_HOME/ +COPY dags $AIRFLOW_HOME/dags -USER airflow -WORKDIR ${AIRFLOW_USER_HOME} -ENTRYPOINT ["/entrypoint.sh"] -CMD ["webserver"] +ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file diff --git a/README.md b/README.md index 922e51a7..a1154198 100644 --- a/README.md +++ b/README.md @@ -1,38 +1,57 @@ -# docker-airflow -[![CI status](https://github.com/puckel/docker-airflow/workflows/CI/badge.svg?branch=master)](https://github.com/puckel/docker-airflow/actions?query=workflow%3ACI+branch%3Amaster+event%3Apush) -[![Docker Build status](https://img.shields.io/docker/build/puckel/docker-airflow?style=plastic)](https://hub.docker.com/r/puckel/docker-airflow/tags?ordering=last_updated) +# docker-airflow 2 +[![Docker Image CI](https://github.com/dataops-sre/docker-airflow2/actions/workflows/ci.yml/badge.svg)](https://github.com/dataops-sre/docker-airflow2/actions/workflows/ci.yml) -[![Docker Hub](https://img.shields.io/badge/docker-ready-blue.svg)](https://hub.docker.com/r/puckel/docker-airflow/) -[![Docker Pulls](https://img.shields.io/docker/pulls/puckel/docker-airflow.svg)]() -[![Docker Stars](https://img.shields.io/docker/stars/puckel/docker-airflow.svg)]() +[![Docker Hub](https://img.shields.io/badge/docker-ready-blue.svg)](https://hub.docker.com/r/dataopssre/docker-airflow2) +[![Docker Pulls](https://img.shields.io/docker/pulls/dataopssre/docker-airflow2.svg)](https://hub.docker.com/r/dataopssre/docker-airflow2) +[![Docker Stars](https://img.shields.io/docker/stars/dataopssre/docker-airflow2.svg)](https://hub.docker.com/r/dataopssre/docker-airflow2) -This repository contains **Dockerfile** of [apache-airflow](https://github.com/apache/incubator-airflow) for [Docker](https://www.docker.com/)'s [automated build](https://registry.hub.docker.com/u/puckel/docker-airflow/) published to the public [Docker Hub Registry](https://registry.hub.docker.com/). +This repository contains **Dockerfile** of [apache-airflow2](https://github.com/apache/airflow) for [Docker](https://www.docker.com/)'s [automated build](https://hub.docker.com/r/dataopssre/docker-airflow2/) published to the public [Docker Hub Registry](https://hub.docker.com/r/dataopssre/docker-airflow2). + +## TL,TR +Use Docker image [dataopssre/docker-airflow2](https://hub.docker.com/r/dataopssre/docker-airflow2) to update your airflow setup + +Helm chart to deploy airflow2 docker image: +``` +helm repo add dataops-sre-airflow https://dataops-sre.github.io/docker-airflow2/ +helm repo update +helm install airflow dataops-sre-airflow/airflow --wait --timeout 300s +``` ## Informations -* Based on Python (3.7-slim-buster) official Image [python:3.7-slim-buster](https://hub.docker.com/_/python/) and uses the official [Postgres](https://hub.docker.com/_/postgres/) as backend and [Redis](https://hub.docker.com/_/redis/) as queue +* Based on official Airflow 2 Image [apache/airflow2:2.2.3-python3.9 +](https://hub.docker.com/_/python/) and uses the official [Postgres](https://hub.docker.com/_/postgres/) as backend and [Redis](https://hub.docker.com/_/redis/) as queue +* Docker entrypoint script is forked from [puckel/docker-airflow](https://github.com/puckel/docker-airflow) * Install [Docker](https://www.docker.com/) * Install [Docker Compose](https://docs.docker.com/compose/install/) -* Following the Airflow release from [Python Package Index](https://pypi.python.org/pypi/apache-airflow) -## Installation -Pull the image from the Docker repository. +## Motivation +This repo is forked form [puckel/docker-airflow](https://github.com/puckel/docker-airflow), the original repo seems not maintained. + +Airflow is been updated to version 2 and release its [official docker image](https://hub.docker.com/r/apache/airflow), you can also find [bitnami airflow image](https://hub.docker.com/r/bitnami/airflow). Nevertheless, puckel's image is still interesting, in the market none of providers offer an Airflow run with LocalExecutor with scheduler in one container, it is extremely usefull when to deploy a simple Airflow to an AWS EKS cluster. With Kubernetes you can resolve Airflow scablity issue by using uniquely KubernetesPodOpetertor in your dags, then we need zero computational power for airflow, it serves pure purpose of scheduler, seperate scheduler and webserver into two different pods is a bit problematic on AWS EKS cluster, we want to keep dags and logs into a Persistant volume, but AWS has some limitation for EBS volume multi attach, which means webserver and scheduler pod has to be scheduled on the same EKS node, it is a bit annoying. Thus puckel's airflow startup script is usefull. + +what this fork do : + +* Disactive by default the login screen in Airflow 2 +* Improve current script to only take into account Airflow environment variables +* Make sure docker compose files works +* Add Airflow2 deployment helm chart and release a public repository in Github + +You can use the helm chart release in this repository, see [here](https://dataops-sre.github.io/docker-airflow2) to deploys airflow2 to a Kubernetes cluster. - docker pull puckel/docker-airflow ## Build Optionally install [Extra Airflow Packages](https://airflow.incubator.apache.org/installation.html#extra-package) and/or python dependencies at build time : - docker build --rm --build-arg AIRFLOW_DEPS="datadog,dask" -t puckel/docker-airflow . - docker build --rm --build-arg PYTHON_DEPS="flask_oauthlib>=0.9" -t puckel/docker-airflow . + docker build --rm --build-arg AIRFLOW_DEPS="datadog,dask" -t dataopssre/docker-airflow2 . + docker build --rm --build-arg PYTHON_DEPS="requests" -t dataopssre/docker-airflow2 . or combined - docker build --rm --build-arg AIRFLOW_DEPS="datadog,dask" --build-arg PYTHON_DEPS="flask_oauthlib>=0.9" -t puckel/docker-airflow . + docker build --rm --build-arg AIRFLOW_DEPS="datadog,dask" --build-arg PYTHON_DEPS="requests" -t dataopssre/docker-airflow2 . -Don't forget to update the airflow images in the docker-compose files to puckel/docker-airflow:latest. ## Usage @@ -40,7 +59,7 @@ By default, docker-airflow runs Airflow with **SequentialExecutor** : docker run -d -p 8080:8080 puckel/docker-airflow webserver -If you want to run another executor, use the other docker-compose.yml files provided in this repository. +If you want to run another executor, use the docker-compose.yml files provided in this repository. For **LocalExecutor** : @@ -50,11 +69,10 @@ For **CeleryExecutor** : docker-compose -f docker-compose-CeleryExecutor.yml up -d -NB : If you want to have DAGs example loaded (default=False), you've to set the following environment variable : +NB : If you want to have DAGs example loaded (default=False), you've to set the following environment variable in docker-compose files : -`LOAD_EX=n` +`AIRFLOW__CORE__LOAD_EXAMPLES=True` - docker run -d -p 8080:8080 -e LOAD_EX=y puckel/docker-airflow If you want to use Ad hoc query, make sure you've configured connections: Go to Admin -> Connections and Edit "postgres_default" set this values (equivalent to values in airflow.cfg/docker-compose*.yml) : @@ -65,26 +83,26 @@ Go to Admin -> Connections and Edit "postgres_default" set this values (equivale For encrypted connection passwords (in Local or Celery Executor), you must have the same fernet_key. By default docker-airflow generates the fernet_key at startup, you have to set an environment variable in the docker-compose (ie: docker-compose-LocalExecutor.yml) file to set the same key accross containers. To generate a fernet_key : - docker run puckel/docker-airflow python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print(FERNET_KEY)" + docker run dataopssre/docker-airflow2 python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print(FERNET_KEY)" ## Configuring Airflow -It's possible to set any configuration value for Airflow from environment variables, which are used over values from the airflow.cfg. +It's possible to set any configuration value for Airflow from environment variables The general rule is the environment variable should be named `AIRFLOW__
__`, for example `AIRFLOW__CORE__SQL_ALCHEMY_CONN` sets the `sql_alchemy_conn` config option in the `[core]` section. -Check out the [Airflow documentation](http://airflow.readthedocs.io/en/latest/howto/set-config.html#setting-configuration-options) for more details +Check out the [Airflow documentation](https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html) for more details You can also define connections via environment variables by prefixing them with `AIRFLOW_CONN_` - for example `AIRFLOW_CONN_POSTGRES_MASTER=postgres://user:password@localhost:5432/master` for a connection called "postgres_master". The value is parsed as a URI. This will work for hooks etc, but won't show up in the "Ad-hoc Query" section unless an (empty) connection is also created in the DB ## Custom Airflow plugins -Airflow allows for custom user-created plugins which are typically found in `${AIRFLOW_HOME}/plugins` folder. Documentation on plugins can be found [here](https://airflow.apache.org/plugins.html) +Airflow allows for custom user-created plugins which are typically found in `${AIRFLOW_HOME}/plugins` folder. Documentation on plugins can be found [here](https://airflow.apache.org/docs/apache-airflow/stable/plugins.html) In order to incorporate plugins into your docker container - Create the plugins folders `plugins/` with your custom plugins. - Mount the folder as a volume by doing either of the following: - - Include the folder as a volume in command-line `-v $(pwd)/plugins/:/usr/local/airflow/plugins` + - Include the folder as a volume in command-line `-v $(pwd)/plugins/:/opt/airflow/plugins` - Use docker-compose-LocalExecutor.yml or docker-compose-CeleryExecutor.yml which contains support for adding the plugins folder as a volume ## Install custom python package @@ -111,20 +129,19 @@ This can be used to scale to a multi node setup using docker swarm. If you want to run other airflow sub-commands, such as `list_dags` or `clear` you can do so like this: - docker run --rm -ti puckel/docker-airflow airflow list_dags + docker run --rm -ti dataopssre/docker-airflow2 airflow dags list or with your docker-compose set up like this: - docker-compose -f docker-compose-CeleryExecutor.yml run --rm webserver airflow list_dags + docker-compose -f docker-compose-CeleryExecutor.yml run --rm webserver airflow dags list You can also use this to run a bash shell or any other command in the same environment that airflow would be run in: - docker run --rm -ti puckel/docker-airflow bash - docker run --rm -ti puckel/docker-airflow ipython + docker run --rm -ti dataopssre/docker-airflow2 bash + docker run --rm -ti dataopssre/docker-airflow2 ipython # Simplified SQL database configuration using PostgreSQL -If the executor type is set to anything else than *SequentialExecutor* you'll need an SQL database. Here is a list of PostgreSQL configuration variables and their default values. They're used to compute the `AIRFLOW__CORE__SQL_ALCHEMY_CONN` and `AIRFLOW__CELERY__RESULT_BACKEND` variables when needed for you if you don't provide them explicitly: diff --git a/charts/airflow/.helmignore b/charts/airflow/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/charts/airflow/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/airflow/Chart.lock b/charts/airflow/Chart.lock new file mode 100644 index 00000000..25618433 --- /dev/null +++ b/charts/airflow/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 10.1.4 +digest: sha256:ab019bce6981e2e913c9399f96f8d1ea7e1ff27043d586f890edc57cbb12b7a8 +generated: "2020-12-17T15:24:54.15255122+01:00" diff --git a/charts/airflow/Chart.yaml b/charts/airflow/Chart.yaml new file mode 100644 index 00000000..0aa90a38 --- /dev/null +++ b/charts/airflow/Chart.yaml @@ -0,0 +1,34 @@ +apiVersion: v2 +name: airflow +description: | + Airflow 2 Helm chart for Kubernetes, it deploys a simple Airflow with + LocalExecutor with airflow scheduler in a same pod. It has login screen + disabled, it supports zipped dags. + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 1.2.1 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +appVersion: 2.2.3 + +dependencies: +- name: postgresql + version: 10.1.4 + repository: https://charts.bitnami.com/bitnami + condition: postgresql.enabled + tags: + - database \ No newline at end of file diff --git a/charts/airflow/charts/postgresql-10.1.4.tgz b/charts/airflow/charts/postgresql-10.1.4.tgz new file mode 100644 index 00000000..5db01cb6 Binary files /dev/null and b/charts/airflow/charts/postgresql-10.1.4.tgz differ diff --git a/charts/airflow/templates/NOTES.txt b/charts/airflow/templates/NOTES.txt new file mode 100644 index 00000000..e0db4302 --- /dev/null +++ b/charts/airflow/templates/NOTES.txt @@ -0,0 +1,21 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ . }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "airflow.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "airflow.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "airflow.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "airflow.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:8080 +{{- end }} diff --git a/charts/airflow/templates/_helpers.tpl b/charts/airflow/templates/_helpers.tpl new file mode 100644 index 00000000..a15c05a5 --- /dev/null +++ b/charts/airflow/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "airflow.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airflow.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airflow.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "airflow.labels" -}} +helm.sh/chart: {{ include "airflow.chart" . }} +{{ include "airflow.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "airflow.selectorLabels" -}} +app.kubernetes.io/name: {{ include "airflow.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "airflow.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "airflow.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/charts/airflow/templates/deployment.yaml b/charts/airflow/templates/deployment.yaml new file mode 100644 index 00000000..eb24f44f --- /dev/null +++ b/charts/airflow/templates/deployment.yaml @@ -0,0 +1,86 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "airflow.fullname" . }} + labels: + {{- include "airflow.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "airflow.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "airflow.selectorLabels" . | nindent 8 }} + spec: + serviceAccountName: {{ include "airflow.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.airflow.image.repository }}:{{ .Values.airflow.image.tag }}" + imagePullPolicy: {{ .Values.airflow.image.imagePullPolicy }} + ports: + - containerPort: 8080 + volumeMounts: + - mountPath: /opt/airflow/logs + name: {{ include "airflow.fullname" . }}-logs-pvc + - mountPath: /opt/airflow/dags + name: {{ include "airflow.fullname" . }}-dags-pvc + args: + - webserver + env: + {{- range $key, $val := .Values.airflow.config }} + - name: {{ $key }} + value: {{ $val | quote }} + {{- end}} + livenessProbe: + failureThreshold: 5 + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: {{ .Values.web.initialDelaySeconds }} + periodSeconds: 60 + successThreshold: 1 + timeoutSeconds: 1 + readinessProbe: + failureThreshold: 5 + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: {{ .Values.web.initialDelaySeconds }} + periodSeconds: 60 + successThreshold: 1 + timeoutSeconds: 1 + {{- with .Values.airflow.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: {{ include "airflow.fullname" . }}-logs-pvc + persistentVolumeClaim: + claimName: {{ include "airflow.fullname" . }}-logs-pvc + - name: {{ include "airflow.fullname" . }}-dags-pvc + persistentVolumeClaim: + claimName: {{ include "airflow.fullname" . }}-dags-pvc + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/airflow/templates/pvc.yaml b/charts/airflow/templates/pvc.yaml new file mode 100644 index 00000000..48131271 --- /dev/null +++ b/charts/airflow/templates/pvc.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "airflow.fullname" . }}-logs-pvc + labels: + {{- include "airflow.labels" . | nindent 4 }} + annotations: + helm.sh/resource-policy: keep +spec: + accessModes: + - {{ .Values.logsPersistence.accessMode }} + resources: + requests: + storage: {{ .Values.logsPersistence.size }} + +--- + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "airflow.fullname" . }}-dags-pvc + labels: + {{- include "airflow.labels" . | nindent 4 }} + annotations: + helm.sh/resource-policy: keep +spec: + accessModes: + - {{ .Values.dagsPersistence.accessMode }} + resources: + requests: + storage: {{ .Values.dagsPersistence.size }} diff --git a/charts/airflow/templates/service.yaml b/charts/airflow/templates/service.yaml new file mode 100644 index 00000000..75bd8b35 --- /dev/null +++ b/charts/airflow/templates/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "airflow.fullname" . }} + labels: + {{- include "airflow.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + protocol: TCP + name: http + selector: + {{- include "airflow.selectorLabels" . | nindent 4 }} diff --git a/charts/airflow/templates/serviceaccount.yaml b/charts/airflow/templates/serviceaccount.yaml new file mode 100644 index 00000000..7c005bf2 --- /dev/null +++ b/charts/airflow/templates/serviceaccount.yaml @@ -0,0 +1,25 @@ +{{- if .Values.serviceAccount.create -}} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "airflow.serviceAccountName" . }} + labels: + {{- include "airflow.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "airflow.serviceAccountName" . }}-rb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: admin +subjects: +- kind: ServiceAccount + name: {{ include "airflow.serviceAccountName" . }} +{{- end }} diff --git a/charts/airflow/templates/tests/test-connection.yaml b/charts/airflow/templates/tests/test-connection.yaml new file mode 100644 index 00000000..5a702274 --- /dev/null +++ b/charts/airflow/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "airflow.fullname" . }}-test-connection" + labels: + {{- include "airflow.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test-success +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "airflow.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/charts/airflow/values.yaml b/charts/airflow/values.yaml new file mode 100644 index 00000000..63225f04 --- /dev/null +++ b/charts/airflow/values.yaml @@ -0,0 +1,80 @@ +# Default values for airflow. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +nameOverride: "" +fullnameOverride: "" + +airflow: + image: + repository: dataopssre/docker-airflow2 + tag: 2.2.3 + imagePullPolicy: IfNotPresent + config: + AIRFLOW__CORE__EXECUTOR: LocalExecutor + AIRFLOW__WEBSERVER__EXPOSE_CONFIG: true + AIRFLOW__CORE__DAG_RUN_CONF_OVERRIDES_PARAMS: true + AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: 30 + AIRFLOW__CORE__FERNET_KEY: 46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= + AIRFLOW__KUBERNETES__NAMESPACE: default + AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@airflow-postgresql:5432/airflow + AIRFLOW__CORE__DAG_DISCOVERY_SAFE_MODE: false + # Don't load airflow dag from db for zipped dags + AIRFLOW__CORE__STORE_DAG_CODE: false + POSTGRES_HOST: airflow-postgresql + ENV: minikube + resources: + requests: + cpu: 150m + memory: 1Gi + +web: + initialStartupDelay: "10" + initialDelaySeconds: "10" + +postgresql: + enabled: true + postgresqlUsername: airflow + postgresqlDatabase: airflow + postgresqlPassword: airflow + +logsPersistence: + accessMode: ReadWriteOnce + size: 1Gi + +dagsPersistence: + accessMode: ReadWriteOnce + size: 1Gi + +serviceAccount: + create: true + annotations: {} + name: airflow + +podAnnotations: {} + +podSecurityContext: + fsGroup: 1000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 8080 + +ingress: + enabled: false + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/config/airflow.cfg b/config/airflow.cfg deleted file mode 100644 index 9e4d5229..00000000 --- a/config/airflow.cfg +++ /dev/null @@ -1,990 +0,0 @@ -[core] -# The folder where your airflow pipelines live, most likely a -# subfolder in a code repository. This path must be absolute. -dags_folder = /usr/local/airflow/dags - -# The folder where airflow should store its log files -# This path must be absolute -base_log_folder = /usr/local/airflow/logs - -# Airflow can store logs remotely in AWS S3, Google Cloud Storage or Elastic Search. -# Set this to True if you want to enable remote logging. -remote_logging = False - -# Users must supply an Airflow connection id that provides access to the storage -# location. -remote_log_conn_id = -remote_base_log_folder = -encrypt_s3_logs = False - -# Logging level -logging_level = INFO - -# Logging level for Flask-appbuilder UI -fab_logging_level = WARN - -# Logging class -# Specify the class that will specify the logging configuration -# This class has to be on the python classpath -# Example: logging_config_class = my.path.default_local_settings.LOGGING_CONFIG -logging_config_class = - -# Flag to enable/disable Colored logs in Console -# Colour the logs when the controlling terminal is a TTY. -colored_console_log = True - -# Log format for when Colored logs is enabled -colored_log_format = [%%(blue)s%%(asctime)s%%(reset)s] {{%%(blue)s%%(filename)s:%%(reset)s%%(lineno)d}} %%(log_color)s%%(levelname)s%%(reset)s - %%(log_color)s%%(message)s%%(reset)s -colored_formatter_class = airflow.utils.log.colored_log.CustomTTYColoredFormatter - -# Format of Log line -log_format = [%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s -simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s - -# Log filename format -log_filename_template = {{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log -log_processor_filename_template = {{ filename }}.log -dag_processor_manager_log_location = /usr/local/airflow/logs/dag_processor_manager/dag_processor_manager.log - -# Name of handler to read task instance logs. -# Default to use task handler. -task_log_reader = task - -# Hostname by providing a path to a callable, which will resolve the hostname. -# The format is "package:function". -# -# For example, default value "socket:getfqdn" means that result from getfqdn() of "socket" -# package will be used as hostname. -# -# No argument should be required in the function specified. -# If using IP address as hostname is preferred, use value ``airflow.utils.net:get_host_ip_address`` -hostname_callable = socket:getfqdn - -# Default timezone in case supplied date times are naive -# can be utc (default), system, or any IANA timezone string (e.g. Europe/Amsterdam) -default_timezone = utc - -# The executor class that airflow should use. Choices include -# SequentialExecutor, LocalExecutor, CeleryExecutor, DaskExecutor, KubernetesExecutor -executor = SequentialExecutor - -# The SqlAlchemy connection string to the metadata database. -# SqlAlchemy supports many different database engine, more information -# their website -# sql_alchemy_conn = sqlite:////tmp/airflow.db - -# The encoding for the databases -sql_engine_encoding = utf-8 - -# If SqlAlchemy should pool database connections. -sql_alchemy_pool_enabled = True - -# The SqlAlchemy pool size is the maximum number of database connections -# in the pool. 0 indicates no limit. -sql_alchemy_pool_size = 5 - -# The maximum overflow size of the pool. -# When the number of checked-out connections reaches the size set in pool_size, -# additional connections will be returned up to this limit. -# When those additional connections are returned to the pool, they are disconnected and discarded. -# It follows then that the total number of simultaneous connections the pool will allow -# is pool_size + max_overflow, -# and the total number of "sleeping" connections the pool will allow is pool_size. -# max_overflow can be set to -1 to indicate no overflow limit; -# no limit will be placed on the total number of concurrent connections. Defaults to 10. -sql_alchemy_max_overflow = 10 - -# The SqlAlchemy pool recycle is the number of seconds a connection -# can be idle in the pool before it is invalidated. This config does -# not apply to sqlite. If the number of DB connections is ever exceeded, -# a lower config value will allow the system to recover faster. -sql_alchemy_pool_recycle = 1800 - -# Check connection at the start of each connection pool checkout. -# Typically, this is a simple statement like "SELECT 1". -# More information here: -# https://docs.sqlalchemy.org/en/13/core/pooling.html#disconnect-handling-pessimistic -sql_alchemy_pool_pre_ping = True - -# The schema to use for the metadata database. -# SqlAlchemy supports databases with the concept of multiple schemas. -sql_alchemy_schema = - -# The amount of parallelism as a setting to the executor. This defines -# the max number of task instances that should run simultaneously -# on this airflow installation -parallelism = 32 - -# The number of task instances allowed to run concurrently by the scheduler -dag_concurrency = 16 - -# Are DAGs paused by default at creation -dags_are_paused_at_creation = True - -# The maximum number of active DAG runs per DAG -max_active_runs_per_dag = 16 - -# Whether to load the examples that ship with Airflow. It's good to -# get started, but you probably want to set this to False in a production -# environment -load_examples = True - -# Where your Airflow plugins are stored -plugins_folder = /usr/local/airflow/plugins - -# Secret key to save connection passwords in the db -fernet_key = $FERNET_KEY - -# Whether to disable pickling dags -donot_pickle = False - -# How long before timing out a python file import -dagbag_import_timeout = 30 - -# How long before timing out a DagFileProcessor, which processes a dag file -dag_file_processor_timeout = 50 - -# The class to use for running task instances in a subprocess -task_runner = StandardTaskRunner - -# If set, tasks without a ``run_as_user`` argument will be run with this user -# Can be used to de-elevate a sudo user running Airflow when executing tasks -default_impersonation = - -# What security module to use (for example kerberos) -security = - -# If set to False enables some unsecure features like Charts and Ad Hoc Queries. -# In 2.0 will default to True. -secure_mode = False - -# Turn unit test mode on (overwrites many configuration options with test -# values at runtime) -unit_test_mode = False - -# Whether to enable pickling for xcom (note that this is insecure and allows for -# RCE exploits). This will be deprecated in Airflow 2.0 (be forced to False). -enable_xcom_pickling = True - -# When a task is killed forcefully, this is the amount of time in seconds that -# it has to cleanup after it is sent a SIGTERM, before it is SIGKILLED -killed_task_cleanup_time = 60 - -# Whether to override params with dag_run.conf. If you pass some key-value pairs -# through ``airflow dags backfill -c`` or -# ``airflow dags trigger -c``, the key-value pairs will override the existing ones in params. -dag_run_conf_overrides_params = False - -# Worker initialisation check to validate Metadata Database connection -worker_precheck = False - -# When discovering DAGs, ignore any files that don't contain the strings ``DAG`` and ``airflow``. -dag_discovery_safe_mode = True - -# The number of retries each task is going to have by default. Can be overridden at dag or task level. -default_task_retries = 0 - -# Whether to serialises DAGs and persist them in DB. -# If set to True, Webserver reads from DB instead of parsing DAG files -# More details: https://airflow.apache.org/docs/stable/dag-serialization.html -store_serialized_dags = False - -# Updating serialized DAG can not be faster than a minimum interval to reduce database write rate. -min_serialized_dag_update_interval = 30 - -# On each dagrun check against defined SLAs -check_slas = True - -[cli] -# In what way should the cli access the API. The LocalClient will use the -# database directly, while the json_client will use the api running on the -# webserver -api_client = airflow.api.client.local_client - -# If you set web_server_url_prefix, do NOT forget to append it here, ex: -# ``endpoint_url = http://localhost:8080/myroot`` -# So api will look like: ``http://localhost:8080/myroot/api/experimental/...`` -endpoint_url = http://localhost:8080 - -[debug] -# Used only with DebugExecutor. If set to True DAG will fail with first -# failed task. Helpful for debugging purposes. -fail_fast = False - -[api] -# How to authenticate users of the API -auth_backend = airflow.api.auth.backend.default - -[lineage] -# what lineage backend to use -backend = - -[atlas] -sasl_enabled = False -host = -port = 21000 -username = -password = - -[operators] -# The default owner assigned to each new operator, unless -# provided explicitly or passed via ``default_args`` -default_owner = airflow -default_cpus = 1 -default_ram = 512 -default_disk = 512 -default_gpus = 0 - -[hive] -# Default mapreduce queue for HiveOperator tasks -default_hive_mapred_queue = - -[webserver] -# The base url of your website as airflow cannot guess what domain or -# cname you are using. This is used in automated emails that -# airflow sends to point links to the right web server -base_url = http://localhost:8080 - -# The ip specified when starting the web server -web_server_host = 0.0.0.0 - -# The port on which to run the web server -web_server_port = 8080 - -# Paths to the SSL certificate and key for the web server. When both are -# provided SSL will be enabled. This does not change the web server port. -web_server_ssl_cert = - -# Paths to the SSL certificate and key for the web server. When both are -# provided SSL will be enabled. This does not change the web server port. -web_server_ssl_key = - -# Number of seconds the webserver waits before killing gunicorn master that doesn't respond -web_server_master_timeout = 120 - -# Number of seconds the gunicorn webserver waits before timing out on a worker -web_server_worker_timeout = 120 - -# Number of workers to refresh at a time. When set to 0, worker refresh is -# disabled. When nonzero, airflow periodically refreshes webserver workers by -# bringing up new ones and killing old ones. -worker_refresh_batch_size = 1 - -# Number of seconds to wait before refreshing a batch of workers. -worker_refresh_interval = 30 - -# Secret key used to run your flask app -# It should be as random as possible -secret_key = temporary_key - -# Number of workers to run the Gunicorn web server -workers = 4 - -# The worker class gunicorn should use. Choices include -# sync (default), eventlet, gevent -worker_class = sync - -# Log files for the gunicorn webserver. '-' means log to stderr. -access_logfile = - - -# Log files for the gunicorn webserver. '-' means log to stderr. -error_logfile = - - -# Expose the configuration file in the web server -expose_config = True - -# Expose hostname in the web server -expose_hostname = True - -# Expose stacktrace in the web server -expose_stacktrace = True - -# Set to true to turn on authentication: -# https://airflow.apache.org/security.html#web-authentication -authenticate = False - -# Filter the list of dags by owner name (requires authentication to be enabled) -filter_by_owner = False - -# Filtering mode. Choices include user (default) and ldapgroup. -# Ldap group filtering requires using the ldap backend -# -# Note that the ldap server needs the "memberOf" overlay to be set up -# in order to user the ldapgroup mode. -owner_mode = user - -# Default DAG view. Valid values are: -# tree, graph, duration, gantt, landing_times -dag_default_view = tree - -# "Default DAG orientation. Valid values are:" -# LR (Left->Right), TB (Top->Bottom), RL (Right->Left), BT (Bottom->Top) -dag_orientation = LR - -# Puts the webserver in demonstration mode; blurs the names of Operators for -# privacy. -demo_mode = False - -# The amount of time (in secs) webserver will wait for initial handshake -# while fetching logs from other worker machine -log_fetch_timeout_sec = 5 - -# Time interval (in secs) to wait before next log fetching. -log_fetch_delay_sec = 2 - -# Distance away from page bottom to enable auto tailing. -log_auto_tailing_offset = 30 - -# Animation speed for auto tailing log display. -log_animation_speed = 1000 - -# By default, the webserver shows paused DAGs. Flip this to hide paused -# DAGs by default -hide_paused_dags_by_default = False - -# Consistent page size across all listing views in the UI -page_size = 100 - -# Use FAB-based webserver with RBAC feature -rbac = False - -# Define the color of navigation bar -navbar_color = #007A87 - -# Default dagrun to show in UI -default_dag_run_display_number = 25 - -# Enable werkzeug ``ProxyFix`` middleware for reverse proxy -enable_proxy_fix = False - -# Number of values to trust for ``X-Forwarded-For``. -# More info: https://werkzeug.palletsprojects.com/en/0.16.x/middleware/proxy_fix/ -proxy_fix_x_for = 1 - -# Number of values to trust for ``X-Forwarded-Proto`` -proxy_fix_x_proto = 1 - -# Number of values to trust for ``X-Forwarded-Host`` -proxy_fix_x_host = 1 - -# Number of values to trust for ``X-Forwarded-Port`` -proxy_fix_x_port = 1 - -# Number of values to trust for ``X-Forwarded-Prefix`` -proxy_fix_x_prefix = 1 - -# Set secure flag on session cookie -cookie_secure = False - -# Set samesite policy on session cookie -cookie_samesite = - -# Default setting for wrap toggle on DAG code and TI log views. -default_wrap = False - -# Allow the UI to be rendered in a frame -x_frame_enabled = True - -# Send anonymous user activity to your analytics tool -# choose from google_analytics, segment, or metarouter -# analytics_tool = - -# Unique ID of your account in the analytics tool -# analytics_id = - -# Update FAB permissions and sync security manager roles -# on webserver startup -update_fab_perms = True - -# Minutes of non-activity before logged out from UI -# 0 means never get forcibly logged out -force_log_out_after = 0 - -# The UI cookie lifetime in days -session_lifetime_days = 30 - -[email] -email_backend = airflow.utils.email.send_email_smtp - -[smtp] - -# If you want airflow to send emails on retries, failure, and you want to use -# the airflow.utils.email.send_email_smtp function, you have to configure an -# smtp server here -smtp_host = localhost -smtp_starttls = True -smtp_ssl = False -# Example: smtp_user = airflow -# smtp_user = -# Example: smtp_password = airflow -# smtp_password = -smtp_port = 25 -smtp_mail_from = airflow@example.com - -[sentry] - -# Sentry (https://docs.sentry.io) integration -sentry_dsn = - -[celery] - -# This section only applies if you are using the CeleryExecutor in -# ``[core]`` section above -# The app name that will be used by celery -celery_app_name = airflow.executors.celery_executor - -# The concurrency that will be used when starting workers with the -# ``airflow celery worker`` command. This defines the number of task instances that -# a worker will take, so size up your workers based on the resources on -# your worker box and the nature of your tasks -worker_concurrency = 16 - -# The maximum and minimum concurrency that will be used when starting workers with the -# ``airflow celery worker`` command (always keep minimum processes, but grow -# to maximum if necessary). Note the value should be max_concurrency,min_concurrency -# Pick these numbers based on resources on worker box and the nature of the task. -# If autoscale option is available, worker_concurrency will be ignored. -# http://docs.celeryproject.org/en/latest/reference/celery.bin.worker.html#cmdoption-celery-worker-autoscale -# Example: worker_autoscale = 16,12 -worker_autoscale = 16,12 - -# When you start an airflow worker, airflow starts a tiny web server -# subprocess to serve the workers local log files to the airflow main -# web server, who then builds pages and sends them to users. This defines -# the port on which the logs are served. It needs to be unused, and open -# visible from the main web server to connect into the workers. -worker_log_server_port = 8793 - -# The Celery broker URL. Celery supports RabbitMQ, Redis and experimentally -# a sqlalchemy database. Refer to the Celery documentation for more -# information. -# http://docs.celeryproject.org/en/latest/userguide/configuration.html#broker-settings -broker_url = redis://redis:6379/1 - -# The Celery result_backend. When a job finishes, it needs to update the -# metadata of the job. Therefore it will post a message on a message bus, -# or insert it into a database (depending of the backend) -# This status is used by the scheduler to update the state of the task -# The use of a database is highly recommended -# http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-result-backend-settings -result_backend = db+postgresql://airflow:airflow@postgres/airflow - -# Celery Flower is a sweet UI for Celery. Airflow has a shortcut to start -# it ``airflow flower``. This defines the IP that Celery Flower runs on -flower_host = 0.0.0.0 - -# The root URL for Flower -# Example: flower_url_prefix = /flower -flower_url_prefix = - -# This defines the port that Celery Flower runs on -flower_port = 5555 - -# Securing Flower with Basic Authentication -# Accepts user:password pairs separated by a comma -# Example: flower_basic_auth = user1:password1,user2:password2 -flower_basic_auth = - -# Default queue that tasks get assigned to and that worker listen on. -default_queue = default - -# How many processes CeleryExecutor uses to sync task state. -# 0 means to use max(1, number of cores - 1) processes. -sync_parallelism = 0 - -# Import path for celery configuration options -celery_config_options = airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG - -# In case of using SSL -ssl_active = False -ssl_key = -ssl_cert = -ssl_cacert = - -# Celery Pool implementation. -# Choices include: prefork (default), eventlet, gevent or solo. -# See: -# https://docs.celeryproject.org/en/latest/userguide/workers.html#concurrency -# https://docs.celeryproject.org/en/latest/userguide/concurrency/eventlet.html -pool = prefork - -# The number of seconds to wait before timing out ``send_task_to_executor`` or -# ``fetch_celery_task_state`` operations. -operation_timeout = 2 - -[celery_broker_transport_options] - -# This section is for specifying options which can be passed to the -# underlying celery broker transport. See: -# http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-broker_transport_options -# The visibility timeout defines the number of seconds to wait for the worker -# to acknowledge the task before the message is redelivered to another worker. -# Make sure to increase the visibility timeout to match the time of the longest -# ETA you're planning to use. -# visibility_timeout is only supported for Redis and SQS celery brokers. -# See: -# http://docs.celeryproject.org/en/master/userguide/configuration.html#std:setting-broker_transport_options -# Example: visibility_timeout = 21600 -# visibility_timeout = - -[dask] - -# This section only applies if you are using the DaskExecutor in -# [core] section above -# The IP address and port of the Dask cluster's scheduler. -cluster_address = 127.0.0.1:8786 - -# TLS/ SSL settings to access a secured Dask scheduler. -tls_ca = -tls_cert = -tls_key = - -[scheduler] -# Task instances listen for external kill signal (when you clear tasks -# from the CLI or the UI), this defines the frequency at which they should -# listen (in seconds). -job_heartbeat_sec = 5 - -# The scheduler constantly tries to trigger new tasks (look at the -# scheduler section in the docs for more information). This defines -# how often the scheduler should run (in seconds). -scheduler_heartbeat_sec = 5 - -# After how much time should the scheduler terminate in seconds -# -1 indicates to run continuously (see also num_runs) -run_duration = -1 - -# The number of times to try to schedule each DAG file -# -1 indicates unlimited number -num_runs = -1 - -# The number of seconds to wait between consecutive DAG file processing -processor_poll_interval = 1 - -# after how much time (seconds) a new DAGs should be picked up from the filesystem -min_file_process_interval = 0 - -# How often (in seconds) to scan the DAGs directory for new files. Default to 5 minutes. -dag_dir_list_interval = 300 - -# How often should stats be printed to the logs. Setting to 0 will disable printing stats -print_stats_interval = 30 - -# If the last scheduler heartbeat happened more than scheduler_health_check_threshold -# ago (in seconds), scheduler is considered unhealthy. -# This is used by the health check in the "/health" endpoint -scheduler_health_check_threshold = 30 -child_process_log_directory = /usr/local/airflow/logs/scheduler - -# Local task jobs periodically heartbeat to the DB. If the job has -# not heartbeat in this many seconds, the scheduler will mark the -# associated task instance as failed and will re-schedule the task. -scheduler_zombie_task_threshold = 300 - -# Turn off scheduler catchup by setting this to False. -# Default behavior is unchanged and -# Command Line Backfills still work, but the scheduler -# will not do scheduler catchup if this is False, -# however it can be set on a per DAG basis in the -# DAG definition (catchup) -catchup_by_default = True - -# This changes the batch size of queries in the scheduling main loop. -# If this is too high, SQL query performance may be impacted by one -# or more of the following: -# - reversion to full table scan -# - complexity of query predicate -# - excessive locking -# Additionally, you may hit the maximum allowable query length for your db. -# Set this to 0 for no limit (not advised) -max_tis_per_query = 512 - -# Statsd (https://github.com/etsy/statsd) integration settings -statsd_on = False -statsd_host = localhost -statsd_port = 8125 -statsd_prefix = airflow - -# If you want to avoid send all the available metrics to StatsD, -# you can configure an allow list of prefixes to send only the metrics that -# start with the elements of the list (e.g: scheduler,executor,dagrun) -statsd_allow_list = - -# The scheduler can run multiple threads in parallel to schedule dags. -# This defines how many threads will run. -max_threads = 2 -authenticate = False - -# Turn off scheduler use of cron intervals by setting this to False. -# DAGs submitted manually in the web UI or with trigger_dag will still run. -use_job_schedule = True - -# Allow externally triggered DagRuns for Execution Dates in the future -# Only has effect if schedule_interval is set to None in DAG -allow_trigger_in_future = False - -[ldap] -# set this to ldaps://: -uri = -user_filter = objectClass=* -user_name_attr = uid -group_member_attr = memberOf -superuser_filter = -data_profiler_filter = -bind_user = cn=Manager,dc=example,dc=com -bind_password = insecure -basedn = dc=example,dc=com -cacert = /etc/ca/ldap_ca.crt -search_scope = LEVEL - -# This setting allows the use of LDAP servers that either return a -# broken schema, or do not return a schema. -ignore_malformed_schema = False - -[mesos] -# Mesos master address which MesosExecutor will connect to. -master = localhost:5050 - -# The framework name which Airflow scheduler will register itself as on mesos -framework_name = Airflow - -# Number of cpu cores required for running one task instance using -# 'airflow run --local -p ' -# command on a mesos slave -task_cpu = 1 - -# Memory in MB required for running one task instance using -# 'airflow run --local -p ' -# command on a mesos slave -task_memory = 256 - -# Enable framework checkpointing for mesos -# See http://mesos.apache.org/documentation/latest/slave-recovery/ -checkpoint = False - -# Failover timeout in milliseconds. -# When checkpointing is enabled and this option is set, Mesos waits -# until the configured timeout for -# the MesosExecutor framework to re-register after a failover. Mesos -# shuts down running tasks if the -# MesosExecutor framework fails to re-register within this timeframe. -# Example: failover_timeout = 604800 -# failover_timeout = - -# Enable framework authentication for mesos -# See http://mesos.apache.org/documentation/latest/configuration/ -authenticate = False - -# Mesos credentials, if authentication is enabled -# Example: default_principal = admin -# default_principal = -# Example: default_secret = admin -# default_secret = - -# Optional Docker Image to run on slave before running the command -# This image should be accessible from mesos slave i.e mesos slave -# should be able to pull this docker image before executing the command. -# Example: docker_image_slave = puckel/docker-airflow -# docker_image_slave = - -[kerberos] -ccache = /tmp/airflow_krb5_ccache - -# gets augmented with fqdn -principal = airflow -reinit_frequency = 3600 -kinit_path = kinit -keytab = airflow.keytab - -[github_enterprise] -api_rev = v3 - -[admin] -# UI to hide sensitive variable fields when set to True -hide_sensitive_variable_fields = True - -[elasticsearch] -# Elasticsearch host -host = - -# Format of the log_id, which is used to query for a given tasks logs -log_id_template = {{dag_id}}-{{task_id}}-{{execution_date}}-{{try_number}} - -# Used to mark the end of a log stream for a task -end_of_log_mark = end_of_log - -# Qualified URL for an elasticsearch frontend (like Kibana) with a template argument for log_id -# Code will construct log_id using the log_id template from the argument above. -# NOTE: The code will prefix the https:// automatically, don't include that here. -frontend = - -# Write the task logs to the stdout of the worker, rather than the default files -write_stdout = False - -# Instead of the default log formatter, write the log lines as JSON -json_format = False - -# Log fields to also attach to the json output, if enabled -json_fields = asctime, filename, lineno, levelname, message - -[elasticsearch_configs] -use_ssl = False -verify_certs = True - -[kubernetes] -# The repository, tag and imagePullPolicy of the Kubernetes Image for the Worker to Run -worker_container_repository = -worker_container_tag = -worker_container_image_pull_policy = IfNotPresent - -# If True (default), worker pods will be deleted upon termination -delete_worker_pods = True - -# Number of Kubernetes Worker Pod creation calls per scheduler loop -worker_pods_creation_batch_size = 1 - -# The Kubernetes namespace where airflow workers should be created. Defaults to ``default`` -namespace = default - -# The name of the Kubernetes ConfigMap containing the Airflow Configuration (this file) -# Example: airflow_configmap = airflow-configmap -airflow_configmap = - -# The name of the Kubernetes ConfigMap containing ``airflow_local_settings.py`` file. -# -# For example: -# -# ``airflow_local_settings_configmap = "airflow-configmap"`` if you have the following ConfigMap. -# -# ``airflow-configmap.yaml``: -# -# .. code-block:: yaml -# -# --- -# apiVersion: v1 -# kind: ConfigMap -# metadata: -# name: airflow-configmap -# data: -# airflow_local_settings.py: | -# def pod_mutation_hook(pod): -# ... -# airflow.cfg: | -# ... -# Example: airflow_local_settings_configmap = airflow-configmap -airflow_local_settings_configmap = - -# For docker image already contains DAGs, this is set to ``True``, and the worker will -# search for dags in dags_folder, -# otherwise use git sync or dags volume claim to mount DAGs -dags_in_image = False - -# For either git sync or volume mounted DAGs, the worker will look in this subpath for DAGs -dags_volume_subpath = - -# For DAGs mounted via a volume claim (mutually exclusive with git-sync and host path) -dags_volume_claim = - -# For volume mounted logs, the worker will look in this subpath for logs -logs_volume_subpath = - -# A shared volume claim for the logs -logs_volume_claim = - -# For DAGs mounted via a hostPath volume (mutually exclusive with volume claim and git-sync) -# Useful in local environment, discouraged in production -dags_volume_host = - -# A hostPath volume for the logs -# Useful in local environment, discouraged in production -logs_volume_host = - -# A list of configMapsRefs to envFrom. If more than one configMap is -# specified, provide a comma separated list: configmap_a,configmap_b -env_from_configmap_ref = - -# A list of secretRefs to envFrom. If more than one secret is -# specified, provide a comma separated list: secret_a,secret_b -env_from_secret_ref = - -# Git credentials and repository for DAGs mounted via Git (mutually exclusive with volume claim) -git_repo = -git_branch = -git_subpath = - -# The specific rev or hash the git_sync init container will checkout -# This becomes GIT_SYNC_REV environment variable in the git_sync init container for worker pods -git_sync_rev = - -# Use git_user and git_password for user authentication or git_ssh_key_secret_name -# and git_ssh_key_secret_key for SSH authentication -git_user = -git_password = -git_sync_root = /git -git_sync_dest = repo - -# Mount point of the volume if git-sync is being used. -# i.e. /usr/local/airflow/dags -git_dags_folder_mount_point = - -# To get Git-sync SSH authentication set up follow this format -# -# ``airflow-secrets.yaml``: -# -# .. code-block:: yaml -# -# --- -# apiVersion: v1 -# kind: Secret -# metadata: -# name: airflow-secrets -# data: -# # key needs to be gitSshKey -# gitSshKey: -# Example: git_ssh_key_secret_name = airflow-secrets -git_ssh_key_secret_name = - -# To get Git-sync SSH authentication set up follow this format -# -# ``airflow-configmap.yaml``: -# -# .. code-block:: yaml -# -# --- -# apiVersion: v1 -# kind: ConfigMap -# metadata: -# name: airflow-configmap -# data: -# known_hosts: | -# github.com ssh-rsa <...> -# airflow.cfg: | -# ... -# Example: git_ssh_known_hosts_configmap_name = airflow-configmap -git_ssh_known_hosts_configmap_name = - -# To give the git_sync init container credentials via a secret, create a secret -# with two fields: GIT_SYNC_USERNAME and GIT_SYNC_PASSWORD (example below) and -# add ``git_sync_credentials_secret = `` to your airflow config under the -# ``kubernetes`` section -# -# Secret Example: -# -# .. code-block:: yaml -# -# --- -# apiVersion: v1 -# kind: Secret -# metadata: -# name: git-credentials -# data: -# GIT_SYNC_USERNAME: -# GIT_SYNC_PASSWORD: -git_sync_credentials_secret = - -# For cloning DAGs from git repositories into volumes: https://github.com/kubernetes/git-sync -git_sync_container_repository = k8s.gcr.io/git-sync -git_sync_container_tag = v3.1.1 -git_sync_init_container_name = git-sync-clone -git_sync_run_as_user = 65533 - -# The name of the Kubernetes service account to be associated with airflow workers, if any. -# Service accounts are required for workers that require access to secrets or cluster resources. -# See the Kubernetes RBAC documentation for more: -# https://kubernetes.io/docs/admin/authorization/rbac/ -worker_service_account_name = - -# Any image pull secrets to be given to worker pods, If more than one secret is -# required, provide a comma separated list: secret_a,secret_b -image_pull_secrets = - -# GCP Service Account Keys to be provided to tasks run on Kubernetes Executors -# Should be supplied in the format: key-name-1:key-path-1,key-name-2:key-path-2 -gcp_service_account_keys = - -# Use the service account kubernetes gives to pods to connect to kubernetes cluster. -# It's intended for clients that expect to be running inside a pod running on kubernetes. -# It will raise an exception if called from a process not running in a kubernetes environment. -in_cluster = True - -# When running with in_cluster=False change the default cluster_context or config_file -# options to Kubernetes client. Leave blank these to use default behaviour like ``kubectl`` has. -# cluster_context = -# config_file = - -# Affinity configuration as a single line formatted JSON object. -# See the affinity model for top-level key names (e.g. ``nodeAffinity``, etc.): -# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#affinity-v1-core -affinity = - -# A list of toleration objects as a single line formatted JSON array -# See: -# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#toleration-v1-core -tolerations = - -# Keyword parameters to pass while calling a kubernetes client core_v1_api methods -# from Kubernetes Executor provided as a single line formatted JSON dictionary string. -# List of supported params are similar for all core_v1_apis, hence a single config -# variable for all apis. -# See: -# https://raw.githubusercontent.com/kubernetes-client/python/master/kubernetes/client/apis/core_v1_api.py -# Note that if no _request_timeout is specified, the kubernetes client will wait indefinitely -# for kubernetes api responses, which will cause the scheduler to hang. -# The timeout is specified as [connect timeout, read timeout] -kube_client_request_args = {{"_request_timeout" : [60,60] }} - -# Specifies the uid to run the first process of the worker pods containers as -run_as_user = - -# Specifies a gid to associate with all containers in the worker pods -# if using a git_ssh_key_secret_name use an fs_group -# that allows for the key to be read, e.g. 65533 -fs_group = - -[kubernetes_node_selectors] - -# The Key-value pairs to be given to worker pods. -# The worker pods will be scheduled to the nodes of the specified key-value pairs. -# Should be supplied in the format: key = value - -[kubernetes_annotations] - -# The Key-value annotations pairs to be given to worker pods. -# Should be supplied in the format: key = value - -[kubernetes_environment_variables] - -# The scheduler sets the following environment variables into your workers. You may define as -# many environment variables as needed and the kubernetes launcher will set them in the launched workers. -# Environment variables in this section are defined as follows -# `` = `` -# -# For example if you wanted to set an environment variable with value `prod` and key -# ``ENVIRONMENT`` you would follow the following format: -# ENVIRONMENT = prod -# -# Additionally you may override worker airflow settings with the ``AIRFLOW__
__`` -# formatting as supported by airflow normally. - -[kubernetes_secrets] - -# The scheduler mounts the following secrets into your workers as they are launched by the -# scheduler. You may define as many secrets as needed and the kubernetes launcher will parse the -# defined secrets and mount them as secret environment variables in the launched workers. -# Secrets in this section are defined as follows -# `` = =`` -# -# For example if you wanted to mount a kubernetes secret key named ``postgres_password`` from the -# kubernetes secret object ``airflow-secret`` as the environment variable ``POSTGRES_PASSWORD`` into -# your workers you would follow the following format: -# ``POSTGRES_PASSWORD = airflow-secret=postgres_credentials`` -# -# Additionally you may override worker airflow settings with the ``AIRFLOW__
__`` -# formatting as supported by airflow normally. - -[kubernetes_labels] - -# The Key-value pairs to be given to worker pods. -# The worker pods will be given these static labels, as well as some additional dynamic labels -# to identify the task. -# Should be supplied in the format: ``key = value`` diff --git a/config/webserver_config.py b/config/webserver_config.py new file mode 100644 index 00000000..baeb4920 --- /dev/null +++ b/config/webserver_config.py @@ -0,0 +1,121 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Default configuration for the Airflow webserver""" +import os + +from flask_appbuilder.security.manager import AUTH_DB + +# from flask_appbuilder.security.manager import AUTH_LDAP +# from flask_appbuilder.security.manager import AUTH_OAUTH +# from flask_appbuilder.security.manager import AUTH_OID +# from flask_appbuilder.security.manager import AUTH_REMOTE_USER + + +basedir = os.path.abspath(os.path.dirname(__file__)) + +# Flask-WTF flag for CSRF +WTF_CSRF_ENABLED = True + +# ---------------------------------------------------- +# AUTHENTICATION CONFIG +# ---------------------------------------------------- +# For details on how to set up each of the following authentication, see +# http://flask-appbuilder.readthedocs.io/en/latest/security.html# +# authentication-methods for details. + +# The authentication type +# AUTH_OID : Is for OpenID +# AUTH_DB : Is for database +# AUTH_LDAP : Is for LDAP +# AUTH_REMOTE_USER : Is for using REMOTE_USER from web server +# AUTH_OAUTH : Is for OAuth +AUTH_TYPE = AUTH_DB + +# Uncomment to setup Full admin role name +# AUTH_ROLE_ADMIN = 'Admin' + +# Uncomment to setup Public role name, no authentication needed +# AUTH_ROLE_PUBLIC = 'Public' + +# Set the Public role to Admin to allow full access. +AUTH_ROLE_PUBLIC = 'Admin' + +# Will allow user self registration +# AUTH_USER_REGISTRATION = True + +# The default user self registration role +# AUTH_USER_REGISTRATION_ROLE = "Public" + +# When using OAuth Auth, uncomment to setup provider(s) info +# Google OAuth example: +# OAUTH_PROVIDERS = [{ +# 'name':'google', +# 'token_key':'access_token', +# 'icon':'fa-google', +# 'remote_app': { +# 'api_base_url':'https://www.googleapis.com/oauth2/v2/', +# 'client_kwargs':{ +# 'scope': 'email profile' +# }, +# 'access_token_url':'https://accounts.google.com/o/oauth2/token', +# 'authorize_url':'https://accounts.google.com/o/oauth2/auth', +# 'request_token_url': None, +# 'client_id': GOOGLE_KEY, +# 'client_secret': GOOGLE_SECRET_KEY, +# } +# }] + +# When using LDAP Auth, setup the ldap server +# AUTH_LDAP_SERVER = "ldap://ldapserver.new" + +# When using OpenID Auth, uncomment to setup OpenID providers. +# example for OpenID authentication +# OPENID_PROVIDERS = [ +# { 'name': 'Yahoo', 'url': 'https://me.yahoo.com' }, +# { 'name': 'AOL', 'url': 'http://openid.aol.com/' }, +# { 'name': 'Flickr', 'url': 'http://www.flickr.com/' }, +# { 'name': 'MyOpenID', 'url': 'https://www.myopenid.com' }] + +# ---------------------------------------------------- +# Theme CONFIG +# ---------------------------------------------------- +# Flask App Builder comes up with a number of predefined themes +# that you can use for Apache Airflow. +# http://flask-appbuilder.readthedocs.io/en/latest/customizing.html#changing-themes +# Please make sure to remove "navbar_color" configuration from airflow.cfg +# in order to fully utilize the theme. (or use that property in conjunction +# with theme) +# APP_THEME = "bootstrap-theme.css" # default bootstrap +# APP_THEME = "amelia.css" +# APP_THEME = "cerulean.css" +# APP_THEME = "cosmo.css" +# APP_THEME = "cyborg.css" +# APP_THEME = "darkly.css" +# APP_THEME = "flatly.css" +# APP_THEME = "journal.css" +# APP_THEME = "lumen.css" +# APP_THEME = "paper.css" +# APP_THEME = "readable.css" +# APP_THEME = "sandstone.css" +# APP_THEME = "simplex.css" +# APP_THEME = "slate.css" +# APP_THEME = "solar.css" +# APP_THEME = "spacelab.css" +# APP_THEME = "superhero.css" +# APP_THEME = "united.css" +# APP_THEME = "yeti.css" diff --git a/dags/tuto.py b/dags/tuto.py index cead2b61..089517d1 100644 --- a/dags/tuto.py +++ b/dags/tuto.py @@ -22,7 +22,8 @@ # 'end_date': datetime(2016, 1, 1), } -dag = DAG("tutorial", default_args=default_args, schedule_interval=timedelta(1)) +dag = DAG("tutorial", default_args=default_args, + schedule_interval=timedelta(1)) # t1, t2 and t3 are examples of tasks created by instantiating operators t1 = BashOperator(task_id="print_date", bash_command="date", dag=dag) diff --git a/docker-compose-CeleryExecutor.yml b/docker-compose-CeleryExecutor.yml index de4f5dac..2b028a73 100644 --- a/docker-compose-CeleryExecutor.yml +++ b/docker-compose-CeleryExecutor.yml @@ -1,7 +1,7 @@ version: '2.1' services: redis: - image: 'redis:5.0.5' + image: 'redis:6.2' # command: redis-server --requirepass redispass postgres: @@ -10,83 +10,85 @@ services: - POSTGRES_USER=airflow - POSTGRES_PASSWORD=airflow - POSTGRES_DB=airflow + ports: + - "5432:5432" # Uncomment these lines to persist data on the local filesystem. # - PGDATA=/var/lib/postgresql/data/pgdata # volumes: # - ./pgdata:/var/lib/postgresql/data/pgdata webserver: - image: puckel/docker-airflow:1.10.9 + build: . restart: always depends_on: - postgres - redis environment: - - LOAD_EX=n - - FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= - - EXECUTOR=Celery + - AIRFLOW__CORE__LOAD_EXAMPLES=false + - AIRFLOW__CORE__FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= + - AIRFLOW__CORE__EXECUTOR=CeleryExecutor # - POSTGRES_USER=airflow # - POSTGRES_PASSWORD=airflow # - POSTGRES_DB=airflow # - REDIS_PASSWORD=redispass volumes: - - ./dags:/usr/local/airflow/dags + - ./dags:/opt/airflow/dags # Uncomment to include custom plugins # - ./plugins:/usr/local/airflow/plugins ports: - "8080:8080" command: webserver healthcheck: - test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"] + test: ["CMD-SHELL", "[ -f /opt/airflow/airflow-webserver.pid ]"] interval: 30s timeout: 30s retries: 3 flower: - image: puckel/docker-airflow:1.10.9 + build: . restart: always depends_on: - redis environment: - - EXECUTOR=Celery + - AIRFLOW__CORE__EXECUTOR=CeleryExecutor # - REDIS_PASSWORD=redispass ports: - "5555:5555" - command: flower + command: airflow celery flower scheduler: - image: puckel/docker-airflow:1.10.9 + build: . restart: always depends_on: - webserver volumes: - - ./dags:/usr/local/airflow/dags + - ./dags:/opt/airflow/dags # Uncomment to include custom plugins # - ./plugins:/usr/local/airflow/plugins environment: - - LOAD_EX=n - - FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= - - EXECUTOR=Celery + - AIRFLOW__CORE__LOAD_EXAMPLES=false + - AIRFLOW__CORE__FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= + - AIRFLOW__CORE__EXECUTOR=CeleryExecutor # - POSTGRES_USER=airflow # - POSTGRES_PASSWORD=airflow # - POSTGRES_DB=airflow # - REDIS_PASSWORD=redispass - command: scheduler + command: airflow scheduler worker: - image: puckel/docker-airflow:1.10.9 + build: . restart: always depends_on: - scheduler volumes: - - ./dags:/usr/local/airflow/dags + - ./dags:/opt/airflow/dags # Uncomment to include custom plugins # - ./plugins:/usr/local/airflow/plugins environment: - - FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= - - EXECUTOR=Celery + - AIRFLOW__CORE__FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= + - AIRFLOW__CORE__EXECUTOR=CeleryExecutor # - POSTGRES_USER=airflow # - POSTGRES_PASSWORD=airflow # - POSTGRES_DB=airflow # - REDIS_PASSWORD=redispass - command: worker + command: airflow celery worker diff --git a/docker-compose-LocalExecutor.yml b/docker-compose-LocalExecutor.yml index 26e9e92e..95dd7659 100644 --- a/docker-compose-LocalExecutor.yml +++ b/docker-compose-LocalExecutor.yml @@ -6,31 +6,33 @@ services: - POSTGRES_USER=airflow - POSTGRES_PASSWORD=airflow - POSTGRES_DB=airflow + ports: + - "5432:5432" logging: options: max-size: 10m max-file: "3" webserver: - image: puckel/docker-airflow:1.10.9 + build: . restart: always depends_on: - postgres environment: - - LOAD_EX=n - - EXECUTOR=Local + - AIRFLOW__CORE__LOAD_EXAMPLES=false + - AIRFLOW__CORE__EXECUTOR=LocalExecutor logging: options: max-size: 10m max-file: "3" volumes: - - ./dags:/usr/local/airflow/dags + - ./dags:/opt/airflow/dags # - ./plugins:/usr/local/airflow/plugins ports: - "8080:8080" command: webserver healthcheck: - test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"] + test: ["CMD-SHELL", "[ -f /opt/airflow/airflow-webserver.pid ]"] interval: 30s timeout: 30s retries: 3 diff --git a/script/entrypoint.sh b/script/entrypoint.sh index 166f4837..f133da38 100755 --- a/script/entrypoint.sh +++ b/script/entrypoint.sh @@ -8,14 +8,9 @@ TRY_LOOP="20" # Global defaults and back-compat -: "${AIRFLOW_HOME:="/usr/local/airflow"}" -: "${AIRFLOW__CORE__FERNET_KEY:=${FERNET_KEY:=$(python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print(FERNET_KEY)")}}" -: "${AIRFLOW__CORE__EXECUTOR:=${EXECUTOR:-Sequential}Executor}" - -# Load DAGs examples (default: Yes) -if [[ -z "$AIRFLOW__CORE__LOAD_EXAMPLES" && "${LOAD_EX:=n}" == n ]]; then - AIRFLOW__CORE__LOAD_EXAMPLES=False -fi +: "${AIRFLOW_HOME:="/opt/airflow"}" +: "${AIRFLOW__CORE__FERNET_KEY:=$(python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print(FERNET_KEY)")}" +: "${AIRFLOW__CORE__EXECUTOR:="SequentialExecutor"}" export \ AIRFLOW_HOME \ @@ -109,27 +104,14 @@ fi case "$1" in webserver) - airflow initdb + airflow db init if [ "$AIRFLOW__CORE__EXECUTOR" = "LocalExecutor" ] || [ "$AIRFLOW__CORE__EXECUTOR" = "SequentialExecutor" ]; then # With the "Local" and "Sequential" executors it should all run in one container. airflow scheduler & fi exec airflow webserver ;; - worker|scheduler) - # Give the webserver time to run initdb. - sleep 10 - exec airflow "$@" - ;; - flower) - sleep 10 - exec airflow "$@" - ;; - version) - exec airflow "$@" - ;; *) - # The command is something like bash, not an airflow subcommand. Just run it in the right environment. exec "$@" ;; -esac +esac \ No newline at end of file