diff --git a/.dockerignore b/.dockerignore index 2fa840fb8..5509e8f7a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,4 @@ -.git +.git* env/ *.pyc *.rdb @@ -14,3 +14,6 @@ Jenkinsfile werf.yaml **/*.dylib **/*.dll +Dockerfile* +.dockerignore +/config/deploy diff --git a/.env.example b/.env.example new file mode 100644 index 000000000..f20c768a2 --- /dev/null +++ b/.env.example @@ -0,0 +1,3 @@ +# Get your individual key from here: https://developer.nlr.gov/signup/ and +# replace the DEMO_KEY with that +NLR_API_KEY=DEMO_KEY diff --git a/.github/scripts/decrypt.sh b/.github/scripts/decrypt.sh deleted file mode 100755 index 32d3ae913..000000000 --- a/.github/scripts/decrypt.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -# transcrypt/transcrypt --flush-credentials -y || true -# transcrypt/transcrypt -c aes-256-cbc -p "$TRANSCRYPT_PASSWORD" -y - - diff --git a/.github/scripts/make_keys.py.sh b/.github/scripts/make_keys.py.sh deleted file mode 100755 index fde27a36e..000000000 --- a/.github/scripts/make_keys.py.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -cp keys.py.template keys.py -echo "pvwatts_api_key = '${NREL_DEV_API_KEY}'" >> keys.py -echo "developer_nrel_gov_key = '${NREL_DEV_API_KEY}'" >> keys.py -echo "Successfully created keys.py" diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 000000000..1e8dfcfae --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,115 @@ +name: Deploy + +on: + push: + branches: "**" + pull_request: + types: + - closed + - labeled + - reopened + - unlabeled + workflow_dispatch: + +concurrency: + # Concurrency group is more complicated in this case because: + # 1. This gets triggered by both `push` and `pull_request` label events, so + # both should use the same git head ref (and not `github.ref`, which may + # be different for PRs). + # 2. Our own deploy process may trigger an `unlabeled` event for removing the + # db-restore label, so separate that so that doesn't cause the previous + # deploy that was finishing up to be canceled. + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }}${{ github.event.action == 'unlabeled' && github.event.label.name == 'deploy-db-restore' && '-removing-ephemeral-label' || '' }} + cancel-in-progress: true + +jobs: + deploy-metadata: + name: Deploy Metadata + runs-on: self-hosted + outputs: + staging-perform-deploy: ${{ steps.staging-metadata.outputs.perform-deploy }} + staging-perform-undeploy: ${{ steps.staging-metadata.outputs.perform-undeploy }} + staging-metadata: ${{ toJSON(steps.staging-metadata.outputs) }} + production-perform-deploy: ${{ steps.production-metadata.outputs.perform-deploy }} + production-metadata: ${{ toJSON(steps.production-metadata.outputs) }} + steps: + - name: Import vault nonsensitive secrets + id: vault-nonsensitive-secrets + uses: TADA/vault-action/nonsensitive-secrets@v1 + with: + template: | + {{ with (datasource "vault" "reopt-api/ci/deploy").data }} + {{ $secrets = coll.Merge (coll.Dict + "container_registry" .container_registry + "production_rancher_project_id" .production_rancher_project_id + "production_url_host" .production_url_host + "staging_rancher_project_id" .staging_rancher_project_id + "staging_url_host_base" .staging_url_host_base + ) $secrets }} + {{ end }} + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} + + - name: Staging Metadata + id: staging-metadata + uses: TADA/deploy-action/metadata@v2 + with: + deploy-env: staging + app-name: reopt-api + rancher-project-id: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).staging_rancher_project_id }} + registry: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).container_registry }} + branch-url-host-base: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).staging_url_host_base }} + branch-db-name-base: reopt_api_staging + + - name: Production Metadata + id: production-metadata + uses: TADA/deploy-action/metadata@v2 + with: + deploy-env: production + app-name: reopt-api + rancher-project-id: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).production_rancher_project_id }} + registry: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).container_registry }} + branch-url-host-base: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).production_url_host }} + + undeploy-staging: + name: Undeploy Staging + needs: + - deploy-metadata + if: ${{ needs.deploy-metadata.outputs.staging-perform-undeploy == 'true' }} + uses: TADA/deploy-action/.github/workflows/undeploy-branch.yml@v2 + with: + metadata: ${{ needs.deploy-metadata.outputs.staging-metadata }} + vault-kubeconfig-path: secret/data/deploy/staging/on-prem-rancher-test-ponderosa-cluster-test-reopt + vault-db-superuser-path: secret/data/reopt-db/staging/db-superuser + secrets: + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} + + deploy-staging: + name: Deploy Staging + needs: + - deploy-metadata + if: ${{ needs.deploy-metadata.outputs.staging-perform-deploy == 'true' }} + uses: TADA/deploy-action/.github/workflows/deploy.yml@v2 + with: + metadata: ${{ needs.deploy-metadata.outputs.staging-metadata }} + vault-registry-credentials-path: secret/data/deploy/common/aws-ecr + vault-kubeconfig-path: secret/data/deploy/staging/on-prem-rancher-test-ponderosa-cluster-test-reopt + secrets: + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} + + deploy-production: + name: Deploy Production + needs: + - deploy-metadata + - deploy-staging + if: ${{ needs.deploy-metadata.outputs.production-perform-deploy == 'true' }} + uses: TADA/deploy-action/.github/workflows/deploy.yml@v2 + with: + metadata: ${{ needs.deploy-metadata.outputs.production-metadata }} + vault-registry-credentials-path: secret/data/deploy/common/aws-ecr + vault-kubeconfig-path: secret/data/deploy/production/on-prem-rancher-ponderosa-cluster-reopt + secrets: + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} diff --git a/.github/workflows/prune-deploy-images.yml b/.github/workflows/prune-deploy-images.yml new file mode 100644 index 000000000..3791dc3f4 --- /dev/null +++ b/.github/workflows/prune-deploy-images.yml @@ -0,0 +1,51 @@ +name: Prune Deploy Images + +on: + schedule: + - cron: "6 6 * * *" # Every day at 11:06 PM MST / 12:06 AM MDT + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + vault-nonsensitive-secrets: + name: Vault Non-Sensitive Secrets + runs-on: self-hosted + outputs: + nonsensitive-secrets: ${{ steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets }} + steps: + - name: Import vault nonsensitive secrets + id: vault-nonsensitive-secrets + uses: TADA/vault-action/nonsensitive-secrets@v1 + with: + template: | + {{ with (datasource "vault" "reopt-api/ci/deploy").data }} + {{ $secrets = coll.Merge (coll.Dict + "container_registry" .container_registry + "production_rancher_project_id" .production_rancher_project_id + "staging_rancher_project_id" .staging_rancher_project_id + ) $secrets }} + {{ end }} + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} + + prune-images: + name: Prune Deploy Images + uses: TADA/deploy-action/.github/workflows/prune-deploy-images.yml@v2 + needs: + - vault-nonsensitive-secrets + with: + vault-registry-credentials-path: secret/data/deploy/common/aws-ecr + registry: ${{ fromJSON(needs.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).container_registry }} + images: | + tada/reopt-api + clusters: | + - vault-kubeconfig-path: secret/data/deploy/staging/on-prem-rancher-test-ponderosa-cluster-test-reopt + rancher-project-id: ${{ fromJSON(needs.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).staging_rancher_project_id }} + - vault-kubeconfig-path: secret/data/deploy/production/on-prem-rancher-ponderosa-cluster-reopt + rancher-project-id: ${{ fromJSON(needs.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).production_rancher_project_id }} + secrets: + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} diff --git a/.github/workflows/pull_request_tests.yml b/.github/workflows/pull_request_tests.yml index 92a96768b..ab241cd22 100644 --- a/.github/workflows/pull_request_tests.yml +++ b/.github/workflows/pull_request_tests.yml @@ -17,17 +17,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - # - name: Decrypt - # env: - # TRANSCRYPT_PASSWORD: ${{ secrets.TRANSCRYPT_PASSWORD }} - # run: ./.github/scripts/decrypt.sh - - name: Make keys.py - env: - NREL_DEV_API_KEY: ${{ secrets.NREL_DEV_API_KEY }} - run: ./.github/scripts/make_keys.py.sh + - uses: actions/checkout@v5 - name: Build containers run: docker compose up -d + env: + NLR_API_KEY: ${{ secrets.NREL_DEV_API_KEY }} - name: Check running containers run: docker ps -a - name: Wait for julia_api diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index a37ea849a..3ddecd0d1 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -13,17 +13,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - # - name: Decrypt - # env: - # TRANSCRYPT_PASSWORD: ${{ secrets.TRANSCRYPT_PASSWORD }} - # run: ./.github/scripts/decrypt.sh - - name: Make keys.py - env: - NREL_DEV_API_KEY: ${{ secrets.NREL_DEV_API_KEY }} - run: ./.github/scripts/make_keys.py.sh + - uses: actions/checkout@v5 - name: Build containers run: docker compose up -d + env: + NLR_API_KEY: ${{ secrets.NREL_DEV_API_KEY }} - name: Check running containers run: docker ps -a - name: Wait for julia_api diff --git a/.github/workflows/restart-celery-julia.yml b/.github/workflows/restart-celery-julia.yml new file mode 100644 index 000000000..a02d36990 --- /dev/null +++ b/.github/workflows/restart-celery-julia.yml @@ -0,0 +1,118 @@ +name: Restart Celery & Julia + +on: + schedule: + - cron: "23 8 * * *" # Every day at 01:23 AM MST / 02:23 AM MDT + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +jobs: + deploy-metadata: + name: Deploy Metadata + runs-on: self-hosted + outputs: + ci-deploy-image: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).ci_deploy_image }} + staging-perform-deploy: ${{ steps.staging-metadata.outputs.perform-deploy }} + staging-perform-undeploy: ${{ steps.staging-metadata.outputs.perform-undeploy }} + staging-metadata: ${{ toJSON(steps.staging-metadata.outputs) }} + production-perform-deploy: ${{ steps.production-metadata.outputs.perform-deploy }} + production-metadata: ${{ toJSON(steps.production-metadata.outputs) }} + steps: + - name: Import vault nonsensitive secrets + id: vault-nonsensitive-secrets + uses: TADA/vault-action/nonsensitive-secrets@v1 + with: + template: | + {{ with (datasource "vault" "reopt-api/ci/deploy").data }} + {{ $secrets = coll.Merge (coll.Dict + "ci_deploy_image" .ci_deploy_image + "container_registry" .container_registry + "production_rancher_project_id" .production_rancher_project_id + "production_url_host" .production_url_host + "staging_rancher_project_id" .staging_rancher_project_id + "staging_url_host_base" .staging_url_host_base + ) $secrets }} + {{ end }} + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} + + - name: Staging Metadata + id: staging-metadata + uses: TADA/deploy-action/metadata@v2 + with: + deploy-env: staging + app-name: reopt-api + rancher-project-id: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).staging_rancher_project_id }} + registry: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).container_registry }} + branch-url-host-base: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).staging_url_host_base }} + branch-db-name-base: reopt_api_staging + + - name: Production Metadata + id: production-metadata + uses: TADA/deploy-action/metadata@v2 + with: + deploy-env: production + app-name: reopt-api + rancher-project-id: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).production_rancher_project_id }} + registry: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).container_registry }} + branch-url-host-base: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).production_url_host }} + + restart-staging: + name: Restart Staging + needs: + - deploy-metadata + if: ${{ needs.deploy-metadata.outputs.staging-perform-deploy == 'true' }} + runs-on: self-hosted + container: + image: ${{ needs.deploy-metadata.outputs.ci-deploy-image }} + env: + NODE_OPTIONS: --use-openssl-ca + steps: + - name: Kubernetes config setup + uses: TADA/deploy-action/kubeconfig@v2 + with: + vault-kubeconfig-path: secret/data/deploy/staging/on-prem-rancher-test-ponderosa-cluster-test-reopt + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} + rancher-project-id: ${{ fromJSON(needs.deploy-metadata.outputs.staging-metadata).rancher-project-id }} + - name: Rollout restart + env: + app_namespace: "${{ fromJSON(needs.deploy-metadata.outputs.staging-metadata).app-namespace }}" + run: | + set -x + kubectl -n "$app_namespace" rollout restart deployment/celery-deployment + kubectl -n "$app_namespace" rollout status deployment/celery-deployment --timeout=10m + kubectl -n "$app_namespace" rollout restart deployment/julia-deployment + kubectl -n "$app_namespace" rollout status deployment/julia-deployment --timeout=10m + + restart-production: + name: Restart Production + needs: + - deploy-metadata + - restart-staging + if: ${{ needs.deploy-metadata.outputs.production-perform-deploy == 'true' }} + runs-on: self-hosted + container: + image: ${{ needs.deploy-metadata.outputs.ci-deploy-image }} + env: + NODE_OPTIONS: --use-openssl-ca + steps: + - name: Kubernetes config setup + uses: TADA/deploy-action/kubeconfig@v2 + with: + vault-kubeconfig-path: secret/data/deploy/production/on-prem-rancher-ponderosa-cluster-reopt + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} + rancher-project-id: ${{ fromJSON(needs.deploy-metadata.outputs.production-metadata).rancher-project-id }} + - name: Rollout restart + env: + app_namespace: "${{ fromJSON(needs.deploy-metadata.outputs.production-metadata).app-namespace }}" + run: | + set -x + kubectl -n "$app_namespace" rollout restart deployment/celery-deployment + kubectl -n "$app_namespace" rollout status deployment/celery-deployment --timeout=10m + kubectl -n "$app_namespace" rollout restart deployment/julia-deployment + kubectl -n "$app_namespace" rollout status deployment/julia-deployment --timeout=10m diff --git a/.github/workflows/undeploy-closed-prs.yml b/.github/workflows/undeploy-closed-prs.yml new file mode 100644 index 000000000..1bbd76d9e --- /dev/null +++ b/.github/workflows/undeploy-closed-prs.yml @@ -0,0 +1,54 @@ +name: Undeploy Closed PRs + +on: + schedule: + - cron: "23 16 * * *" # Every day at 09:23 AM MST / 10:23 AM MDT + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + deploy-metadata: + name: Deploy Metadata + runs-on: self-hosted + outputs: + staging-metadata: ${{ toJSON(steps.staging-metadata.outputs) }} + steps: + - name: Import vault nonsensitive secrets + id: vault-nonsensitive-secrets + uses: TADA/vault-action/nonsensitive-secrets@v1 + with: + template: | + {{ with (datasource "vault" "reopt-api/ci/deploy").data }} + {{ $secrets = coll.Merge (coll.Dict + "staging_rancher_project_id" .staging_rancher_project_id + "staging_url_host_base" .staging_url_host_base + ) $secrets }} + {{ end }} + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} + + - name: Staging Metadata + id: staging-metadata + uses: TADA/deploy-action/metadata@v2 + with: + deploy-env: staging + app-name: reopt-api + rancher-project-id: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).staging_rancher_project_id }} + branch-url-host-base: ${{ fromJSON(steps.vault-nonsensitive-secrets.outputs.nonsensitive-secrets).staging_url_host_base }} + branch-db-name-base: reopt_api_staging + + undeploy-staging: + name: Undeploy Staging + needs: + - deploy-metadata + uses: TADA/deploy-action/.github/workflows/undeploy-closed-prs.yml@v2 + with: + base-metadata: ${{ needs.deploy-metadata.outputs.staging-metadata }} + vault-kubeconfig-path: secret/data/deploy/staging/on-prem-rancher-test-ponderosa-cluster-test-reopt + vault-db-superuser-path: secret/data/reopt-db/staging/db-superuser + secrets: + vault-role-id: ${{ secrets.VAULT_ROLE_ID }} + vault-secret-id: ${{ secrets.VAULT_SECRET_ID }} diff --git a/.gitignore b/.gitignore index aba3fad8d..94edd4b03 100755 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ __pycache__/ *$py.class keys.py +.env .idea input_files/* !input_files/**/ @@ -123,3 +124,6 @@ julia_src/Dockerfile.xpress docker-compose.xpress.yml julia_src/solver_setup.sh compare_run_*.json + +/config/deploy/external +/config/deploy/tmp diff --git a/.gomplate.yaml b/.gomplate.yaml new file mode 100644 index 000000000..89659b153 --- /dev/null +++ b/.gomplate.yaml @@ -0,0 +1,3 @@ +datasources: + vault: + url: "vault:///secret/data" diff --git a/.helm/namespaces/app.yaml b/.helm/namespaces/app.yaml deleted file mode 100644 index 7e88af61b..000000000 --- a/.helm/namespaces/app.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: {{ .Env.DEPLOY_APP_NAMESPACE_NAME }} - annotations: - field.cattle.io/projectId: {{ .Env.RANCHER_PROJECT_ID }} - field.cattle.io/resourceQuota: '{"limit":{"pods":"{{ .Env.DEPLOY_APP_NAMESPACE_POD_LIMIT }}"}}' diff --git a/.helm/namespaces/shared-resources.yaml b/.helm/namespaces/shared-resources.yaml deleted file mode 100644 index 7e1be5241..000000000 --- a/.helm/namespaces/shared-resources.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: {{ .Env.DEPLOY_SHARED_RESOURCES_NAMESPACE_NAME }} - annotations: - field.cattle.io/projectId: {{ .Env.RANCHER_PROJECT_ID }} - field.cattle.io/resourceQuota: '{"limit":{"pods":"{{ .Env.DEPLOY_SHARED_RESOURCES_NAMESPACE_POD_LIMIT }}"}}' diff --git a/.helm/secret-values.development.yaml b/.helm/secret-values.development.yaml deleted file mode 100644 index e8bf8a455..000000000 --- a/.helm/secret-values.development.yaml +++ /dev/null @@ -1,2 +0,0 @@ -secrets: - redis_password: 10002e0d33ad1785006b074cda33268e9021e5a649773438a3556c0679aad0162ff4d80dec917ca4582168b2a633c5f905ed diff --git a/.helm/secret-values.production.yaml b/.helm/secret-values.production.yaml deleted file mode 100644 index 414a71a19..000000000 --- a/.helm/secret-values.production.yaml +++ /dev/null @@ -1,2 +0,0 @@ -secrets: - redis_password: 10009752210666abab8694970f80e8248197733947c60c9ba27f5287e5839b74db8b293b8ebcb18a173132ee928133b62f04 diff --git a/.helm/secret-values.staging.yaml b/.helm/secret-values.staging.yaml deleted file mode 100644 index 1c053f4ae..000000000 --- a/.helm/secret-values.staging.yaml +++ /dev/null @@ -1,2 +0,0 @@ -secrets: - redis_password: 10006ed43b5d95373fd966af13014d65342e6fc31658633262ca150fe1ec338ee3f2cde10def52af5336b19f0268ff445e74 diff --git a/.helm/secret-values.yaml b/.helm/secret-values.yaml deleted file mode 100644 index e3a719463..000000000 --- a/.helm/secret-values.yaml +++ /dev/null @@ -1,2 +0,0 @@ -secrets: - redis_password: 1000fb94d50114b50c0457a01b0fc1b9245b79c3780d3f3b21ea587d89311204933e diff --git a/.helm/secret/development-keys.py b/.helm/secret/development-keys.py deleted file mode 100644 index 66224418d..000000000 --- a/.helm/secret/development-keys.py +++ /dev/null @@ -1 +0,0 @@ -1000a287d4991b6127153631eee92ee306ab1a68dc7a9bdb8524a3de50ca2a666e9efb35319928d708a9da1f06efb7b15b0601339cb7995ddd6b3a0ffa00aa5597abdb6f553608acdc8e21fd3c6146f17d7f98ca01295a58b45a05e2a35e019fc23f7457409ec8d0a76cc4abb3e973348ac7b24e6d965769ebdd6104394b473c83b0725339f9346e0055d303178e5d855572027bb6b7ea4b871af5cc9367186d4baad982656a36805a122d9be9276f4bc91f57c3a373e0d9ef19084b8871ec8a4dea5f4e930f0097c7f1bb454f20bd8809cec4a96825304ca6061140551e792a983538fccde5137a069e0db683fba346b82b064bcf19abbba922a4a868425d1c47983a43f55046c0ad6e4cb56278aa59a2153141bf97e61a3b307ce8a325e816856604808d0b70d129366bb0ee7d333ae777164afa80fbe9c0101637f05094fb157da0935f5a580a3e5dc0ddf08c3d17e8d3256acb231af9ac4c6f8f013fb2f6497b640c3bf3ee2d2e4cc5ba2b396584faf6b5957948956799481ab6888fcc036ecb70425117ab15931e66184448f5e294e6c54d1b9d2142ec552d18b899f5c3e24bf2870e49a709e46355b62eb9a436baee47aad53a6504ee47d8c2e54724ed5b1120a222361e18bf7162eabae1d87c613bd87ef4b9d578bae3ffca241eb2a89cb90d7d7648767819a89bfaf96eb6ec53dac17271dd030706cec9ddac2454459a25d40384f7f5026fd085df6e16f2eaf1c49bd307e2fe62bc95f76d79fe4c71b8776fe60bd92e4a370b1cd2707bdbee5576fa0139c8277a47160fddc92bb167efc41726acebc32e63fa6fb6896b0102b179da146be946193996e22e2271574e70bf9d4191c9410536416611cf358e23acc324db381228aa03a9119d2dad8fad493ee3570302eb2d6df8dc78d1fec6fac406ce55ee6ba2b36ade3a68e47a5a2b76d563b47b583cc2d8e258bded75ea8edba123a95fd88eac3f4f5a45520f1fb417311c6f469e580150e0f4bb96e35f1cbec5ccd0d2d330a1ec799504bec44cfa256f9977ea55e1877f760f05e5457ef10cd6b2a34fd42ab5c02ad7fd6658f3d38fcb6a144cf27ee0581a90803e136155e9e891a60fcf2eba3a9d60e23a346e251cfc8a60a35c636f01b273179c3d514b4610f541237661b134a052159597f8e312856f22567831afbd9893dad5664a65474146abb8c3028b9520a8d7df4ae1424a976903aa14abe433b41d283b5e00e9b2d0b5e5eabe301bf16e8f22896b3455e53b914d90709b1241245c6606ae3b981a1c diff --git a/.helm/secret/production-keys.py b/.helm/secret/production-keys.py deleted file mode 100644 index 21f436ab9..000000000 --- a/.helm/secret/production-keys.py +++ /dev/null @@ -1 +0,0 @@ -100025a03b71714752567c2de575616c8b7579c7dadc9b11159121ab5ac4540b5066127f7806996f427e595a4b0704b5704c3a188a7a4dc7f38ec0276018e45e5476168601ff37467ed008284468c45601eac86636cfc0deb1df467202a9049c9130e963b39eaad74e6e3165c7d8adb1ff1ffb78ff61ece5ca7db4f9ed4eb81a5692987eff93ad3a7b45b6c0f9440ab0841fee83840a59f2c3c4423bdf45bd675d202156e7a216a6b5effcdfa0db1c05407ce36686b115498e823e47117707dd298ecf0e6b7594ea56a4e3720038ccd77ffdc69f917929e99c2e168056e46c318c269a0ca39fa75e97cb78b168f4fd8c55a4fea8f49086c3ef2081d27b310f899c10a7ae7ce7a9c9f44af09e7de46ee934ac849c0779cfb576f02124f40889f075b8e33427fd16e31b2a1171b89145993c1a958ca71fde968818cf18aebcad411dcf2bc749b4ec78d4e2a1ab777629851da1a3e91fe79b8049b5f6cae106667bd30321225d7c94489f1d59512472477e8a8127e6591992afd15cf379a70c5110790ab27429fcaa54eeaed5935854451bf88dea6ba2d0b6b7e77797e2d5053309d0c610dba06d51ba94afe35115a88e0664809c38f7906846f8e12dfc79c430de26039011c57cef6351a23b0751e91f1b33ea86e02d9e2779a7a79ec4045b8bc505d4076ab5814a9eba2c6155c54701a42810101c3c7b2579f5501976af502e88f6eca17db31bb0cd5baa846552fcd0c3e1b1d706919479122d543f4c0fa6b7bd5dba77d3794f30c989bd84b8f64cb692df5ca3e4e3f365bfefddaa2a37dad56f9aabebaaafa12ccc285a0cefe4d711f2b990141b759d8303c2288f59a8fe19d0f9988a012f161581723f8d0cea748f215889f9ba4cad109d042f488547a8fb16b2d742c5172a53654d609e0da25d163619349a6734cfa6407cbd928e579637be614753311b3c6eb65bc453eb666ad628b0279f4f1ada9d708e11c5d104cda2ce8dde01e40462ac89f3183997e54f28010acd8c6bebf9ea34d7882b4f021add6f1534426b85d06058664a63b331170ee54413ac047b2d18a233e0e5fbd2199058d594366c142e1d07c41a4efa98593847f2bde01c652b975a0be21be155a1d2f1628cae8ef3fb69a4340f1ec4f87299f127b065a8b19fe11b102adbad45e337ec1a8bd8e6bb30a5521466e86c603537f7549c4f4caa26160f25984e5065ca44c8a85d diff --git a/.helm/secret/staging-keys.py b/.helm/secret/staging-keys.py deleted file mode 100644 index 8e527bd04..000000000 --- a/.helm/secret/staging-keys.py +++ /dev/null @@ -1 +0,0 @@ -1000c06d9305e04dc9f413f0087fc91909d40984d91ddfbfba505c33325695725cbce0d378841dda920fb9a88a9782b89090103ff193ce519db8912bf14b85bc4750deb4a6a81b84022b71c87b6d7a769625b1a8d7086cea3674db349856efe37a3b5b14e212cd5f817762055fe9fb2b34965bdd16fdba30ece9f1953e0fac0aa920e1d8835bc3b55f7a3a17a233260b44b51683a742a301bc4e775f8543c507d73f28ff4cf51f4a359322bd3dfcdcf17aa6cc56faca4d9aef3690b3b48e08dc5af9cd9846a6f9f28e91447ddc0951cf31bd07565314f4823e10c7f89476f32f9327e81b539118ff4c19d394c366b6ec248d9b3270b85d0dfe0e604b8b88863d303265a3b91490a47d517cadd08670d3a6744dc5c21b2b509ac65d2bd359a64aaa25e9c9744b6da1baab3d3413c510f1cfdb424bb00b810477c74cb9a094c0a59c773f8b13c8b2b27e0e052a66817d874bb995491568a1cdd5c64760345c9db3f8bc53553944eebd26b1ce0a001148980a765c8f38ed05388bf8eb40a20f15fd3508f14525986ea2e0877427de86cb7683747372559a11a3c546e7273afcb08871119df76689f22ce688a0a8fe145a4d0cf7397e80ce14486ebc646428f6789a2e94308adbe780c39f711734e00e1b9ea93f0dfdb5da0d9f8e4afe389e68dea05bbdf14b9a4a8986419da2ad0759845487fce519358a7b0009375fcf585f1e077ef177ab25544097ce6e739c9db8a6b49b6bc1989041264047dec69d70d3e854796f070fc1696f810105700270eb847c7346b51574fde2bf87e71039a440a9c235abb5ccb0f6ea5eec63cb3323b9080eec85a84574f531b316a800ba56b735d3ed678690a2cbde5f6448d022fa4c41a75f7d5ebcb20e3e0f2e4a10a97645c3e61c0ca7ff5519174919495ba8d69b9c8b510cceb152b32b8af714903101139224f66fc695dc2464f51b31545540b36b0296ba106444e121007bbda0d1e30a4fc7c26fb926c665d098c71ba686e19f9844b7ea8ad9ddafb40776dd2f68e965b8607f253349262b20799dc4d041d35f3730e494673fe896a334944101758435c9cd934a36d724f47729ac132ca1717ff657b5a579e0f8eeea0549a22eea69dd4c1cedf29a59998bd16667164a458623676c06b05f5565997868cbfec3a8fe727d56d7d5f2e1882e5e146baa07d41f0551755f96844118b121eab3927086702c523a1d77ac6fdf6ca11e4e7b0c67a404331a80db3cfc136e8b5b7646d5cdf477652f8012fdd0a2fa7716744c223ea423a1ce00252fe170368590d90adc459398347dcf03f25e05acbbbe605d7ce2f1045625c2c89db1fd2e3df89228b7960237546ac1063441e5d28871576fb5770c05bd508fb704f541b77e933cb243cd46f83b81d44401dfd267e299d0ff223ebc000feee0713dfba95d1fef13de4bf4662cc89b05559ecd08cdaff2c602ff93159d38f1867275c89c6a4073a23bb8c305c87cc008c0086d1b7d3dbe2b413835a3552af21edce243005e357073d72d469e1b0a302a60 diff --git a/.helm/templates/base-config-map.yaml b/.helm/templates/base-config-map.yaml deleted file mode 100644 index 13d7a4dae..000000000 --- a/.helm/templates/base-config-map.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Chart.Name }}-base-config-map -data: - APP_ENV: {{ .Values.appEnv | quote }} - {{ if .Values.branchName }} - BRANCH_NAME: {{ .Values.branchName | quote }} - {{ end }} - {{ if .Values.dbName }} - DB_NAME: {{ .Values.dbName | quote }} - # Use the database name as the queue name so that separate branches or - # databases on staging remain in separate queues. - APP_QUEUE_NAME: {{ .Values.dbName | quote }} - {{ else }} - # In production (or where the database name isn't explicitly overridden), - # just use the app env as the queue name. - APP_QUEUE_NAME: {{ .Values.appEnv | quote }} - {{ end }} - DJANGO_SETTINGS_MODULE: {{ .Values.djangoSettingsModule | quote }} - SOLVER: xpress - JULIA_HOST: {{ tpl .Values.juliaHost . | quote }} - REDIS_HOST: {{ tpl .Values.redisHost . | quote }} - REDIS_PORT: {{ .Values.redisPort | quote }} - K8S_DEPLOY: "true" diff --git a/.helm/templates/celery-deployment.yaml b/.helm/templates/celery-deployment.yaml deleted file mode 100644 index 14daae8ad..000000000 --- a/.helm/templates/celery-deployment.yaml +++ /dev/null @@ -1,103 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Chart.Name }}-celery-deployment - labels: - app: {{ .Chart.Name }}-celery -spec: - replicas: {{ .Values.celeryReplicas }} - selector: - matchLabels: - app: {{ .Chart.Name }}-celery - template: - metadata: - labels: - app: {{ .Chart.Name }}-celery - appImageTagChecksum: {{ index .Values.werf.image "reopt-api" | sha1sum }} - spec: - topologySpreadConstraints: - - maxSkew: 1 - topologyKey: kubernetes.io/hostname - whenUnsatisfiable: ScheduleAnyway - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - {{ .Chart.Name }}-celery - - key: appImageTagChecksum - operator: In - values: - - {{ index .Values.werf.image "reopt-api" | sha1sum }} - imagePullSecrets: - - name: {{ .Chart.Name }}-ecr-image-pull-secret - volumes: - - name: {{ .Chart.Name }}-secrets-volume - secret: - secretName: {{ .Chart.Name }}-secrets - initContainers: - - name: {{ .Chart.Name }}-ready-wait - image: {{ index .Values.werf.image "reopt-api" }} - args: ["bin/ready-wait"] - envFrom: - - configMapRef: - name: {{ .Chart.Name }}-base-config-map - volumeMounts: - - name: {{ .Chart.Name }}-secrets-volume - readOnly: true - mountPath: /opt/reopt/keys.py - subPath: {{ .Values.appEnv }}-keys.py - containers: - - name: {{ .Chart.Name }}-celery - image: {{ index .Values.werf.image "reopt-api" }} - args: ["bin/worker"] - envFrom: - - configMapRef: - name: {{ .Chart.Name }}-base-config-map - env: - - name: JULIA_HOST - value: "localhost" - volumeMounts: - - name: {{ .Chart.Name }}-secrets-volume - readOnly: true - mountPath: /opt/reopt/keys.py - subPath: {{ .Values.appEnv }}-keys.py -# readinessProbe: -# exec: -# command: ["pgrep", "-f", "bin/celery"] -# periodSeconds: 5 -# timeoutSeconds: 3 -# failureThreshold: 3 -# livenessProbe: -# exec: -# command: ["pgrep", "-f", "bin/celery"] -# initialDelaySeconds: 30 -# periodSeconds: 60 -# timeoutSeconds: 30 -# failureThreshold: 10 - resources: - requests: - cpu: {{ .Values.celeryCpuRequest | quote }} - memory: {{ .Values.celeryMemoryRequest | quote }} - limits: - cpu: {{ .Values.celeryCpuLimit | quote }} - memory: {{ .Values.celeryMemoryLimit | quote }} - - - name: {{ .Chart.Name }}-julia - image: {{ index .Values.werf.image "julia-api" }} - args: ["julia", "--project=/opt/julia_src", "http.jl"] - ports: - - containerPort: 8081 - env: - - name: JULIA_NUM_THREADS - value: "4" - envFrom: - - configMapRef: - name: {{ .Chart.Name }}-base-config-map - resources: - requests: - cpu: {{ .Values.juliaCpuRequest | quote }} - memory: {{ .Values.juliaMemoryRequest | quote }} - limits: - cpu: {{ .Values.juliaCpuLimit | quote }} - memory: {{ .Values.juliaMemoryLimit | quote }} diff --git a/.helm/templates/db-migrate-job.yaml b/.helm/templates/db-migrate-job.yaml deleted file mode 100644 index 55966b26b..000000000 --- a/.helm/templates/db-migrate-job.yaml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ .Chart.Name }}-db-migrate-job-{{ .Release.Revision }} -spec: - backoffLimit: 0 - template: - spec: - restartPolicy: Never - imagePullSecrets: - - name: {{ .Chart.Name }}-ecr-image-pull-secret - volumes: - - name: {{ .Chart.Name }}-secrets-volume - secret: - secretName: {{ .Chart.Name }}-secrets - containers: - - name: {{ .Chart.Name }}-job - image: {{ index .Values.werf.image "reopt-api" }} - args: ["bin/migrate"] - envFrom: - - configMapRef: - name: {{ .Chart.Name }}-base-config-map - volumeMounts: - - name: {{ .Chart.Name }}-secrets-volume - readOnly: true - mountPath: /opt/reopt/keys.py - subPath: {{ .Values.appEnv }}-keys.py diff --git a/.helm/templates/django-deployment.yaml b/.helm/templates/django-deployment.yaml deleted file mode 100644 index 26a0d89a3..000000000 --- a/.helm/templates/django-deployment.yaml +++ /dev/null @@ -1,91 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Chart.Name }}-django-deployment - labels: - app: {{ .Chart.Name }}-django -spec: - replicas: {{ .Values.djangoReplicas }} - selector: - matchLabels: - app: {{ .Chart.Name }}-django - template: - metadata: - labels: - app: {{ .Chart.Name }}-django - appImageTagChecksum: {{ index .Values.werf.image "reopt-api" | sha1sum }} - spec: - topologySpreadConstraints: - - maxSkew: 1 - topologyKey: kubernetes.io/hostname - whenUnsatisfiable: ScheduleAnyway - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - {{ .Chart.Name }}-django - - key: appImageTagChecksum - operator: In - values: - - {{ index .Values.werf.image "reopt-api" | sha1sum }} - imagePullSecrets: - - name: {{ .Chart.Name }}-ecr-image-pull-secret - volumes: - - name: {{ .Chart.Name }}-secrets-volume - secret: - secretName: {{ .Chart.Name }}-secrets - initContainers: - - name: {{ .Chart.Name }}-ready-wait - image: {{ index .Values.werf.image "reopt-api" }} - args: ["bin/ready-wait"] - envFrom: - - configMapRef: - name: {{ .Chart.Name }}-base-config-map - volumeMounts: - - name: {{ .Chart.Name }}-secrets-volume - readOnly: true - mountPath: /opt/reopt/keys.py - subPath: {{ .Values.appEnv }}-keys.py - containers: - - name: {{ .Chart.Name }}-django - image: {{ index .Values.werf.image "reopt-api" }} - args: ["bin/server"] - ports: - - containerPort: 8000 - envFrom: - - configMapRef: - name: {{ .Chart.Name }}-base-config-map - volumeMounts: - - name: {{ .Chart.Name }}-secrets-volume - readOnly: true - mountPath: /opt/reopt/keys.py - subPath: {{ .Values.appEnv }}-keys.py -# readinessProbe: -# httpGet: -# path: /_health -# port: 8000 -# httpHeaders: -# - name: X-Forwarded-Proto -# value: https -# periodSeconds: 5 -# timeoutSeconds: 60 -# failureThreshold: 10 -# livenessProbe: -# httpGet: -# path: /_health -# port: 8000 -# httpHeaders: -# - name: X-Forwarded-Proto -# value: https -# initialDelaySeconds: 30 -# periodSeconds: 60 -# timeoutSeconds: 60 -# failureThreshold: 10 - resources: - requests: - cpu: {{ .Values.djangoCpuRequest | quote }} - memory: {{ .Values.djangoMemoryRequest | quote }} - limits: - cpu: {{ .Values.djangoCpuLimit | quote }} - memory: {{ .Values.djangoMemoryLimit | quote }} diff --git a/.helm/templates/django-ingress.yaml b/.helm/templates/django-ingress.yaml deleted file mode 100644 index 8c3d5428c..000000000 --- a/.helm/templates/django-ingress.yaml +++ /dev/null @@ -1,40 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: {{ .Chart.Name }}-django-ingress - annotations: - # Size should be kept in sync with "client_max_body_size" in - # config/templates/nginx/site.conf.tmpl. - nginx.ingress.kubernetes.io/proxy-body-size: "2048m" - nginx.ingress.kubernetes.io/proxy-read-timeout: "800" - nginx.ingress.kubernetes.io/proxy-connect-timeout: "800" -spec: - tls: - - hosts: - - {{ .Values.ingressHost }} - {{ if .Values.tempIngressHost }} - - {{ .Values.tempIngressHost }} - {{ end }} - rules: - - host: {{ .Values.ingressHost }} - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: {{ .Chart.Name }}-django-service - port: - number: 8000 - {{ if .Values.tempIngressHost }} - - host: {{ .Values.tempIngressHost }} - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: {{ .Chart.Name }}-django-service - port: - number: 8000 - {{ end }} diff --git a/.helm/templates/django-service.yaml b/.helm/templates/django-service.yaml deleted file mode 100644 index d6c649772..000000000 --- a/.helm/templates/django-service.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Chart.Name }}-django-service -spec: - selector: - app: {{ .Chart.Name }}-django - ports: - - protocol: TCP - port: 8000 diff --git a/.helm/templates/ecr-image-pull-secret.yaml b/.helm/templates/ecr-image-pull-secret.yaml deleted file mode 100644 index 6f4f759a1..000000000 --- a/.helm/templates/ecr-image-pull-secret.yaml +++ /dev/null @@ -1,108 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ .Chart.Name }}-ecr-login-renew-service-account - annotations: - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-weight": "1" ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ .Chart.Name }}-ecr-login-renew-role - annotations: - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-weight": "1" -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["create", "update", "get", "delete"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ .Chart.Name }}-ecr-login-renew-role-binding - annotations: - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-weight": "1" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ .Chart.Name }}-ecr-login-renew-role -subjects: - - kind: ServiceAccount - name: {{ .Chart.Name }}-ecr-login-renew-service-account ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Chart.Name }}-ecr-login-renew-config-map - annotations: - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-weight": "1" -data: - DOCKER_SECRET_NAME: {{ .Chart.Name }}-ecr-image-pull-secret - TARGET_NAMESPACE: {{ .Release.Namespace | quote }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: {{ .Chart.Name }}-ecr-login-renew-secrets - annotations: - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-weight": "1" -type: Opaque -data: - AWS_REGION: {{ .Values.ecrAwsRegion | b64enc | quote }} - AWS_ACCESS_KEY_ID: {{ .Values.ecrAwsAccessKeyId | b64enc | quote }} - AWS_SECRET_ACCESS_KEY: {{ .Values.ecrAwsSecretAccessKey | b64enc | quote }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ .Chart.Name }}-ecr-login-renew-initial-setup - annotations: - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-weight": "2" -spec: - template: - spec: - restartPolicy: OnFailure - dnsConfig: - options: - - name: ndots - value: "1" - serviceAccountName: {{ .Chart.Name }}-ecr-login-renew-service-account - containers: - - name: {{ .Chart.Name }}-ecr-login-renew - image: ghcr.io/nabsul/k8s-ecr-login-renew:v1.7.1 - envFrom: - - configMapRef: - name: {{ .Chart.Name }}-ecr-login-renew-config-map - - secretRef: - name: {{ .Chart.Name }}-ecr-login-renew-secrets ---- -apiVersion: batch/v1 -kind: CronJob -metadata: - name: {{ .Chart.Name }}-ecr-login-renew-cron-job -spec: - schedule: "37 */6 * * *" - jobTemplate: - spec: - template: - spec: - restartPolicy: OnFailure - dnsConfig: - options: - - name: ndots - value: "1" - serviceAccountName: {{ .Chart.Name }}-ecr-login-renew-service-account - containers: - - name: {{ .Chart.Name }}-ecr-login-renew - image: ghcr.io/nabsul/k8s-ecr-login-renew:v1.7.1 - envFrom: - - configMapRef: - name: {{ .Chart.Name }}-ecr-login-renew-config-map - - secretRef: - name: {{ .Chart.Name }}-ecr-login-renew-secrets diff --git a/.helm/templates/julia-deployment.yaml b/.helm/templates/julia-deployment.yaml deleted file mode 100644 index dea2880a6..000000000 --- a/.helm/templates/julia-deployment.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Chart.Name }}-julia-deployment - labels: - app: {{ .Chart.Name }}-julia -spec: - replicas: {{ .Values.juliaReplicas }} - selector: - matchLabels: - app: {{ .Chart.Name }}-julia - template: - metadata: - labels: - app: {{ .Chart.Name }}-julia - appImageTagChecksum: {{ index .Values.werf.image "julia-api" | sha1sum }} - spec: - topologySpreadConstraints: - - maxSkew: 1 - topologyKey: kubernetes.io/hostname - whenUnsatisfiable: ScheduleAnyway - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - {{ .Chart.Name }}-julia - - key: appImageTagChecksum - operator: In - values: - - {{ index .Values.werf.image "julia-api" | sha1sum }} - imagePullSecrets: - - name: {{ .Chart.Name }}-ecr-image-pull-secret - containers: - - name: {{ .Chart.Name }}-julia - image: {{ index .Values.werf.image "julia-api" }} - args: ["julia", "--project=/opt/julia_src", "http.jl"] - ports: - - containerPort: 8081 - env: - - name: JULIA_NUM_THREADS - value: "4" - envFrom: - - configMapRef: - name: {{ .Chart.Name }}-base-config-map - readinessProbe: - httpGet: - path: /health - port: 8081 - periodSeconds: 5 - timeoutSeconds: 60 - failureThreshold: 10 - resources: - requests: - cpu: {{ .Values.juliaCpuRequest | quote }} - memory: "2000Mi" - limits: - cpu: {{ .Values.juliaCpuLimit | quote }} - memory: "2000Mi" diff --git a/.helm/templates/julia-service.yaml b/.helm/templates/julia-service.yaml deleted file mode 100644 index 8bd9be4ef..000000000 --- a/.helm/templates/julia-service.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Chart.Name }}-julia-service -spec: - selector: - app: {{ .Chart.Name }}-julia - ports: - - protocol: TCP - port: 8081 diff --git a/.helm/templates/redis-service.yaml b/.helm/templates/redis-service.yaml deleted file mode 100644 index ce7665369..000000000 --- a/.helm/templates/redis-service.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Chart.Name }}-redis-service - annotations: - "helm.sh/resource-policy": keep -spec: - selector: - app: {{ .Chart.Name }}-redis - ports: - - protocol: TCP - port: 6379 diff --git a/.helm/templates/redis-stateful-set.yaml b/.helm/templates/redis-stateful-set.yaml deleted file mode 100644 index 339705613..000000000 --- a/.helm/templates/redis-stateful-set.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{ .Chart.Name }}-redis-stateful-set - labels: - app: {{ .Chart.Name }}-redis - annotations: - "helm.sh/resource-policy": keep -spec: - replicas: 1 - selector: - matchLabels: - app: {{ .Chart.Name }}-redis - serviceName: {{ .Chart.Name }}-redis - template: - metadata: - labels: - app: {{ .Chart.Name }}-redis - spec: - containers: - - name: {{ .Chart.Name }}-redis - image: redis:6.0.8-alpine - args: ["sh", "-c", "redis-server --requirepass $$REDIS_PASSWORD --save 900 1 --save 300 10 --save 60 10000 --appendonly yes --maxmemory 2048mb --maxmemory-policy allkeys-lru"] - # volumeMounts: - # - name: {{ .Chart.Name }}-redis-data-volume - # mountPath: /data - env: - - name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Chart.Name }}-secrets - key: redis_password - # volumeClaimTemplates: - # - metadata: - # name: {{ .Chart.Name }}-redis-data-volume - # spec: - # storageClassName: {{ .Values.redisDataVolumeStorageClassName | quote }} - # accessModes: ["ReadWriteOnce"] - # resources: - # requests: - # storage: 1.5Gi diff --git a/.helm/templates/secrets.yaml b/.helm/templates/secrets.yaml deleted file mode 100644 index 707e59311..000000000 --- a/.helm/templates/secrets.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: {{ .Chart.Name }}-secrets -type: Opaque -data: - development-keys.py: {{ werf_secret_file "development-keys.py" | b64enc | quote }} - staging-keys.py: {{ werf_secret_file "staging-keys.py" | b64enc | quote }} - production-keys.py: {{ werf_secret_file "production-keys.py" | b64enc | quote }} - redis_password: {{ .Values.secrets.redis_password | b64enc | quote }} diff --git a/.helm/values.deploy.yaml b/.helm/values.deploy.yaml deleted file mode 100644 index e8bd27bfa..000000000 --- a/.helm/values.deploy.yaml +++ /dev/null @@ -1 +0,0 @@ -redisDataVolumeStorageClassName: netappnfs diff --git a/.helm/values.development.yaml b/.helm/values.development.yaml deleted file mode 100644 index e69de29bb..000000000 diff --git a/.helm/values.production.yaml b/.helm/values.production.yaml deleted file mode 100644 index 46abcafb6..000000000 --- a/.helm/values.production.yaml +++ /dev/null @@ -1,16 +0,0 @@ -appEnv: production -djangoSettingsModule: reopt_api.production_settings -djangoReplicas: 10 -djangoCpuRequest: "1000m" -djangoCpuLimit: "2000m" -djangoMemoryRequest: "2000Mi" -djangoMemoryLimit: "2000Mi" -celeryReplicas: 10 -juliaReplicas: 5 -juliaCpuRequest: "2000m" -juliaCpuLimit: "4000m" -juliaMemoryRequest: "12000Mi" -juliaMemoryLimit: "12000Mi" -juliaDeploymentStrategy: - rollingUpdate: - maxSurge: "0%" diff --git a/.helm/values.staging.yaml b/.helm/values.staging.yaml deleted file mode 100644 index d7cfc7107..000000000 --- a/.helm/values.staging.yaml +++ /dev/null @@ -1,2 +0,0 @@ -appEnv: staging -djangoSettingsModule: reopt_api.staging_settings \ No newline at end of file diff --git a/.helm/values.yaml b/.helm/values.yaml deleted file mode 100644 index b6dae9237..000000000 --- a/.helm/values.yaml +++ /dev/null @@ -1,25 +0,0 @@ -ecrAwsRegion: us-east-2 -ecrAwsAccessKeyId: -ecrAwsSecretAccessKey: -ingressHost: localhost -appEnv: development -djangoSettingsModule: reopt_api.dev_settings -redisDataVolumeStorageClassName: -juliaHost: "{{ .Chart.Name }}-julia-service" -redisHost: "{{ .Chart.Name }}-redis-service" -redisPort: 6379 -djangoReplicas: 1 -djangoCpuRequest: "500m" -djangoCpuLimit: "2000m" -djangoMemoryRequest: "700Mi" -djangoMemoryLimit: "700Mi" -celeryReplicas: 1 -celeryCpuRequest: "100m" -celeryCpuLimit: "1000m" -celeryMemoryRequest: "1600Mi" -celeryMemoryLimit: "1600Mi" -juliaReplicas: 1 -juliaCpuRequest: "500m" -juliaCpuLimit: "2000m" -juliaMemoryRequest: "6000Mi" -juliaMemoryLimit: "6000Mi" diff --git a/Capfile b/Capfile deleted file mode 100644 index 50fae920c..000000000 --- a/Capfile +++ /dev/null @@ -1,21 +0,0 @@ -# Load DSL and set up stages -require "capistrano/setup" - -# Include default deployment tasks -require "capistrano/deploy" - -# Load the SCM plugin -require "capistrano/scm/git" -install_plugin Capistrano::SCM::Git - -# Includes additional plugins -require "capistrano/rbenv" -require "capistrano/bundler" -require "capistrano/file-permissions" -require "captastic/subdomains" -require "captastic/foreman" -require "captastic/nginx" -require "capistrano/tada-defaults" - -# Load custom tasks from `lib/capistrano/tasks` if you have any defined -Dir.glob("lib/capistrano/tasks/*.rake").each { |r| import r } \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index c5c064371..07f5ee0a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,19 +8,21 @@ ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt ENV SRC_DIR=/opt/reopt/reo/src ENV LD_LIBRARY_PATH="/opt/reopt/reo/src:${LD_LIBRARY_PATH}" -# Copy all code -ENV PYTHONDONTWRITEBYTECODE=1 -COPY . /opt/reopt - # Install python packages +ENV PYTHONDONTWRITEBYTECODE=1 +COPY requirements.txt /opt/reopt/ WORKDIR /opt/reopt RUN ["pip", "install", "-r", "requirements.txt"] # Conditionally install EVI-EnLitePy and pydantic (dependency) if EVI-EnLitePy has been cloned via git submodule +COPY EVI-EnLitePy /opt/reopt/ RUN if [ -d "/opt/reopt/EVI-EnLitePy" ] && [ "$(ls -A /opt/reopt/EVI-EnLitePy)" ]; then \ cd /opt/reopt/EVI-EnLitePy && pip install -e .; \ pip install pydantic; \ fi +# Copy the rest of the app code. +COPY . /opt/reopt + EXPOSE 8000 ENTRYPOINT ["/bin/bash", "-c"] diff --git a/Gemfile b/Gemfile deleted file mode 100644 index 4745ba95e..000000000 --- a/Gemfile +++ /dev/null @@ -1,17 +0,0 @@ -source "https://rubygems.org" - -# Process management -gem "foreman", "~> 0.87.0" - -group :development do - # Deployment - gem "capistrano", "~> 3.12.0" - gem "capistrano-rbenv", "~> 2.1.4" - gem "capistrano-bundler", "~> 1.6.0" - gem "capistrano-file-permissions", "~> 1.0.0" - gem "capistrano-tada-defaults", :git => "https://github.nrel.gov/TADA/capistrano-tada-defaults.git" - gem "captastic", :git => "https://github.nrel.gov/TADA/captastic.git" - gem "captastic-nginx", :git => "https://github.nrel.gov/TADA/captastic-nginx.git" - gem "captastic-foreman", :git => "https://github.nrel.gov/TADA/captastic-foreman.git" - gem "captastic-subdomains", :git => "https://github.nrel.gov/TADA/captastic-subdomains.git" -end \ No newline at end of file diff --git a/Gemfile.lock b/Gemfile.lock deleted file mode 100644 index f55d163fa..000000000 --- a/Gemfile.lock +++ /dev/null @@ -1,79 +0,0 @@ -GIT - remote: https://github.nrel.gov/TADA/capistrano-tada-defaults.git - revision: b9921ac074f5b5f13b6f34fd3c70ab0e6e13cc37 - specs: - capistrano-tada-defaults (0.2.0) - -GIT - remote: https://github.nrel.gov/TADA/captastic-foreman.git - revision: fc5ca792a9673e5f401717ee1c4c92a35d2d9702 - specs: - captastic-foreman (0.2.1) - captastic - inifile - -GIT - remote: https://github.nrel.gov/TADA/captastic-nginx.git - revision: 8198edc9e1b9378f5912c4c1a025adbf46182200 - specs: - captastic-nginx (0.1.0) - -GIT - remote: https://github.nrel.gov/TADA/captastic-subdomains.git - revision: 77e0965208cefdb95640130858f9dd1fbc80373b - specs: - captastic-subdomains (0.1.2) - -GIT - remote: https://github.nrel.gov/TADA/captastic.git - revision: f51c58441b98c9d969ec2df46d16141d1fff728f - specs: - captastic (0.1.2) - -GEM - remote: https://rubygems.org/ - specs: - airbrussh (1.4.0) - sshkit (>= 1.6.1, != 1.7.0) - capistrano (3.12.0) - airbrussh (>= 1.0.0) - i18n - rake (>= 10.0.0) - sshkit (>= 1.9.0) - capistrano-bundler (1.6.0) - capistrano (~> 3.1) - capistrano-file-permissions (1.0.0) - capistrano (~> 3.0) - capistrano-rbenv (2.1.6) - capistrano (~> 3.1) - sshkit (~> 1.3) - concurrent-ruby (1.1.6) - foreman (0.87.0) - i18n (1.8.2) - concurrent-ruby (~> 1.0) - inifile (3.0.0) - net-scp (2.0.0) - net-ssh (>= 2.6.5, < 6.0.0) - net-ssh (5.2.0) - rake (13.0.1) - sshkit (1.20.0) - net-scp (>= 1.1.2) - net-ssh (>= 2.8.0) - -PLATFORMS - ruby - -DEPENDENCIES - capistrano (~> 3.12.0) - capistrano-bundler (~> 1.6.0) - capistrano-file-permissions (~> 1.0.0) - capistrano-rbenv (~> 2.1.4) - capistrano-tada-defaults! - captastic! - captastic-foreman! - captastic-nginx! - captastic-subdomains! - foreman (~> 0.87.0) - -BUNDLED WITH - 1.17.2 \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index b6a699bf2..000000000 --- a/Jenkinsfile +++ /dev/null @@ -1,213 +0,0 @@ -@Library("tada-jenkins-library") _ - -pipeline { - agent none - options { - disableConcurrentBuilds() - buildDiscarder(logRotator(daysToKeepStr: "365")) - } - - environment { - DEPLOY_IMAGE_REPO_DOMAIN = credentials("reopt-api-image-repo-domain") - APP_IMAGE_REPO_DOMAIN = credentials("reopt-api-app-image-repo-domain") - DEVELOPMENT_BASE_DOMAIN = credentials("reopt-api-development-base-domain") - DEVELOPMENT_TEMP_BASE_DOMAIN = credentials("reopt-api-development-temp-base-domain") - STAGING_BASE_DOMAIN = credentials("reopt-api-staging-base-domain") - STAGING_TEMP_BASE_DOMAIN = credentials("reopt-api-staging-temp-base-domain") - PRODUCTION_DOMAIN = credentials("reopt-api-production-domain") - XPRESS_LICENSE_HOST = credentials("reopt-api-xpress-license-host") - NREL_ROOT_CERT_URL_ROOT = credentials("reopt-api-nrel-root-cert-url-root") - } - - parameters { - booleanParam( - name: "DEVELOPMENT_DEPLOY", - defaultValue: false, - description: "Development Deploy: Deploy to development.", - ) - - booleanParam( - name: "STAGING_DEPLOY", - defaultValue: false, - description: "Staging Deploy: Deploy to staging for a non-master branch (master will always be deployed).", - ) - } - - stages { - stage("app-agent") { - agent { - dockerfile { - filename "Dockerfile" - additionalBuildArgs "--pull --build-arg XPRESS_LICENSE_HOST='${XPRESS_LICENSE_HOST}' --build-arg NREL_ROOT_CERT_URL_ROOT='${NREL_ROOT_CERT_URL_ROOT}'" - } - } - - stages { - stage("tests") { - steps { - sh "echo 'Container built and running.'" - } - } - } - } - - stage("deploy-agent") { - agent { - docker { - image "${DEPLOY_IMAGE_REPO_DOMAIN}/tada-public/tada-jenkins-kube-deploy:werf-1.2" - args tadaDockerInDockerArgs() - } - } - - environment { - TMPDIR = tadaDockerInDockerTmp() - WERF_REPO = "${APP_IMAGE_REPO_DOMAIN}/tada/reopt-api" - WERF_LOG_VERBOSE = "true" - WERF_SYNCHRONIZATION = ":local" - XPRESS_LICENSE_HOST = credentials("reopt-api-xpress-license-host") - LICENSESERVER_URL = credentials("reopt-api-xpress-licenseserver-url") - XPRESS_INSTALLED = "False" - NREL_ROOT_CERT_URL_ROOT = credentials("reopt-api-nrel-root-cert-url-root") - } - - stages { - stage("deploy") { - stages { - // stage("solver setup") { - // steps { - // dir("julia_src/licenseserver") { - // git url: env.LICENSESERVER_URL - // } - // sh "cp julia_src/licenseserver/Dockerfile.xpress julia_src/" - // } - // } - - stage("lint") { - steps { - withCredentials([string(credentialsId: "reopt-api-werf-secret-key", variable: "WERF_SECRET_KEY")]) { - sh "werf helm lint" - } - } - } - - stage("build") { - steps { - withDockerRegistry(url: "https://${env.WERF_REPO}", credentialsId: "ecr:us-east-2:aws-nrel-tada-ci") { - sh "werf build" - } - } - } - - stage("deploy-development") { - when { expression { params.DEVELOPMENT_DEPLOY } } - - environment { - DEPLOY_ENV = "development" - DEPLOY_SHARED_RESOURCES_NAMESPACE_POD_LIMIT = "6" - DEPLOY_APP_NAMESPACE_POD_LIMIT = "20" - } - - steps { - withDockerRegistry(url: "https://${env.WERF_REPO}", credentialsId: "ecr:us-east-2:aws-nrel-tada-ci") { - withCredentials([aws(credentialsId: "aws-nrel-tada-ci")]) { - withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod4"]) { - tadaWithWerfNamespaces(rancherProject: "reopt-api-dev", primaryBranch: "master", dbBaseName: "reopt_api_development", baseDomain: "${DEVELOPMENT_BASE_DOMAIN}") { - withCredentials([string(credentialsId: "reopt-api-werf-secret-key", variable: "WERF_SECRET_KEY")]) { - sh """ - werf converge \ - --values=./.helm/values.deploy.yaml \ - --values=./.helm/values.${DEPLOY_ENV}.yaml \ - --secret-values=./.helm/secret-values.${DEPLOY_ENV}.yaml \ - --set='ecrAwsAccessKeyId=${AWS_ACCESS_KEY_ID}' \ - --set='ecrAwsSecretAccessKey=${AWS_SECRET_ACCESS_KEY}' \ - --set='branchName=${BRANCH_NAME}' \ - --set='ingressHost=${DEPLOY_BRANCH_DOMAIN}' \ - --set='tempIngressHost=${tadaDeployBranchDomain(baseDomain: env.DEVELOPMENT_TEMP_BASE_DOMAIN, primaryBranch: "master")}' \ - --set='dbName=${DEPLOY_BRANCH_DB_NAME}' - """ - } - } - } - } - } - } - } - - stage("deploy-staging") { - when { expression { params.STAGING_DEPLOY || env.BRANCH_NAME == "master" } } - - environment { - DEPLOY_ENV = "staging" - DEPLOY_SHARED_RESOURCES_NAMESPACE_POD_LIMIT = "6" - DEPLOY_APP_NAMESPACE_POD_LIMIT = "20" - } - - steps { - withDockerRegistry(url: "https://${env.WERF_REPO}", credentialsId: "ecr:us-east-2:aws-nrel-tada-ci") { - withCredentials([aws(credentialsId: "aws-nrel-tada-ci")]) { - withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod4"]) { - tadaWithWerfNamespaces(rancherProject: "reopt-api-staging", primaryBranch: "master", dbBaseName: "reopt_api_staging", baseDomain: "${STAGING_BASE_DOMAIN}") { - withCredentials([string(credentialsId: "reopt-api-werf-secret-key", variable: "WERF_SECRET_KEY")]) { - sh """ - werf converge \ - --values=./.helm/values.deploy.yaml \ - --values=./.helm/values.${DEPLOY_ENV}.yaml \ - --secret-values=./.helm/secret-values.${DEPLOY_ENV}.yaml \ - --set='ecrAwsAccessKeyId=${AWS_ACCESS_KEY_ID}' \ - --set='ecrAwsSecretAccessKey=${AWS_SECRET_ACCESS_KEY}' \ - --set='branchName=${BRANCH_NAME}' \ - --set='ingressHost=${DEPLOY_BRANCH_DOMAIN}' \ - --set='tempIngressHost=${tadaDeployBranchDomain(baseDomain: env.STAGING_TEMP_BASE_DOMAIN, primaryBranch: "master")}' \ - --set='dbName=${DEPLOY_BRANCH_DB_NAME}' - """ - } - } - } - } - } - } - } - - stage("deploy-production") { - when { branch "master" } - - environment { - DEPLOY_ENV = "production" - DEPLOY_SHARED_RESOURCES_NAMESPACE_POD_LIMIT = "6" - DEPLOY_APP_NAMESPACE_POD_LIMIT = "70" - } - - steps { - withDockerRegistry(url: "https://${env.WERF_REPO}", credentialsId: "ecr:us-east-2:aws-nrel-tada-ci") { - withCredentials([aws(credentialsId: "aws-nrel-tada-ci")]) { - withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod5"]) { - tadaWithWerfNamespaces(rancherProject: "reopt-api-production", primaryBranch: "master") { - withCredentials([string(credentialsId: "reopt-api-werf-secret-key", variable: "WERF_SECRET_KEY")]) { - sh """ - werf converge \ - --values=./.helm/values.deploy.yaml \ - --values=./.helm/values.${DEPLOY_ENV}.yaml \ - --secret-values=./.helm/secret-values.${DEPLOY_ENV}.yaml \ - --set='ecrAwsAccessKeyId=${AWS_ACCESS_KEY_ID}' \ - --set='ecrAwsSecretAccessKey=${AWS_SECRET_ACCESS_KEY}' \ - --set='ingressHost=${PRODUCTION_DOMAIN}' - """ - } - } - } - } - } - } - } - } - } - } - } - } - - post { - always { - tadaSendNotifications() - } - } -} diff --git a/Jenkinsfile-deploy b/Jenkinsfile-deploy deleted file mode 100644 index ec9fed809..000000000 --- a/Jenkinsfile-deploy +++ /dev/null @@ -1,77 +0,0 @@ -@Library("tada-jenkins-library") _ - -properties([ - parameters([ - choice( - name: "PARAM_STAGE", - choices: "development\nstaging\nproduction", - description: "Where do you want to deploy to?" - ), - - [ - $class: "GitParameterDefinition", - name: "PARAM_BRANCH", - type: "PT_BRANCH", - defaultValue: "origin/master", - sortMode: "ASCENDING_SMART", - selectedValue: "DEFAULT", - quickFilterEnabled: true, - ], - ]) -]) - -pipeline { - agent { - docker { - image "ruby:2.7.2-buster" - } - } - options { - disableConcurrentBuilds() - } - - environment { - DEV_URL = credentials("reopt-api-dev-url") - STAGE_URL = credentials("reopt-api-stage-url") - STAGE1_URL = credentials("reopt-api1-stage-url") - STAGE2_URL = credentials("reopt-api2-stage-url") - STAGE_BASEDOMAIN_URL = credentials("reopt-api-stage-base-url") - PROD_URL = credentials("reopt-api-prod-url") - PROD1_URL = credentials("reopt-api1-prod-url") - PROD2_URL = credentials("reopt-api2-prod-url") - XPRESSDIR = "/opt/xpressmp" - } - - stages { - stage("checkout-deploy-branch") { - steps { - tadaCheckoutDeployBranch("https://github.com/NREL/REopt_API.git") - } - } - - stage("deploy") { - steps { - script { - currentBuild.description = "Stage: $PARAM_STAGE Branch: $PARAM_BRANCH" - - sh "bundle install" - sshagent(credentials: ["jenkins-ssh"]) { - if(env.PARAM_STAGE == "development") { - // TODO: Remove this if we setup branched deployments for - // development (with real DNS subdomain support). - sh "bundle exec cap ${PARAM_STAGE} deploy --trace DEV_BRANCH=${PARAM_BRANCH} DEBUG_DEPLOY=true" - } else { - sh "bundle exec cap ${PARAM_STAGE} deploy --trace BRANCH=${PARAM_BRANCH} DEBUG_DEPLOY=true" - } - } - } - } - } - } - - post { - always { - tadaSendNotifications() - } - } -} diff --git a/Jenkinsfile-deploy-c110p b/Jenkinsfile-deploy-c110p deleted file mode 100644 index 68f7136f4..000000000 --- a/Jenkinsfile-deploy-c110p +++ /dev/null @@ -1,60 +0,0 @@ -@Library("tada-jenkins-library") _ - -properties([ - parameters([ - [ - $class: "GitParameterDefinition", - name: "PARAM_BRANCH", - type: "PT_BRANCH", - defaultValue: "origin/develop", - sortMode: "ASCENDING_SMART", - selectedValue: "DEFAULT", - quickFilterEnabled: true, - description: "Which branch do you want to deploy?" - ], - ]) -]) - -pipeline { - agent { - docker { - image "ruby:2.7.2-buster" - } - } - options { - disableConcurrentBuilds() - } - - environment { - C110P_URL = credentials("reopt-api-c110p-url") - XPRESSDIR = "/opt/xpressmp" - PARAM_STAGE = "internal_c110p" - } - - stages { - stage("checkout-deploy-branch") { - steps { - tadaCheckoutDeployBranch("https://github.com/NREL/REopt_API.git") - } - } - - stage("deploy") { - steps { - script { - currentBuild.description = "Stage: $PARAM_STAGE Branch: $PARAM_BRANCH" - - sh "bundle install" - sshagent(credentials: ["jenkins-ssh"]) { - sh "bundle exec cap ${PARAM_STAGE} deploy --trace BRANCH=${PARAM_BRANCH} DEBUG_DEPLOY=true" - } - } - } - } - } - - post { - always { - tadaSendNotifications() - } - } -} diff --git a/Jenkinsfile-image-cleanup b/Jenkinsfile-image-cleanup deleted file mode 100644 index 10b5c0951..000000000 --- a/Jenkinsfile-image-cleanup +++ /dev/null @@ -1,59 +0,0 @@ -@Library("tada-jenkins-library") _ - -pipeline { - agent none - options { - disableConcurrentBuilds() - buildDiscarder(logRotator(daysToKeepStr: "30")) - } - triggers { - cron(env.BRANCH_NAME == "main" ? "H H(0-5) * * *" : "") - } - - environment { - DEPLOY_IMAGE_REPO_DOMAIN = credentials("reopt-api-image-repo-domain") - APP_IMAGE_REPO_DOMAIN = credentials("reopt-api-app-image-repo-domain") - } - - stages { - stage("deploy-agent") { - agent { - docker { - image "${DEPLOY_IMAGE_REPO_DOMAIN}/tada-public/tada-jenkins-kube-deploy:werf-1.2" - args tadaDockerInDockerArgs() - } - } - - environment { - TMPDIR = tadaDockerInDockerTmp() - WERF_REPO = "${APP_IMAGE_REPO_DOMAIN}/tada/reopt-api" - WERF_LOG_VERBOSE = "true" - WERF_SYNCHRONIZATION = ":local" - XPRESS_LICENSE_HOST = credentials("reopt-api-xpress-license-host") - NREL_ROOT_CERT_URL_ROOT = credentials("reopt-api-nrel-root-cert-url-root") - } - - stages { - stage("cleanup") { - steps { - withDockerRegistry(url: "https://${env.WERF_REPO}", credentialsId: "ecr:us-east-2:aws-nrel-tada-ci") { - withCredentials([aws(credentialsId: "aws-nrel-tada-ci")]) { - tadaRancherAllProjectNamespacesKubeConfig(projects: [[credentialsId: "kubeconfig-nrel-reopt-prod4", rancherProject: "reopt-api-dev"], [credentialsId: "kubeconfig-nrel-reopt-prod4", rancherProject: "reopt-api-staging"], [credentialsId: "kubeconfig-nrel-reopt-prod5", rancherProject: "reopt-api-production"]]) { - withCredentials([gitUsernamePassword(credentialsId: "github-nrel-gov-admin")]) { - sh "werf cleanup --scan-context-namespace-only" - } - } - } - } - } - } - } - } - } - - post { - always { - tadaSendNotifications() - } - } -} diff --git a/Jenkinsfile-restart-celery-julia b/Jenkinsfile-restart-celery-julia deleted file mode 100644 index a463d10dc..000000000 --- a/Jenkinsfile-restart-celery-julia +++ /dev/null @@ -1,114 +0,0 @@ -@Library("tada-jenkins-library") _ - -pipeline { - agent any - environment { - TZ = 'America/Denver' // MST/MDT (handles daylight saving automatically) - DEPLOY_IMAGE_REPO_DOMAIN = credentials("reopt-api-image-repo-domain") - DEVELOPMENT_BASE_DOMAIN = credentials("reopt-api-development-base-domain") - DEVELOPMENT_TEMP_BASE_DOMAIN = credentials("reopt-api-development-temp-base-domain") - STAGING_BASE_DOMAIN = credentials("reopt-api-staging-base-domain") - STAGING_TEMP_BASE_DOMAIN = credentials("reopt-api-staging-temp-base-domain") - PRODUCTION_DOMAIN = credentials("reopt-api-production-domain") - XPRESS_LICENSE_HOST = credentials("reopt-api-xpress-license-host") - NREL_ROOT_CERT_URL_ROOT = credentials("reopt-api-nrel-root-cert-url-root") - } - triggers { - cron('0 1 * * *') // 1:00 AM MST/MDT - } - - parameters { - booleanParam( - name: "DEVELOPMENT_DEPLOY", - defaultValue: false, - description: "Development Deploy: Deploy to development.", - ) - - booleanParam( - name: "STAGING_DEPLOY", - defaultValue: false, - description: "Staging Deploy: Deploy to staging for a non-master branch (master will always be deployed).", - ) - } - - stages { - stage("deploy-agent") { - agent { - docker { - image "${DEPLOY_IMAGE_REPO_DOMAIN}/tada-public/tada-jenkins-kube-deploy:werf-1.2" - args tadaDockerInDockerArgs() - } - } - - environment { - TMPDIR = tadaDockerInDockerTmp() - } - - stages { - stage('Rolling Restart Celery and Julia') { - stages { - stage("deploy-development") { - when { expression { params.DEVELOPMENT_DEPLOY } } - - environment { - DEPLOY_ENV = "development" - } - - steps { - withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod4"]) { - tadaWithWerfEnv(rancherProject: "reopt-api-dev", primaryBranch: "master", dbBaseName: "reopt_api_development", baseDomain: "${DEVELOPMENT_BASE_DOMAIN}") { - sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-celery-deployment" - sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-celery-deployment --timeout=10m" - - sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-julia-deployment" - sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-julia-deployment --timeout=10m" - } - } - } - } - - stage("deploy-staging") { - when { expression { params.STAGING_DEPLOY || env.BRANCH_NAME == "master" } } - - environment { - DEPLOY_ENV = "staging" - } - - steps { - withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod4"]) { - tadaWithWerfEnv(rancherProject: "reopt-api-staging", primaryBranch: "master", dbBaseName: "reopt_api_staging", baseDomain: "${STAGING_BASE_DOMAIN}") { - sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-celery-deployment" - sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-celery-deployment --timeout=10m" - - sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-julia-deployment" - sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-julia-deployment --timeout=10m" - } - } - } - } - - stage("deploy-production") { - when { branch "master" } - - environment { - DEPLOY_ENV = "production" - } - - steps { - withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod5"]) { - tadaWithWerfEnv(rancherProject: "reopt-api-production", primaryBranch: "master") { - sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-celery-deployment" - sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-celery-deployment --timeout=10m" - - sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-julia-deployment" - sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-julia-deployment --timeout=10m" - } - } - } - } - } - } - } - } - } -} diff --git a/Jenkinsfile-undeploy-c110p b/Jenkinsfile-undeploy-c110p deleted file mode 100644 index 8ff8d304e..000000000 --- a/Jenkinsfile-undeploy-c110p +++ /dev/null @@ -1,48 +0,0 @@ -@Library("tada-jenkins-library") _ - -properties([ - parameters([ - string( - name: "PARAM_BRANCHES", - description: "Enter a space-deliminted list of branches you wish to undeploy." - ), - ]) -]) - -pipeline { - agent { - docker { - image "ruby:2.7.2-buster" - } - } - options { - disableConcurrentBuilds() - } - - environment { - C110P_URL = credentials("reopt-api-c110p-url") - XPRESSDIR = "/opt/xpressmp" - PARAM_STAGE = "internal_c110p" - } - - stages { - stage("undeploy") { - steps { - script { - currentBuild.description = "Stage: $PARAM_STAGE Branches: $PARAM_BRANCHES" - - sh "bundle install" - sshagent(credentials: ["jenkins-ssh"]) { - sh "for branch_name in ${PARAM_BRANCHES}; do bundle exec cap ${PARAM_STAGE} undeploy --trace BRANCH=\$branch_name CONFIRM_UNDEPLOY=true DEBUG_DEPLOY=true; done" - } - } - } - } - } - - post { - always { - tadaSendNotifications() - } - } -} diff --git a/Jenkinsfile-undeploy-develop b/Jenkinsfile-undeploy-develop deleted file mode 100644 index b0c9f9a86..000000000 --- a/Jenkinsfile-undeploy-develop +++ /dev/null @@ -1,76 +0,0 @@ -@Library("tada-jenkins-library") _ - -pipeline { - agent none - options { - disableConcurrentBuilds() - buildDiscarder(logRotator(daysToKeepStr: "30")) - } - triggers { - cron(env.BRANCH_NAME == "master" ? "H * * * *" : "") - } - - environment { - DEPLOY_IMAGE_REPO_DOMAIN = credentials("reopt-api-image-repo-domain") - } - - parameters { - booleanParam( - name: "UNDEPLOY_DELETED_BRANCHES", - defaultValue: true, - description: "Undeploy branches that have been deleted from develop.", - ) - booleanParam( - name: "UNDEPLOY_MERGED_BRANCHES", - defaultValue: true, - description: "Undeploy branches that have been merged into the master branch (but not deleted) from develop.", - ) - string( - name: "UNDEPLOY_BRANCH_NAMES", - defaultValue: "", - description: "Undeploy the listed branch names (space separated for multiple branches) from develop.", - ) - } - - stages { - stage("deploy-agent") { - agent { - docker { - image "${DEPLOY_IMAGE_REPO_DOMAIN}/tada-public/tada-jenkins-kube-deploy:werf-1.2" - args tadaDockerInDockerArgs() - } - } - - environment { - TMPDIR = tadaDockerInDockerTmp() - WERF_LOG_VERBOSE = "true" - WERF_SYNCHRONIZATION = ":local" - DEPLOY_ENV = "development" - RANCHER_PROJECT = "reopt-api-dev" - DB_BASE_NAME = "reopt_api_develop" - XPRESS_LICENSE_HOST = credentials("reopt-api-xpress-license-host") - NREL_ROOT_CERT_URL_ROOT = credentials("reopt-api-nrel-root-cert-url-root") - } - - stages { - stage("undeploy-branches") { - steps { - withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod4"]) { - tadaUndeployEachBranch(rancherProject: env.RANCHER_PROJECT, undeployDeletedBranches: params.UNDEPLOY_DELETED_BRANCHES, undeployMergedBranches: params.UNDEPLOY_MERGED_BRANCHES, undeployBranchNames: params.UNDEPLOY_BRANCH_NAMES, primaryBranch: "master") { - tadaWithWerfEnv(rancherProject: env.RANCHER_PROJECT, dbBaseName: env.DB_BASE_NAME) { - tadaUndeployBranch() - } - } - } - } - } - } - } - } - - post { - always { - tadaSendNotifications() - } - } -} diff --git a/Jenkinsfile-undeploy-rhel b/Jenkinsfile-undeploy-rhel deleted file mode 100644 index ade1bbb90..000000000 --- a/Jenkinsfile-undeploy-rhel +++ /dev/null @@ -1,57 +0,0 @@ -@Library("tada-jenkins-library") _ - -properties([ - parameters([ - choice( - name: "PARAM_STAGE", - choices: "development\nstaging", - description: "Where do you want to deploy to?" - ), - - string( - name: "PARAM_BRANCHES", - description: "Enter a space-deliminted list of branches you wish to undeploy." - ), - ]) -]) - -pipeline { - agent { - docker { - image "ruby:2.7.2-buster" - } - } - options { - disableConcurrentBuilds() - } - - environment { - DEV_URL = credentials("reopt-api-dev-url") - STAGE_URL = credentials("reopt-api-stage-url") - STAGE1_URL = credentials("reopt-api1-stage-url") - STAGE2_URL = credentials("reopt-api2-stage-url") - STAGE_BASEDOMAIN_URL = credentials("reopt-api-stage-base-url") - XPRESSDIR = "/opt/xpressmp" - } - - stages { - stage("undeploy") { - steps { - script { - currentBuild.description = "Stage: $PARAM_STAGE Branches: $PARAM_BRANCHES" - - sh "bundle install" - sshagent(credentials: ["jenkins-ssh"]) { - sh "for branch_name in ${PARAM_BRANCHES}; do bundle exec cap ${PARAM_STAGE} undeploy --trace BRANCH=\$branch_name CONFIRM_UNDEPLOY=true DEBUG_DEPLOY=true; done" - } - } - } - } - } - - post { - always { - tadaSendNotifications() - } - } -} diff --git a/Jenkinsfile-undeploy-staging b/Jenkinsfile-undeploy-staging deleted file mode 100644 index 548e0045b..000000000 --- a/Jenkinsfile-undeploy-staging +++ /dev/null @@ -1,76 +0,0 @@ -@Library("tada-jenkins-library") _ - -pipeline { - agent none - options { - disableConcurrentBuilds() - buildDiscarder(logRotator(daysToKeepStr: "30")) - } - triggers { - cron(env.BRANCH_NAME == "master" ? "H * * * *" : "") - } - - environment { - DEPLOY_IMAGE_REPO_DOMAIN = credentials("reopt-api-image-repo-domain") - } - - parameters { - booleanParam( - name: "UNDEPLOY_DELETED_BRANCHES", - defaultValue: true, - description: "Undeploy branches that have been deleted from staging.", - ) - booleanParam( - name: "UNDEPLOY_MERGED_BRANCHES", - defaultValue: true, - description: "Undeploy branches that have been merged into the master branch (but not deleted) from staging.", - ) - string( - name: "UNDEPLOY_BRANCH_NAMES", - defaultValue: "", - description: "Undeploy the listed branch names (space separated for multiple branches) from staging.", - ) - } - - stages { - stage("deploy-agent") { - agent { - docker { - image "${DEPLOY_IMAGE_REPO_DOMAIN}/tada-public/tada-jenkins-kube-deploy:werf-1.2" - args tadaDockerInDockerArgs() - } - } - - environment { - TMPDIR = tadaDockerInDockerTmp() - WERF_LOG_VERBOSE = "true" - WERF_SYNCHRONIZATION = ":local" - DEPLOY_ENV = "staging" - RANCHER_PROJECT = "reopt-api-staging" - DB_BASE_NAME = "reopt_api_staging" - XPRESS_LICENSE_HOST = credentials("reopt-api-xpress-license-host") - NREL_ROOT_CERT_URL_ROOT = credentials("reopt-api-nrel-root-cert-url-root") - } - - stages { - stage("undeploy-branches") { - steps { - withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod4"]) { - tadaUndeployEachBranch(rancherProject: env.RANCHER_PROJECT, undeployDeletedBranches: params.UNDEPLOY_DELETED_BRANCHES, undeployMergedBranches: params.UNDEPLOY_MERGED_BRANCHES, undeployBranchNames: params.UNDEPLOY_BRANCH_NAMES, primaryBranch: "master") { - tadaWithWerfEnv(rancherProject: env.RANCHER_PROJECT, dbBaseName: env.DB_BASE_NAME) { - tadaUndeployBranch() - } - } - } - } - } - } - } - } - - post { - always { - tadaSendNotifications() - } - } -} diff --git a/Procfile b/Procfile deleted file mode 100644 index ed6b995fe..000000000 --- a/Procfile +++ /dev/null @@ -1,2 +0,0 @@ -web: $DEPLOY_CURRENT_PATH/bin/server -worker: $DEPLOY_CURRENT_PATH/bin/worker \ No newline at end of file diff --git a/bin/ready-wait b/bin/ready-wait deleted file mode 100755 index 9146cf996..000000000 --- a/bin/ready-wait +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -set -Eeuxo pipefail - -# Used for Kubernetes initContainers to wait for the database to be migrated -# before allowing the app to spin up and be part of the live pods. -while true; do - # Look for pending migrations by viewing the migration plan and looking for - # any pending migrations identified by starting with "[ ]" (already run - # migrations will be identified by "[X]". Keep waiting until all migrations - # are run, and there are no more pending migrations. - # - # A little hacky, but based on info at https://stackoverflow.com/a/31847406 - # - # If the app is upgraded to Django 3.1, then this could maybe be made simpler - # with the new "migrate --check" support: - # https://docs.djangoproject.com/en/3.1/ref/django-admin/#cmdoption-migrate-check - migrations=$(python manage.py showmigrations --plan) - { set +x; } 2>/dev/null - pending_migrations=$(echo "$migrations" | grep '\[ \]' || true) - if [ -z "$pending_migrations" ]; then - echo "Migrations up to date." - break - else - echo "Waiting for migrations to complete:" - echo "$pending_migrations" - sleep 3 - fi - set -x -done diff --git a/checkdb.py b/checkdb.py index 8dbe50f31..c66eef155 100644 --- a/checkdb.py +++ b/checkdb.py @@ -1,35 +1,16 @@ import psycopg2 import os -from keys import * +from keys_env import * from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT env = os.getenv('APP_ENV') -dbname = os.getenv("DB_NAME") - -if env == 'development': - dbhost = dev_database_host - dbuser = dev_user - dbpass = dev_user_password -elif env == 'staging': - dbhost = staging_database_host - dbuser = staging_user - dbpass = staging_user_password -elif env == 'production': - dbhost = prod_database_host - dbuser = production_user - dbpass = production_user_password -else: - dbname = dev_database_name - dbhost = dev_database_host - dbuser = dev_user - dbpass = dev_user_password if env != 'production': conn = psycopg2.connect( dbname="postgres", - user=dbuser, - password=dbpass, - host=dbhost, + user=db_username, + password=db_password, + host=db_host, ) conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) @@ -40,20 +21,20 @@ for dbn in cur.fetchall(): dbnames.append(dbn[0]) - if dbname not in dbnames: - cur.execute("CREATE DATABASE {};".format(dbname)) + if db_name not in dbnames: + cur.execute("CREATE DATABASE {};".format(db_name)) conn.commit() cur.close() conn.close() conn = psycopg2.connect( - dbname=dbname, - user=dbuser, - password=dbpass, - host=dbhost, + dbname=db_name, + user=db_username, + password=db_password, + host=db_host, ) cur = conn.cursor() cur.execute("CREATE SCHEMA reopt_api;") - cur.execute("ALTER SCHEMA reopt_api OWNER TO {};".format(dbuser)) + cur.execute("ALTER SCHEMA reopt_api OWNER TO {};".format(db_username)) conn.commit() cur.close() diff --git a/config/deploy.rb b/config/deploy.rb deleted file mode 100644 index 30995a2d4..000000000 --- a/config/deploy.rb +++ /dev/null @@ -1,55 +0,0 @@ -# config valid only for current version of Capistrano -lock "~> 3.12.0" - -set :application, "reopt_api" -set :repo_url, "https://github.com/NREL/REopt_API.git" - -# Set the base deployment directory. -set :deploy_to_base, "/srv/data/apps" - -# Set the user the web app runs as. -set :foreman_user, "www-data-local" -set :file_permissions_users, ["www-data-local"] - -# Symlink other directories across deploys. -set :linked_dirs, fetch(:linked_dirs, []).push("static/files", "tmp") - -# Allow the web user to write files for Xpress -set :file_permissions_paths, fetch(:file_permissions_paths, []).push("static/files") - -# Allow the web user to write files for Wind Data -set :file_permissions_paths, fetch(:file_permissions_paths, []).push("input_files") - -namespace :app do - task :pip_install do - on roles(:app) do - within release_path do - execute "virtualenv", "env", "--python=/bin/python3" - execute "./env/bin/pip3", "install", "-r", "requirements.txt" - execute "./env/bin/python", "-c", "'import julia; julia.install()'" - execute ". /opt/xpressmp/bin/xpvars.sh && julia --project='#{release_path}/julia_envs/Xpress/' -e 'import Pkg; Pkg.instantiate(); include(\"#{release_path}/julia_envs/Xpress/build_julia_image.jl\"); build_julia_image(\"#{release_path}\")'" - end - end - end - - task :keys do - on roles(:app) do - execute "ln", "-snf", "/etc/reopt-api-secrets/keys.py", "#{release_path}/keys.py" - end - end - - task :migrate do - on roles(:db) do - within release_path do - with "PATH" => "#{release_path}/env/bin:$PATH", "VIRTUAL_ENV" => "#{release_path}/env", "DJANGO_SETTINGS_MODULE" => fetch(:django_settings_module) do - execute "./env/bin/python", "manage.py", "migrate", "--noinput" - execute "./env/bin/python", "manage.py", "collectstatic", "--noinput" - end - end - end - end - - before "deploy:updated", "app:pip_install" - before "deploy:updated", "app:keys" - after "deploy:updated", "app:migrate" -end \ No newline at end of file diff --git a/config/deploy/AppKbldConfig.pkl b/config/deploy/AppKbldConfig.pkl new file mode 100644 index 000000000..53e750ed5 --- /dev/null +++ b/config/deploy/AppKbldConfig.pkl @@ -0,0 +1,56 @@ +extends "@tadaSharedKube/KbldConfig.pkl" + +import "pkl:json" + +local deployMetadata = import("@tadaSharedKube/tadaDeployMetadata.pkl").deployMetadata + +local const secretsString = read?("env:DEPLOY_SECRETS") +local const secrets = new json.Parser {}.parse(secretsString ?? "{}") + +// Define the docker images that need to be built and the registries to push +// them to. For integration with `kbld` during deployments. +sources { + new { + image = "reopt-api" + path = "." + docker { + build { + file = "Dockerfile" + pull = true + buildkit = true + rawOptions { + "--build-arg" + "NREL_ROOT_CERT_URL_ROOT=\(secrets.`SECRET_NLR_ROOT_CERT_URL_ROOT`)" + } + } + } + } + + new { + image = "julia-api" + path = "julia_src" + docker { + build { + file = "Dockerfile" + pull = true + buildkit = true + rawOptions { + "--build-arg" + "NREL_ROOT_CERT_URL_ROOT=\(secrets.`SECRET_NLR_ROOT_CERT_URL_ROOT`)" + } + } + } + } +} + +destinations { + new { + ["image"] = "reopt-api" + ["newImage"] = "\(secrets.`SECRET_CONTAINER_REGISTRY`)/tada/reopt-api" + } + + new { + ["image"] = "julia-api" + ["newImage"] = "\(secrets.`SECRET_CONTAINER_REGISTRY`)/tada/reopt-api" + } +} diff --git a/config/deploy/PklProject b/config/deploy/PklProject new file mode 100644 index 000000000..fcef1daad --- /dev/null +++ b/config/deploy/PklProject @@ -0,0 +1,8 @@ +amends "pkl:Project" + +dependencies { + ["tadaSharedKube"] = import("./external/tada-shared-kube/PklProject") + ["k8s"] { + uri = "package://pkg.pkl-lang.org/pkl-k8s/k8s@1.4.1" + } +} diff --git a/config/deploy/PklProject.deps.json b/config/deploy/PklProject.deps.json new file mode 100644 index 000000000..41f0d1adb --- /dev/null +++ b/config/deploy/PklProject.deps.json @@ -0,0 +1,24 @@ +{ + "schemaVersion": 1, + "resolvedDependencies": { + "package://pkg.pkl-lang.org/pkl-k8s/k8s@1": { + "type": "remote", + "uri": "projectpackage://pkg.pkl-lang.org/pkl-k8s/k8s@1.4.1", + "checksums": { + "sha256": "0520f82219c074d2f4b69ab9c57a4a1c1d0b3c4acc1e5d12749e72e0d567d9bf" + } + }, + "package://github.nrel.gov/TADA/tada-shared-kube@0": { + "type": "local", + "uri": "projectpackage://github.nrel.gov/TADA/tada-shared-kube@0.1.0", + "path": "external/tada-shared-kube" + }, + "package://pkg.pkl-lang.org/pkl-pantry/pkl.experimental.deepToTyped@1": { + "type": "remote", + "uri": "projectpackage://pkg.pkl-lang.org/pkl-pantry/pkl.experimental.deepToTyped@1.0.1", + "checksums": { + "sha256": "3a1fce17191cab6bcce6cca3cd86b2d50a3489927678fdffb4551a6afedeac32" + } + } + } +} diff --git a/config/deploy/WebDeployment.pkl b/config/deploy/WebDeployment.pkl new file mode 100644 index 000000000..73e73eeee --- /dev/null +++ b/config/deploy/WebDeployment.pkl @@ -0,0 +1,399 @@ +import "./AppKbldConfig.pkl" +import "@k8s/K8sResource.pkl" +import "@k8s/api/core/v1/EnvFromSource.pkl" +import "@k8s/api/apps/v1/StatefulSet.pkl" +import "@tadaSharedKube/TadaAppNamespace.pkl" +import "@tadaSharedKube/TadaEcrLoginRenewConfigMap.pkl" +import "@tadaSharedKube/TadaEcrLoginRenewCronJob.pkl" +import "@tadaSharedKube/TadaEcrLoginRenewInitialSetupJob.pkl" +import "@tadaSharedKube/TadaEcrLoginRenewRoleBinding.pkl" +import "@tadaSharedKube/TadaEcrLoginRenewRole.pkl" +import "@tadaSharedKube/TadaEcrLoginRenewSecrets.pkl" +import "@tadaSharedKube/TadaEcrLoginRenewServiceAccount.pkl" +import "@tadaSharedKube/TadaWebConfigMap.pkl" +import "@tadaSharedKube/TadaWebDbMigrateJob.pkl" +import "@tadaSharedKube/TadaWebDbMigrateSecrets.pkl" +import "@tadaSharedKube/TadaWebDeployment.pkl" +import "@tadaSharedKube/TadaWebIngress.pkl" +import "@tadaSharedKube/TadaWebPodDisruptionBudget.pkl" +import "@tadaSharedKube/TadaWebSecrets.pkl" +import "@tadaSharedKube/TadaWebService.pkl" + +local deployMetadata = import("@tadaSharedKube/tadaDeployMetadata.pkl").deployMetadata +local containerImage = "reopt-api" + +resources: Listing = new { + new TadaAppNamespace { + when (deployMetadata.`deploy-env` == "production") { + tadaRancherResourceQuotaPodLimit = 70 + } else { + tadaRancherResourceQuotaPodLimit = 20 + } + } + + new AppKbldConfig {} + + // To be able to pull images from our ECR repos in our on-premise (non-AWS) + // clusters. + local ecrLoginRenewConfigMapResource = new TadaEcrLoginRenewConfigMap {} + ecrLoginRenewConfigMapResource + new TadaEcrLoginRenewCronJob {} + new TadaEcrLoginRenewInitialSetupJob {} + new TadaEcrLoginRenewRoleBinding {} + new TadaEcrLoginRenewRole {} + new TadaEcrLoginRenewSecrets {} + new TadaEcrLoginRenewServiceAccount {} + + local webConfigMap = new TadaWebConfigMap { + data { + ["APP_ENV"] = deployMetadata.`deploy-env` + ["BRANCH_NAME"] = deployMetadata.`branch-name` + ["JULIA_HOST"] = "julia-service.\(deployMetadata.`app-namespace`).svc.cluster.local" + ["REDIS_HOST"] = "redis-service.\(deployMetadata.`app-namespace`).svc.cluster.local" + ["K8S_DEPLOY"] = "true" + + when (deployMetadata.`branch-is-primary` == "false") { + ["DB_NAME"] = deployMetadata.`branch-db-name` + // Use the database name as the queue name so that separate branches or + // databases on staging remain in separate queues. + ["APP_QUEUE_NAME"] = deployMetadata.`branch-db-name` + } else { + // In production (or where the database name isn't explicitly + // overridden), just use the app env as the queue name. + ["APP_QUEUE_NAME"] = deployMetadata.`deploy-env` + } + } + } + webConfigMap + + local webSecrets = new TadaWebSecrets {} + webSecrets + + new TadaWebDbMigrateSecrets {} + + new TadaWebDbMigrateJob { + tadaContainerImage = containerImage + tadaImagePullSecretName = ecrLoginRenewConfigMapResource.imagePullSecretName + + spec { + template { + spec { + containers { + [[ name == "web" ]] { + args = new { + "bin/migrate" + } + } + } + } + } + } + } + + // Deployment for the Django APIs. + local djangoDeployment = new TadaWebDeployment { + tadaName = "django" + tadaContainerImage = containerImage + tadaImagePullSecretName = ecrLoginRenewConfigMapResource.imagePullSecretName + + tadaResourceLimitCpu = "2000m" + + when (deployMetadata.`deploy-env` == "production") { + tadaReplicaCount = 10 + tadaResourceRequestCpu = "1000m" + tadaResourceLimitMemory = 2000.mib + tadaResourceRequestMemory = 2000.mib + } else { + tadaResourceRequestCpu = "500m" + tadaResourceRequestMemory = 800.mib + tadaResourceLimitMemory = 1000.mib + } + + tadaContainerPort = 8000 + tadaHealthProbePath = "/_health" + + spec { + template { + spec { + containers { + [[ name == tadaName ]] { + args = new { + "bin/server" + } + } + } + } + } + } + } + djangoDeployment + + // Deployment for the standalone Julia containers. These have lower memory + // limits since they're only used for lighterweight API requests from the + // Django app. The primary Julia containers are separate pods on the Celery + // deployment. + local juliaDeployment = new TadaWebDeployment { + tadaName = "julia" + tadaContainerImage = "julia-api" + tadaImagePullSecretName = ecrLoginRenewConfigMapResource.imagePullSecretName + + tadaResourceLimitCpu = "4000m" + tadaResourceLimitMemory = 2000.mib + tadaResourceRequestMemory = 2000.mib + + when (deployMetadata.`deploy-env` == "production") { + tadaReplicaCount = 5 + tadaResourceRequestCpu = "2000m" + } else { + tadaResourceRequestCpu = "500m" + } + + tadaContainerPort = 8081 + tadaHealthProbePath = "/health" + + spec { + template { + spec { + containers { + [[ name == tadaName ]] { + args = new { + "julia" "--project=/opt/julia_src" "http.jl" + } + + // Overwrite to remove default web-secrets from list, since Julia + // container doesn't need secrets access. + envFrom = new Listing { + new { + configMapRef { + name = webConfigMap.metadata.name + } + } + } + + env { + new { + name = "JULIA_NUM_THREADS" + value = "4" + } + } + } + } + } + } + } + } + juliaDeployment + + // Deployment for the Celery workers, which also includes Julia pods deployed + // in tandem so there's always a one-to-one relationship between Celery + // worker pods and a dedicated Julia pod. + new TadaWebDeployment { + tadaName = "celery" + tadaContainerImage = containerImage + tadaImagePullSecretName = ecrLoginRenewConfigMapResource.imagePullSecretName + + tadaResourceLimitCpu = "1000m" + tadaResourceRequestCpu = "100m" + tadaResourceLimitMemory = 1600.mib + tadaResourceRequestMemory = 1600.mib + + when (deployMetadata.`deploy-env` == "production") { + tadaReplicaCount = 10 + } + + spec { + template { + spec { + containers { + [[ name == tadaName ]] { + args = new { + "bin/worker" + } + + env { + new { + name = "JULIA_HOST" + value = "localhost" + } + } + + startupProbe { + httpGet = null + exec { + command = new { + "pgrep" "-f" "bin/celery" + } + } + } + + readinessProbe { + httpGet = null + exec { + command = new { + "pgrep" "-f" "bin/celery" + } + } + } + + livenessProbe { + httpGet = null + exec { + command = new { + "pgrep" "-f" "bin/celery" + } + } + } + } + + // Add an extra pod for the Julia container. We will leverage the + // standalone Julia deployment's container definition, but tweaked + // for different resource limits, since these are the more resource + // intenstive areas. + (juliaDeployment.spec.template.spec.containers[0]) { + resources { + requests { + when (deployMetadata.`deploy-env` == "production") { + ["memory"] = 12000.mib + } else { + ["memory"] = 6000.mib + } + } + + limits { + when (deployMetadata.`deploy-env` == "production") { + ["memory"] = 12000.mib + } else { + ["memory"] = 6000.mib + } + } + } + } + } + } + } + } + } + + new StatefulSet { + metadata { + name = "redis-stateful-set" + + labels { + ["app"] = "redis" + } + + annotations { + ["kapp.k14s.io/change-group"] = "change-group/deployment" + } + } + + spec { + selector { + matchLabels { + ["app"] = "redis" + } + } + + replicas = 1 + + template { + metadata { + labels { + ["app"] = "redis" + } + } + + spec { + imagePullSecrets { + new { + name = ecrLoginRenewConfigMapResource.imagePullSecretName + } + } + + containers { + new { + name = "redis" + + image = "public.ecr.aws/docker/library/redis:6.2.21-alpine" + + args = new { + "sh" "-c" "redis-server --requirepass $$SECRET_REDIS_PASSWORD --save 900 1 --save 300 10 --save 60 10000 --appendonly yes --maxmemory 2048mb --maxmemory-policy allkeys-lru" + } + + env { + new { + name = "SECRET_REDIS_PASSWORD" + valueFrom { + secretKeyRef { + name = webSecrets.metadata.name + key = "SECRET_REDIS_PASSWORD" + } + } + } + } + } + } + } + } + } + } + + local djangoService = new TadaWebService { + tadaName = "django" + tadaPort = 8000 + } + djangoService + + local juliaService = new TadaWebService { + tadaName = "julia" + tadaPort = 8081 + } + juliaService + + new TadaWebService { + tadaName = "redis" + tadaPort = 6379 + } + + new TadaWebIngress { + tadaName = "django" + tadaServicePort = 8000 + + // Configure the ingress to listen on the old `*.nrel.gov` domains (in + // addition to the new `nlr.gov` domains). We can remove this once the + // nrel.gov domain is fully gone. + local nrelHost = deployMetadata.`url-host`.replaceLast(".nlr.gov", ".nrel.gov") + spec { + rules { + new { + host = nrelHost!! + http { + paths { + new { + path = "/" + pathType = "Prefix" + backend { + service { + name = djangoService.metadata.name + port { + number = 8000 + } + } + } + } + } + } + } + } + } + } + + new TadaWebPodDisruptionBudget {} +} + +output { + value = resources + renderer = (K8sResource.output.renderer as YamlRenderer) { + isStream = true + } +} diff --git a/config/deploy/development.rb b/config/deploy/development.rb deleted file mode 100644 index fc224d491..000000000 --- a/config/deploy/development.rb +++ /dev/null @@ -1,9 +0,0 @@ -server ENV.fetch("DEV_URL"), :user => "deploy", :roles => ["web", "app", "db"] - -set :app_env, "development" -set :django_settings_module, "reopt_api.dev_settings" -set :base_domain, ENV.fetch("DEV_URL") - -# TODO: Remove this if we get more flexible branched deployments setup (with -# our normal DNS subdomain support). -set :branch, ENV.fetch("DEV_BRANCH").gsub("origin/", "") \ No newline at end of file diff --git a/config/deploy/internal_c110p.rb b/config/deploy/internal_c110p.rb deleted file mode 100644 index 14344b71e..000000000 --- a/config/deploy/internal_c110p.rb +++ /dev/null @@ -1,5 +0,0 @@ -server ENV.fetch("C110P_URL"), :user => "deploy", :roles => ["web", "app", "db"] - -set :app_env, "internal_c110p" -set :django_settings_module, "reopt_api.internal_c110p_settings" -set :base_domain, ENV.fetch("C110P_URL") \ No newline at end of file diff --git a/config/deploy/production.rb b/config/deploy/production.rb deleted file mode 100644 index bcf8e4f1f..000000000 --- a/config/deploy/production.rb +++ /dev/null @@ -1,7 +0,0 @@ -server ENV.fetch("PROD1_URL"), :user => "deploy", :roles => ["web", "app", "db"] -server ENV.fetch("PROD2_URL"), :user => "deploy", :roles => ["web", "app"] - -set :app_env, "production" -set :django_settings_module, "reopt_api.production_settings" -set :base_domain, ENV.fetch("PROD_URL") -set :base_domain_aliases, [ENV.fetch("PROD1_URL"), ENV.fetch("PROD2_URL")] \ No newline at end of file diff --git a/config/deploy/render b/config/deploy/render new file mode 100755 index 000000000..ad5afac88 --- /dev/null +++ b/config/deploy/render @@ -0,0 +1,62 @@ +#!/usr/bin/env ruby + +require "json" +require "open3" +require "rake" +require "shellwords" + +# Install dependencies. +sh "vendir sync --chdir #{Shellwords.escape(__dir__)} --locked 1>&2" +sh "pkl project resolve --working-dir #{Shellwords.escape(__dir__)} 1>&2" + +# Parse metadata generated from GitHub Actions. +deploy_metadata = JSON.parse(ENV.fetch("DEPLOY_METADATA", "{}")) +deploy_env = deploy_metadata.fetch("deploy-env", "development") + +# Generate decrypted secrets file. +output, status = Open3.capture2( + {"DEPLOY_ENV" => deploy_env}, + "gomplate", + "--config", File.expand_path("../../.gomplate.yaml", __dir__), + "--file", File.expand_path("../../config/vault_secrets.json.tmpl", __dir__) +) +raise "gomplate error: #{output.inspect}" unless status.success? +ENV["DEPLOY_SECRETS"] = output + +# Generate decrypted secrets file for the database migration environment (which +# contain extra values we don't normally want to expose). +output, status = Open3.capture2( + {"DEPLOY_ENV" => deploy_env, "DB_MIGRATE" => "true"}, + "gomplate", + "--config", File.expand_path("../../.gomplate.yaml", __dir__), + "--file", File.expand_path("../../config/vault_secrets.json.tmpl", __dir__) +) +raise "gomplate error: #{output.inspect}" unless status.success? +ENV["DEPLOY_DB_MIGRATE_SECRETS"] = output + +# Generate decrypted secrets file for the database migration environment (which +# contain extra values we don't normally want to expose). +output, status = Open3.capture2( + {"DEPLOY_ENV" => deploy_env, "ECR_LOGIN_RENEW" => "true"}, + "gomplate", + "--config", File.expand_path("../../.gomplate.yaml", __dir__), + "--file", File.expand_path("../../config/vault_secrets.json.tmpl", __dir__) +) +raise "gomplate error: #{output.inspect}" unless status.success? +ENV["DEPLOY_ECR_LOGIN_RENEW_SECRETS"] = output + +# Render the Kubernetes YAML output. +case ENV["RENDER_JOB"] +when "run-job" + sh "pkl", "eval", + "--working-dir", __dir__, + "--format", "yaml", + File.expand_path("./RunJob.pkl", __dir__) +when nil + sh "pkl", "eval", + "--working-dir", __dir__, + "--format", "yaml", + File.expand_path("./WebDeployment.pkl", __dir__) +else + raise "Unknown RENDER_JOB=#{ENV["RENDER_JOB"].inspect}" +end diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb deleted file mode 100644 index fd85d18cd..000000000 --- a/config/deploy/staging.rb +++ /dev/null @@ -1,6 +0,0 @@ -server ENV.fetch("STAGE1_URL"), :user => "deploy", :roles => ["web", "app", "db"] - -set :app_env, "staging" -set :django_settings_module, "reopt_api.staging_settings" -set :base_domain, ENV.fetch("STAGE_BASEDOMAIN_URL") -set :base_domain_aliases, [ENV.fetch("STAGE_URL"), ENV.fetch("STAGE1_URL"), ENV.fetch("STAGE2_URL")] \ No newline at end of file diff --git a/config/deploy/templates/foreman/env.erb b/config/deploy/templates/foreman/env.erb deleted file mode 100644 index f4d8de2b3..000000000 --- a/config/deploy/templates/foreman/env.erb +++ /dev/null @@ -1,12 +0,0 @@ -APP_ENV=<%= fetch(:app_env) %> -# Set the queue name so that it's unique for separate branches on staging and -# development (by using "deploy_name") and so it's also separate for each -# individual server (by using "hostname"). This ensures branches have their own -# queues and also servers have their own queues (since currently the processing -# has to stay on a single server). -APP_QUEUE_NAME="<%= fetch(:deploy_name) %>-<%= host.hostname %>" -DEPLOY_CURRENT_PATH=<%= current_path %> -LD_PRELOAD=/usr/local/julia-1.3.1/lib/julia/libstdc++.so.6 -SOLVER=xpress -XPRESSDIR=/opt/xpressmp -JULIA_PROJECT="<%= current_path %>/julia_envs/Xpress" \ No newline at end of file diff --git a/config/deploy/templates/nginx/proxy_settings.conf.erb b/config/deploy/templates/nginx/proxy_settings.conf.erb deleted file mode 100644 index c5aa774b2..000000000 --- a/config/deploy/templates/nginx/proxy_settings.conf.erb +++ /dev/null @@ -1,26 +0,0 @@ -# Allow response streaming -proxy_buffering off; - -# Set original IP -proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - -# Proxy over HTTP 1.1 so keepalive connections to the backend are supported -proxy_http_version 1.1; -proxy_set_header Connection ""; - -# Pass along the original Host header, rather than the upstream name used in -# proxy_pass. -proxy_set_header Host $host; - -# Disable redirect rewriting. -proxy_redirect off; - -# Increase timeout for longer response times. -# -# This value should be be kept in sync with the xpress/mosel run timeout -# (defined in reo/models.py), and the gunicorn timeout (defined in -# config/gunicorn.conf.py). -# -# This timeout should be greater than the gunicorn timeout, to give the app an -# opportunity to handle timeouts more gracefully. -proxy_read_timeout 450s; \ No newline at end of file diff --git a/config/deploy/templates/nginx/site.conf.erb b/config/deploy/templates/nginx/site.conf.erb deleted file mode 100644 index 1a44c6c6d..000000000 --- a/config/deploy/templates/nginx/site.conf.erb +++ /dev/null @@ -1,139 +0,0 @@ -# Security headers -# https://www.owasp.org/index.php/OWASP_Secure_Headers_Project#tab=Headers -<% if(fetch(:app_env) == "production") %> - more_set_headers "Strict-Transport-Security: max-age=31536000; includeSubDomains; preload"; -<% else %> - more_set_headers "Strict-Transport-Security: max-age=300; preload"; -<% end %> - -more_set_headers "X-XSS-Protection: 1; mode=block"; -more_set_headers "X-Frame-Options: DENY"; -more_set_headers "X-Content-Type-Options: nosniff"; - -upstream <%= fetch(:deploy_name) %>-http { - server 127.0.0.1:80; - keepalive 10; -} - -upstream <%= fetch(:deploy_name) %>-django_backend { - server unix:<%= current_path %>/tmp/gunicorn.sock; - keepalive 10; -} - -# The following settings can only be set once, globally in the http scope, so -# don't apply them to branched deployments. -<% if(!fetch(:subdomain)) %> -# Set far-future cache expiration headers for all JS, CSS, and image -# responses. -# -# This assumes all of these resources should be cache-busted by changing the -# filename when the contents change (which everything should be doing). -map $sent_http_content_type $expires { - default off; - <% if(fetch(:app_env) != "development") %> - application/javascript max; - text/css max; - ~image/ max; - <% end %> -} -expires $expires; -<% end %> - -# SSL Terminator. -server { - listen 443 ssl; - listen [::]:443 ssl; - server_name <%= fetch(:all_domains).join(" ") %>; - - <% if(fetch(:base_domain) =~ /vagrant$/) %> - ssl_certificate /etc/ssl/vagrant.crt; - ssl_certificate_key /etc/ssl/vagrant.key; - <% elsif(fetch(:stage) == :internal_c110p) %> - <% if(fetch(:subdomain)) %> - ssl_certificate /etc/ssl/star_<%= host.hostname %>.crt; - ssl_certificate_key /etc/ssl/star_<%= host.hostname %>.key; - <% else %> - ssl_certificate /etc/ssl/<%= host.hostname %>.crt; - ssl_certificate_key /etc/ssl/<%= host.hostname %>.key; - <% end %> - <% else %> - ssl_certificate /etc/httpd/ssl/<%= host.hostname %>.pem; - ssl_certificate_key /etc/httpd/ssl/<%= host.hostname %>.key; - <% end %> - - access_log <%= current_path %>/log/access.log combined_extended; - error_log <%= current_path %>/log/error.log; - - # Default proxy settings. - include <%= fetch(:compiled_config_dir) %>/nginx/proxy_settings.conf; - - # Pass along original HTTPS protocol and port information. - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Port $server_port; - - location / { - proxy_pass http://<%= fetch(:deploy_name) %>-http; - } -} - -server { - listen 80; - listen [::]:80; - server_name <%= fetch(:all_domains).join(" ") %>; - - # Redirect everything to HTTPS. - # - # Since we're sitting behind an SSL terminator all of our requests will be on - # port 80, so we need to check the forwarded information. - if ($http_x_forwarded_proto != "https") { - return 301 https://$host$request_uri; - } - - root <%= current_path %>/static; - - access_log <%= current_path %>/log/access.log combined_extended; - error_log <%= current_path %>/log/error.log; - - # gzip configuration - gzip on; - gzip_comp_level 2; - gzip_http_version 1.0; - gzip_types text/xml application/xml text/javascript application/javascript application/x-javascript application/json text/csv; - gzip_proxied any; - - # Default proxy settings. - include <%= fetch(:compiled_config_dir) %>/nginx/proxy_settings.conf; - - # Set original IP and forwarded details - proxy_set_header X-Forwarded-Host $host; - proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; - proxy_set_header X-Forwarded-Port $http_x_forwarded_port; - - # Workaround for Django requiring a trailing slash on the POST endpoint. - rewrite ^/v1/job$ /v1/job/ last; - - <% if(fetch(:app_env) != "production") %> - # Don't allow search engine indexing of our staging sites. - location = /robots.txt { - return 200 "User-agent: *\nDisallow: /"; - } - <% end %> - - location / { - # Serve static files with nginx, everything else with Puma. - try_files $uri/index.html $uri.html $uri @app; - } - - location @app { - <% if(fetch(:app_env) != "production") %> - set $lazy_hydra_start_script "sudo supervisorctl start '<%= fetch(:deploy_name) %>:*'"; - set $lazy_hydra_stop_script "sudo supervisorctl stop '<%= fetch(:deploy_name) %>:*'"; - set $lazy_hydra_health_url "unix:<%= current_path %>/tmp/gunicorn.sock:/"; - set $lazy_hydra_startup_timeout 90; # 90 seconds - this app can take longer than normal to spin up. - set $lazy_hydra_inactivity_timeout 10800; # 3 hours - include "lazy-hydra/hook.conf"; - <% end %> - - proxy_pass http://<%= fetch(:deploy_name) %>-django_backend; - } -} \ No newline at end of file diff --git a/config/deploy/vendir.lock.yml b/config/deploy/vendir.lock.yml new file mode 100644 index 000000000..d8b107d81 --- /dev/null +++ b/config/deploy/vendir.lock.yml @@ -0,0 +1,9 @@ +apiVersion: vendir.k14s.io/v1alpha1 +directories: +- contents: + - git: + commitTitle: 'Merge pull request #29 from TADA/main...' + sha: 2cd9ca01cdf325f2daaa608b8b91c7de4a5d2b8c + path: tada-shared-kube + path: external +kind: LockConfig diff --git a/config/deploy/vendir.yml b/config/deploy/vendir.yml new file mode 100644 index 000000000..9bbc9b46b --- /dev/null +++ b/config/deploy/vendir.yml @@ -0,0 +1,9 @@ +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config +directories: + - path: external + contents: + - path: tada-shared-kube + git: + url: https://github.nrel.gov/TADA/tada-shared-kube.git + ref: origin/v2 diff --git a/config/gunicorn.conf.py b/config/gunicorn.conf.py index 879818060..695d13aa3 100644 --- a/config/gunicorn.conf.py +++ b/config/gunicorn.conf.py @@ -1,20 +1,13 @@ import os import multiprocessing -# Bind to unix socket that nginx will proxy to. if os.environ.get('TEST') is None: - if os.environ.get('K8S_DEPLOY') is None: - bind = 'unix:' + os.path.join(os.path.abspath(os.path.dirname(os.path.dirname(__file__))), 'tmp/gunicorn.sock') - else: - bind = "0.0.0.0:8000" + bind = "0.0.0.0:8000" else: bind = "127.0.0.1:8000" # Based the number of workers on the number of CPU cores. -if os.environ.get('K8S_DEPLOY') is None: - workers = multiprocessing.cpu_count() -else: - workers = 4 +workers = 4 # Note that the app currently has threading issues, so we explicitly want a # non-thread worker process model. @@ -27,24 +20,12 @@ # Increase timeout for longer response times. # # This value should be be kept in sync with the xpress/mosel run timeout -# (defined in reo/models.py), and the nginx timeout (defined in -# config/deploy/templates/nginx/proxy_settings.conf.erb). +# (defined in reo/models.py). # -# This timeout should be greater than the xpress timeout, but less than the -# nginx timeout, to give the app an opportunity to handle timeouts more -# gracefully. +# This timeout should be greater than the xpress timeout to give the app an +# opportunity to handle timeouts more gracefully. timeout = 435 # Set the appropriate DJANGO_SETTINGS_MODULE environment variable based on the # current environment. -env = os.environ['APP_ENV'] -if env == 'development': - raw_env = ['DJANGO_SETTINGS_MODULE=reopt_api.dev_settings'] -elif env == 'staging': - raw_env = ['DJANGO_SETTINGS_MODULE=reopt_api.staging_settings'] -elif env == 'production': - raw_env = ['DJANGO_SETTINGS_MODULE=reopt_api.production_settings'] -elif env == 'internal_c110p': - raw_env = ['DJANGO_SETTINGS_MODULE=reopt_api.internal_c110p_settings'] -else: - raw_env = ['DJANGO_SETTINGS_MODULE=reopt_api.dev_settings'] +raw_env = ['DJANGO_SETTINGS_MODULE=reopt_api.settings'] diff --git a/config/vault_secrets.json.tmpl b/config/vault_secrets.json.tmpl new file mode 100644 index 000000000..9c656fbda --- /dev/null +++ b/config/vault_secrets.json.tmpl @@ -0,0 +1,52 @@ +{{/* Build a JSON file containing secrets fetched from our Vault server. The needed secrets may vary depending on the environment (eg, different secrets are needed for development vs production). */}} + +{{ $deploy_env := (env.Getenv "DEPLOY_ENV" (env.Getenv "APP_ENV" "development")) }} +{{ $secrets := coll.Dict }} + +{{ if (ne (env.Getenv "ECR_LOGIN_RENEW") "true") }} + {{ with (datasource "vault" "reopt-api/common/build").data }} + {{ $secrets = coll.Merge (coll.Dict + "SECRET_NLR_ROOT_CERT_URL_ROOT" .nlr_root_cert_url_root + ) $secrets }} + {{ end }} + + {{ with (datasource "vault" "reopt-api/ci/deploy").data }} + {{ $secrets = coll.Merge (coll.Dict + "SECRET_CONTAINER_REGISTRY" .container_registry + ) $secrets }} + {{ end }} + + {{ if (or (eq $deploy_env "staging") (eq $deploy_env "production")) }} + {{ with (datasource "vault" (printf "reopt-api/%s/web" $deploy_env)).data }} + {{ $secrets = coll.Merge (coll.Dict + "SECRET_ASHRAE_TMY_NLR_API_KEY" .ashrae_tmy_nlr_api_key + "SECRET_DB_HOST" .db_host + "SECRET_DB_NAME" .db_name + "SECRET_DEVELOPER_NLR_GOV_API_KEY" .developer_nlr_gov_api_key + "SECRET_DJANGO_SECRET_KEY" .django_secret_key + "SECRET_PVWATTS_NLR_API_KEY" .pvwatts_nlr_api_key + "SECRET_REDIS_PASSWORD" .redis_password + "SECRET_ROLLBAR_ACCESS_TOKEN" .rollbar_access_token + ) $secrets }} + {{ end }} + + {{ with (datasource "vault" (printf "reopt-api/%s/db-migrate" $deploy_env)).data }} + {{ $secrets = coll.Merge (coll.Dict + "SECRET_DB_PASSWORD" .db_password + "SECRET_DB_USERNAME" .db_username + ) $secrets }} + {{ end }} + {{ end }} +{{ end }} + +{{ if (eq (env.Getenv "ECR_LOGIN_RENEW") "true") }} + {{ with (datasource "vault" "deploy/common/aws-ecr").data }} + {{ $secrets = coll.Merge (coll.Dict + "AWS_ACCESS_KEY_ID" .aws_access_key_id + "AWS_SECRET_ACCESS_KEY" .aws_secret_access_key + "AWS_REGION" .aws_region + ) $secrets }} + {{ end }} +{{ end }} + +{{ $secrets | data.ToJSON }} diff --git a/docker-compose.nginx.yml b/docker-compose.nginx.yml deleted file mode 100644 index 78648fbc7..000000000 --- a/docker-compose.nginx.yml +++ /dev/null @@ -1,90 +0,0 @@ -version: "2.1" - -services: - - redis-nginx: - container_name: redis-nginx - image: redis - command: redis-server - expose: - - 6379 - - db-nginx: - container_name: db-nginx - image: postgres - restart: always - volumes: - - pgdata:/var/lib/postgresql/data - environment: - - POSTGRES_USER=reopt - - POSTGRES_PASSWORD=reopt - - POSTGRES_DB=reopt - ports: - - 5432:5432 - - celery-nginx: - container_name: celery-nginx - build: - context: . - image: base-api-image:latest - command: > - "celery -A reopt_api worker -l info" - environment: - - APP_ENV=local - - SQL_HOST=db-nginx - - SQL_PORT=5432 - - REDIS_HOST=redis-nginx - - SOLVER=xpress - - JULIA_HOST=julia-nginx - volumes: - - .:/opt/reopt - depends_on: - - db-nginx - - redis-nginx - - julia-nginx - - django-nginx: - image: base-api-image:latest - container_name: django-nginx - command: > - "python manage.py migrate - && /opt/reopt/bin/wait-for-it.bash -t 0 julia-nginx:8081 -- python manage.py runserver 0.0.0.0:8000" - environment: - - APP_ENV=local - - SQL_HOST=db-nginx - - SQL_PORT=5432 - - REDIS_HOST=redis-nginx - - SOLVER=xpress - - JULIA_HOST=julia-nginx - depends_on: - - db-nginx - - redis-nginx - - julia-nginx - - celery-nginx - expose: - - 8000 - volumes: - - .:/opt/reopt - - nginx: - build: ./nginx - ports: - - 80:80 - depends_on: - - django-nginx - restart: "on-failure" - - julia-nginx: - container_name: julia-nginx - build: - context: julia_src/ - command: bash ./run_julia_servers.sh 8 - environment: - - JULIA_NUM_THREADS=2 - expose: - - 8081 - volumes: - - ./julia_src:/opt/julia_src - -volumes: - pgdata: diff --git a/docker-compose.nojulia.yml b/docker-compose.nojulia.yml index 556fe0b9e..f78c9603d 100644 --- a/docker-compose.nojulia.yml +++ b/docker-compose.nojulia.yml @@ -32,10 +32,18 @@ services: "celery -A reopt_api worker -l info" environment: - APP_ENV=local - - SQL_HOST=db - - SQL_PORT=5432 + - DB_HOST=db + - DB_NAME=reopt + - DB_USERNAME=reopt + - DB_PASSWORD=reopt + - DB_SEARCH_PATH=public - REDIS_HOST=redis + - REDIS_PASSWORD= - JULIA_HOST=host.docker.internal + - NLR_API_KEY + env_file: + - path: .env + required: false volumes: - .:/opt/reopt depends_on: @@ -50,10 +58,18 @@ services: && python manage.py runserver 0.0.0.0:8000" environment: - APP_ENV=local - - SQL_HOST=db - - SQL_PORT=5432 + - DB_HOST=db + - DB_NAME=reopt + - DB_USERNAME=reopt + - DB_PASSWORD=reopt + - DB_SEARCH_PATH=public - REDIS_HOST=redis + - REDIS_PASSWORD= - JULIA_HOST=host.docker.internal + - NLR_API_KEY + env_file: + - path: .env + required: false depends_on: - db - redis diff --git a/docker-compose.yml b/docker-compose.yml index 49d3a3d32..ed3796cb6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,9 +25,17 @@ services: "celery -A reopt_api worker -l info" environment: - APP_ENV=local - - SQL_HOST=db - - SQL_PORT=5432 + - DB_HOST=db + - DB_NAME=reopt + - DB_USERNAME=reopt + - DB_PASSWORD=reopt + - DB_SEARCH_PATH=public - REDIS_HOST=redis + - REDIS_PASSWORD= + - NLR_API_KEY + env_file: + - path: .env + required: false volumes: - .:/opt/reopt depends_on: @@ -42,9 +50,17 @@ services: && /opt/reopt/bin/wait-for-it.bash -t 0 julia:8081 -- python manage.py runserver 0.0.0.0:8000" environment: - APP_ENV=local - - SQL_HOST=db - - SQL_PORT=5432 + - DB_HOST=db + - DB_NAME=reopt + - DB_USERNAME=reopt + - DB_PASSWORD=reopt + - DB_SEARCH_PATH=public - REDIS_HOST=redis + - REDIS_PASSWORD= + - NLR_API_KEY + env_file: + - path: .env + required: false depends_on: - db - redis diff --git a/keys.py.template b/keys.py.template deleted file mode 100755 index 7c72f9f33..000000000 --- a/keys.py.template +++ /dev/null @@ -1,29 +0,0 @@ - -rollbar_access_token = 'test' - -#get your individual key from here: https://developer.nrel.gov/docs/api-key/ and replace the DEMO_KEY with that -pvwatts_api_key = 'DEMO_KEY' -developer_nrel_gov_key = 'DEMO_KEY' - - -secret_key_='secret_key_test' - -dev_database_host = 'localhost' -dev_database_name = 'reopt' -dev_user = 'reopt_api' -dev_user_password = 'reopt' -dev_redis_host = 'localhost' -dev_redis_password = 'password' - -# items that don't have to be changed by users (needed for NREL API servers) -staging_redis_host = "" -staging_redis_password = "" -staging_database_host = "" -staging_database_name = "" -production_redis_host = "" -production_redis_password = "" -prod_database_host = "" -prod_database_name = "" -production_user = "" -production_user_password = "" - diff --git a/keys_env.py b/keys_env.py new file mode 100644 index 000000000..eae9d776d --- /dev/null +++ b/keys_env.py @@ -0,0 +1,19 @@ +import os + +# To adjust settings, set environment variables (either through a .env file, +# docker-compose.yml or other means). + +pvwatts_api_key = os.getenv('SECRET_PVWATTS_NLR_API_KEY', os.getenv('NLR_API_KEY', 'DEMO_KEY')) +developer_nrel_gov_key = os.getenv('SECRET_DEVELOPER_NLR_GOV_API_KEY', os.getenv('NLR_API_KEY', 'DEMO_KEY')) +ashrae_tmy_key = os.getenv('SECRET_ASHRAE_TMY_NLR_API_KEY', os.getenv('NLR_API_KEY', 'DEMO_KEY')) + +secret_key_ = os.getenv('SECRET_DJANGO_SECRET_KEY', 'secret_key_test') + +db_host = os.getenv('DB_HOST', os.getenv('SECRET_DB_HOST', 'localhost')) +db_name = os.getenv('DB_NAME', os.getenv('SECRET_DB_NAME', 'reopt')) +db_username = os.getenv('DB_USERNAME', os.getenv('SECRET_DB_USERNAME', 'reopt_api')) +db_password = os.getenv('DB_PASSWORD', os.getenv('SECRET_DB_PASSWORD', 'reopt')) +db_search_path = os.getenv('DB_SEARCH_PATH', os.getenv('SECRET_DB_SEARCH_PATH', 'reopt_api')) +redis_host = os.getenv('REDIS_HOST', os.getenv('SECRET_REDIS_HOST', 'localhost')) +redis_password = os.getenv('REDIS_PASSWORD', os.getenv('SECRET_REDIS_PASSWORD', 'password')) +rollbar_access_token = os.getenv('SECRET_ROLLBAR_ACCESS_TOKEN', 'test') diff --git a/manage.py b/manage.py index 4dec556fe..e00f2ce5e 100644 --- a/manage.py +++ b/manage.py @@ -4,7 +4,7 @@ import sys if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "reopt_api.dev_settings") + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "reopt_api.settings") from django.core.management import execute_from_command_line diff --git a/nginx/Dockerfile b/nginx/Dockerfile deleted file mode 100644 index c86307f32..000000000 --- a/nginx/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM nginx:1.19.0-alpine - -RUN rm /etc/nginx/conf.d/default.conf -COPY nginx.conf /etc/nginx/conf.d diff --git a/nginx/nginx.conf b/nginx/nginx.conf deleted file mode 100644 index f5ab4d720..000000000 --- a/nginx/nginx.conf +++ /dev/null @@ -1,18 +0,0 @@ -upstream reopt_api { - server django-nginx:8000; -} - -server { - - listen 80; - - location / { - proxy_pass http://reopt_api; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $host; - proxy_redirect off; - client_max_body_size 30M; - proxy_read_timeout 600s; - } - -} diff --git a/reo/src/pvwatts.py b/reo/src/pvwatts.py index 83e98b3d0..29faf4947 100644 --- a/reo/src/pvwatts.py +++ b/reo/src/pvwatts.py @@ -1,7 +1,7 @@ # REoptĀ®, Copyright (c) Alliance for Sustainable Energy, LLC. See also https://github.com/NREL/REopt_API/blob/master/LICENSE. import requests import json -import keys +import keys_env import logging from reo.exceptions import PVWattsDownloadError log = logging.getLogger(__name__) @@ -39,7 +39,7 @@ class PVWatts: def __init__(self, url_base="https://developer.nrel.gov/api/pvwatts/v6.json", - key=keys.developer_nrel_gov_key, + key=keys_env.developer_nrel_gov_key, azimuth=180, system_capacity=1, losses=0.14, diff --git a/reo/src/wind_resource.py b/reo/src/wind_resource.py index a8d041a89..5d4df21c1 100644 --- a/reo/src/wind_resource.py +++ b/reo/src/wind_resource.py @@ -10,7 +10,7 @@ from requests.adapters import HTTPAdapter import json -from keys import developer_nrel_gov_key +from keys_env import developer_nrel_gov_key import logging log = logging.getLogger(__name__) """ diff --git a/reopt_api/celery.py b/reopt_api/celery.py index e76436d3e..c8b889734 100644 --- a/reopt_api/celery.py +++ b/reopt_api/celery.py @@ -4,48 +4,10 @@ import logging from celery import Celery from celery.signals import after_setup_logger -from keys import * +from keys_env import * # set the default Django settings module for the 'celery' program. -try: - env = os.environ['APP_ENV'] - - if env == 'internal_c110p': - raw_env = 'reopt_api.internal_c110p_settings' - redis_host = ':' + dev_redis_password + '@localhost' - elif env == 'development': - raw_env = 'reopt_api.dev_settings' - if os.environ.get('K8S_DEPLOY') is None: - redis_host = ':' + dev_redis_password + '@' + dev_database_host - else: - redis_host = ':' + dev_redis_password + '@' + dev_redis_host - elif env == 'staging': - raw_env = 'reopt_api.staging_settings' - if os.environ.get('K8S_DEPLOY') is None: - redis_host = ':' + staging_redis_password + '@' + staging_database_host - else: - redis_host = ':' + staging_redis_password + '@' + staging_redis_host - elif env == 'production': - raw_env = 'reopt_api.production_settings' - if os.environ.get('K8S_DEPLOY') is None: - redis_host = ':' + production_redis_password + '@' + prod_database_host - else: - redis_host = ':' + production_redis_password + '@' + production_redis_host - else: - raw_env = 'reopt_api.dev_settings' - redis_host = os.environ.get('REDIS_HOST', 'localhost') - -except KeyError: - """ - This catch is necessary for running celery from command line when testing/developing locally. - APP_ENV is defined in config/deploy/[development, production, staging].rb files for servers. - For testing and local development, APP_ENV *can* be defined in .env file (see README.md), - which `honcho` or `foreman` loads before running Procfile. - """ - raw_env = 'reopt_api.dev_settings' - redis_host = os.environ.get('REDIS_HOST', 'localhost') - -os.environ.setdefault('DJANGO_SETTINGS_MODULE', raw_env) +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'reopt_api.settings') app = Celery('reopt_api') @@ -58,7 +20,11 @@ # should have a `CELERY_` prefix. app.config_from_object('django.conf:settings', namespace='CELERY') -app.conf.broker_url = 'redis://' + redis_host + ':6379/0' +redis_auth = '' +if redis_password: + redis_auth = ':' + redis_password + '@' + +app.conf.broker_url = 'redis://' + redis_auth + redis_host + ':6379/0' # Create separate queues for each server (naming each queue after the server's # hostname). Since the worker jobs currently all have to be processes on the diff --git a/reopt_api/dev_settings.py b/reopt_api/dev_settings.py deleted file mode 100644 index ffb94d8a4..000000000 --- a/reopt_api/dev_settings.py +++ /dev/null @@ -1,165 +0,0 @@ -# REoptĀ®, Copyright (c) Alliance for Sustainable Energy, LLC. See also https://github.com/NREL/REopt_API/blob/master/LICENSE. -from keys import * -import sys -import os -import django -import rollbar -""" -Django settings for reopt_api project. - -Generated by 'django-admin startproject' using Django 2.2. - -For more information on this file, see -https://docs.djangoproject.com/en/2.2/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/2.2/ref/settings/ -""" - -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) - -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = secret_key_ - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = False - -ALLOWED_HOSTS = ['*'] - -# Application definition - -INSTALLED_APPS = ( - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'reo', - 'summary', - 'tastypie', - 'proforma', - 'resilience_stats', - 'futurecosts', - 'django_celery_results', - 'django_extensions', - 'reoptjl', - 'ghpghx' - ) - -MIDDLEWARE_CLASSES = ( - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'rollbar.contrib.django.middleware.RollbarNotifierMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django.middleware.security.SecurityMiddleware', -) - -ROOT_URLCONF = 'reopt_api.urls' - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - ], - }, - }, -] - - -WSGI_APPLICATION = 'reopt_api.wsgi.application' - -ROLLBAR = { - 'access_token': rollbar_access_token, - 'environment': 'development', - 'root': BASE_DIR, - 'enabled':True -} - -# Database -if 'test' in sys.argv or os.environ.get('APP_ENV') == 'local': - ROLLBAR['enabled'] = False - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': 'reopt', - 'USER': 'reopt', - 'PASSWORD': 'reopt', - 'OPTIONS': { - 'options': '-c search_path=public' - }, - "HOST": os.environ.get("SQL_HOST", "localhost"), - "PORT": os.environ.get("SQL_PORT", "5432"), - } -} -else: - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'HOST': dev_database_host, - 'NAME': dev_database_name, - 'OPTIONS': { - 'options': '-c search_path=reopt_api' - }, - 'USER': dev_user, - 'PASSWORD': dev_user_password, - } -} - -# Internationalization -# https://docs.djangoproject.com/en/1.8/topics/i18n/ -rollbar.init(**ROLLBAR) - -LANGUAGE_CODE = 'en-us' - -TIME_ZONE = 'UTC' - -USE_I18N = True - -USE_L10N = True - -USE_TZ = True - -# Results backend -CELERY_RESULT_BACKEND = 'django-db' - -# celery task registration -CELERY_IMPORTS = ( - 'reo.api', - 'reo.scenario', - 'reo.process_results', - 'reo.src.run_jump_model', - 'resilience_stats.outage_simulator_LF', - 'futurecosts.api', - 'futurecosts.tasks', - 'reoptjl.api', - 'reoptjl.src.run_jump_model' -) - -if 'test' in sys.argv: - CELERY_TASK_ALWAYS_EAGER = True - CELERY_TASK_EAGER_PROPAGATES_EXCEPTIONS = False - -CELERY_WORKER_MAX_MEMORY_PER_CHILD = 4000000 # 4 GB - -# Static files (used for Proforma xlsx) -STATIC_ROOT = os.path.join(BASE_DIR, 'static') -STATIC_URL = '/static/' - -APPEND_SLASH = False -TASTYPIE_ALLOW_MISSING_SLASH = True -DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "reopt_api.dev_settings") -django.setup() diff --git a/reopt_api/internal_c110p_settings.py b/reopt_api/internal_c110p_settings.py deleted file mode 100755 index d0f41995e..000000000 --- a/reopt_api/internal_c110p_settings.py +++ /dev/null @@ -1,135 +0,0 @@ -# REoptĀ®, Copyright (c) Alliance for Sustainable Energy, LLC. See also https://github.com/NREL/REopt_API/blob/master/LICENSE. -from keys import * -import sys -""" -Django settings for reopt_api project. - -Generated by 'django-admin startproject' using Django 1.8. - -For more information on this file, see -https://docs.djangoproject.com/en/2.2/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/2.2/ref/settings/ -""" - -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) -import os -import django - -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = secret_key_ - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True - -ALLOWED_HOSTS = [] - -# Application definition -INSTALLED_APPS = ( - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'reo', - 'summary', - 'tastypie', - 'proforma', - 'resilience_stats', - 'django_celery_results', - 'django_extensions', - 'ghpghx' - ) - -MIDDLEWARE_CLASSES = ( - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django.middleware.security.SecurityMiddleware', -) - -ROOT_URLCONF = 'reopt_api.urls' - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - ], - }, - }, -] - - -WSGI_APPLICATION = 'reopt_api.wsgi.application' - -# Database -# https://docs.djangoproject.com/en/2.2/ref/settings/#databases - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'HOST': 'localhost', - 'NAME': 'reopt_development', - 'OPTIONS': { - 'options': '-c search_path=reopt_api' - }, - 'USER': dev_user, - 'PASSWORD': dev_user_password, - } -} - -# Internationalization -# https://docs.djangoproject.com/en/2.2/topics/i18n/ - -LANGUAGE_CODE = 'en-us' - -TIME_ZONE = 'UTC' - -USE_I18N = True - -USE_L10N = True - -USE_TZ = True - -# Results backend -CELERY_RESULT_BACKEND = 'django-db' -CELERY_WORKER_MAX_MEMORY_PER_CHILD = 6000000 # 6 GB - -# celery task registration -CELERY_IMPORTS = ( - 'reo.api', - 'reo.scenario', - 'reo.process_results', - 'reo.src.run_jump_model', - 'resilience_stats.outage_simulator_LF', - 'django_extensions', - 'ghpghx' -) - -if 'test' in sys.argv: - CELERY_TASK_ALWAYS_EAGER = True - CELERY_TASK_EAGER_PROPAGATES_EXCEPTIONS = False - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/2.2/howto/static-files/ -STATIC_ROOT = os.path.join(BASE_DIR, 'static') -STATIC_URL = '/static/' - -APPEND_SLASH = False -TASTYPIE_ALLOW_MISSING_SLASH = True - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "reopt_api.internal_c110p_settings") -django.setup() diff --git a/reopt_api/production_settings.py b/reopt_api/settings.py similarity index 78% rename from reopt_api/production_settings.py rename to reopt_api/settings.py index 2574c6557..035ee1ade 100644 --- a/reopt_api/production_settings.py +++ b/reopt_api/settings.py @@ -1,8 +1,9 @@ # REoptĀ®, Copyright (c) Alliance for Sustainable Energy, LLC. See also https://github.com/NREL/REopt_API/blob/master/LICENSE. -from keys import * +from keys_env import * import os import django import rollbar +import sys """ Django settings for reopt_api project. @@ -16,6 +17,8 @@ https://docs.djangoproject.com/en/2.2/ref/settings/ """ +APP_ENV = os.getenv('APP_ENV', os.getenv('DEPLOY_ENV', 'development')) + # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -87,15 +90,23 @@ DATABASES = { 'default':{ 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'HOST': prod_database_host, - 'NAME': prod_database_name, + 'HOST': db_host, + 'NAME': db_name, 'OPTIONS': { - 'options': '-c search_path=reopt_api' + 'options': '-c search_path=' + db_search_path }, - 'USER': production_user, - 'PASSWORD': production_user_password, + 'USER': db_username, + 'PASSWORD': db_password, } } +if 'test' in sys.argv or APP_ENV == 'local': + DATABASES['default']['NAME'] = 'reopt' + DATABASES['default']['USER'] = 'reopt' + DATABASES['default']['PASSWORD'] = 'reopt' + DATABASES['default']['OPTIONS'] = { + 'options': '-c search_path=public' + } + # Internationalization # https://docs.djangoproject.com/en/2.2/topics/i18n/ @@ -113,7 +124,15 @@ # Results backend CELERY_RESULT_BACKEND = 'django-db' -CELERY_WORKER_MAX_MEMORY_PER_CHILD = 4000000 # 4 GB +if APP_ENV == 'staging': + CELERY_WORKER_MAX_MEMORY_PER_CHILD = 6000000 # 6 GB +else: + CELERY_WORKER_MAX_MEMORY_PER_CHILD = 4000000 # 4 GB + +if 'test' in sys.argv: + CELERY_TASK_ALWAYS_EAGER = True + CELERY_TASK_EAGER_PROPAGATES_EXCEPTIONS = False + # we have been resetting celery workers by max memory due to a memory growth problem. # this problem may be fixed by removing PyJulia, but can't view Rancher metrics yet. # Once we can confirm that we no longer have a memory grwoth issue we can disable this setting. @@ -141,14 +160,19 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/2.2/howto/static-files/ STATIC_ROOT = os.path.join(BASE_DIR, 'static') -STATIC_URL = '/' +if APP_ENV == 'development': + STATIC_URL = '/static/' +else: + STATIC_URL = '/' ROLLBAR = { 'access_token': rollbar_access_token, - 'environment': 'production', + 'environment': APP_ENV, 'root': BASE_DIR, 'branch': os.environ.get('BRANCH_NAME') } +if 'test' in sys.argv or APP_ENV == 'local': + ROLLBAR['enabled'] = False rollbar.init(**ROLLBAR) @@ -156,5 +180,5 @@ TASTYPIE_ALLOW_MISSING_SLASH = True DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "reopt_api.production_settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "reopt_api.settings") django.setup() diff --git a/reopt_api/staging_settings.py b/reopt_api/staging_settings.py deleted file mode 100755 index af4f61d4b..000000000 --- a/reopt_api/staging_settings.py +++ /dev/null @@ -1,159 +0,0 @@ -# REoptĀ®, Copyright (c) Alliance for Sustainable Energy, LLC. See also https://github.com/NREL/REopt_API/blob/master/LICENSE. -from keys import * -import os -import django -import rollbar -""" -Django settings for reopt_api project. - -Generated by 'django-admin startproject' using Django 2.2. - -For more information on this file, see -https://docs.djangoproject.com/en/2.2/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/2.2/ref/settings/ -""" - -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) - -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = secret_key_ - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = False - -ALLOWED_HOSTS = ['*'] - - -# Application definition - -INSTALLED_APPS = ( - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'reo', - 'summary', - 'tastypie', - 'proforma', - 'resilience_stats', - 'futurecosts', - 'django_celery_results', - 'django_extensions', - 'reoptjl', - 'ghpghx' -) - -MIDDLEWARE_CLASSES = ( - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'rollbar.contrib.django.middleware.RollbarNotifierMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django.middleware.security.SecurityMiddleware', -) - -ROOT_URLCONF = 'reopt_api.urls' - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - ], - }, - }, -] - - -WSGI_APPLICATION = 'reopt_api.wsgi.application' - -# Database -# https://docs.djangoproject.com/en/2.2/ref/settings/#databases - -DATABASES = { - 'default':{ - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'HOST': staging_database_host, - 'NAME': staging_database_name, - 'OPTIONS': { - 'options': '-c search_path=reopt_api' - }, - 'USER': staging_user, - 'PASSWORD': staging_user_password, - } -} - -# Internationalization -# https://docs.djangoproject.com/en/2.2/topics/i18n/ - -LANGUAGE_CODE = 'en-us' - -TIME_ZONE = 'UTC' - -USE_I18N = True - -USE_L10N = True - -USE_TZ = True - -# Results backend -CELERY_RESULT_BACKEND = 'django-db' - -CELERY_WORKER_MAX_MEMORY_PER_CHILD = 4000000 # 6 GB -# we have been resetting celery workers by max memory due to a memory growth problem. -# this problem may be fixed by removing PyJulia, but can't view Rancher metrics yet. -# Once we can confirm that we no longer have a memory grwoth issue we can disable this setting. -# It has to be set according to the RAM available. - -# limit number of concurrent workers -CELERY_WORKER_CONCURRENCY = 1 -# controlling number of celery workers with number of celery pods - -# celery task registration -CELERY_IMPORTS = ( - 'reo.api', - 'reo.scenario', - 'reo.process_results', - 'reo.src.run_jump_model', - 'resilience_stats.outage_simulator_LF', - 'futurecosts.api', - 'futurecosts.tasks', - 'django_extensions', - 'reoptjl.api', - 'reoptjl.src.run_jump_model', - 'ghpghx' -) - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/2.2/howto/static-files/ -STATIC_ROOT = os.path.join(BASE_DIR, 'static') -STATIC_URL = '/' - -ROLLBAR = { - 'access_token': rollbar_access_token, - 'environment': 'staging', - 'root': BASE_DIR, - 'branch': os.environ.get('BRANCH_NAME') -} - -rollbar.init(**ROLLBAR) - -APPEND_SLASH = False -TASTYPIE_ALLOW_MISSING_SLASH = True -DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "reopt_api.staging_settings") -django.setup() diff --git a/reoptjl/api.py b/reoptjl/api.py index 6d75a30b0..3508941ec 100644 --- a/reoptjl/api.py +++ b/reoptjl/api.py @@ -17,7 +17,7 @@ from ghpghx.models import GHPGHXInputs from django.core.exceptions import ValidationError from reoptjl.models import APIMeta -import keys +import keys_env log = logging.getLogger(__name__) @@ -74,7 +74,7 @@ def obj_get_list(self, bundle, **kwargs): def obj_create(self, bundle, **kwargs): run_uuid = str(uuid.uuid4()) # Attempt to get POSTed api_key assigned to APIMeta.api_key (or try method below for user_uuid) - api_key = keys.developer_nrel_gov_key #bundle.request.GET.get("api_key", "") + api_key = keys_env.developer_nrel_gov_key #bundle.request.GET.get("api_key", "") if 'user_uuid' in bundle.data.keys(): if type(bundle.data['user_uuid']) == str: diff --git a/transcrypt/.editorconfig b/transcrypt/.editorconfig deleted file mode 100644 index a35ef6e8a..000000000 --- a/transcrypt/.editorconfig +++ /dev/null @@ -1,16 +0,0 @@ -root = true - -[*] -charset = utf-8 -end_of_line = lf -insert_final_newline = true -trim_trailing_whitespace = true - -[*.md] -indent_size = 4 -indent_style = space -trim_trailing_whitespace = false - -[transcrypt] -indent_style = tab -tab_width = 4 diff --git a/transcrypt/.gitattributes b/transcrypt/.gitattributes deleted file mode 100644 index db8bb76a8..000000000 --- a/transcrypt/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -sensitive_file filter=crypt diff=crypt diff --git a/transcrypt/INSTALL.md b/transcrypt/INSTALL.md deleted file mode 100644 index 2e7350bd4..000000000 --- a/transcrypt/INSTALL.md +++ /dev/null @@ -1,61 +0,0 @@ -Install transcrypt -================== - -The requirements to run transcrypt are minimal: - -* Bash -* Git -* OpenSSL - -You also need access to the _transcrypt_ script itself... - -Manual Installation -------------------- - -You can add transcrypt directly to your repository, or just put it somewhere in -your $PATH: - - $ git clone https://github.com/elasticdog/transcrypt.git - $ cd transcrypt/ - $ sudo ln -s ${PWD}/transcrypt /usr/local/bin/transcrypt - -Installation via Packages -------------------------- - -A number of packages are available for installing transcrypt directly on your -system via its native package manager. Some of these packages also include man -page documentation as well as shell auto-completion scripts. - -### Arch Linux - -If you're on Arch Linux, you can build/install transcrypt using the -[provided PKGBUILD](https://github.com/elasticdog/transcrypt/blob/master/contrib/packaging/pacman/PKGBUILD): - - $ git clone https://github.com/elasticdog/transcrypt.git - $ cd transcrypt/contrib/packaging/pacman/ - $ makepkg -sic - -### Heroku - -If you're running software on Heroku, you can integrate transcrypt into your -slug compilation phase by using the -[transcrypt buildpack](https://github.com/perplexes/heroku-buildpack-transcrypt), -developed by [Colin Curtin](https://github.com/perplexes). - -### NixOS - -If you're on NixOS, you can install transcrypt directly via -[Nix](https://nixos.org/nix/): - - $ nix-env -iA nixos.gitAndTools.transcrypt - -> _**Note:** -> The [transcrypt derivation](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/version-management/git-and-tools/transcrypt/default.nix) -> was added in Oct 2015, so it is not available on the 15.09 channel._ - -### OS X - -If you're on OS X, you can install transcrypt directly via -[Homebrew](http://brew.sh/): - - $ brew install transcrypt diff --git a/transcrypt/LICENSE b/transcrypt/LICENSE deleted file mode 100644 index 5ed94e0bb..000000000 --- a/transcrypt/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -Copyright (c) 2014-2019, Aaron Bull Schaefer -Copyright (c) 2011, Woody Gilk - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/transcrypt/README.md b/transcrypt/README.md deleted file mode 100644 index 15136f1ce..000000000 --- a/transcrypt/README.md +++ /dev/null @@ -1,296 +0,0 @@ -# transcrypt - -A script to configure transparent encryption of sensitive files stored in a Git -repository. Files that you choose will be automatically encrypted when you -commit them, and automatically decrypted when you check them out. The process -will degrade gracefully, so even people without your encryption password can -safely commit changes to the repository's non-encrypted files. - -transcrypt protects your data when it's pushed to remotes that you may not -directly control (e.g., GitHub, Dropbox clones, etc.), while still allowing you -to work normally on your local working copy. You can conveniently store things -like passwords and private keys within your repository and not have to share -them with your entire team or complicate your workflow. - -## Overview - -transcrypt is in the same vein as existing projects like -[git-crypt](https://github.com/AGWA/git-crypt) and -[git-encrypt](https://github.com/shadowhand/git-encrypt), which follow Git's -documentation regarding the use of clean/smudge filters for encryption. In -comparison to those other projects, transcrypt makes substantial improvements in -the areas of usability and safety. - -- transcrypt is just a Bash script and does not require compilation -- transcrypt uses OpenSSL's symmetric cipher routines rather than implementing - its own crypto -- transcrypt does not have to remain installed after the initial repository - configuration -- transcrypt generates a unique salt for each encrypted file -- transcrypt uses safety checks to avoid clobbering or duplicating configuration - data -- transcrypt facilitates setting up additional clones as well as rekeying -- transcrypt adds an alias `git ls-crypt` to list all encrypted files - -### Salt Generation - -The _decryption -> encryption_ process on an unchanged file must be -deterministic for everything to work transparently. To do that, the same salt -must be used each time we encrypt the same file. Rather than use a static salt -common to all files, transcrypt first has OpenSSL generate an HMAC-SHA256 -cryptographic hash-based message authentication code for each decrypted file -(keyed with a combination of the filename and transcrypt password), and then -uses the last 16 bytes of that HMAC for the file's unique salt. When the content -of the file changes, so does the salt. Since an -[HMAC has been proven to be a PRF](http://cseweb.ucsd.edu/~mihir/papers/hmac-new.html), -this method of salt selection does not leak information about the original -contents, but is still deterministic. - -## Usage - -The requirements to run transcrypt are minimal: - -- Bash -- Git -- OpenSSL - -...and optionally: - -- GnuPG - for secure configuration import/export - -You also need access to the _transcrypt_ script itself. You can add it directly -to your repository, or just put it somewhere in your \$PATH: - - $ git clone https://github.com/elasticdog/transcrypt.git - $ cd transcrypt/ - $ sudo ln -s ${PWD}/transcrypt /usr/local/bin/transcrypt - -#### Installation via Packages - -A number of packages are available for installing transcrypt directly on your -system via its native package manager. Some of these packages also include man -page documentation as well as shell auto-completion scripts. - -- Arch Linux -- Heroku (via [Buildpacks](https://devcenter.heroku.com/articles/buildpacks)) -- NixOS -- OS X (via [Homebrew](http://brew.sh/)) - -...see the [INSTALL document](INSTALL.md) for more details. - -### Initialize an Unconfigured Repository - -transcrypt will interactively prompt you for the required information, all you -have to do run the script within a Git repository: - - $ cd / - $ transcrypt - -If you already know the values you want to use, you can specify them directly -using the command line options. Run `transcrypt --help` for more details. - -### Designate a File to be Encrypted - -Once a repository has been configured with transcrypt, you can designate for -files to be encrypted by applying the "crypt" filter and diff to a -[pattern](https://www.kernel.org/pub/software/scm/git/docs/gitignore.html#_pattern_format) -in the top-level _[.gitattributes](http://git-scm.com/docs/gitattributes)_ -config. If that pattern matches a file in your repository, the file will be -transparently encrypted once you stage and commit it: - - $ cd / - $ echo 'sensitive_file filter=crypt diff=crypt' >> .gitattributes - $ git add .gitattributes sensitive_file - $ git commit -m 'Add encrypted version of a sensitive file' - -The _.gitattributes_ file should be committed and tracked along with everything -else in your repository so clones will be aware of what is encrypted. Make sure -you don't accidentally add a pattern that would encrypt this file :-) - -> For your reference, if you find the above description confusing, you'll find -> that this repository has been configured following these exact steps. - -### Listing the Currently Encrypted Files - -For convenience, transcrypt also adds a Git alias to allow you to list all of -the currently encrypted files in a repository: - - $ git ls-crypt - sensitive_file - -Alternatively, you can use the `--list` command line option: - - $ transcrypt --list - sensitive_file - -You can also use this to verify your _.gitattributes_ patterns when designating -new files to be encrypted, as the alias will list pattern matches as long as -everything has been staged (via `git add`). - -After committing things, but before you push to a remote repository, you can -validate that files are encrypted as expected by viewing them in their raw form: - - $ git show HEAD: --no-textconv - -The `` in the above command must be relative to the _top-level_ of -the repository. Alternatively, you can use the `--show-raw` command line option -and provide a path relative to your current directory: - - $ transcrypt --show-raw sensitive_file - -### Initialize a Clone of a Configured Repository - -If you have just cloned a repository containing files that are encrypted, you'll -want to configure transcrypt with the same cipher and password as the origin -repository. The owner of the origin repository can dump the credentials for you -by running the `--display` command line option: - - $ transcrypt --display - The current repository was configured using transcrypt v0.2.0 - and has the following configuration: - - CIPHER: aes-256-cbc - PASSWORD: correct horse battery staple - - Copy and paste the following command to initialize a cloned repository: - - transcrypt -c aes-256-cbc -p 'correct horse battery staple' - -Once transcrypt has stored the matching credentials, it will force a checkout of -any exising encrypted files in order to decrypt them. - -### Rekeying - -Periodically, you may want to change the encryption cipher or password used to -encrypt the files in your repository. You can do that easily with transcrypt's -rekey option: - - $ transcrypt --rekey - -> As a warning, rekeying will remove your ability to see historical diffs of the -> encrypted files in plain text. Changes made with the new key will still be -> visible, and you can always see the historical diffs in encrypted form by -> disabling the text conversion filters: -> -> $ git log --patch --no-textconv - -After rekeying, all clones of your repository should flush their transcrypt -credentials, fetch and merge the new encrypted files via Git, and then -re-configure transcrypt with the new credentials. - - $ transcrypt --flush-credentials - $ git fetch origin - $ git merge origin/master - $ transcrypt -c aes-256-cbc -p 'the-new-password' - -### Command Line Options - -Completion scripts for both Bash and Zsh are included in the _contrib/_ -directory. - - transcrypt [option...] - - -c, --cipher=CIPHER - the symmetric cipher to utilize for encryption; - defaults to aes-256-cbc - - -p, --password=PASSWORD - the password to derive the key from; - defaults to 30 random base64 characters - - -y, --yes - assume yes and accept defaults for non-specified options - - -d, --display - display the current repository's cipher and password - - -r, --rekey - re-encrypt all encrypted files using new credentials - - -f, --flush-credentials - remove the locally cached encryption credentials and re-encrypt - any files that had been previously decrypted - - -F, --force - ignore whether the git directory is clean, proceed with the - possibility that uncommitted changes are overwritten - - -u, --uninstall - remove all transcrypt configuration from the repository and - leave files in the current working copy decrypted - - -l, --list - list all of the transparently encrypted files in the repository, - relative to the top-level directory - - -s, --show-raw=FILE - show the raw file as stored in the git commit object; use this - to check if files are encrypted as expected - - -e, --export-gpg=RECIPIENT - export the repository's cipher and password to a file encrypted - for a gpg recipient - - -i, --import-gpg=FILE - import the password and cipher from a gpg encrypted file - - -v, --version - print the version information - - -h, --help - view this help message - -## Caveats - -### Overhead - -The method of using filters to selectively encrypt/decrypt files does add some -overhead to Git by regularly forking OpenSSL processes and removing Git's -ability to efficiently cache file changes. That said, it's not too different -from tracking binary files, and when used as intended, transcrypt should not -noticeably impact performance. There are much better options if your goal is to -encrypt the entire repository. - -### Localhost - -Note that the configuration and encryption information is stored in plain text -within the repository's _.git/config_ file. This prevents them from being -transferred to remote clones, but they are not protected from inquisitive users -on your local machine. - -For safety, you may prefer to only have the credentials stored when actually -updating encrypted files, and then flush them with `--flush-credentials` once -you're done (make sure you have the credentials backed up elsewhere!). This will -also revert any decrypted files back to their encrypted form in your local -working copy. - -### Cipher Selection - -Last up, regarding the default cipher choice of `aes-256-cbc`...there aren't any -fantastic alternatives without pulling in outside dependencies. Ideally, we -would use an authenticated cipher mode like `id-aes256-GCM` by default, but -there are a couple of issues: - -1. I'd like to support OS X out of the box, and unfortunately they are the - lowest common denominator when it comes to OpenSSL. For whatever reason, they - still include OpenSSL 0.9.8y rather than a newer release. Unfortunately, - GCM-based ciphers weren't added until OpenSSL 1.0.1 (back in early 2012). - -2. Even with newer versions of OpenSSL, the authenticated cipher modes - [don't work exactly right](http://openssl.6102.n7.nabble.com/id-aes256-GCM-command-line-encrypt-decrypt-fail-td27187.html) - when utilizing the command line `openssl enc`. - -I'm contemplating if transcrypt should append an HMAC to the `aes-256-cbc` -ciphertext to provide authentication, or if we should live with the -[malleability issues](http://www.jakoblell.com/blog/2013/12/22/practical-malleability-attack-against-cbc-encrypted-luks-partitions/) -as a known limitation. Essentially, malicious comitters without the transcrypt -password could potentially manipulate the plaintext in limited ways (given that -the attacker knows the original plaintext). Honestly, I'm not sure if the added -complexity here would be worth it given transcrypt's use case. - -## License - -transcrypt is provided under the terms of the -[MIT License](https://en.wikipedia.org/wiki/MIT_License). - -Copyright © 2014-2019, [Aaron Bull Schaefer](mailto:aaron@elasticdog.com). diff --git a/transcrypt/contrib/bash/transcrypt b/transcrypt/contrib/bash/transcrypt deleted file mode 100644 index 9a49f485a..000000000 --- a/transcrypt/contrib/bash/transcrypt +++ /dev/null @@ -1,55 +0,0 @@ -# completion script for transcrypt - -_files_and_dirs() { - local IFS=$'\n' - local LASTCHAR=' ' - - COMPREPLY=( $(compgen -o plusdirs -f -- "${COMP_WORDS[COMP_CWORD]}") ) - - if [[ ${#COMPREPLY[@]} -eq 1 ]]; then - [[ -d "$COMPREPLY" ]] && LASTCHAR='/' - COMPREPLY=$(printf '%q%s' "$COMPREPLY" "$LASTCHAR") - else - for ((i=0; i < ${#COMPREPLY[@]}; i++)); do - [[ -d "${COMPREPLY[$i]}" ]] && COMPREPLY[$i]=${COMPREPLY[$i]}/ - done - fi -} - -_transcrypt() { - local cur prev opts - COMPREPLY=() - cur="${COMP_WORDS[COMP_CWORD]}" - prev="${COMP_WORDS[COMP_CWORD-1]}" - opts="-c -p -y -d -r -f -F -u -l -s -e -i -v -h \ - --cipher --password --yes --display --rekey --flush-credentials --force --uninstall --list --show-raw --export-gpg --import-gpg --version --help" - - case "${prev}" in - -c | --cipher) - local ciphers=$(openssl list-cipher-commands) - COMPREPLY=( $(compgen -W "${ciphers}" -- ${cur}) ) - return 0 - ;; - -p | --password) - return 0 - ;; - -s | --show-raw) - _files_and_dirs - return 0 - ;; - -e | --export-gpg) - return 0 - ;; - -i | --import-gpg) - _files_and_dirs - return 0 - ;; - *) - ;; - esac - - COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) - COMPREPLY=$(printf '%q%s' "$COMPREPLY" ' ') -} - -complete -o nospace -F _transcrypt transcrypt diff --git a/transcrypt/contrib/packaging/pacman/.gitignore b/transcrypt/contrib/packaging/pacman/.gitignore deleted file mode 100644 index 11fca2d4d..000000000 --- a/transcrypt/contrib/packaging/pacman/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -# ignore everything -* - -# except these files -!.gitignore -!PKGBUILD diff --git a/transcrypt/contrib/packaging/pacman/PKGBUILD b/transcrypt/contrib/packaging/pacman/PKGBUILD deleted file mode 100644 index 92976650d..000000000 --- a/transcrypt/contrib/packaging/pacman/PKGBUILD +++ /dev/null @@ -1,23 +0,0 @@ -# Maintainer: Aaron Bull Schaefer -pkgname=transcrypt -pkgver=2.0.0 -pkgrel=1 -pkgdesc='A script to configure transparent encryption of files within a Git repository' -arch=('any') -url='https://github.com/elasticdog/transcrypt' -license=('MIT') -depends=('git' 'openssl') -optdepends=('gnupg: config import/export support') -source=("https://github.com/elasticdog/${pkgname}/archive/v${pkgver}.tar.gz") -sha256sums=('12b891bcee50c71f5ee00c3c3e992c591ad6146ece3d3c5efa065d966a010d65') - -package() { - cd "${pkgname}-${pkgver}/" - - install -m 755 -D transcrypt "${pkgdir}/usr/bin/transcrypt" - install -m 644 -D man/transcrypt.1 "${pkgdir}/usr/share/man/man1/transcrypt.1" - install -m 644 -D LICENSE "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE" - - install -m 644 -D contrib/bash/transcrypt "${pkgdir}/usr/share/bash-completion/completions/transcrypt" - install -m 644 -D contrib/zsh/_transcrypt "${pkgdir}/usr/share/zsh/site-functions/_transcrypt" -} diff --git a/transcrypt/contrib/zsh/_transcrypt b/transcrypt/contrib/zsh/_transcrypt deleted file mode 100644 index 6bf4a92a4..000000000 --- a/transcrypt/contrib/zsh/_transcrypt +++ /dev/null @@ -1,37 +0,0 @@ -#compdef transcrypt - -_transcrypt() { - local curcontext="$curcontext" state line - typeset -A opt_args - - _arguments \ - '(- 1 *)'{-l,--list}'[list encrypted files]' \ - '(- 1 *)'{-s,--show-raw=}'[show raw file]:file:->file' \ - '(- 1 *)'{-e,--export-gpg=}'[export config to gpg recipient]:recipient:' \ - '(- 1 *)'{-v,--version}'[print version]' \ - '(- 1 *)'{-h,--help}'[view help message]' \ - '(-c --cipher -d --display -f --flush-credentials -u --uninstall)'{-c,--cipher=}'[specify encryption cipher]:cipher:->cipher' \ - '(-p --password -d --display -f --flush-credentials -u --uninstall)'{-p,--password=}'[specify encryption password]:password:' \ - '(-y --yes)'{-y,--yes}'[assume yes and accept defaults]' \ - '(-d --display -p --password -c --cipher -r --rekey -u --uninstall)'{-d,--display}'[display current credentials]' \ - '(-r --rekey -d --display -f --flush-credentials -u --uninstall)'{-r,--rekey}'[rekey all encrypted files]' \ - '(-f --flush-credentials -c --cipher -p --password -r --rekey -u --uninstall)'{-f,--flush-credentials}'[flush cached credentials]' \ - '(-F --force -d --display -u --uninstall)'{-F,--force}'[ignore repository clean state]' \ - '(-u --uninstall -c --cipher -d --display -f --flush-credentials -p --password -r --rekey)'{-u,--uninstall}'[uninstall transcrypt]' \ - '(-i --import-gpg -c --cipher -p --password -d --display -f --flush-credentials -u --uninstall)'{-i,--import-gpg=}'[import config from gpg file]:file:->file' \ - && return 0 - - case $state in - cipher) - ciphers=( ${(f)"$(_call_program available-ciphers openssl list-cipher-commands)"} ) - _describe -t available-ciphers 'available ciphers' ciphers - ;; - file) - _path_files - ;; - esac -} - -_transcrypt "$@" - -return 1 diff --git a/transcrypt/man/transcrypt.1 b/transcrypt/man/transcrypt.1 deleted file mode 100644 index 7b1d9798a..000000000 --- a/transcrypt/man/transcrypt.1 +++ /dev/null @@ -1,118 +0,0 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "TRANSCRYPT" "1" "August 2016" "" "" -. -.SH "NAME" -\fBtranscrypt\fR \- transparently encrypt files within a git repository -. -.SH "SYNOPSIS" -\fBtranscrypt\fR [\fIoptions\fR\.\.\.] -. -.SH "DESCRIPTION" -transcrypt will configure a Git repository to support the transparent encryption/decryption of files by utilizing OpenSSL\'s symmetric cipher routines and Git\'s built\-in clean/smudge filters\. It will also add a Git alias "ls\-crypt" to list all transparently encrypted files within the repository\. -. -.P -The transcrypt source code and full documentation may be downloaded from \fIhttps://github\.com/elasticdog/transcrypt\fR\. -. -.SH "OPTIONS" -. -.TP -\fB\-c\fR, \fB\-\-cipher\fR=\fIcipher\fR -the symmetric cipher to utilize for encryption; defaults to aes\-256\-cbc -. -.TP -\fB\-p\fR, \fB\-\-password\fR=\fIpassword\fR -the password to derive the key from; defaults to 30 random base64 characters -. -.TP -\fB\-y\fR, \fB\-\-yes\fR -assume yes and accept defaults for non\-specified options -. -.TP -\fB\-d\fR, \fB\-\-display\fR -display the current repository\'s cipher and password -. -.TP -\fB\-r\fR, \fB\-\-rekey\fR -re\-encrypt all encrypted files using new credentials -. -.TP -\fB\-f\fR, \fB\-\-flush\-credentials\fR -remove the locally cached encryption credentials and re\-encrypt any files that had been previously decrypted -. -.TP -\fB\-F\fR, \fB\-\-force\fR -ignore whether the git directory is clean, proceed with the possibility that uncommitted changes are overwritten -. -.TP -\fB\-u\fR, \fB\-\-uninstall\fR -remove all transcrypt configuration from the repository and leave files in the current working copy decrypted -. -.TP -\fB\-l\fR, \fB\-\-list\fR -list all of the transparently encrypted files in the repository, relative to the top\-level directory -. -.TP -\fB\-s\fR, \fB\-\-show\-raw\fR=\fIfile\fR -show the raw file as stored in the git commit object; use this to check if files are encrypted as expected -. -.TP -\fB\-e\fR, \fB\-\-export\-gpg\fR=\fIrecipient\fR -export the repository\'s cipher and password to a file encrypted for a gpg recipient -. -.TP -\fB\-i\fR, \fB\-\-import\-gpg\fR=\fIfile\fR -import the password and cipher from a gpg encrypted file -. -.TP -\fB\-v\fR, \fB\-\-version\fR -print the version information -. -.TP -\fB\-h\fR, \fB\-\-help\fR -view this help message -. -.SH "EXAMPLES" -To initialize a Git repository to support transparent encryption, just change into the repo and run the transcrypt script\. transcrypt will prompt you interactively for all required information if the corresponding option flags were not given\. -. -.IP "" 4 -. -.nf - -$ cd / -$ transcrypt -. -.fi -. -.IP "" 0 -. -.P -Once a repository has been configured with transcrypt, you can transparently encrypt files by applying the "crypt" filter and diff to a pattern in the top\-level \fI\.gitattributes\fR config\. If that pattern matches a file in your repository, the file will be transparently encrypted once you stage and commit it: -. -.IP "" 4 -. -.nf - -$ echo \'sensitive_file filter=crypt diff=crypt\' >> \.gitattributes -$ git add \.gitattributes sensitive_file -$ git commit \-m \'Add encrypted version of a sensitive file\' -. -.fi -. -.IP "" 0 -. -.P -See the gitattributes(5) man page for more information\. -. -.P -If you have just cloned a repository containing files that are encrypted, you\'ll want to configure transcrypt with the same cipher and password as the origin repository\. Once transcrypt has stored the matching credentials, it will force a checkout of any existing encrypted files in order to decrypt them\. -. -.P -If the origin repository has just rekeyed, all clones should flush their transcrypt credentials, fetch and merge the new encrypted files via Git, and then re\-configure transcrypt with the new credentials\. -. -.SH "AUTHOR" -Aaron Bull Schaefer -. -.SH "SEE ALSO" -enc(1), gitattributes(5) diff --git a/transcrypt/man/transcrypt.1.ronn b/transcrypt/man/transcrypt.1.ronn deleted file mode 100644 index fb6a7d03b..000000000 --- a/transcrypt/man/transcrypt.1.ronn +++ /dev/null @@ -1,107 +0,0 @@ -transcrypt(1) -- transparently encrypt files within a git repository -==================================================================== - -## SYNOPSIS - -`transcrypt` [...] - -## DESCRIPTION - -transcrypt will configure a Git repository to support the transparent -encryption/decryption of files by utilizing OpenSSL's symmetric cipher routines -and Git's built-in clean/smudge filters. It will also add a Git alias -"ls-crypt" to list all transparently encrypted files within the repository. - -The transcrypt source code and full documentation may be downloaded from -. - -## OPTIONS - - * `-c`, `--cipher`=: - the symmetric cipher to utilize for encryption; - defaults to aes-256-cbc - - * `-p`, `--password`=: - the password to derive the key from; - defaults to 30 random base64 characters - - * `-y`, `--yes`: - assume yes and accept defaults for non-specified options - - * `-d`, `--display`: - display the current repository's cipher and password - - * `-r`, `--rekey`: - re-encrypt all encrypted files using new credentials - - * `-f`, `--flush-credentials`: - remove the locally cached encryption credentials - and re-encrypt any files that had been previously decrypted - - * `-F`, `--force`: - ignore whether the git directory is clean, proceed with the - possibility that uncommitted changes are overwritten - - * `-u`, `--uninstall`: - remove all transcrypt configuration from the repository - and leave files in the current working copy decrypted - - * `-l`, `--list`: - list all of the transparently encrypted files in the repository, - relative to the top-level directory - - * `-s`, `--show-raw`=: - show the raw file as stored in the git commit object; - use this to check if files are encrypted as expected - - * `-e`, `--export-gpg`=: - export the repository's cipher and password to a file encrypted - for a gpg recipient - - * `-i`, `--import-gpg`=: - import the password and cipher from a gpg encrypted file - - * `-v`, `--version`: - print the version information - - * `-h`, `--help`: - view this help message - -## EXAMPLES - -To initialize a Git repository to support transparent encryption, just change -into the repo and run the transcrypt script. transcrypt will prompt you -interactively for all required information if the corresponding option flags -were not given. - - $ cd / - $ transcrypt - -Once a repository has been configured with transcrypt, you can transparently -encrypt files by applying the "crypt" filter and diff to a pattern in the -top-level _.gitattributes_ config. If that pattern matches a file in your -repository, the file will be transparently encrypted once you stage and commit -it: - - $ echo 'sensitive_file filter=crypt diff=crypt' >> .gitattributes - $ git add .gitattributes sensitive_file - $ git commit -m 'Add encrypted version of a sensitive file' - -See the gitattributes(5) man page for more information. - -If you have just cloned a repository containing files that are encrypted, -you'll want to configure transcrypt with the same cipher and password as the -origin repository. Once transcrypt has stored the matching credentials, it will -force a checkout of any existing encrypted files in order to decrypt them. - -If the origin repository has just rekeyed, all clones should flush their -transcrypt credentials, fetch and merge the new encrypted files via Git, and -then re-configure transcrypt with the new credentials. - -## AUTHOR - -Aaron Bull Schaefer <aaron@elasticdog.com> - -## SEE ALSO - -enc(1), gitattributes(5) diff --git a/transcrypt/transcrypt b/transcrypt/transcrypt deleted file mode 100755 index ef5261a79..000000000 --- a/transcrypt/transcrypt +++ /dev/null @@ -1,887 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# -# transcrypt - https://github.com/elasticdog/transcrypt -# -# A script to configure transparent encryption of sensitive files stored in -# a Git repository. It utilizes OpenSSL's symmetric cipher routines and follows -# the gitattributes(5) man page regarding the use of filters. -# -# Copyright (c) 2014-2019 Aaron Bull Schaefer -# This source code is provided under the terms of the MIT License -# that can be be found in the LICENSE file. -# - -##### CONSTANTS - -# the release version of this script -readonly VERSION='2.0.0' - -# the default cipher to utilize -readonly DEFAULT_CIPHER='aes-256-cbc' - -##### FUNCTIONS - -# print a canonicalized absolute pathname -realpath() { - local path=$1 - - # make path absolute - local abspath=$path - if [[ -n ${abspath##/*} ]]; then - abspath=$(pwd -P)/$abspath - fi - - # canonicalize path - local dirname= - if [[ -d $abspath ]]; then - dirname=$(cd "$abspath" && pwd -P) - abspath=$dirname - elif [[ -e $abspath ]]; then - dirname=$(cd "${abspath%/*}/" 2>/dev/null && pwd -P) - abspath=$dirname/${abspath##*/} - fi - - if [[ -d $dirname && -e $abspath ]]; then - printf '%s\n' "$abspath" - else - printf 'invalid path: %s\n' "$path" >&2 - exit 1 - fi -} - -# establish repository metadata and directory handling -gather_repo_metadata() { - # whether or not transcrypt is already configured - readonly CONFIGURED=$(git config --get --local transcrypt.version 2>/dev/null) - - # the current git repository's top-level directory - readonly REPO=$(git rev-parse --show-toplevel 2>/dev/null) - - # whether or not a HEAD revision exists - readonly HEAD_EXISTS=$(git rev-parse --verify --quiet HEAD 2>/dev/null) - - # https://github.com/RichiH/vcsh - # whether or not the git repository is running under vcsh - readonly IS_VCSH=$(git config --get --local --bool vcsh.vcsh 2>/dev/null) - - # whether or not the git repository is bare - readonly IS_BARE=$(git rev-parse --is-bare-repository 2>/dev/null || printf 'false') - - # the current git repository's .git directory - local RELATIVE_GIT_DIR - RELATIVE_GIT_DIR=$(git rev-parse --git-dir 2>/dev/null || printf '') - readonly GIT_DIR=$(realpath "$RELATIVE_GIT_DIR" 2>/dev/null) - - # the current git repository's gitattributes file - local CORE_ATTRIBUTES - CORE_ATTRIBUTES=$(git config --get --local --path core.attributesFile 2>/dev/null || printf '') - if [[ $CORE_ATTRIBUTES ]]; then - readonly GIT_ATTRIBUTES=$CORE_ATTRIBUTES - elif [[ $IS_BARE == 'true' ]] || [[ $IS_VCSH == 'true' ]]; then - readonly GIT_ATTRIBUTES="${GIT_DIR}/info/attributes" - else - readonly GIT_ATTRIBUTES="${REPO}/.gitattributes" - fi -} - -# print a message to stderr -warn() { - local fmt="$1" - shift - # shellcheck disable=SC2059 - printf "transcrypt: $fmt\n" "$@" >&2 -} - -# print a message to stderr and exit with either -# the given status or that of the most recent command -die() { - local st="$?" - if [[ "$1" != *[^0-9]* ]]; then - st="$1" - shift - fi - warn "$@" - exit "$st" -} - -# verify that all requirements have been met -run_safety_checks() { - # validate that we're in a git repository - [[ $GIT_DIR ]] || die 'you are not currently in a git repository; did you forget to run "git init"?' - - # exit if transcrypt is not in the required state - if [[ $requires_existing_config ]] && [[ ! $CONFIGURED ]]; then - die 1 'the current repository is not configured' - elif [[ ! $requires_existing_config ]] && [[ $CONFIGURED ]]; then - die 1 'the current repository is already configured; see --display' - fi - - # check for dependencies - for cmd in {column,grep,mktemp,openssl,sed,tee}; do - command -v $cmd >/dev/null || die 'required command "%s" was not found' "$cmd" - done - - # ensure the repository is clean (if it has a HEAD revision) so we can force - # checkout files without the destruction of uncommitted changes - if [[ $requires_clean_repo ]] && [[ $HEAD_EXISTS ]] && [[ $IS_BARE == 'false' ]]; then - # check if the repo is dirty - if ! git diff-index --quiet HEAD --; then - die 1 'the repo is dirty; commit or stash your changes before running transcrypt' - fi - fi -} - -# unset the cipher variable if it is not supported by openssl -validate_cipher() { - local list_cipher_commands - if openssl list-cipher-commands &>/dev/null; then - # OpenSSL < v1.1.0 - list_cipher_commands='openssl list-cipher-commands' - else - # OpenSSL >= v1.1.0 - list_cipher_commands='openssl list -cipher-commands' - fi - - local supported - supported=$($list_cipher_commands | tr -s ' ' '\n' | grep --line-regexp "$cipher") || true - if [[ ! $supported ]]; then - if [[ $interactive ]]; then - printf '"%s" is not a valid cipher; choose one of the following:\n\n' "$cipher" - $list_cipher_commands | column -c 80 - printf '\n' - cipher='' - else - # shellcheck disable=SC2016 - die 1 '"%s" is not a valid cipher; see `%s`' "$cipher" "$list_cipher_commands" - fi - fi -} - -# ensure we have a cipher to encrypt with -get_cipher() { - while [[ ! $cipher ]]; do - local answer= - if [[ $interactive ]]; then - printf 'Encrypt using which cipher? [%s] ' "$DEFAULT_CIPHER" - read -r answer - fi - - # use the default cipher if the user gave no answer; - # otherwise verify the given cipher is supported by openssl - if [[ ! $answer ]]; then - cipher=$DEFAULT_CIPHER - else - cipher=$answer - validate_cipher - fi - done -} - -# ensure we have a password to encrypt with -get_password() { - while [[ ! $password ]]; do - local answer= - if [[ $interactive ]]; then - printf 'Generate a random password? [Y/n] ' - read -r -n 1 -s answer - printf '\n' - fi - - # generate a random password if the user answered yes; - # otherwise prompt the user for a password - if [[ $answer =~ $YES_REGEX ]] || [[ ! $answer ]]; then - local password_length=30 - local random_base64 - random_base64=$(openssl rand -base64 $password_length) - password=$random_base64 - else - printf 'Password: ' - read -r password - [[ $password ]] || printf 'no password was specified\n' - fi - done -} - -# confirm the transcrypt configuration -confirm_configuration() { - local answer= - - printf '\nRepository metadata:\n\n' - [[ ! $REPO ]] || printf ' GIT_WORK_TREE: %s\n' "$REPO" - printf ' GIT_DIR: %s\n' "$GIT_DIR" - printf ' GIT_ATTRIBUTES: %s\n\n' "$GIT_ATTRIBUTES" - printf 'The following configuration will be saved:\n\n' - printf ' CIPHER: %s\n' "$cipher" - printf ' PASSWORD: %s\n\n' "$password" - printf 'Does this look correct? [Y/n] ' - read -r -n 1 -s answer - - # exit if the user did not confirm - if [[ $answer =~ $YES_REGEX ]] || [[ ! $answer ]]; then - printf '\n\n' - else - printf '\n' - die 1 'configuration has been aborted' - fi -} - -# confirm the rekey configuration -confirm_rekey() { - local answer= - - printf '\nRepository metadata:\n\n' - [[ ! $REPO ]] || printf ' GIT_WORK_TREE: %s\n' "$REPO" - printf ' GIT_DIR: %s\n' "$GIT_DIR" - printf ' GIT_ATTRIBUTES: %s\n\n' "$GIT_ATTRIBUTES" - printf 'The following configuration will be saved:\n\n' - printf ' CIPHER: %s\n' "$cipher" - printf ' PASSWORD: %s\n\n' "$password" - printf 'You are about to re-encrypt all encrypted files using new credentials.\n' - printf 'Once you do this, their historical diffs will no longer display in plain text.\n\n' - printf 'Proceed with rekey? [y/N] ' - read -r answer - - # only rekey if the user explicitly confirmed - if [[ $answer =~ $YES_REGEX ]]; then - printf '\n' - else - die 1 'rekeying has been aborted' - fi -} - -# automatically stage rekeyed files in preparation for the user to commit them -stage_rekeyed_files() { - local encrypted_files - encrypted_files=$(git ls-crypt) - if [[ $encrypted_files ]] && [[ $IS_BARE == 'false' ]]; then - # touch all encrypted files to prevent stale stat info - cd "$REPO" || die 1 'could not change into the "%s" directory' "$REPO" - # shellcheck disable=SC2086 - touch $encrypted_files - # shellcheck disable=SC2086 - git update-index --add -- $encrypted_files - - printf '*** rekeyed files have been staged ***\n' - printf '*** COMMIT THESE CHANGES RIGHT AWAY! ***\n\n' - fi -} - -# save helper scripts under the repository's git directory -save_helper_scripts() { - mkdir -p "${GIT_DIR}/crypt" - - # The `decryption -> encryption` process on an unchanged file must be - # deterministic for everything to work transparently. To do that, the same - # salt must be used each time we encrypt the same file. An HMAC has been - # proven to be a PRF, so we generate an HMAC-SHA256 for each decrypted file - # (keyed with a combination of the filename and transcrypt password), and - # then use the last 16 bytes of that HMAC for the file's unique salt. - - cat <<-'EOF' >"${GIT_DIR}/crypt/clean" - #!/usr/bin/env bash - filename=$1 - # ignore empty files - if [[ -s $filename ]]; then - # cache STDIN to test if it's already encrypted - tempfile=$(mktemp 2>/dev/null || mktemp -t tmp) - trap 'rm -f "$tempfile"' EXIT - tee "$tempfile" &>/dev/null - # the first bytes of an encrypted file are always "Salted" in Base64 - read -n 8 firstbytes <"$tempfile" - if [[ $firstbytes == "U2FsdGVk" ]]; then - cat "$tempfile" - else - cipher=$(git config --get --local transcrypt.cipher) - password=$(git config --get --local transcrypt.password) - salt=$(openssl dgst -hmac "${filename}:${password}" -sha256 "$filename" | tr -d '\r\n' | tail -c 16) - ENC_PASS=$password openssl enc -$cipher -md MD5 -pass env:ENC_PASS -e -a -S "$salt" -in "$tempfile" - fi - fi - EOF - - cat <<-'EOF' >"${GIT_DIR}/crypt/smudge" - #!/usr/bin/env bash - tempfile=$(mktemp 2>/dev/null || mktemp -t tmp) - trap 'rm -f "$tempfile"' EXIT - cipher=$(git config --get --local transcrypt.cipher) - password=$(git config --get --local transcrypt.password) - tee "$tempfile" | ENC_PASS=$password openssl enc -$cipher -md MD5 -pass env:ENC_PASS -d -a 2>/dev/null || cat "$tempfile" - EOF - - cat <<-'EOF' >"${GIT_DIR}/crypt/textconv" - #!/usr/bin/env bash - filename=$1 - # ignore empty files - if [[ -s $filename ]]; then - cipher=$(git config --get --local transcrypt.cipher) - password=$(git config --get --local transcrypt.password) - ENC_PASS=$password openssl enc -$cipher -md MD5 -pass env:ENC_PASS -d -a -in "$filename" 2>/dev/null || cat "$filename" - fi - EOF - - # make scripts executable - for script in {clean,smudge,textconv}; do - chmod 0755 "${GIT_DIR}/crypt/${script}" - done -} - -# write the configuration to the repository's git config -save_configuration() { - save_helper_scripts - - # write the encryption info - git config transcrypt.version "$VERSION" - git config transcrypt.cipher "$cipher" - git config transcrypt.password "$password" - - # write the filter settings - if [[ -d $(git rev-parse --git-common-dir) ]]; then - # this allows us to support multiple working trees via git-worktree - # ...but the --git-common-dir flag was only added in November 2014 - # shellcheck disable=SC2016 - git config filter.crypt.clean '"$(git rev-parse --git-common-dir)"/crypt/clean %f' - # shellcheck disable=SC2016 - git config filter.crypt.smudge '"$(git rev-parse --git-common-dir)"/crypt/smudge' - # shellcheck disable=SC2016 - git config diff.crypt.textconv '"$(git rev-parse --git-common-dir)"/crypt/textconv' - else - # shellcheck disable=SC2016 - git config filter.crypt.clean '"$(git rev-parse --git-dir)"/crypt/clean %f' - # shellcheck disable=SC2016 - git config filter.crypt.smudge '"$(git rev-parse --git-dir)"/crypt/smudge' - # shellcheck disable=SC2016 - git config diff.crypt.textconv '"$(git rev-parse --git-dir)"/crypt/textconv' - fi - git config filter.crypt.required 'true' - git config diff.crypt.cachetextconv 'true' - git config diff.crypt.binary 'true' - git config merge.renormalize 'true' - - # add a git alias for listing encrypted files - git config alias.ls-crypt "!git ls-files | git check-attr --stdin filter | awk 'BEGIN { FS = \":\" }; /crypt$/{ print \$1 }'" -} - -# display the current configuration settings -display_configuration() { - local current_cipher - current_cipher=$(git config --get --local transcrypt.cipher) - local current_password - current_password=$(git config --get --local transcrypt.password) - local escaped_password=${current_password//\'/\'\\\'\'} - - printf 'The current repository was configured using transcrypt version %s\n' "$CONFIGURED" - printf 'and has the following configuration:\n\n' - [[ ! $REPO ]] || printf ' GIT_WORK_TREE: %s\n' "$REPO" - printf ' GIT_DIR: %s\n' "$GIT_DIR" - printf ' GIT_ATTRIBUTES: %s\n\n' "$GIT_ATTRIBUTES" - printf ' CIPHER: %s\n' "$current_cipher" - printf ' PASSWORD: %s\n\n' "$current_password" - printf 'Copy and paste the following command to initialize a cloned repository:\n\n' - printf " transcrypt -c %s -p '%s'\n" "$current_cipher" "$escaped_password" -} - -# remove transcrypt-related settings from the repository's git config -clean_gitconfig() { - git config --remove-section transcrypt 2>/dev/null || true - git config --remove-section filter.crypt 2>/dev/null || true - git config --remove-section diff.crypt 2>/dev/null || true - git config --unset merge.renormalize - - # remove the merge section if it's now empty - local merge_values - merge_values=$(git config --get-regex --local 'merge\..*') || true - if [[ ! $merge_values ]]; then - git config --remove-section merge 2>/dev/null || true - fi -} - -# force the checkout of any files with the crypt filter applied to them; -# this will decrypt existing encrypted files if you've just cloned a repository, -# or it will encrypt locally decrypted files if you've just flushed the credentials -force_checkout() { - # make sure a HEAD revision exists - if [[ $HEAD_EXISTS ]] && [[ $IS_BARE == 'false' ]]; then - # this would normally delete uncommitted changes in the working directory, - # but we already made sure the repo was clean during the safety checks - local encrypted_files - encrypted_files=$(git ls-crypt) - cd "$REPO" || die 1 'could not change into the "%s" directory' "$REPO" - IFS=$'\n' - for file in $encrypted_files; do - rm "$file" - git checkout --force HEAD -- "$file" >/dev/null - done - unset IFS - fi -} - -# remove the locally cached encryption credentials and -# re-encrypt any files that had been previously decrypted -flush_credentials() { - local answer= - - if [[ $interactive ]]; then - printf 'You are about to flush the local credentials; make sure you have saved them elsewhere.\n' - printf 'All previously decrypted files will revert to their encrypted form.\n\n' - printf 'Proceed with credential flush? [y/N] ' - read -r answer - printf '\n' - else - # although destructive, we should support the --yes option - answer='y' - fi - - # only flush if the user explicitly confirmed - if [[ $answer =~ $YES_REGEX ]]; then - clean_gitconfig - - # re-encrypt any files that had been previously decrypted - force_checkout - - printf 'The local transcrypt credentials have been successfully flushed.\n' - else - die 1 'flushing of credentials has been aborted' - fi -} - -# remove all transcrypt configuration from the repository -uninstall_transcrypt() { - local answer= - - if [[ $interactive ]]; then - printf 'You are about to remove all transcrypt configuration from your repository.\n' - printf 'All previously encrypted files will remain decrypted in this working copy.\n\n' - printf 'Proceed with uninstall? [y/N] ' - read -r answer - printf '\n' - else - # although destructive, we should support the --yes option - answer='y' - fi - - # only uninstall if the user explicitly confirmed - if [[ $answer =~ $YES_REGEX ]]; then - clean_gitconfig - - # remove helper scripts - for script in {clean,smudge,textconv}; do - [[ ! -f "${GIT_DIR}/crypt/${script}" ]] || rm "${GIT_DIR}/crypt/${script}" - done - [[ ! -d "${GIT_DIR}/crypt" ]] || rmdir "${GIT_DIR}/crypt" - - # touch all encrypted files to prevent stale stat info - local encrypted_files - encrypted_files=$(git ls-crypt) - if [[ $encrypted_files ]] && [[ $IS_BARE == 'false' ]]; then - cd "$REPO" || die 1 'could not change into the "%s" directory' "$REPO" - # shellcheck disable=SC2086 - touch $encrypted_files - fi - - # remove the `git ls-crypt` alias - git config --unset alias.ls-crypt - - # remove the alias section if it's now empty - local alias_values - alias_values=$(git config --get-regex --local 'alias\..*') || true - if [[ ! $alias_values ]]; then - git config --remove-section alias 2>/dev/null || true - fi - - # remove any defined crypt patterns in gitattributes - case $OSTYPE in - darwin*) - /usr/bin/sed -i '' '/filter=crypt diff=crypt[ \t]*$/d' "$GIT_ATTRIBUTES" - ;; - linux*) - sed -i '/filter=crypt diff=crypt[ \t]*$/d' "$GIT_ATTRIBUTES" - ;; - esac - - printf 'The transcrypt configuration has been completely removed from the repository.\n' - else - die 1 'uninstallation has been aborted' - fi -} - -# list all of the currently encrypted files in the repository -list_files() { - if [[ $IS_BARE == 'false' ]]; then - cd "$REPO" || die 1 'could not change into the "%s" directory' "$REPO" - git ls-files | git check-attr --stdin filter | awk 'BEGIN { FS = ":" }; /crypt$/{ print $1 }' - fi -} - -# show the raw file as stored in the git commit object -show_raw_file() { - if [[ -f $show_file ]]; then - # ensure the file is currently being tracked - local escaped_file=${show_file//\//\\\/} - if git ls-files --others -- "$show_file" | awk "/${escaped_file}/{ exit 1 }"; then - file_paths=$(git ls-tree --name-only --full-name HEAD "$show_file") - else - die 1 'the file "%s" is not currently being tracked by git' "$show_file" - fi - elif [[ $show_file == '*' ]]; then - file_paths=$(git ls-crypt) - else - die 1 'the file "%s" does not exist' "$show_file" - fi - - IFS=$'\n' - for file in $file_paths; do - printf '==> %s <==\n' "$file" >&2 - git --no-pager show HEAD:"$file" --no-textconv - printf '\n' >&2 - done - unset IFS -} - -# export password and cipher to a gpg encrypted file -export_gpg() { - # check for dependencies - command -v gpg >/dev/null || die 'required command "gpg" was not found' - - # ensure the recipient key exists - if ! gpg --list-keys "$gpg_recipient" 2>/dev/null; then - die 1 'GPG recipient key "%s" does not exist' "$gpg_recipient" - fi - - local current_cipher - current_cipher=$(git config --get --local transcrypt.cipher) - local current_password - current_password=$(git config --get --local transcrypt.password) - mkdir -p "${GIT_DIR}/crypt" - - local gpg_encrypt_cmd="gpg --batch --recipient $gpg_recipient --trust-model always --yes --armor --quiet --encrypt -" - printf 'password=%s\ncipher=%s\n' "$current_password" "$current_cipher" | $gpg_encrypt_cmd >"${GIT_DIR}/crypt/${gpg_recipient}.asc" - printf "The transcrypt configuration has been encrypted and exported to:\n%s/crypt/%s.asc\n" "$GIT_DIR" "$gpg_recipient" -} - -# import password and cipher from a gpg encrypted file -import_gpg() { - # check for dependencies - command -v gpg >/dev/null || die 'required command "gpg" was not found' - - local path - if [[ -f "${GIT_DIR}/crypt/${gpg_import_file}" ]]; then - path="${GIT_DIR}/crypt/${gpg_import_file}" - elif [[ -f "${GIT_DIR}/crypt/${gpg_import_file}.asc" ]]; then - path="${GIT_DIR}/crypt/${gpg_import_file}.asc" - elif [[ ! -f $gpg_import_file ]]; then - die 1 'the file "%s" does not exist' "$gpg_import_file" - else - path="$gpg_import_file" - fi - - local configuration='' - local safety_counter=0 # fix for intermittent 'no secret key' decryption failures - while [[ ! $configuration ]]; do - configuration=$(gpg --batch --quiet --decrypt "$path") - - safety_counter=$((safety_counter + 1)) - if [[ $safety_counter -eq 3 ]]; then - die 1 'unable to decrypt the file "%s"' "$path" - fi - done - - cipher=$(printf '%s' "$configuration" | grep '^cipher' | cut -d'=' -f 2-) - password=$(printf '%s' "$configuration" | grep '^password' | cut -d'=' -f 2-) -} - -# print this script's usage message to stderr -usage() { - cat <<-EOF >&2 - usage: transcrypt [-c CIPHER] [-p PASSWORD] [-h] - EOF -} - -# print this script's help message to stdout -help() { - cat <<-EOF - NAME - transcrypt -- transparently encrypt files within a git repository - - SYNOPSIS - transcrypt [options...] - - DESCRIPTION - - transcrypt will configure a Git repository to support the transparent - encryption/decryption of files by utilizing OpenSSL's symmetric cipher - routines and Git's built-in clean/smudge filters. It will also add a - Git alias "ls-crypt" to list all transparently encrypted files within - the repository. - - The transcrypt source code and full documentation may be downloaded - from https://github.com/elasticdog/transcrypt. - - OPTIONS - -c, --cipher=CIPHER - the symmetric cipher to utilize for encryption; - defaults to aes-256-cbc - - -p, --password=PASSWORD - the password to derive the key from; - defaults to 30 random base64 characters - - -y, --yes - assume yes and accept defaults for non-specified options - - -d, --display - display the current repository's cipher and password - - -r, --rekey - re-encrypt all encrypted files using new credentials - - -f, --flush-credentials - remove the locally cached encryption credentials and re-encrypt - any files that had been previously decrypted - - -F, --force - ignore whether the git directory is clean, proceed with the - possibility that uncommitted changes are overwritten - - -u, --uninstall - remove all transcrypt configuration from the repository and - leave files in the current working copy decrypted - - -l, --list - list all of the transparently encrypted files in the repository, - relative to the top-level directory - - -s, --show-raw=FILE - show the raw file as stored in the git commit object; use this - to check if files are encrypted as expected - - -e, --export-gpg=RECIPIENT - export the repository's cipher and password to a file encrypted - for a gpg recipient - - -i, --import-gpg=FILE - import the password and cipher from a gpg encrypted file - - -v, --version - print the version information - - -h, --help - view this help message - - EXAMPLES - - To initialize a Git repository to support transparent encryption, just - change into the repo and run the transcrypt script. transcrypt will - prompt you interactively for all required information if the corre- - sponding option flags were not given. - - $ cd / - $ transcrypt - - Once a repository has been configured with transcrypt, you can trans- - parently encrypt files by applying the "crypt" filter and diff to a - pattern in the top-level .gitattributes config. If that pattern matches - a file in your repository, the file will be transparently encrypted - once you stage and commit it: - - $ echo 'sensitive_file filter=crypt diff=crypt' >> .gitattributes - $ git add .gitattributes sensitive_file - $ git commit -m 'Add encrypted version of a sensitive file' - - See the gitattributes(5) man page for more information. - - If you have just cloned a repository containing files that are - encrypted, you'll want to configure transcrypt with the same cipher and - password as the origin repository. Once transcrypt has stored the - matching credentials, it will force a checkout of any existing - encrypted files in order to decrypt them. - - If the origin repository has just rekeyed, all clones should flush - their transcrypt credentials, fetch and merge the new encrypted files - via Git, and then re-configure transcrypt with the new credentials. - - AUTHOR - Aaron Bull Schaefer - - SEE ALSO - enc(1), gitattributes(5) - EOF -} - -##### MAIN - -# reset all variables that might be set -cipher='' -display_config='' -flush_creds='' -gpg_import_file='' -gpg_recipient='' -interactive='true' -list='' -password='' -rekey='' -show_file='' -uninstall='' - -# used to bypass certain safety checks -requires_existing_config='' -requires_clean_repo='true' - -# parse command line options -while [[ "${1:-}" != '' ]]; do - case $1 in - -c | --cipher) - cipher=$2 - shift - ;; - --cipher=*) - cipher=${1#*=} - ;; - -p | --password) - password=$2 - shift - ;; - --password=*) - password=${1#*=} - ;; - -y | --yes) - interactive='' - ;; - -d | --display) - display_config='true' - requires_existing_config='true' - requires_clean_repo='' - ;; - -r | --rekey) - rekey='true' - requires_existing_config='true' - ;; - -f | --flush-credentials) - flush_creds='true' - requires_existing_config='true' - ;; - -F | --force) - requires_clean_repo='' - ;; - -u | --uninstall) - uninstall='true' - requires_existing_config='true' - requires_clean_repo='' - ;; - -l | --list) - list='true' - ;; - -s | --show-raw) - show_file=$2 - show_raw_file - exit 0 - ;; - --show-raw=*) - show_file=${1#*=} - show_raw_file - exit 0 - ;; - -e | --export-gpg) - gpg_recipient=$2 - requires_existing_config='true' - requires_clean_repo='' - shift - ;; - --export-gpg=*) - gpg_recipient=${1#*=} - requires_existing_config='true' - requires_clean_repo='' - ;; - -i | --import-gpg) - gpg_import_file=$2 - shift - ;; - --import-gpg=*) - gpg_import_file=${1#*=} - ;; - -v | --version) - printf 'transcrypt %s\n' "$VERSION" - exit 0 - ;; - -h | --help | -\?) - help - exit 0 - ;; - --*) - warn 'unknown option -- %s' "${1#--}" - usage - exit 1 - ;; - *) - warn 'unknown option -- %s' "${1#-}" - usage - exit 1 - ;; - esac - shift -done - -gather_repo_metadata - -# always run our safety checks -run_safety_checks - -# regular expression used to test user input -readonly YES_REGEX='^[Yy]$' - -# in order to keep behavior consistent no matter what order the options were -# specified in, we must run these here rather than in the case statement above -if [[ $list ]]; then - list_files - exit 0 -elif [[ $uninstall ]]; then - uninstall_transcrypt - exit 0 -elif [[ $display_config ]] && [[ $flush_creds ]]; then - display_configuration - printf '\n' - flush_credentials - exit 0 -elif [[ $display_config ]]; then - display_configuration - exit 0 -elif [[ $flush_creds ]]; then - flush_credentials - exit 0 -elif [[ $gpg_recipient ]]; then - export_gpg - exit 0 -elif [[ $gpg_import_file ]]; then - import_gpg -elif [[ $cipher ]]; then - validate_cipher -fi - -# perform function calls to configure transcrypt -get_cipher -get_password - -if [[ $rekey ]] && [[ $interactive ]]; then - confirm_rekey -elif [[ $interactive ]]; then - confirm_configuration -fi - -save_configuration - -if [[ $rekey ]]; then - stage_rekeyed_files -else - force_checkout -fi - -# ensure the git attributes file exists -if [[ ! -f $GIT_ATTRIBUTES ]]; then - mkdir -p "${GIT_ATTRIBUTES%/*}" - printf '#pattern filter=crypt diff=crypt\n' >"$GIT_ATTRIBUTES" -fi - -printf 'The repository has been successfully configured by transcrypt.\n' - -exit 0 diff --git a/werf-giterminism.yaml b/werf-giterminism.yaml deleted file mode 100644 index f1802bc8a..000000000 --- a/werf-giterminism.yaml +++ /dev/null @@ -1,12 +0,0 @@ -giterminismConfigVersion: 1 -config: - goTemplateRendering: - allowEnvVariables: - - XPRESS_LICENSE_HOST - - NREL_ROOT_CERT_URL_ROOT - # dockerfile: - # allowUncommitted: - # - julia_src/Dockerfile.xpress - # allowContextAddFiles: - # - julia_src/Dockerfile.xpress - # - julia_src/licenseserver diff --git a/werf.yaml b/werf.yaml deleted file mode 100644 index 0e9b7a396..000000000 --- a/werf.yaml +++ /dev/null @@ -1,19 +0,0 @@ -project: reopt-api -configVersion: 1 ---- -image: reopt-api -context: . -dockerfile: Dockerfile -args: - XPRESS_LICENSE_HOST: {{ env "XPRESS_LICENSE_HOST" | quote }} - NREL_ROOT_CERT_URL_ROOT: {{ env "NREL_ROOT_CERT_URL_ROOT" | quote }} ---- -image: julia-api -context: julia_src/ -dockerfile: Dockerfile -# contextAddFiles: -# - Dockerfile.xpress -# - licenseserver -args: - XPRESS_LICENSE_HOST: {{ env "XPRESS_LICENSE_HOST" | quote }} - NREL_ROOT_CERT_URL_ROOT: {{ env "NREL_ROOT_CERT_URL_ROOT" | quote }}