From de34caf7916d7b2caae390f012251c37b876d62c Mon Sep 17 00:00:00 2001 From: merveillevaneck Date: Tue, 5 May 2026 15:25:20 +0200 Subject: [PATCH 1/4] deployment config --- .gitignore | 2 ++ deploy.sh | 33 +++++++++++++++++++++++++++++++++ install.sh | 18 ++++++++++++++++++ install_runner.sh | 10 ++++++++++ resources/admins_encrypted.json | 2 +- resources/prod_config.edn | 20 ++++++++++++++++++++ runner_config.txt | 4 ++++ source-be.service | 12 ++++++++++++ start.sh | 9 ++++----- 9 files changed, 104 insertions(+), 6 deletions(-) create mode 100755 deploy.sh create mode 100755 install.sh create mode 100755 install_runner.sh create mode 100644 resources/prod_config.edn create mode 100644 runner_config.txt create mode 100644 source-be.service diff --git a/.gitignore b/.gitignore index 11809f1..18254b0 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ resources/admins_encrypted.json .db admins.json +admins_encrypted.json +prod.env diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 0000000..1cfd94f --- /dev/null +++ b/deploy.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +cd ~/Developer/source-be-deploy + +echo "Pulling changes..." +git pull + +echo "Starting compilation..." +clojure -T:build uber + +echo "Creating archive..." +mkdir source-be-deploy +cp start.sh source-be.service target/source-be-standalone.jar source-be-deploy +cp -r resources source-be-deploy +cp prod.env source-be-deploy/.env +tar -czvf source-be-deploy.tar.gz ./source-be-deploy + +echo "Copying archive..." +sshpass -e scp -r ~/Developer/source-be-deploy/source-be-deploy.tar.gz deploy@api.wearesource.earth:/home/deploy +echo "Done!" + +echo "Setting up prod server..." +sshpass -e ssh deploy@api.wearesource.earth " + echo "Extracting archive..." + tar -xvzf source-be-deploy.tar.gz + + echo "Restarting service..." + sudo systemctl restart source-fe.service +" + +echo "Cleaning up..." +rm -rf source-be-deploy +rm source-be-deploy.tar.gz diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..a0739be --- /dev/null +++ b/install.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +echo "Installing service..." +chmod +x deploy.sh +chmod +x install_runner.sh +chmod +x migrate.sh + +./deploy.sh + +sshpass -e ssh deploy@api.wearesource.earth " + cd /home/deploy/source-be-deploy + chmod +x start.sh + + sudo mv source-be.service /etc/systemd/system + sudo systemctl daemon-reload + sudo systemctl enable source-be.service + sudo systemctl start source-be.service +" diff --git a/install_runner.sh b/install_runner.sh new file mode 100755 index 0000000..21ec95d --- /dev/null +++ b/install_runner.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +mkdir -p ~/.config/source/actions-runner +cd ~/.config/source/actions-runner + +curl -o actions-runner-linux-x64-2.334.0.tar.gz -L https://github.com/actions/runner/releases/download/v2.334.0/actions-runner-linux-x64-2.334.0.tar.gz + +tar xzf ./actions-runner-linux-x64-2.334.0.tar.gz + +./config.sh --url https://github.com/modulr-software/source --token AMFR4MIUDHJ2SIGAWODFXYDJ7HEWO diff --git a/resources/admins_encrypted.json b/resources/admins_encrypted.json index 1418c11..b7c25a1 100644 --- a/resources/admins_encrypted.json +++ b/resources/admins_encrypted.json @@ -1 +1 @@ -1f849fdbcaaa4d5cf82b56107541fa49174399ba9cc32f67fd7eceb7f0847b8050f85973837676f7d7f8d92fa4d6cacf9e175a8fd6fff154212d1187c2888ec39e5d813d2a247e571991dc61e3ab567e796a92e0fbb7a4ee4ff62567a2c405d3008ce6fa89e6b85f7515aff0420cf59ed7e6112e232dec6952b2165f46a167d875b46ec1d1fa4f2a035112fe739607040da8f90bc8abd8f40b798020b6046535a4415e373bc1be463b447cb3218d88419b05d47d99cd053e16f985610cbe9565db3192db3c2810958176f2ebcbd8808a55e2c7e970f2f12b201ce39babdfd1ffe1cf22910395aeb8ff16a5af98896765046a0514c23b1f4fee34bc1a7ad293406151deba329de10c71b919038c165f0c848b60de52a4737ed0f27793c4bc85e7a3c1373e8982280fd53edddad4fca12eeaf212633a55c63ef457185887477ab6068957502a76932f9a267afb03f18ccacf53e52faea2fc56527fedc89570261403b9ade0a6e43e2244223a998338ca623e291ed714b133913e870f1fc07dff801e4450a0a67ac328b6b22f18ffd3ac4704a8e8d6e63dd1940a1c02b6f9c4183358585dc600857fdc0b8a72309e4edb41d1cd2fa2f450b4c6ff4dca70a2a09b9fcafac195493dc43d4d25062bee0b7b29e4fb89c592c7d0eee06724b8fd60c501a5ed49cde6a25679f86c207527c6cbc52c6c7f4215a2d1c857a4de92d14d016b \ No newline at end of file +5bd8411f3674b8719e1488bc12d9e71307b43ce574c53bc011b682b0d3b25b10ed4c4eba80bfead10d1b1e463eeee56f32f6f1e34dcfe88e6e172832302612e65711530e0874609b2790b5121efe9d795d6eca09880cc4dd7135dd831bfdf410b85596064f64b96acaa9dc0145884ec7785bae91cd4e6895a3665f044dffe8b6739c9dd71ca10845eebb88da7086e5edd856c49476ca7fa783ce77c33edf17b8bed499d8d978fa192eff0f1011f3bc3bbf31ec9f53052a0d765862045f7d0f5a9bb147abc7e2e874cc19b2481c4988db753d49969c6000ef754e86053af4b8df4b175741c6c5e75672d393e59f7fd333e7e4488f3f2473d4b415859163b3dd70e0d01b8e25b573ae7af3115240e846112ef34e90f13b405fa534af78664427602e35aacb29ab51720c9ea07de6aee2d65e66fee6b396d1387e4b541b6a9d01687be7914e04ef1412a07d8f93ce7aa90cd2e18d34ca945628b3ac4b0c7c37a5a47e33c0166c70dd34632f3491a2c14274cf32fcb0ed65ffc9ff111639fad7ee9d67196a08bb70d2987494f84d35a06253bd8a555aa642c7e978585223d80c93ec9685adf644f46c4c3657e429e22d1221 diff --git a/resources/prod_config.edn b/resources/prod_config.edn new file mode 100644 index 0000000..697c9d6 --- /dev/null +++ b/resources/prod_config.edn @@ -0,0 +1,20 @@ +{:supersecretkey #env SUPER_SECRET_KEY + :admins-path "resources/admins.json" + :admins-encrypted-path "resources/admins_encrypted.json" + :cors-origin #or [#env CORS_ORIGIN "https://localhost:4000"] + :base-url #or [#env BASE_URL "http://localhost:3000"] + :port #or [#env PORT 3000] + :env #or [#env ENV "prod"] + :database {:url #or [#env DATABASE_URL "postgres://localhost"] + :type "postgresql"} + :email {:address #or [#env SUPPORT_ADDRESS "lee@wearesource.earth"] + :username #env EMAIL_USERNAME + :password #env EMAIL_PASSWORD} + :oauth2 {:google {:authorization-uri "https://accounts.google.com/o/oauth2/auth" + :access-token-uri "https://oauth2.googleapis.com/token" + :redirect-uri #or [#env GOOGLE_REDIRECT_URI "https://mervstation.tail4f070.ts.net/sourcebe/oauth2/google/callback"] + :client-id #env GOOGLE_CLIENT_ID + :client-secret #env GOOGLE_CLIENT_SECRET + :access-query-param :access_token + :scope ["https://www.googleapis.com/auth/userinfo.email"] + :grant-type "authorization_code"}}} diff --git a/runner_config.txt b/runner_config.txt new file mode 100644 index 0000000..a3b7b78 --- /dev/null +++ b/runner_config.txt @@ -0,0 +1,4 @@ + + + +Y diff --git a/source-be.service b/source-be.service new file mode 100644 index 0000000..18abab1 --- /dev/null +++ b/source-be.service @@ -0,0 +1,12 @@ +[Unit] +Description=Source Back End Server Startup Service +After=postgresql.service + +[Service] +Type=simple +ExecStart=/home/deploy/source-be-deploy/start.sh +Restart=on-failure +RestartSec=5s + +[Install] +WantedBy=multi-user.target diff --git a/start.sh b/start.sh index 9169394..d460e04 100755 --- a/start.sh +++ b/start.sh @@ -1,8 +1,7 @@ #!/bin/bash -export $(grep '.*' .env | xargs) -echo "Running migrations before startup..." -clojure -M:migrate +cd /home/deploy/source-be-deploy +export $(grep '.*' .env | xargs) -echo "Starting server..." -$JAVA_CMD -jar target/source-be-standalone.jar +echo "Starting backend server..." +/home/deploy/.jenv/shims/java -jar source-be-standalone.jar From f11277a1b5415f12d69ae221a76c3f1d84d30486 Mon Sep 17 00:00:00 2001 From: merveillevaneck Date: Tue, 5 May 2026 15:40:49 +0200 Subject: [PATCH 2/4] added actions runner install script --- install_runner.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install_runner.sh b/install_runner.sh index 21ec95d..d86f7b0 100755 --- a/install_runner.sh +++ b/install_runner.sh @@ -1,10 +1,10 @@ #!/bin/bash -mkdir -p ~/.config/source/actions-runner -cd ~/.config/source/actions-runner +mkdir -p ~/.config/source-be/actions-runner +cd ~/.config/source-be/actions-runner curl -o actions-runner-linux-x64-2.334.0.tar.gz -L https://github.com/actions/runner/releases/download/v2.334.0/actions-runner-linux-x64-2.334.0.tar.gz tar xzf ./actions-runner-linux-x64-2.334.0.tar.gz -./config.sh --url https://github.com/modulr-software/source --token AMFR4MIUDHJ2SIGAWODFXYDJ7HEWO +./config.sh --url https://github.com/modulr-software/source-be --token AMFR4ML2O5TLDW2O5JR6KCTJ7H7U2 From c1c9188749bcbb4e53b106427d09aac9e7c53d2a Mon Sep 17 00:00:00 2001 From: merveillevaneck Date: Tue, 5 May 2026 15:43:14 +0200 Subject: [PATCH 3/4] added workflow --- .github/workflows/prod-deploy.yml | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/.github/workflows/prod-deploy.yml b/.github/workflows/prod-deploy.yml index f282b89..d652c65 100644 --- a/.github/workflows/prod-deploy.yml +++ b/.github/workflows/prod-deploy.yml @@ -6,18 +6,8 @@ on: jobs: deploy: name: Deploy to Prod - runs-on: ubuntu-latest - concurrency: deploy-group # optional: ensure only one action runs at a time + runs-on: self-hosted + #concurrency: deploy-group # optional: ensure only one action runs at a time steps: - - name: Execute remote SSH commands using password - uses: appleboy/ssh-action@v1 - with: - host: ${{ secrets.HOST }} - username: ${{ secrets.USERNAME }} - password: ${{ secrets.PASSWORD }} - script: | - cd /home/deploy/source-be - git checkout main - git pull - ./build.sh - sudo /bin/systemctl restart source-be + - name: Execute deploy script + run: cd ~/Developer/source-be-deploy && ./deploy.sh From 8ef16c9e814fd41974310fe2c2bcfa3108549f7e Mon Sep 17 00:00:00 2001 From: merveillevaneck Date: Wed, 6 May 2026 10:39:25 +0200 Subject: [PATCH 4/4] updated config files and added files necessary for migrations to run --- deploy.sh | 4 +- install.sh | 1 + migrate.sh | 1 + resources/admins_encrypted.json | 1 - resources/{prod_config.edn => config.edn} | 2 +- resources/dev_config.edn | 22 --------- resources/staging_config.edn | 20 -------- src/source/config.clj | 4 +- start.sh | 2 + test/source_be/cache_test.clj | 43 ----------------- test/source_be/google_test.clj | 59 ----------------------- 11 files changed, 8 insertions(+), 151 deletions(-) delete mode 100644 resources/admins_encrypted.json rename resources/{prod_config.edn => config.edn} (93%) delete mode 100644 resources/dev_config.edn delete mode 100644 resources/staging_config.edn delete mode 100644 test/source_be/cache_test.clj delete mode 100644 test/source_be/google_test.clj diff --git a/deploy.sh b/deploy.sh index 1cfd94f..a87589d 100755 --- a/deploy.sh +++ b/deploy.sh @@ -10,8 +10,8 @@ clojure -T:build uber echo "Creating archive..." mkdir source-be-deploy -cp start.sh source-be.service target/source-be-standalone.jar source-be-deploy -cp -r resources source-be-deploy +cp start.sh migrate.sh source-be.service deps.edn target/source-be-standalone.jar source-be-deploy +cp -r resources src source-be-deploy cp prod.env source-be-deploy/.env tar -czvf source-be-deploy.tar.gz ./source-be-deploy diff --git a/install.sh b/install.sh index a0739be..cfbc20e 100755 --- a/install.sh +++ b/install.sh @@ -10,6 +10,7 @@ chmod +x migrate.sh sshpass -e ssh deploy@api.wearesource.earth " cd /home/deploy/source-be-deploy chmod +x start.sh + chmod +x migrate.sh sudo mv source-be.service /etc/systemd/system sudo systemctl daemon-reload diff --git a/migrate.sh b/migrate.sh index 24715f2..6138385 100755 --- a/migrate.sh +++ b/migrate.sh @@ -2,4 +2,5 @@ export $(grep '.*' .env | xargs) +echo "Running migrations..." clojure -M:migrate "$@" diff --git a/resources/admins_encrypted.json b/resources/admins_encrypted.json deleted file mode 100644 index b7c25a1..0000000 --- a/resources/admins_encrypted.json +++ /dev/null @@ -1 +0,0 @@ -5bd8411f3674b8719e1488bc12d9e71307b43ce574c53bc011b682b0d3b25b10ed4c4eba80bfead10d1b1e463eeee56f32f6f1e34dcfe88e6e172832302612e65711530e0874609b2790b5121efe9d795d6eca09880cc4dd7135dd831bfdf410b85596064f64b96acaa9dc0145884ec7785bae91cd4e6895a3665f044dffe8b6739c9dd71ca10845eebb88da7086e5edd856c49476ca7fa783ce77c33edf17b8bed499d8d978fa192eff0f1011f3bc3bbf31ec9f53052a0d765862045f7d0f5a9bb147abc7e2e874cc19b2481c4988db753d49969c6000ef754e86053af4b8df4b175741c6c5e75672d393e59f7fd333e7e4488f3f2473d4b415859163b3dd70e0d01b8e25b573ae7af3115240e846112ef34e90f13b405fa534af78664427602e35aacb29ab51720c9ea07de6aee2d65e66fee6b396d1387e4b541b6a9d01687be7914e04ef1412a07d8f93ce7aa90cd2e18d34ca945628b3ac4b0c7c37a5a47e33c0166c70dd34632f3491a2c14274cf32fcb0ed65ffc9ff111639fad7ee9d67196a08bb70d2987494f84d35a06253bd8a555aa642c7e978585223d80c93ec9685adf644f46c4c3657e429e22d1221 diff --git a/resources/prod_config.edn b/resources/config.edn similarity index 93% rename from resources/prod_config.edn rename to resources/config.edn index 697c9d6..e28d267 100644 --- a/resources/prod_config.edn +++ b/resources/config.edn @@ -12,7 +12,7 @@ :password #env EMAIL_PASSWORD} :oauth2 {:google {:authorization-uri "https://accounts.google.com/o/oauth2/auth" :access-token-uri "https://oauth2.googleapis.com/token" - :redirect-uri #or [#env GOOGLE_REDIRECT_URI "https://mervstation.tail4f070.ts.net/sourcebe/oauth2/google/callback"] + :redirect-uri #or [#env GOOGLE_REDIRECT_URI "https://api.wearesource.earth/oauth2/google/callback"] :client-id #env GOOGLE_CLIENT_ID :client-secret #env GOOGLE_CLIENT_SECRET :access-query-param :access_token diff --git a/resources/dev_config.edn b/resources/dev_config.edn deleted file mode 100644 index 5f2aed9..0000000 --- a/resources/dev_config.edn +++ /dev/null @@ -1,22 +0,0 @@ -{:supersecretkey #env SUPER_SECRET_KEY - :admins-path "resources/admins.json" - :admins-encrypted-path "resources/admins_encrypted.json" - :cors-origin #or [#env CORS_ORIGIN "http://localhost:3001"] - :base-url #or [#env BASE_URL "https://localhost:3000"] - :port #or [#env PORT 3000] - :env #or [#env ENV "dev"] - :database {:url #or [#env DATABASE_URL "postgres://localhost"] - :type "postgresql" - :staging #or [#env STAGING_DATABASE_URL "postgres://username:password@localhost:16380"] - :prod #or [#env PROD_DATABASE_URL "postgres://username:password@localhost:16380"]} - :email {:address #or [#env SUPPORT_ADDRESS "lee@wearesource.earth"] - :username #env EMAIL_USERNAME - :password #env EMAIL_PASSWORD} - :oauth2 {:google {:authorization-uri "https://accounts.google.com/o/oauth2/auth" - :access-token-uri "https://oauth2.googleapis.com/token" - :redirect-uri #or [#env GOOGLE_REDIRECT_URI "http://localhost:3000/oauth2/google/callback"] - :client-id #env GOOGLE_CLIENT_ID - :client-secret #env GOOGLE_CLIENT_SECRET - :access-query-param :access_token - :scope ["https://www.googleapis.com/auth/userinfo.email"] - :grant-type "authorization_code"}}} diff --git a/resources/staging_config.edn b/resources/staging_config.edn deleted file mode 100644 index 3df88ec..0000000 --- a/resources/staging_config.edn +++ /dev/null @@ -1,20 +0,0 @@ -{:supersecretkey #env SUPER_SECRET_KEY - :admins-path "resources/admins.json" - :admins-encrypted-path "resources/admins_encrypted.json" - :cors-origin #or [#env CORS_ORIGIN "https://mervstation.tail4f070.ts.net"] - :base-url #or [#env BASE_URL "https://mervstation.tail4f070.ts.net/sourcebe"] - :port #or [#env PORT 3998] - :env #or [#env ENV "staging"] - :database {:url #or [#env DATABASE_URL "postgres://localhost"] - :type "postgresql"} - :email {:address #or [#env SUPPORT_ADDRESS "lee@wearesource.earth"] - :username #env EMAIL_USERNAME - :password #env EMAIL_PASSWORD} - :oauth2 {:google {:authorization-uri "https://accounts.google.com/o/oauth2/auth" - :access-token-uri "https://oauth2.googleapis.com/token" - :redirect-uri #or [#env GOOGLE_REDIRECT_URI "https://mervstation.tail4f070.ts.net/sourcebe/oauth2/google/callback"] - :client-id #env GOOGLE_CLIENT_ID - :client-secret #env GOOGLE_CLIENT_SECRET - :access-query-param :access_token - :scope ["https://www.googleapis.com/auth/userinfo.email"] - :grant-type "authorization_code"}}} diff --git a/src/source/config.clj b/src/source/config.clj index 63c3315..fbba9f4 100644 --- a/src/source/config.clj +++ b/src/source/config.clj @@ -31,9 +31,7 @@ [:oauth2 [:map-of keyword? oauth2-provider-schema]]]) (defn- load-config [] - (let [environment (get (System/getenv) "ENV") - environment (if (nil? environment) "dev" environment) - config (aero/read-config (io/resource (str environment "_config.edn"))) + (let [config (aero/read-config (io/resource "config.edn")) decoded (m/decode schema config mt/string-transformer)] (when-not (m/validate schema decoded) (println (->> decoded diff --git a/start.sh b/start.sh index d460e04..b3c46af 100755 --- a/start.sh +++ b/start.sh @@ -3,5 +3,7 @@ cd /home/deploy/source-be-deploy export $(grep '.*' .env | xargs) +./migrate.sh up + echo "Starting backend server..." /home/deploy/.jenv/shims/java -jar source-be-standalone.jar diff --git a/test/source_be/cache_test.clj b/test/source_be/cache_test.clj deleted file mode 100644 index 80b9f8a..0000000 --- a/test/source_be/cache_test.clj +++ /dev/null @@ -1,43 +0,0 @@ -(ns source-be.cache-test - (:require [clojure.test :refer :all] - [source.cache :as cache])) - -(deftest use-cache - (testing "adding, retrieving and removing values in cache by uuid" - (let [google-test-cache (cache/create-cache) - emails-test-cache (cache/create-cache) - google-result (do - (cache/add-item google-test-cache {:uri "http://yeet.com/test"} "testuuid") - (cache/get-item google-test-cache "testuuid")) - emails-result (do - (cache/add-item emails-test-cache "test@toast.com" "testuuid") - (cache/get-item emails-test-cache "testuuid"))] - - (is (= (count (cache/get-all-items google-test-cache)) 1)) - (is (= (count (cache/get-all-items emails-test-cache)) 1)) - (is (= google-result {:uri "http://yeet.com/test"})) - (is (= emails-result "test@toast.com")) - - (cache/add-item google-test-cache {:uri "http://delete.com/"} "todelete") - (cache/add-item google-test-cache {:uri "http://dontdelete.com/"}) - (is (= (count (cache/get-all-items google-test-cache)) 3)) - - (cache/add-item emails-test-cache "delete@delete.com" "todelete") - (cache/add-item emails-test-cache "dontdelete@dontdelete.com") - (is (= (count (cache/get-all-items emails-test-cache)) 3)) - - (let [google-removed (do - (cache/remove-item google-test-cache "testuuid") - (cache/get-item google-test-cache "testuuid")) - email-removed (do - (cache/remove-item emails-test-cache "testuuid") - (cache/get-item emails-test-cache "testuuid"))] - - (is (= (count (cache/get-all-items google-test-cache)) 2)) - (is (= (count (cache/get-all-items emails-test-cache)) 2)) - - (is (not (some? google-removed))) - (is (not (some? email-removed))))))) - - -(run-tests) diff --git a/test/source_be/google_test.clj b/test/source_be/google_test.clj deleted file mode 100644 index 1232137..0000000 --- a/test/source_be/google_test.clj +++ /dev/null @@ -1,59 +0,0 @@ -(ns source-be.google-test - (:require [clojure.test :refer :all] - [source.cache :as cache] - [source.oauth2.google.core :as google] - [clj-oauth2.client :as oauth2])) - -(defn oauth2-get [uri headers] - (let [auth-header (get-in - headers - [:headers "Authorization"])] - (if (and (some? uri) (= "Bearer access-token" auth-header)) - {:body "{\"email\":\"toast@toast.com\"}"} - {:body nil}))) - -(defn oauth2-get-access-token - [endpoint - & [params {expected-state :state expected-scope :scope}]] - (when (some? (:code params)) - {:access-token "access-token"})) - -(defn google-auth-flow-test-case - [mock-code - expected-email - get-access-token-override - get-override] - - (with-redefs [oauth2/get-access-token get-access-token-override - oauth2/get get-override] - - (let [auth-reqs-service (cache/create-cache) - auth-req (google/-auth-uri auth-reqs-service)] - - (is (some? (:uri auth-req))) - (is (some? (:uuid auth-req))) - (is (= (count (cache/get-all-items auth-reqs-service)) 1)) - - (let [email (google/-google-session-user - auth-reqs-service - (:uuid auth-req) - mock-code)] - - (is (= expected-email email)))))) - -(deftest google-auth-flow-test - (testing "testing the google oauth2 flow" - - (testing "successful auth flow" - (google-auth-flow-test-case - {:code "a1b2c3"} - "toast@toast.com" - oauth2-get-access-token - oauth2-get)) - - (testing "missing code verifier" - (google-auth-flow-test-case - {} - nil - oauth2-get-access-token - oauth2-get))))