From baf38e88f8eeb2cbcfa645b25dfddf7b06369797 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Fri, 24 Oct 2025 15:09:53 -0700 Subject: [PATCH 01/35] initial service file commit --- card_data/infrastructure/dagster.service | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 card_data/infrastructure/dagster.service diff --git a/card_data/infrastructure/dagster.service b/card_data/infrastructure/dagster.service new file mode 100644 index 00000000..64765859 --- /dev/null +++ b/card_data/infrastructure/dagster.service @@ -0,0 +1,20 @@ +[Unit] +Description=Dagster Development Server +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=ubuntu +WorkingDirectory=/home/ubuntu/card_data/card_data +Environment="AWS_DEFAULT_REGION=us-west-2" +Environment="PATH=/home/ubuntu/card_data/card_data/.venv/bin:/usr/local/bin:/usr/bin:/bin" +ExecStartPre=/home/ubuntu/wait-for-rds.sh +ExecStart=/home/ubuntu/start-dagster.sh +Restart=on-failure +RestartSec=10 +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target \ No newline at end of file From 58980ea692ab86a318071bb24b3a4368405a1fc4 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Fri, 24 Oct 2025 15:13:49 -0700 Subject: [PATCH 02/35] initial commit --- card_data/infrastructure/wait-for-rds.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 card_data/infrastructure/wait-for-rds.sh diff --git a/card_data/infrastructure/wait-for-rds.sh b/card_data/infrastructure/wait-for-rds.sh new file mode 100644 index 00000000..062caf8c --- /dev/null +++ b/card_data/infrastructure/wait-for-rds.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +MAX_TRIES=20 +COUNT=0 + +RDS_HOST="..rds.amazonaws.com" +RDS_PORT=5432 + +echo "Checking if RDS is available..." + +while [ $COUNT -lt $MAX_TRIES ]; do + if nc -z -w5 $RDS_HOST $RDS_PORT 2>/dev/null; then + echo "RDS is available!" + exit 0 + fi + COUNT=$((COUNT + 1)) + echo "Attempt $COUNT/$MAX_TRIES - RDS not ready yet..." + sleep 10 +done + +echo "RDS did not become available in time" +exit 1 \ No newline at end of file From 572bf86b48bd102e073ef54519c4b86c9af36cf2 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Fri, 24 Oct 2025 15:14:41 -0700 Subject: [PATCH 03/35] initial dagster config commit --- card_data/dagster.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 card_data/dagster.yaml diff --git a/card_data/dagster.yaml b/card_data/dagster.yaml new file mode 100644 index 00000000..bad89452 --- /dev/null +++ b/card_data/dagster.yaml @@ -0,0 +1,12 @@ +storage: + postgres: + postgres_db: + username: postgres + password: + env: AWS_RDS_PASSWORD + hostname: + env: AWS_RDS_HOSTNAME + db_name: postgres + port: 5432 + params: + sslmode: require \ No newline at end of file From ecb172df35475eb638a1d393f0fb51bb33a2e114 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 26 Oct 2025 10:35:48 -0700 Subject: [PATCH 04/35] adding footnotes extension --- mkdocs.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index b757902b..fd486cb9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -51,8 +51,7 @@ repo_url: https://github.com/digitalghost-dev/poke-cli markdown_extensions: - admonition - - pymdownx.superfences - - pymdownx.details + - footnotes - attr_list - pymdownx.emoji: emoji_index: !!python/name:material.extensions.emoji.twemoji @@ -61,6 +60,8 @@ markdown_extensions: anchor_linenums: true line_spans: __span pygments_lang_class: true + - pymdownx.superfences + - pymdownx.details - pymdownx.inlinehilite - pymdownx.snippets From 473d84565e30c5cf0636183581a04d8e7a2405c4 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 26 Oct 2025 10:38:51 -0700 Subject: [PATCH 05/35] updating version numbers --- .github/workflows/ci.yml | 2 +- .goreleaser.yml | 2 +- Dockerfile | 2 +- nfpm.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f20639bc..c899b4d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ on: - main env: - VERSION_NUMBER: 'v1.7.2' + VERSION_NUMBER: 'v1.7.3' DOCKERHUB_REGISTRY_NAME: 'digitalghostdev/poke-cli' AWS_REGION: 'us-west-2' diff --git a/.goreleaser.yml b/.goreleaser.yml index da490082..38286ad4 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -14,7 +14,7 @@ builds: - windows - darwin ldflags: - - -s -w -X main.version=v1.7.2 + - -s -w -X main.version=v1.7.3 archives: - formats: [ 'zip' ] diff --git a/Dockerfile b/Dockerfile index b7599edd..bb94b478 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN go mod download COPY . . -RUN go build -ldflags "-X main.version=v1.7.2" -o poke-cli . +RUN go build -ldflags "-X main.version=v1.7.3" -o poke-cli . # build 2 FROM --platform=$BUILDPLATFORM alpine:3.22 diff --git a/nfpm.yaml b/nfpm.yaml index 8957c785..9949708c 100644 --- a/nfpm.yaml +++ b/nfpm.yaml @@ -1,7 +1,7 @@ name: "poke-cli" arch: "arm64" platform: "linux" -version: "v1.7.2" +version: "v1.7.3" section: "default" version_schema: semver maintainer: "Christian S" From 60e2df1768c4db5eb941a35231dc42bea486611f Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 26 Oct 2025 10:40:26 -0700 Subject: [PATCH 06/35] updating page weight --- docs/Infrastructure_Guide/dagster.md | 44 ++++++++++++++++++++++++-- docs/Infrastructure_Guide/dbt.md | 4 +-- docs/Infrastructure_Guide/python.md | 4 +-- docs/Infrastructure_Guide/terraform.md | 4 +-- 4 files changed, 48 insertions(+), 8 deletions(-) diff --git a/docs/Infrastructure_Guide/dagster.md b/docs/Infrastructure_Guide/dagster.md index 192c20ac..d51d2067 100644 --- a/docs/Infrastructure_Guide/dagster.md +++ b/docs/Infrastructure_Guide/dagster.md @@ -1,8 +1,8 @@ --- -weight: 6 +weight: 7 --- -# 6. Dagster +# 7. Dagster !!! question "What is Dagster?" @@ -51,3 +51,43 @@ This project uses a directory named `pipelines` to store all the Dagster files: └── configuration.yml ``` +## + +## Automating Startup with `systemd` +In order to save on costs, the EC2 and RDS instances are scheduled to start and stop once each day. To automate the +starting of the Dagster webservice, `systemd`, along with a couple of shell scripts, will be used to create this +automation. + +This project, under `card_data/infrastructure/` has a file for starting the Dagster web service called `start-dagster.sh` +and is set up to be used. + + +```shell +aws rds describe-db-instances \ +--region us-west-2 \ +--query 'DBInstances[*].[DBInstanceIdentifier,Endpoint.Address,Endpoint.Port]' \ +--output table +``` + +```shell +aws secretsmanager create-secret \ + --name dagster/supabase-creds \ + --secret-string '{"password":"your_password","user":"your_user"}' \ + --region us-west-2 +``` + +```shell +nano /home/ubuntu/wait-for-rds.sh + +nano /home/ubuntu/start-dagster.sh + +sudo nano /etc/systemd/system/dagster.service +``` + +```shell +chmod +x /home/ubuntu/start-dagster.sh + +chmod +x /home/ubuntu/wait-for-rds.sh +``` + +create ` \ No newline at end of file diff --git a/docs/Infrastructure_Guide/dbt.md b/docs/Infrastructure_Guide/dbt.md index e78dcb8e..e81f2ccc 100644 --- a/docs/Infrastructure_Guide/dbt.md +++ b/docs/Infrastructure_Guide/dbt.md @@ -1,8 +1,8 @@ --- -weight: 5 +weight: 6 --- -# 5. dbt +# 6. dbt !!! question "What is dbt?" diff --git a/docs/Infrastructure_Guide/python.md b/docs/Infrastructure_Guide/python.md index d68b68e7..ec283be6 100644 --- a/docs/Infrastructure_Guide/python.md +++ b/docs/Infrastructure_Guide/python.md @@ -1,8 +1,8 @@ --- -weight: 4 +weight: 5 --- -# 4. Python +# 5. Python ## Installing uv _uv is the main package and project manager used in this project._ diff --git a/docs/Infrastructure_Guide/terraform.md b/docs/Infrastructure_Guide/terraform.md index 4a4802fa..76dd0c3b 100644 --- a/docs/Infrastructure_Guide/terraform.md +++ b/docs/Infrastructure_Guide/terraform.md @@ -1,8 +1,8 @@ --- -weight: 3 +weight: 4 --- -# 3. Terraform +# 4. Terraform !!! question "What is Terraform?" From 401c463b0f3164f5ccb79029643466017728c3ba Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 26 Oct 2025 10:40:39 -0700 Subject: [PATCH 07/35] updating test data --- testdata/main_latest_flag.golden | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testdata/main_latest_flag.golden b/testdata/main_latest_flag.golden index 35897396..5f94b689 100644 --- a/testdata/main_latest_flag.golden +++ b/testdata/main_latest_flag.golden @@ -1,6 +1,6 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ ┃ ┃ ┃ Latest available version: ┃ -┃ • v1.7.1 ┃ +┃ • v1.7.2 ┃ ┃ ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ From 6075b5a2e22ee706fc02493b510e8e7a5d3f8b06 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 26 Oct 2025 10:47:47 -0700 Subject: [PATCH 08/35] adding instructions for Elastic IPs and EventBridge --- docs/Infrastructure_Guide/aws.md | 109 ++++++++++++++++++++++++++++++- 1 file changed, 106 insertions(+), 3 deletions(-) diff --git a/docs/Infrastructure_Guide/aws.md b/docs/Infrastructure_Guide/aws.md index d64cb273..64e75203 100644 --- a/docs/Infrastructure_Guide/aws.md +++ b/docs/Infrastructure_Guide/aws.md @@ -1,8 +1,8 @@ --- -weight: 2 +weight: 3 --- -# 2. AWS +# 3. AWS Amazon Web Services was the chosen cloud vendor for hosting this project's infrastructure. @@ -21,6 +21,9 @@ Amazon Web Services was the chosen cloud vendor for hosting this project's infra * [VPC](#vpc) * [RDS](#rds) * [EC2](#ec2) +* [Elastic IPs](#elastic-ips) +* [EventBridge](#eventbridge) +* [Secrets Manager](#secrets-manager) !!! note @@ -226,4 +229,104 @@ Once connected to the virtual machine, run the following commands to get everyth * `source ~/.bashrc` - to load variables in current session. 3. Verify Dagster and Connectivity * `dg dev --host 0.0.0.0 --port 3000` - * In the browser, visit `http://:3000` \ No newline at end of file + * In the browser, visit `http://:3000` + +--- + +## Elastic IPs +An Elastic IP is a static public IPv4 address in AWS that can be assigned to an EC2 instance. + +1. Visit the [EC2 console](https://console.aws.amazon.com/ec2). +2. On the left, under **Network & Security**, click on **Elastic IPs**. +3. In the upper-right, click on **Allocate Elastic IP Address**. +4. On the configuration screen, leave all as default, provide a tag if needed, then click on **Allocate**. +5. After the static IP address is allocated, select the IP address, then on the **Actions** dropdown, and finally, select **Associate Elastic IP Address**. +6. On the next page, choose instance as the **Resource Types**. +7. Select the previously created instance and its private IP. +8. Then, click on **Associate**. + +--- + +## EventBridge +AWS [EventBridge](https://aws.amazon.com/eventbridge/) is a serverless event bus that can use rules to automatically +trigger actions—such as starting or stopping RDS and EC2 instances—based on scheduled times or specific events. + +EventBridge helps save on costs by stopping and starting the RDS and EC2 instance during off-peak hours. In the case +of this project, it would be between the hours of 11:00PM and 6:00AM PST. This is optional but recommended. + +### Create Policy +Once the role is created, a custom policy needs to be created to allow the role access to start/stop the targeted service. + +1. Visit the [IAM console home](https://us-east-1.console.aws.amazon.com/iam/). +2. On the left, click on **Policies**. +3. In the upper-right, click on **Create Policy**. +4. In the **Policy Editor** section, choose the **JSON** editor and paste in the following (Update `Action` to match the targeted service): +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:StartInstances", + "ec2:StopInstances" + ], + "Resource": "arn:aws:ec2:us-west-2:*:instance/*" + } + ] +} +``` +5. Click **Next**. On the following screen, enter a name for the policy. For example, `ec2-start-stop`. +6. Add a description and/or tags. Click on **Create Policy**. + +### Create Roles +EventBridge needs a role to connect and access other AWS resources. This project uses separate roles for each service. +The instructions below discuss the creation of a single role and a single attached policy. + +1. Visit the [IAM console home](https://us-east-1.console.aws.amazon.com/iam/). +2. On the left, click on **Roles**. +3. In the upper-right, click on **Create Role**. +4. For **Trusted Entity Type**, choose **AWS Service**. +5. For **Use Case**, search and choose **EC2**. Leave at default selected option. Click **Next**. +6. For **Permissions Policies**, search and choose the custom policy created in the previous step. +7. Enter descriptive role name. For example, `EC2-StartStop-Scheduler-Role`. +8. Enter a description and/or tags. Then, click **Create Role**. + +### Create Schedule + +1. Visit the [EventBridge console home](https://console.aws.amazon.com/events). + * Ensure AWS is in the correct region as the rest of the project's services. + * If an EventBridge schedule is created in a different region than, say, an EC2 instance, it will not find it. +2. On the right, under **Scheduler**, click on **Schedules**. +3. Click on **Create Schedule**. + +#### Specify Schedule Details + +1. Specify Schedule Detail Page + * Provide a name and description. + * For **Schedule Pattern**, choose **Recurring Schedule**. + * Choose the correct timezone and select **Cron-Base Schedule**. + * The `cron` expression only needs to run once a day. So, a schedule for starting the instance could be `0 6 * * ? *` - (run at 6am). + * Add a **Flexible Time Window** if needed. + * The **Timeframe** section can be left blank. Click **Next**. +2. Select Target Page + * Under **Target Detail**, select **All APIs** and search for and select EC2. + * In the API search box, search for `StartInstances` or `StopInstances`. + * In the next window, grab the **Instance ID** for the EC2 instead created earlier and paste it in to the JSON input. + ```json + { + "InstanceIds": [ + "i-01dwf82r21c9da71f" + ] + } + ``` + * Click **Next**. +3. Settings Page + * Enable schedule state. + * Choose `NONE` for **Action after Schedule Completion**. + * Choose `None` for **Retry Policy and DLQ** + * Customize encryption if needed. + * In the **Permissions** section, choose **Use Existing Role**, then select the previously created role associated with this schedule’s service. + * Click **Next**. +4. Review and Create Schedule Page + * Review all the configuration changes then click **Create Schedule**. \ No newline at end of file From 85d5c5c9bc474da0688252e16c08c7da07c62273 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Tue, 28 Oct 2025 07:29:14 -0700 Subject: [PATCH 09/35] adding project creation steps --- docs/Infrastructure_Guide/supabase.md | 29 ++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/docs/Infrastructure_Guide/supabase.md b/docs/Infrastructure_Guide/supabase.md index 72536afa..f4be559e 100644 --- a/docs/Infrastructure_Guide/supabase.md +++ b/docs/Infrastructure_Guide/supabase.md @@ -1,10 +1,33 @@ --- -weight: 7 +weight: 2 --- -# 7. Supabase +# 2. Supabase ## Create an Account Visit the Supabase [sign-up page](https://supabase.com/dashboard/sign-up) to create an account. -Signing in with GitHub is the easiest method. \ No newline at end of file +Signing in with GitHub is the easiest method. + +## Retrieve PostgreSQL Connection String + +1. Create an organization. + * Provide an organization name. + * Type of organization can be **Personal**. + * Free plan is enough. + * Click **Create**. +2. Create a new project. + * Provide a project name. + * Create or generate a database password. + * Select the best region closest to users. + * The **Security Options** and **Advanced Configuration** options can be left to default. + * Click **Create**. +3. On the **Project Overview** page, in the top part of the page, click on **Connect**. +4. Under the **Connection String** section, change the **Method** dropdown to **Transaction Pooler**. +5. The connection string will be provided in the following format: +```shell +postgresql://postgres.[USERNAME]:[YOUR-PASSWORD]@aws-0-us-east-2.pooler.supabase.com:6543/postgres +``` +6. Note the connection string for later instructions such as creating a secret of the string in AWS Secrets Manager.[^1] + +[^1]: Used in section: [3. AWS // Secrets Manager](aws.md#secrets-manager). \ No newline at end of file From 8abe5245c9a2c0e0efadca4d1df01099088ad1c7 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Tue, 28 Oct 2025 23:27:14 -0700 Subject: [PATCH 10/35] adding cloud deployment page --- docs/Infrastructure_Guide/cloud-deployment.md | 170 ++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 docs/Infrastructure_Guide/cloud-deployment.md diff --git a/docs/Infrastructure_Guide/cloud-deployment.md b/docs/Infrastructure_Guide/cloud-deployment.md new file mode 100644 index 00000000..2fe60965 --- /dev/null +++ b/docs/Infrastructure_Guide/cloud-deployment.md @@ -0,0 +1,170 @@ +--- +weight: 5 +--- + +# 5 // Cloud Deployment +Once the services are created and configured in AWS, the virtual machine can be set up with the needed +tools/libraries to run the data pipelines in Dagster. + +## Installing Tools and Libraries +Connect to the virtual machine and run the following commands to get everything set up: + +1. Install AWS CLI + * Download via `curl`: + ```shell + curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" + ``` + * Install `unzip` program: + ```shell + sudo apt install unzip + ``` + * Unzip the compressed folder: + ```shell + unzip awscliv2.zip + ``` + * Run the installer: + ```shell + sudo ./aws/install + ``` + * Set the default region: + ```shell + export AWS_DEFAULT_REGION=us-west-2 + # or set it to ~./bashrc file + echo 'export AWS_DEFAULT_REGION=us-east-1' >> ~/.bashrc + source ~/.bashrc + ``` + * Run an `aws` command such as `secretsmanager` to verify AWS connectivity: + ```shell + aws secretsmanager list-secrets + ``` +2. Clone Repository + * Create a new directory: + ```shell + git init + ``` + * Change into new directory: + ```shell + cd + ``` + * Add the remote repository: + ```shell + git remote add -f origin https://github.com/digitalghost-dev/poke-cli/ + ``` + * Edit the `git` config file to turn on sparse checkout: + ```shell + git config core.sparseCheckout true + ``` + * Tell `git` which directory to check out. Then, pull that directory. + ```shell + echo "card_data/" >> .git/info/sparse-checkout + ``` + * Pull the repo into the local directory + ```shell + git pull origin main + ``` + * Verify that `card_data/` directory was created. + ```shell + ls + ``` +3. Install Tools + * Install `uv` for Python: + ```shell + curl -LsSf https://astral.sh/uv/0.7.21/install.sh | sh + ``` + * Add to `PATH`: + ```shell + `source $HOME/.local/bin/env + ``` + * Install libraries from `pyproject.toml` file: + ```shell + uv sync + ``` + * Activate virtual environment: + ```shell + source .venv/bin/activate + ``` + * Create `dagster.yaml` file: + ```bash + mkdir -p ~/.dagster && cat > ~/.dagster/dagster.yaml << 'EOF' + storage: + postgres: + postgres_db: + username: postgres + password: "rds-password" + hostname: "rds-hostname" + db_name: postgres + port: 5432 + params: + sslmode: require + EOF + ``` + * Set environment variables: + * `echo 'export DAGSTER_HOME="$HOME/.dagster"' >> ~/.bashrc` + * `echo 'export SUPABASE_USER=""' >> ~/.bashrc` + * `echo 'export SUPABASE_PASSWORD=""' >> ~/.bashrc` + * `source ~/.bashrc` - to load variables in current session. +4. Verify Dagster and Connectivity + * `dg dev --host 0.0.0.0 --port 3000` + * In the browser, visit `http://:3000` + +--- + +## Automating Startup with `systemd` +_Optional_ + +In order to save on costs, the EC2 and RDS instances are scheduled to start and +stop once each day with AWS EventBridge. To automate the starting of the Dagster webservice, +`systemd`, along with a couple of shell scripts, will be used to create this automation. + +### Service Files + +The `card_data/infrastructure/` directory has the following files: + +1. `dagster.service` - the main `systemd` file for defining the Dagster service and environment. +2. `wait-for-rds.sh` - stored as `ExecStartPre` in `dagster.service` to check if the RDS instance is available. +3. `start-dagster.sh` - If the RDS instance is ready, this will run and start the Dagster web service. + +Although the files are included in this repository, they need to be moved or created in a specific directory on +the Linux virtual machine. + +#### Move Files + +```shell +/home/ubuntu/wait-for-rds.sh + +/home/ubuntu/start-dagster.sh + +/etc/systemd/system/dagster.service +``` + +#### Create Files +First, create `dagster.service`: + +* Run `nano /etc/systemd/system/dagster.service` then copy and paste the following code in the editor: + +??? note "dagster.service" + + ```bash + [Unit] + Description=Dagster Development Server + After=network-online.target + Wants=network-online.target + + [Service] + Type=simple + User=ubuntu + WorkingDirectory=/home/ubuntu/card_data/card_data + Environment="AWS_DEFAULT_REGION=us-west-2" + Environment="PATH=/home/ubuntu/card_data/card_data/.venv/bin:/usr/local/bin:/usr/bin:/bin" + ExecStartPre=/home/ubuntu/wait-for-rds.sh + ExecStart=/home/ubuntu/start-dagster.sh + Restart=on-failure + RestartSec=10 + StandardOutput=journal + StandardError=journal + + [Install] + WantedBy=multi-user.target + ``` + +### Start Service From e9d9a92dddcf455faa3c219c9b2062507f2ed35c Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Wed, 29 Oct 2025 10:19:43 -0700 Subject: [PATCH 11/35] removing files --- docs/Infrastructure_Guide/dagster.md | 93 ---------------------------- docs/Infrastructure_Guide/dbt.md | 61 ------------------ docs/Infrastructure_Guide/python.md | 23 ------- 3 files changed, 177 deletions(-) delete mode 100644 docs/Infrastructure_Guide/dagster.md delete mode 100644 docs/Infrastructure_Guide/dbt.md delete mode 100644 docs/Infrastructure_Guide/python.md diff --git a/docs/Infrastructure_Guide/dagster.md b/docs/Infrastructure_Guide/dagster.md deleted file mode 100644 index d51d2067..00000000 --- a/docs/Infrastructure_Guide/dagster.md +++ /dev/null @@ -1,93 +0,0 @@ ---- -weight: 7 ---- - -# 7. Dagster - -!!! question "What is Dagster?" - - Dagster is an open-source data orchestration tool that helps you build, run, and monitor your data pipelines. - It’s designed to make working with data workflows more reliable and maintainable, giving you clear visibility - into each step and making it easier to catch issues before they cause problems. - - View more [about Dagster](https://dagster.io/platform-overview) - -## Installation -Dagster and its components needed for the project can be installed with `uv`: ` - -```bash -uv add dagster dagster-webserver dagster-dg-cli dagster-postgres>=0.27.3 dagster-dbt -``` - -## Project Layout -In my experience, Dagster needed a specific directory structure in order for the program to find all necessary files. -This project uses a directory named `pipelines` to store all the Dagster files: - -``` -. -└── pipelines/ - ├── defs/ - │ ├── extract/ - │ │ └── extract_data.py - │ ├── load/ - │ │ └── load_data.py - │ └── transformation/ - │ └── transform_data.py - ├── poke_cli_dbt/ - │ ├── logs - │ ├── macros/ - │ │ ├── create_relationships.sql - │ │ └── create_rls.sql - │ ├── models/ - │ │ ├── cards.sql - │ │ ├── series.sql - │ │ ├── sets.sql - │ │ └── sources.yml - │ ├── target - │ ├── dbt_project.yml - │ └── profiles.yml - └── soda/ - ├── checks.yml - └── configuration.yml -``` - -## - -## Automating Startup with `systemd` -In order to save on costs, the EC2 and RDS instances are scheduled to start and stop once each day. To automate the -starting of the Dagster webservice, `systemd`, along with a couple of shell scripts, will be used to create this -automation. - -This project, under `card_data/infrastructure/` has a file for starting the Dagster web service called `start-dagster.sh` -and is set up to be used. - - -```shell -aws rds describe-db-instances \ ---region us-west-2 \ ---query 'DBInstances[*].[DBInstanceIdentifier,Endpoint.Address,Endpoint.Port]' \ ---output table -``` - -```shell -aws secretsmanager create-secret \ - --name dagster/supabase-creds \ - --secret-string '{"password":"your_password","user":"your_user"}' \ - --region us-west-2 -``` - -```shell -nano /home/ubuntu/wait-for-rds.sh - -nano /home/ubuntu/start-dagster.sh - -sudo nano /etc/systemd/system/dagster.service -``` - -```shell -chmod +x /home/ubuntu/start-dagster.sh - -chmod +x /home/ubuntu/wait-for-rds.sh -``` - -create ` \ No newline at end of file diff --git a/docs/Infrastructure_Guide/dbt.md b/docs/Infrastructure_Guide/dbt.md deleted file mode 100644 index e81f2ccc..00000000 --- a/docs/Infrastructure_Guide/dbt.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -weight: 6 ---- - -# 6. dbt - -!!! question "What is dbt?" - - dbt (data build tool) is a command-line tool that enables data analysts and engineers to transform data - in warehouses using SQL. The tool allows users to write modular SQL queries as "models" that build upon - each other, automatically managing dependencies and enabling version control, testing, and documentation - of data transformations. dbt compiles SQL models into executable queries and runs them in the proper order, - turning raw data into analysis-ready datasets. - - View more [about dbt](https://www.getdbt.com/product/what-is-dbt) - -## Installation & Initialization - -Install with `uv`: -```bash -uv add dbt -``` - -Initialize a `dbt` project in the `card_data` directory: -```bash -dbt init -``` - -Follow the prompts to finish setting up the `dbt` project. - -## Models -Models are the pieces of SQL code that run when using that `dbt build` command that _build_ the -tables to the destination schema. In this project, that would the `public` schema in the PostgreSQL -database on Supabase. - -The `public` schema is the public facing schema that exposes the API to the data in the tables. - -## Sources -Create a `source.yml` file under the `models/` directory. More info on [sources here](https://docs.getdbt.com/docs/build/sources). - -This file is used to declare and configure the raw data sources. These tables are the foundation for -the dbt models but are not managed by dbt itself. - -For example: -```yaml -sources: - - name: staging - description: "Staging schema containing raw data loaded from extract pipeline" - tables: - - name: series - description: "Pokemon card series data" - columns: - - name: id - description: "Unique series identifier" - - name: name - description: "Series name" - - name: logo - description: "Series logo URL" -``` - -The above `yml` defines the structure for the raw `series` table from the `staging` schema. \ No newline at end of file diff --git a/docs/Infrastructure_Guide/python.md b/docs/Infrastructure_Guide/python.md deleted file mode 100644 index ec283be6..00000000 --- a/docs/Infrastructure_Guide/python.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -weight: 5 ---- - -# 5. Python - -## Installing uv -_uv is the main package and project manager used in this project._ - -Learn more about [uv](https://docs.astral.sh/uv/). - -1. Install via their [installation script](https://docs.astral.sh/uv/getting-started/installation/): - ```bash - curl -LsSf https://astral.sh/uv/install.sh | sh - ``` - or brew: - ```bash - brew install uv - ``` -2. Install Python with `uv`: - ```bash - uv python install 3.12 - ``` \ No newline at end of file From ea93523bdfea7cd7c16911409fc0c8122dcfbec4 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Wed, 29 Oct 2025 10:20:12 -0700 Subject: [PATCH 12/35] adding file path comment --- card_data/infrastructure/dagster.service | 1 + 1 file changed, 1 insertion(+) diff --git a/card_data/infrastructure/dagster.service b/card_data/infrastructure/dagster.service index 64765859..e2cffb81 100644 --- a/card_data/infrastructure/dagster.service +++ b/card_data/infrastructure/dagster.service @@ -1,3 +1,4 @@ +# /etc/systemd/system/dagster.service [Unit] Description=Dagster Development Server After=network-online.target From 6814339b36af30282daf0f7ed2cd32e18ecb5170 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Wed, 29 Oct 2025 10:25:39 -0700 Subject: [PATCH 13/35] updating weights and headers --- docs/Infrastructure_Guide/index.md | 2 +- docs/Infrastructure_Guide/supabase.md | 4 ++-- docs/Infrastructure_Guide/terraform.md | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/Infrastructure_Guide/index.md b/docs/Infrastructure_Guide/index.md index 16290f14..115afb55 100644 --- a/docs/Infrastructure_Guide/index.md +++ b/docs/Infrastructure_Guide/index.md @@ -2,7 +2,7 @@ weight: 1 --- -# 1. Overview +# 1 // Overview This section serves as a knowledge base for the project’s backend infrastructure. It was created for a few purposes: diff --git a/docs/Infrastructure_Guide/supabase.md b/docs/Infrastructure_Guide/supabase.md index f4be559e..a9cf547d 100644 --- a/docs/Infrastructure_Guide/supabase.md +++ b/docs/Infrastructure_Guide/supabase.md @@ -2,7 +2,7 @@ weight: 2 --- -# 2. Supabase +# 2 // Supabase ## Create an Account @@ -28,6 +28,6 @@ Signing in with GitHub is the easiest method. ```shell postgresql://postgres.[USERNAME]:[YOUR-PASSWORD]@aws-0-us-east-2.pooler.supabase.com:6543/postgres ``` -6. Note the connection string for later instructions such as creating a secret of the string in AWS Secrets Manager.[^1] +6. Note the connection string for later instructions such as creating a secret of the string in AWS Secrets Manager[^1]. [^1]: Used in section: [3. AWS // Secrets Manager](aws.md#secrets-manager). \ No newline at end of file diff --git a/docs/Infrastructure_Guide/terraform.md b/docs/Infrastructure_Guide/terraform.md index 76dd0c3b..c0db5d1d 100644 --- a/docs/Infrastructure_Guide/terraform.md +++ b/docs/Infrastructure_Guide/terraform.md @@ -1,8 +1,8 @@ --- -weight: 4 +weight: 6 --- -# 4. Terraform +# 6 // Terraform !!! question "What is Terraform?" From 4d2425d0b9d1691c7ec064ad67c5426fd45710ca Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Wed, 29 Oct 2025 17:04:03 -0700 Subject: [PATCH 14/35] updating version numbers --- card_data/pipelines/poke_cli_dbt/dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/card_data/pipelines/poke_cli_dbt/dbt_project.yml b/card_data/pipelines/poke_cli_dbt/dbt_project.yml index 4c7f2162..74a0c6c4 100644 --- a/card_data/pipelines/poke_cli_dbt/dbt_project.yml +++ b/card_data/pipelines/poke_cli_dbt/dbt_project.yml @@ -1,5 +1,5 @@ name: 'poke_cli_dbt' -version: '1.7.2' +version: '1.7.3' profile: 'poke_cli_dbt' From b720f6bef84c50ef4ba25abbdbef98a5fa83d8af Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Thu, 30 Oct 2025 09:30:04 -0700 Subject: [PATCH 15/35] adding custom .svg image --- docs/stylesheets/extra.css | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 docs/stylesheets/extra.css diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css new file mode 100644 index 00000000..8d5cd906 --- /dev/null +++ b/docs/stylesheets/extra.css @@ -0,0 +1,20 @@ +:root { + --md-admonition-icon--container: url('data:image/svg+xml;charset=utf-8,') +} + +.md-typeset .admonition.container, +.md-typeset details.container { + border-color: rgb(52, 152, 219); +} + +.md-typeset .container > .admonition-title, +.md-typeset .container > summary { + background-color: rgba(52, 152, 219, 0.1); +} + +.md-typeset .container > .admonition-title::before, +.md-typeset .container > summary::before { + background-color: rgb(52, 152, 219); + -webkit-mask-image: var(--md-admonition-icon--container); + mask-image: var(--md-admonition-icon--container); +} \ No newline at end of file From 343a8fb0eabb4f95f5c6ffb8fc926dca750935c6 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Thu, 30 Oct 2025 09:30:28 -0700 Subject: [PATCH 16/35] adding stylesheet extension --- mkdocs.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index fd486cb9..2f856f13 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -73,4 +73,7 @@ extra: plugins: - search - mkdocs-nav-weight: - section_renamed: false \ No newline at end of file + section_renamed: false + +extra_css: + - stylesheets/extra.css \ No newline at end of file From 031ed58deac784532c7b03f78ea460d0f5359daf Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Thu, 30 Oct 2025 09:31:23 -0700 Subject: [PATCH 17/35] initial commit --- card_data/infrastructure/start-dagster.sh | 42 +++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 card_data/infrastructure/start-dagster.sh diff --git a/card_data/infrastructure/start-dagster.sh b/card_data/infrastructure/start-dagster.sh new file mode 100644 index 00000000..3f6afae1 --- /dev/null +++ b/card_data/infrastructure/start-dagster.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Fetch secrets from AWS Secrets Manager +SUPABASE_SECRETS=$(aws secretsmanager get-secret-value \ + --secret-id dagster/supabase-creds \ + --region us-west-2 \ + --query SecretString \ + --output text) + +AWS_RDS_SECRETS_PW=$(aws secretsmanager get-secret-value \ + --secret-id '' \ + --region us-west-2 \ + --query SecretString \ + --output text) + +AWS_RDS_SECRETS_HN=$(aws secretsmanager get-secret-value \ + --secret-id rds-hostname \ + --region us-west-2 \ + --query SecretString \ + --output text) + +# Extract values +SUPABASE_PASSWORD=$(echo "$SUPABASE_SECRETS" | jq -r '.password') +export SUPABASE_PASSWORD + +SUPABASE_USER=$(echo "$SUPABASE_SECRETS" | jq -r '.user') +export SUPABASE_USER + +AWS_RDS_PASSWORD=$(echo "$AWS_RDS_SECRETS_PW" | jq -r '.password') +export AWS_RDS_PASSWORD + +AWS_RDS_HOSTNAME=$(echo "$AWS_RDS_SECRETS_HN" | jq -r '.hostname') +export AWS_RDS_HOSTNAME + +DAGSTER_HOME=/home/ubuntu/card_data/card_data/ +export DAGSTER_HOME + +# Activate the virtual environment +source /home/ubuntu/card_data/card_data/.venv/bin/activate + +# Start Dagster +exec dg dev --host 0.0.0.0 --port 3000 \ No newline at end of file From 7a4e8cea6f49c92c44dfdedbe334de50a684bebd Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Fri, 31 Oct 2025 22:43:01 -0700 Subject: [PATCH 18/35] updating version numbers --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c952c057..1f939c28 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ pokemon-logo

Pokémon CLI

version-label - docker-image-size + docker-image-size ci-status-badge
@@ -94,11 +94,11 @@ Cloudsmith is a fully cloud-based service that lets you easily create, store, an 3. Choose how to interact with the container: * Run a single command and exit: ```bash - docker run --rm -it digitalghostdev/poke-cli:v1.7.2 [subcommand] flag] + docker run --rm -it digitalghostdev/poke-cli:v1.7.3 [subcommand] flag] ``` * Enter the container and use its shell: ```bash - docker run --rm -it --name poke-cli --entrypoint /bin/sh digitalghostdev/poke-cli:v1.7.2 -c "cd /app && exec sh" + docker run --rm -it --name poke-cli --entrypoint /bin/sh digitalghostdev/poke-cli:v1.7.3 -c "cd /app && exec sh" # placed into the /app directory, run the program with './poke-cli' # example: ./poke-cli ability swift-swim ``` From c0fc72f07067585df2c5607f8795d4ca47dab26f Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sat, 1 Nov 2025 20:47:35 -0700 Subject: [PATCH 19/35] Updating to Go v1.24.6 (#192) --- .github/workflows/ci.yml | 2 +- .github/workflows/go_test.yml | 2 +- Dockerfile | 2 +- go.mod | 4 ++-- go.sum | 26 ++++++++++++++++++++++++-- 5 files changed, 29 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c899b4d6..68ff96c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,7 +73,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '1.24.5' + go-version: '1.24.6' - name: Build Go Binary env: diff --git a/.github/workflows/go_test.yml b/.github/workflows/go_test.yml index 7922a97d..03f60fe5 100644 --- a/.github/workflows/go_test.yml +++ b/.github/workflows/go_test.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '1.24.5' + go-version: '1.24.6' - name: Install dependencies run: | diff --git a/Dockerfile b/Dockerfile index bb94b478..3b3f34f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # build 1 -FROM golang:1.24.5-alpine3.22 AS build +FROM golang:1.24.6-alpine3.22 AS build WORKDIR /app diff --git a/go.mod b/go.mod index ae183f2e..b87ff272 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/digitalghost-dev/poke-cli -go 1.24.5 +go 1.24.6 require ( github.com/charmbracelet/bubbles v0.21.0 @@ -13,6 +13,7 @@ require ( github.com/disintegration/imaging v1.6.2 github.com/stretchr/testify v1.10.0 golang.org/x/text v0.27.0 + modernc.org/sqlite v1.39.0 ) require ( @@ -49,7 +50,6 @@ require ( modernc.org/libc v1.66.3 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect - modernc.org/sqlite v1.39.0 // indirect ) // v1.3.4 was pushed as test and not a real version. diff --git a/go.sum b/go.sum index 688379a3..2d84ed27 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= @@ -79,13 +81,13 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.28.0 h1:gdem5JW1OLS4FbkWgLO+7ZeFzYtL3xClb97GaUzYMFE= golang.org/x/image v0.28.0/go.mod h1:GUJYXtnGKEUgggyzh+Vxt+AviiCcyiwpsl8iQ8MvwGY= +golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= +golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -95,15 +97,35 @@ golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= +golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.26.2 h1:991HMkLjJzYBIfha6ECZdjrIYz2/1ayr+FL8GN+CNzM= +modernc.org/cc/v4 v4.26.2/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU= +modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE= +modernc.org/fileutil v1.3.8 h1:qtzNm7ED75pd1C7WgAGcK4edm4fvhtBsEiI/0NQ54YM= +modernc.org/fileutil v1.3.8/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= modernc.org/libc v1.66.3 h1:cfCbjTUcdsKyyZZfEUKfoHcP3S0Wkvz3jgSzByEWVCQ= modernc.org/libc v1.66.3/go.mod h1:XD9zO8kt59cANKvHPXpx7yS2ELPheAey0vjIuZOhOU8= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= modernc.org/sqlite v1.39.0 h1:6bwu9Ooim0yVYA7IZn9demiQk/Ejp0BtTjBWFLymSeY= modernc.org/sqlite v1.39.0/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= From 7f4f7918fafc4cf3b2d1df1e1b8e3d6544651820 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 2 Nov 2025 11:20:16 -0800 Subject: [PATCH 20/35] updating secret name --- card_data/infrastructure/start-dagster.sh | 2 +- card_data/pipelines/utils/secret_retriever.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/card_data/infrastructure/start-dagster.sh b/card_data/infrastructure/start-dagster.sh index 3f6afae1..24e9e3a4 100644 --- a/card_data/infrastructure/start-dagster.sh +++ b/card_data/infrastructure/start-dagster.sh @@ -2,7 +2,7 @@ # Fetch secrets from AWS Secrets Manager SUPABASE_SECRETS=$(aws secretsmanager get-secret-value \ - --secret-id dagster/supabase-creds \ + --secret-id supabase \ --region us-west-2 \ --query SecretString \ --output text) diff --git a/card_data/pipelines/utils/secret_retriever.py b/card_data/pipelines/utils/secret_retriever.py index 62ca7877..598eb3a7 100644 --- a/card_data/pipelines/utils/secret_retriever.py +++ b/card_data/pipelines/utils/secret_retriever.py @@ -10,7 +10,7 @@ def fetch_secret() -> str: cache_config = SecretCacheConfig() cache = SecretCache(config=cache_config, client=client) - secret = cache.get_secret_string("supabase-data") + secret = cache.get_secret_string("supabase") # convert to dictionary secret_dict = json.loads(secret) From 36d3d4b2876c2374396bfd1246116505df3c875b Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 2 Nov 2025 11:38:40 -0800 Subject: [PATCH 21/35] adding systemd instructions --- docs/Infrastructure_Guide/cloud-deployment.md | 150 ++++++++++++++++-- 1 file changed, 141 insertions(+), 9 deletions(-) diff --git a/docs/Infrastructure_Guide/cloud-deployment.md b/docs/Infrastructure_Guide/cloud-deployment.md index 2fe60965..7dd07c06 100644 --- a/docs/Infrastructure_Guide/cloud-deployment.md +++ b/docs/Infrastructure_Guide/cloud-deployment.md @@ -83,7 +83,7 @@ Connect to the virtual machine and run the following commands to get everything ```shell source .venv/bin/activate ``` - * Create `dagster.yaml` file: + * Create `dagster.yaml` file (replace with correct password and hostname): ```bash mkdir -p ~/.dagster && cat > ~/.dagster/dagster.yaml << 'EOF' storage: @@ -127,24 +127,30 @@ The `card_data/infrastructure/` directory has the following files: Although the files are included in this repository, they need to be moved or created in a specific directory on the Linux virtual machine. -#### Move Files +#### Copy Files +Copy or move the files from the checked out repository to the proper directory on the Linux machine (_the files must first +be edited to match project specific configuration. Such as the proper RDS instance name in `wait-for-rds.sh`_): ```shell -/home/ubuntu/wait-for-rds.sh +cp card_data/card_data/infrastructure/wait-for-rds.sh /home/ubuntu/ -/home/ubuntu/start-dagster.sh +cp card_data/card_data/infrastructure/start-dagster.sh /home/ubuntu/ -/etc/systemd/system/dagster.service +cp card_data/card_data/infrastructure/dagster.service /etc/systemd/system/ ``` #### Create Files -First, create `dagster.service`: +The files can also be recreated. Update the files below with project specific configuration then +run the `cat` or `tee` commands listed below. -* Run `nano /etc/systemd/system/dagster.service` then copy and paste the following code in the editor: +First, create `dagster.service` -??? note "dagster.service" +* Run the following shell command to create the file (_edit any differing details such as AWS region_): - ```bash +??? container "dagster.service" + + ```shell + sudo tee /etc/systemd/system/dagster.service > /dev/null << 'EOF' [Unit] Description=Dagster Development Server After=network-online.target @@ -165,6 +171,132 @@ First, create `dagster.service`: [Install] WantedBy=multi-user.target + EOF + && echo "File created successfully" + ``` + +Second, create `wait-for-rds.sh` + +* Retrieve RDS instance name: + ```shell + aws rds describe-db-instances \ + --region us-west-2 \ + --query 'DBInstances[*].[DBInstanceIdentifier,Endpoint.Address,Endpoint.Port]' \ + --output table + ``` + +* Run the following shell command to create the file (_replace with correct instance id_): + +??? container "wait-for-rds.sh" + + ```shell + cat > /home/ubuntu/wait-for-rds.sh << 'EOF' + #!/bin/bash + + MAX_TRIES=20 + COUNT=0 + + RDS_HOST="..rds.amazonaws.com" + RDS_PORT=5432 + + echo "Checking if RDS is available..." + + while [ $COUNT -lt $MAX_TRIES ]; do + if nc -z -w5 $RDS_HOST $RDS_PORT 2>/dev/null; then + echo "RDS is available!" + exit 0 + fi + COUNT=$((COUNT + 1)) + echo "Attempt $COUNT/$MAX_TRIES - RDS not ready yet..." + sleep 10 + done + + echo "RDS did not become available in time" + exit 1 + EOF + ``` + +Last, create `start-dagster.sh` + +* Retrieve RDS secret name from Secrets Manager. AWS auto-creates a secret for RDS. + ```shell + aws secretsmanager list-secrets \ + --filters Key=name,Values=rds\! \ + | jq -r '.SecretList[].Name' + ``` + +??? container "start-dagster.sh" + + ```shell + cat > /home/ubuntu/start-dagster.sh << 'EOF' + #!/bin/bash + + # Fetch secrets from AWS Secrets Manager + SUPABASE_SECRETS=$(aws secretsmanager get-secret-value \ + --secret-id supabase \ + --region us-west-2 \ + --query SecretString \ + --output text) + + AWS_RDS_SECRETS_PW=$(aws secretsmanager get-secret-value \ + --secret-id '' \ + --region us-west-2 \ + --query SecretString \ + --output text) + + AWS_RDS_SECRETS_HN=$(aws secretsmanager get-secret-value \ + --secret-id rds-hostname \ + --region us-west-2 \ + --query SecretString \ + --output text) + + # Extract values + SUPABASE_PASSWORD=$(echo "$SUPABASE_SECRETS" | jq -r '.password') + export SUPABASE_PASSWORD + + SUPABASE_USER=$(echo "$SUPABASE_SECRETS" | jq -r '.user') + export SUPABASE_USER + + AWS_RDS_PASSWORD=$(echo "$AWS_RDS_SECRETS_PW" | jq -r '.password') + export AWS_RDS_PASSWORD + + AWS_RDS_HOSTNAME=$(echo "$AWS_RDS_SECRETS_HN" | jq -r '.hostname') + export AWS_RDS_HOSTNAME + + DAGSTER_HOME=/home/ubuntu/card_data/card_data/ + export DAGSTER_HOME + + # Activate the virtual environment + source /home/ubuntu/card_data/card_data/.venv/bin/activate + + # Start Dagster + exec dg dev --host 0.0.0.0 --port 3000 + EOF ``` ### Start Service + +Apply, enable on boot, and start the service: + +```shell +# Reload systemd to recognize the new service +sudo systemctl daemon-reload + +# Enable it to start on boot +sudo systemctl enable dagster.service + +# Start/stop +sudo systemctl start dagster.service +``` + +Show the status of service running: + +```shell +sudo systemctl status dagster.service +``` + +View live logs: + +```shell +sudo journalctl -u dagster.service -f +``` \ No newline at end of file From 035d6a88bab6075998a4c8f5e675f33e85df951b Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 2 Nov 2025 11:45:05 -0800 Subject: [PATCH 22/35] initial commit --- docs/Infrastructure_Guide/local-deployment.md | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 docs/Infrastructure_Guide/local-deployment.md diff --git a/docs/Infrastructure_Guide/local-deployment.md b/docs/Infrastructure_Guide/local-deployment.md new file mode 100644 index 00000000..0da41cb8 --- /dev/null +++ b/docs/Infrastructure_Guide/local-deployment.md @@ -0,0 +1,137 @@ +--- +weight: 3 +--- + +# 3 // Local Deployment +This page explains how to set up Python, dbt, Dagster, and other tools for running the data pipelines +locally. The [4. AWS](aws.md) section will show how to deploy this solution on the cloud. + +## Python + +### Installing uv +_uv is the main package and project manager used in this project._ + +Learn more about [uv](https://docs.astral.sh/uv/). + +1. Install via their [installation script](https://docs.astral.sh/uv/getting-started/installation/): + ```bash + curl -LsSf https://astral.sh/uv/install.sh | sh + ``` + or brew: + ```bash + brew install uv + ``` +2. Install Python with `uv`: + ```bash + uv python install 3.12 + ``` + +## dbt + +!!! question "What is dbt?" + + dbt (data build tool) is a command-line tool that enables data analysts and engineers to transform data + in warehouses using SQL. The tool allows users to write modular SQL queries as "models" that build upon + each other, automatically managing dependencies and enabling version control, testing, and documentation + of data transformations. dbt compiles SQL models into executable queries and runs them in the proper order, + turning raw data into analysis-ready datasets. + + View more [about dbt](https://www.getdbt.com/product/what-is-dbt) + +### Installation & Initialization + +Install with `uv`: +```bash +uv add dbt +``` + +Initialize a `dbt` project in the `card_data` directory: +```bash +dbt init +``` + +Follow the prompts to finish setting up the `dbt` project. + +### Models +Models are the pieces of SQL code that run when using that `dbt build` command that _build_ the +tables to the destination schema. In this project, that would the `public` schema in the PostgreSQL +database on Supabase. + +The `public` schema is the public facing schema that exposes the API to the data in the tables. + +### Sources +Create a `source.yml` file under the `models/` directory. +More info on [sources here](https://docs.getdbt.com/docs/build/sources). + +This file is used to declare and configure the raw data sources. These tables are the foundation for +the dbt models but are not managed by dbt itself. + +For example: +```yaml +sources: + - name: staging + description: "Staging schema containing raw data loaded from extract pipeline" + tables: + - name: series + description: "Pokemon card series data" + columns: + - name: id + description: "Unique series identifier" + - name: name + description: "Series name" + - name: logo + description: "Series logo URL" +``` + +The above `yml` defines the structure for the raw `series` table from the `staging` schema. + +--- + +## Dagster + +!!! question "What is Dagster?" + + Dagster is an open-source data orchestration tool that helps you build, run, and monitor your data pipelines. + It’s designed to make working with data workflows more reliable and maintainable, giving you clear visibility + into each step and making it easier to catch issues before they cause problems. + + View more [about Dagster](https://dagster.io/platform-overview) + +### Installation +Dagster and its components needed for the project can be installed with `uv`: ` + +```bash +uv add dagster dagster-webserver dagster-dg-cli dagster-postgres>=0.27.3 dagster-dbt +``` + +## Project Layout +In my experience, Dagster needed a specific directory structure in order for the program to find all necessary files. +This project uses a directory named `pipelines` to store all the Dagster files: + +``` +. +└── pipelines/ + ├── defs/ + │ ├── extract/ + │ │ └── extract_data.py + │ ├── load/ + │ │ └── load_data.py + │ └── transformation/ + │ └── transform_data.py + ├── poke_cli_dbt/ + │ ├── logs + │ ├── macros/ + │ │ ├── create_relationships.sql + │ │ └── create_rls.sql + │ ├── models/ + │ │ ├── cards.sql + │ │ ├── series.sql + │ │ ├── sets.sql + │ │ └── sources.yml + │ ├── target + │ ├── dbt_project.yml + │ └── profiles.yml + └── soda/ + ├── checks.yml + └── configuration.yml +``` \ No newline at end of file From 4e7179eb8068e902fb7d1d01d8c8993abd667516 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 2 Nov 2025 11:48:22 -0800 Subject: [PATCH 23/35] adding secrets manager section --- docs/Infrastructure_Guide/aws.md | 94 ++++++++++++++++---------------- 1 file changed, 48 insertions(+), 46 deletions(-) diff --git a/docs/Infrastructure_Guide/aws.md b/docs/Infrastructure_Guide/aws.md index 64e75203..74196e75 100644 --- a/docs/Infrastructure_Guide/aws.md +++ b/docs/Infrastructure_Guide/aws.md @@ -1,9 +1,10 @@ --- -weight: 3 +weight: 4 --- -# 3. AWS -Amazon Web Services was the chosen cloud vendor for hosting this project's infrastructure. +# 4 // AWS +Amazon Web Services was the chosen cloud vendor for hosting this project's infrastructure. This takes the +local deployment and moves it into the cloud. !!! question "What is AWS?" @@ -174,7 +175,7 @@ AWS EC2 (Elastic Compute Cloud) is a cloud service that provides resizable virtu 8. Under **Configure Storage**, leave as default. 9. Under **Advanced Details**, lease as default. -### Connect to Instance +### Verify Connection to Instance 1. First, configure a trusted connection to the previously created RDS instance. * Visit the [RDS console](https://console.aws.amazon.com/rds/home). * Click on the RDS instance previously created. @@ -188,49 +189,10 @@ AWS EC2 (Elastic Compute Cloud) is a cloud service that provides resizable virtu * Instructions on how to connect will be provided and `ssh` command will be provided. For example: * `ssh -i "dagster-vm-key-pair.pem" ubuntu@ec2-..compute.amazonaws.com` * **Note:** Run this command in the directory of the `.pem` file. - * **Note:** Since the virtual machine was created with the default VPC security group, make sure the **Inbound Rules** of the security allows your IP address to connect. + * **Note:** Since the virtual machine was created with the default VPC security group, + make sure the **Inbound Rules** of the security allows your IP address to connect. * The terminal should show an Ubuntu welcome screen once connected. -### Configure Instance -Once connected to the virtual machine, run the following commands to get everything set up: - -1. Clone repository - * Create a new directory: `git init ` - * `cd ` - * `git remote add -f origin https://github.com/digitalghost-dev/poke-cli/` - * `git config core.sparseCheckout true` - * `echo "card_data/" >> .git/info/sparse-checkout` - * `git pull origin main` - * `ls` - verify that `card_data/` directory was created. -2. Install tools - * Install `uv` for Python: `curl -LsSf https://astral.sh/uv/0.7.21/install.sh | sh` - * Add to `PATH`: `source $HOME/.local/bin/env` - * Install libraries from `pyproject.toml` file: `uv sync` - * Activate virtual environment: `source .venv/bin/activate` - * Create `dagster.yaml` file: - ```bash - mkdir -p ~/.dagster && cat > ~/.dagster/dagster.yaml << 'EOF' - storage: - postgres: - postgres_db: - username: postgres - password: "rds-password" - hostname: "rds-hostname" - db_name: postgres - port: 5432 - params: - sslmode: require - EOF - ``` - * Set environment variables: - * `echo 'export DAGSTER_HOME="$HOME/.dagster"' >> ~/.bashrc` - * `echo 'export SUPABASE_USER="supabase_user"' >> ~/.bashrc` - * `echo 'export SUPABASE_PASSWORD="supabase_password"' >> ~/.bashrc` - * `source ~/.bashrc` - to load variables in current session. -3. Verify Dagster and Connectivity - * `dg dev --host 0.0.0.0 --port 3000` - * In the browser, visit `http://:3000` - --- ## Elastic IPs @@ -248,6 +210,8 @@ An Elastic IP is a static public IPv4 address in AWS that can be assigned to an --- ## EventBridge +_Optional_ + AWS [EventBridge](https://aws.amazon.com/eventbridge/) is a serverless event bus that can use rules to automatically trigger actions—such as starting or stopping RDS and EC2 instances—based on scheduled times or specific events. @@ -329,4 +293,42 @@ The instructions below discuss the creation of a single role and a single attach * In the **Permissions** section, choose **Use Existing Role**, then select the previously created role associated with this schedule’s service. * Click **Next**. 4. Review and Create Schedule Page - * Review all the configuration changes then click **Create Schedule**. \ No newline at end of file + * Review all the configuration changes then click **Create Schedule**. + +--- + +## Secrets Manager +AWS Secrets Manager is a service for storing and managing sensitive information like database credentials, API keys, and passwords. +It provides encryption, automatic rotation, IAM-based access control, and integrates with RDS and other AWS services to manage +credentials programmatically. + +This project uses + +### Supabase Secrets +The EC2 instance for Dagster needs to authenticate with Supabase to write the card data. + +Create a secret for Supabase credentials[^1]: + +```shell +aws secretsmanager create-secret \ +--name supabase \ +--secret-string '{"password":"supabase-password","user":"supabase-user","database_uri":"postgresql://postgres.:@aws-0-us-east-2.pooler.supabase.com:6543/postgres"}' \ +--region us-west-2 +``` + +!!! note + + The connection string from Supabase has the username and password in it as such: + ```shell + postgresql://:@aws-0-us-east-2.pooler.supabase.com:6543/postgres + ``` + + The username includes the `postgres` prefix like `postgres.rest-of-user-name`. + +!!! note "Reminder" + + The EC2 instance has the `EC2-SecretsManager-Role` IAM role attached, which grants + permissions to create and retrieve secrets from AWS Secrets Manager. This uses an + IAM role (not an IAM user), so the instance receives temporary credentials automatically. + +[^1]: Can be retrieved by creating a project as shown in [2 // Supabase](supabase.md). \ No newline at end of file From 32db79adf70c776d3e2a6e7d50ff0c0dbf3a3988 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 2 Nov 2025 11:57:38 -0800 Subject: [PATCH 24/35] fixing formatting --- docs/Infrastructure_Guide/terraform.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/Infrastructure_Guide/terraform.md b/docs/Infrastructure_Guide/terraform.md index c0db5d1d..ba0cb908 100644 --- a/docs/Infrastructure_Guide/terraform.md +++ b/docs/Infrastructure_Guide/terraform.md @@ -63,6 +63,7 @@ terraformer import aws --regions us-west-2 --resources rds --profile terraform-u ``` Where: + * `--regions `is where the RDS instance is located. * `--resources` is the resources under RDS * `--profile` is the `terraform-user` profile from `~/.aws/credentials` From f75ccce3540f6b5f8b495a072f4d9b6e88db4517 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 2 Nov 2025 13:39:18 -0800 Subject: [PATCH 25/35] fixing incorrect function call (#193) --- cmd/berry/berry.go | 2 +- cmd/utils/validateargs.go | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/cmd/berry/berry.go b/cmd/berry/berry.go index e452ff6e..74d2ca98 100644 --- a/cmd/berry/berry.go +++ b/cmd/berry/berry.go @@ -39,7 +39,7 @@ func BerryCommand() (string, error) { } // Validate arguments - if err := utils.ValidateTypesArgs(os.Args); err != nil { + if err := utils.ValidateBerryArgs(os.Args); err != nil { output.WriteString(err.Error()) return output.String(), err } diff --git a/cmd/utils/validateargs.go b/cmd/utils/validateargs.go index 41b42bef..0d85a309 100644 --- a/cmd/utils/validateargs.go +++ b/cmd/utils/validateargs.go @@ -2,8 +2,9 @@ package utils import ( "fmt" - "github.com/digitalghost-dev/poke-cli/styling" "strings" + + "github.com/digitalghost-dev/poke-cli/styling" ) // checkLength checks if the number of arguments is lower than the max value. Helper Function. @@ -41,6 +42,18 @@ func ValidateAbilityArgs(args []string) error { return nil } +func ValidateBerryArgs(args []string) error { + if err := checkLength(args, 3); err != nil { + return err + } + + if err := checkNoOtherOptions(args, 3, ""); err != nil { + return err + } + + return nil +} + // ValidateItemArgs validates the command line arguments func ValidateItemArgs(args []string) error { if err := checkLength(args, 3); err != nil { From dd8929a5369b06a422cb169cf3485f70a25b4864 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 2 Nov 2025 13:50:17 -0800 Subject: [PATCH 26/35] removing hardcoded `baseURL` (#194) --- flags/pokemonflagset.go | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/flags/pokemonflagset.go b/flags/pokemonflagset.go index 37ebba95..25db3175 100644 --- a/flags/pokemonflagset.go +++ b/flags/pokemonflagset.go @@ -135,8 +135,7 @@ func AbilitiesFlag(w io.Writer, endpoint string, pokemonName string) error { } func DefenseFlag(w io.Writer, endpoint string, pokemonName string) error { - baseURL := "https://pokeapi.co/api/v2/" - pokemonStruct, _, _ := connections.PokemonApiCall(endpoint, pokemonName, baseURL) + pokemonStruct, _, _ := connections.PokemonApiCall(endpoint, pokemonName, connections.APIURL) // Print the header from header func _, err := fmt.Fprintln(w, header("Type Defenses")) @@ -150,7 +149,7 @@ func DefenseFlag(w io.Writer, endpoint string, pokemonName string) error { typeData := make(map[string]structs.TypesJSONStruct) for _, pokeType := range pokemonStruct.Types { - typeStruct, _, _ := connections.TypesApiCall("type", pokeType.Type.Name, baseURL) + typeStruct, _, _ := connections.TypesApiCall("type", pokeType.Type.Name, connections.APIURL) typeData[pokeType.Type.Name] = typeStruct } @@ -320,8 +319,7 @@ func DefenseFlag(w io.Writer, endpoint string, pokemonName string) error { } func ImageFlag(w io.Writer, endpoint string, pokemonName string, size string) error { - baseURL := "https://pokeapi.co/api/v2/" - pokemonStruct, _, _ := connections.PokemonApiCall(endpoint, pokemonName, baseURL) + pokemonStruct, _, _ := connections.PokemonApiCall(endpoint, pokemonName, connections.APIURL) // Print the header from header func _, err := fmt.Fprintln(w, header("Image")) @@ -410,8 +408,7 @@ func ImageFlag(w io.Writer, endpoint string, pokemonName string, size string) er } func MovesFlag(w io.Writer, endpoint string, pokemonName string) error { - baseURL := "https://pokeapi.co/api/v2/" - pokemonStruct, _, _ := connections.PokemonApiCall(endpoint, pokemonName, baseURL) + pokemonStruct, _, _ := connections.PokemonApiCall(endpoint, pokemonName, connections.APIURL) _, err := fmt.Fprintln(w, header("Learnable Moves")) if err != nil { @@ -446,7 +443,7 @@ func MovesFlag(w io.Writer, endpoint string, pokemonName string) error { go func(moveName string, level int) { defer wg.Done() - moveStruct, _, err := connections.MoveApiCall("move", moveName, baseURL) + moveStruct, _, err := connections.MoveApiCall("move", moveName, connections.APIURL) if err != nil { errorsChan <- fmt.Errorf("error fetching move %s: %v", moveName, err) return @@ -556,8 +553,7 @@ func MovesFlag(w io.Writer, endpoint string, pokemonName string) error { } func StatsFlag(w io.Writer, endpoint string, pokemonName string) error { - baseURL := "https://pokeapi.co/api/v2/" - pokemonStruct, _, _ := connections.PokemonApiCall(endpoint, pokemonName, baseURL) + pokemonStruct, _, _ := connections.PokemonApiCall(endpoint, pokemonName, connections.APIURL) // Print the header from header func _, err := fmt.Fprintln(w, header("Base Stats")) @@ -652,8 +648,7 @@ func StatsFlag(w io.Writer, endpoint string, pokemonName string) error { } func TypesFlag(w io.Writer, endpoint string, pokemonName string) error { - baseURL := "https://pokeapi.co/api/v2/" - pokemonStruct, _, _ := connections.PokemonApiCall(endpoint, pokemonName, baseURL) + pokemonStruct, _, _ := connections.PokemonApiCall(endpoint, pokemonName, connections.APIURL) // Print the header from header func _, err := fmt.Fprintln(w, header("Typing")) From 6402b82c608936f5a43497ec3607420c913fb8b5 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 2 Nov 2025 13:58:44 -0800 Subject: [PATCH 27/35] adding exit code to error handling (#195) --- cmd/search/search.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/search/search.go b/cmd/search/search.go index 1f5efbc7..b99370a0 100644 --- a/cmd/search/search.go +++ b/cmd/search/search.go @@ -39,6 +39,7 @@ func SearchCommand() { p := tea.NewProgram(initialModel()) if _, err := p.Run(); err != nil { fmt.Println("could not start program:", err) + os.Exit(1) } } From 8d8cde83180e5d2e5c69b023364645c8d1e28527 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 2 Nov 2025 14:18:10 -0800 Subject: [PATCH 28/35] updating modernc.org/sqlite dependency --- go.mod | 6 +++--- go.sum | 32 ++++++++++++++++---------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index b87ff272..5b5957b2 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/disintegration/imaging v1.6.2 github.com/stretchr/testify v1.10.0 golang.org/x/text v0.27.0 - modernc.org/sqlite v1.39.0 + modernc.org/sqlite v1.39.1 ) require ( @@ -45,9 +45,9 @@ require ( golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect golang.org/x/image v0.28.0 // indirect golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.34.0 // indirect + golang.org/x/sys v0.36.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - modernc.org/libc v1.66.3 // indirect + modernc.org/libc v1.66.10 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect ) diff --git a/go.sum b/go.sum index 2d84ed27..1cb34172 100644 --- a/go.sum +++ b/go.sum @@ -86,35 +86,35 @@ golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAf golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.28.0 h1:gdem5JW1OLS4FbkWgLO+7ZeFzYtL3xClb97GaUzYMFE= golang.org/x/image v0.28.0/go.mod h1:GUJYXtnGKEUgggyzh+Vxt+AviiCcyiwpsl8iQ8MvwGY= -golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= -golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= +golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= -golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= -golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= -golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -modernc.org/cc/v4 v4.26.2 h1:991HMkLjJzYBIfha6ECZdjrIYz2/1ayr+FL8GN+CNzM= -modernc.org/cc/v4 v4.26.2/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= -modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU= -modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE= -modernc.org/fileutil v1.3.8 h1:qtzNm7ED75pd1C7WgAGcK4edm4fvhtBsEiI/0NQ54YM= -modernc.org/fileutil v1.3.8/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= +modernc.org/cc/v4 v4.26.5 h1:xM3bX7Mve6G8K8b+T11ReenJOT+BmVqQj0FY5T4+5Y4= +modernc.org/cc/v4 v4.26.5/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.28.1 h1:wPKYn5EC/mYTqBO373jKjvX2n+3+aK7+sICCv4Fjy1A= +modernc.org/ccgo/v4 v4.28.1/go.mod h1:uD+4RnfrVgE6ec9NGguUNdhqzNIeeomeXf6CL0GTE5Q= +modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA= +modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.66.3 h1:cfCbjTUcdsKyyZZfEUKfoHcP3S0Wkvz3jgSzByEWVCQ= -modernc.org/libc v1.66.3/go.mod h1:XD9zO8kt59cANKvHPXpx7yS2ELPheAey0vjIuZOhOU8= +modernc.org/libc v1.66.10 h1:yZkb3YeLx4oynyR+iUsXsybsX4Ubx7MQlSYEw4yj59A= +modernc.org/libc v1.66.10/go.mod h1:8vGSEwvoUoltr4dlywvHqjtAqHBaw0j1jI7iFBTAr2I= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= @@ -123,8 +123,8 @@ modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.39.0 h1:6bwu9Ooim0yVYA7IZn9demiQk/Ejp0BtTjBWFLymSeY= -modernc.org/sqlite v1.39.0/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E= +modernc.org/sqlite v1.39.1 h1:H+/wGFzuSCIEVCvXYVHX5RQglwhMOvtHSv+VtidL2r4= +modernc.org/sqlite v1.39.1/go.mod h1:9fjQZ0mB1LLP0GYrp39oOJXx/I2sxEnZtzCmEQIKvGE= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= From 91ed75df3d3634fb2f5600a042ce7f52531e8500 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Sun, 2 Nov 2025 14:20:06 -0800 Subject: [PATCH 29/35] updating function call (#193) --- cmd/utils/validateargs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/utils/validateargs.go b/cmd/utils/validateargs.go index 0d85a309..c76f194e 100644 --- a/cmd/utils/validateargs.go +++ b/cmd/utils/validateargs.go @@ -47,7 +47,7 @@ func ValidateBerryArgs(args []string) error { return err } - if err := checkNoOtherOptions(args, 3, ""); err != nil { + if err := checkNoOtherOptions(args, 3, ""); err != nil { return err } From 229062ae61ad4e986edeecb9c4a18361061aefac Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Mon, 3 Nov 2025 08:16:51 -0800 Subject: [PATCH 30/35] adding prerequisite steps --- docs/Infrastructure_Guide/cloud-deployment.md | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/docs/Infrastructure_Guide/cloud-deployment.md b/docs/Infrastructure_Guide/cloud-deployment.md index 7dd07c06..e39478ac 100644 --- a/docs/Infrastructure_Guide/cloud-deployment.md +++ b/docs/Infrastructure_Guide/cloud-deployment.md @@ -73,7 +73,7 @@ Connect to the virtual machine and run the following commands to get everything ``` * Add to `PATH`: ```shell - `source $HOME/.local/bin/env + `source $HOME/.local/bin/env` ``` * Install libraries from `pyproject.toml` file: ```shell @@ -127,6 +127,32 @@ The `card_data/infrastructure/` directory has the following files: Although the files are included in this repository, they need to be moved or created in a specific directory on the Linux virtual machine. +##### Prerequisites + +Before copying or creating the scripts, ensure the following system tools are installed. +These are required by the shell scripts: + +- **`netcat` (nc)**: Used by `wait-for-rds.sh` to check RDS availability +- **`jq`**: Used by `start-dagster.sh` to parse JSON from AWS Secrets Manager + +For **Debian/Ubuntu** systems, install with: + +```shell +sudo apt update && sudo apt install -y netcat jq +``` + +For other platforms, use the appropriate package manager: + +- **RHEL/CentOS/Amazon Linux**: `sudo yum install -y nc jq` or `sudo dnf install -y nc jq` +- **macOS**: `brew install netcat jq` +- **Alpine Linux**: `apk add netcat-openbsd jq` + +!!! warning + + Without these tools installed, the scripts will fail with errors like + "command not found" when systemd attempts to run them. + + #### Copy Files Copy or move the files from the checked out repository to the proper directory on the Linux machine (_the files must first be edited to match project specific configuration. Such as the proper RDS instance name in `wait-for-rds.sh`_): @@ -162,6 +188,10 @@ First, create `dagster.service` WorkingDirectory=/home/ubuntu/card_data/card_data Environment="AWS_DEFAULT_REGION=us-west-2" Environment="PATH=/home/ubuntu/card_data/card_data/.venv/bin:/usr/local/bin:/usr/bin:/bin" + NoNewPrivileges=true + PrivateTmp=true + ProtectSystem=strict + ProtectHome=read-only ExecStartPre=/home/ubuntu/wait-for-rds.sh ExecStart=/home/ubuntu/start-dagster.sh Restart=on-failure From 49fd819419e1f48f0364008e0270323b2b6c8f48 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Mon, 3 Nov 2025 08:22:15 -0800 Subject: [PATCH 31/35] restrict container styling to admonitions and details only --- docs/stylesheets/extra.css | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css index 8d5cd906..6744cef9 100644 --- a/docs/stylesheets/extra.css +++ b/docs/stylesheets/extra.css @@ -7,13 +7,13 @@ border-color: rgb(52, 152, 219); } -.md-typeset .container > .admonition-title, -.md-typeset .container > summary { +.md-typeset .admonition.container > .admonition-title, +.md-typeset details.container > summary { background-color: rgba(52, 152, 219, 0.1); } -.md-typeset .container > .admonition-title::before, -.md-typeset .container > summary::before { +.md-typeset .admonition.container > .admonition-title::before, +.md-typeset details.container > summary::before { background-color: rgb(52, 152, 219); -webkit-mask-image: var(--md-admonition-icon--container); mask-image: var(--md-admonition-icon--container); From 672e5e0de42730083d04ab43324b32830faf4f70 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Mon, 3 Nov 2025 08:23:30 -0800 Subject: [PATCH 32/35] fixing typo --- docs/Infrastructure_Guide/aws.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/Infrastructure_Guide/aws.md b/docs/Infrastructure_Guide/aws.md index 74196e75..bf4b8b0a 100644 --- a/docs/Infrastructure_Guide/aws.md +++ b/docs/Infrastructure_Guide/aws.md @@ -276,7 +276,7 @@ The instructions below discuss the creation of a single role and a single attach 2. Select Target Page * Under **Target Detail**, select **All APIs** and search for and select EC2. * In the API search box, search for `StartInstances` or `StopInstances`. - * In the next window, grab the **Instance ID** for the EC2 instead created earlier and paste it in to the JSON input. + * In the next window, grab the **Instance ID** for the EC2 instance created earlier and paste it in to the JSON input. ```json { "InstanceIds": [ @@ -302,8 +302,6 @@ AWS Secrets Manager is a service for storing and managing sensitive information It provides encryption, automatic rotation, IAM-based access control, and integrates with RDS and other AWS services to manage credentials programmatically. -This project uses - ### Supabase Secrets The EC2 instance for Dagster needs to authenticate with Supabase to write the card data. From 11fd7eeecc4403aeeec1fc705a2e0b03d1491135 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Mon, 3 Nov 2025 08:24:42 -0800 Subject: [PATCH 33/35] add security hardening directives to dagster service --- card_data/infrastructure/dagster.service | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/card_data/infrastructure/dagster.service b/card_data/infrastructure/dagster.service index e2cffb81..2c2a655a 100644 --- a/card_data/infrastructure/dagster.service +++ b/card_data/infrastructure/dagster.service @@ -10,6 +10,10 @@ User=ubuntu WorkingDirectory=/home/ubuntu/card_data/card_data Environment="AWS_DEFAULT_REGION=us-west-2" Environment="PATH=/home/ubuntu/card_data/card_data/.venv/bin:/usr/local/bin:/usr/bin:/bin" +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=read-only ExecStartPre=/home/ubuntu/wait-for-rds.sh ExecStart=/home/ubuntu/start-dagster.sh Restart=on-failure From 7f3dc99dca12a2219111778fe2cdce1cbdf74298 Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Mon, 3 Nov 2025 09:01:19 -0800 Subject: [PATCH 34/35] fixing formatting inconsistencies, reflecting new `start-dagster.sh` changes --- docs/Infrastructure_Guide/cloud-deployment.md | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/docs/Infrastructure_Guide/cloud-deployment.md b/docs/Infrastructure_Guide/cloud-deployment.md index e39478ac..6d2828df 100644 --- a/docs/Infrastructure_Guide/cloud-deployment.md +++ b/docs/Infrastructure_Guide/cloud-deployment.md @@ -30,7 +30,7 @@ Connect to the virtual machine and run the following commands to get everything ```shell export AWS_DEFAULT_REGION=us-west-2 # or set it to ~./bashrc file - echo 'export AWS_DEFAULT_REGION=us-east-1' >> ~/.bashrc + echo 'export AWS_DEFAULT_REGION=us-west-1' >> ~/.bashrc source ~/.bashrc ``` * Run an `aws` command such as `secretsmanager` to verify AWS connectivity: @@ -73,7 +73,7 @@ Connect to the virtual machine and run the following commands to get everything ``` * Add to `PATH`: ```shell - `source $HOME/.local/bin/env` + source $HOME/.local/bin/env ``` * Install libraries from `pyproject.toml` file: ```shell @@ -282,15 +282,31 @@ Last, create `start-dagster.sh` # Extract values SUPABASE_PASSWORD=$(echo "$SUPABASE_SECRETS" | jq -r '.password') + if [ -z "$SUPABASE_PASSWORD" ] || [ "$SUPABASE_PASSWORD" = "null" ]; then + echo "ERROR: missing SUPABASE_PASSWORD from supabase secret" >&2 + exit 1 + fi export SUPABASE_PASSWORD - + SUPABASE_USER=$(echo "$SUPABASE_SECRETS" | jq -r '.user') + if [ -z "$SUPABASE_USER" ] || [ "$SUPABASE_USER" = "null" ]; then + echo "ERROR: missing SUPABASE_USER from supabase secret" >&2 + exit 1 + fi export SUPABASE_USER - + AWS_RDS_PASSWORD=$(echo "$AWS_RDS_SECRETS_PW" | jq -r '.password') + if [ -z "$AWS_RDS_PASSWORD" ] || [ "$AWS_RDS_PASSWORD" = "null" ]; then + echo "ERROR: missing AWS_RDS_PASSWORD from RDS secret" >&2 + exit 1 + fi export AWS_RDS_PASSWORD - + AWS_RDS_HOSTNAME=$(echo "$AWS_RDS_SECRETS_HN" | jq -r '.hostname') + if [ -z "$AWS_RDS_HOSTNAME" ] || [ "$AWS_RDS_HOSTNAME" = "null" ]; then + echo "ERROR: missing AWS_RDS_HOSTNAME from rds-hostname secret" >&2 + exit 1 + fi export AWS_RDS_HOSTNAME DAGSTER_HOME=/home/ubuntu/card_data/card_data/ From 95ab90f7c946ca9d423e99dd6fa113fa965e5d7b Mon Sep 17 00:00:00 2001 From: Christian Sanchez Date: Mon, 3 Nov 2025 09:02:35 -0800 Subject: [PATCH 35/35] validate secrets before starting Dagster --- card_data/infrastructure/start-dagster.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/card_data/infrastructure/start-dagster.sh b/card_data/infrastructure/start-dagster.sh index 24e9e3a4..4157baee 100644 --- a/card_data/infrastructure/start-dagster.sh +++ b/card_data/infrastructure/start-dagster.sh @@ -21,15 +21,31 @@ AWS_RDS_SECRETS_HN=$(aws secretsmanager get-secret-value \ # Extract values SUPABASE_PASSWORD=$(echo "$SUPABASE_SECRETS" | jq -r '.password') +if [ -z "$SUPABASE_PASSWORD" ] || [ "$SUPABASE_PASSWORD" = "null" ]; then + echo "ERROR: missing SUPABASE_PASSWORD from supabase secret" >&2 + exit 1 +fi export SUPABASE_PASSWORD SUPABASE_USER=$(echo "$SUPABASE_SECRETS" | jq -r '.user') +if [ -z "$SUPABASE_USER" ] || [ "$SUPABASE_USER" = "null" ]; then + echo "ERROR: missing SUPABASE_USER from supabase secret" >&2 + exit 1 +fi export SUPABASE_USER AWS_RDS_PASSWORD=$(echo "$AWS_RDS_SECRETS_PW" | jq -r '.password') +if [ -z "$AWS_RDS_PASSWORD" ] || [ "$AWS_RDS_PASSWORD" = "null" ]; then + echo "ERROR: missing AWS_RDS_PASSWORD from RDS secret" >&2 + exit 1 +fi export AWS_RDS_PASSWORD AWS_RDS_HOSTNAME=$(echo "$AWS_RDS_SECRETS_HN" | jq -r '.hostname') +if [ -z "$AWS_RDS_HOSTNAME" ] || [ "$AWS_RDS_HOSTNAME" = "null" ]; then + echo "ERROR: missing AWS_RDS_HOSTNAME from rds-hostname secret" >&2 + exit 1 +fi export AWS_RDS_HOSTNAME DAGSTER_HOME=/home/ubuntu/card_data/card_data/