diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index a98b15e..df0a0bb 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -19,6 +19,13 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Install ruff + run: pip install ruff + + - name: Lint with ruff + run: ruff check . + + test: name: Run Tests runs-on: ubuntu-latest @@ -28,6 +35,20 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + pip install -r requirements.txt + pip install pytest pytest-cov + + - name: Run tests + run: | + pytest --cov=app --cov-fail-under=80 + build: name: Build Docker Image runs-on: ubuntu-latest @@ -40,3 +61,30 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=latest,enable=${{ github.event_name == 'release' }} + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: docker/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/markdown2pdf.yml b/.github/workflows/markdown2pdf.yml index fcb4671..4177b58 100644 --- a/.github/workflows/markdown2pdf.yml +++ b/.github/workflows/markdown2pdf.yml @@ -19,7 +19,7 @@ jobs: - name: Replace links run: | - cp README.md README_WITH_LINKS.md + cp README.md README_WITH_LINKS.md sed -i -e "s#\(^\!\[[^]]\+\](\)\(images/\)#\1$URL/\2#g" README_WITH_LINKS.md for file in sources/*; do sed -i -e "s#($file)#($URL/$file)#g" README_WITH_LINKS.md ; done diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d337d10..57bec7b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,57 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: trailing-whitespace + # Prevent committing large files + - id: check-added-large-files + args: ['--maxkb=1024'] + # Check YAML files for syntax errors (excluding helm charts) + - id: check-yaml + exclude: ^helm/ + + - repo: https://github.com/pycqa/isort + rev: 7.0.0 + hooks: + # sort imports in python files (black style) + - id: isort + types : [python] + args : ["--profile", "black"] + + - repo: https://github.com/gitleaks/gitleaks + rev: v8.30.0 + hooks: + # Prevent committing secrets + - id: gitleaks + + - repo: https://github.com/thoughtworks/talisman + rev: 'v1.37.0' + hooks: + # security issue hook (secrets / sensitive info) + - id: talisman-commit + + - repo: https://github.com/trufflesecurity/trufflehog + rev: v3.93.3 + hooks: + # security issue hook (secrets / sensitive info) + - id: trufflehog + + + - repo : https://github.com/psf/black + rev: 26.1.0 + hooks: + # code style + - id: black + types : [python] + + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.15.0 + hooks: + # Run the linter. + - id: ruff-check + args: [ --fix ] + # Run the formatter. + - id: ruff-format +ci: + autoupdate_schedule: weekly diff --git a/.talismanrc b/.talismanrc new file mode 100644 index 0000000..6d6247e --- /dev/null +++ b/.talismanrc @@ -0,0 +1,5 @@ +fileignoreconfig: +- filename: .github/workflows/ci-cd.yml + checksum: 1c4380ae921f6f86f8bf2f0a0264daa884006659beeb8831bfbb11cd0d3d3e8f + ignore_detectors: [] +version: "1.0" diff --git a/docker/Dockerfile b/Dockerfile similarity index 100% rename from docker/Dockerfile rename to Dockerfile diff --git a/ExerciseAnswers.md b/ExerciseAnswers.md new file mode 100644 index 0000000..07b1930 --- /dev/null +++ b/ExerciseAnswers.md @@ -0,0 +1,75 @@ +# 3. Exercises + +## 3.0 Contributing + +Done + +## 3.1 Add pre-commit Hooks + +Done + + +## 3.2 Add a New Endpoint + +Done + + +## 3.3 Add a CI Pipeline + +Done + + +## 3.4 Deploy on a K8s "production" Cluster + +Done + + + +## 3.5 Questions + +1. The auto-scaling did not work as expected. What could be the possible reasons? + +### What happened + +I load-tested the endpoint with: + +```bash +hey -z 30s http://minikube.test/GitOps-Starter/api/items +``` + +In parallel, I watched pods and the HPA: + +```bash +kubectl get pods -n default -w +kubectl get hpa -n default -w +``` + +At first, autoscaling didn’t happen because the HPA had no CPU metrics available. In `kubectl get hpa -w`, the CPU and memory metrics showed up as unknown (unknown/10% and unknown/80%), HPA could not observe utilization, so it had nothing to scale on. + +After I enabled the metrics server, the HPA immediately started reporting real values and scaling kicked in. I saw CPU jump above my target (`cpu: 240%/10%`) and the HPA scaled up in steps until it hit my configured limit (`maxReplicas: 10`): `New size: 4`, then `New size: 8`, then `New size: 10`. Later, when the load stopped and CPU dropped (`cpu: 2%/10%`–`3%/10%`), it eventually scaled down again (`Scaled down replica set ... from 10 to 4`, reason: `All metrics below target`). + +I used events to confirm the scaling decisions: + +```bash +kubectl get events -n default --sort-by=.lastTimestamp \ +| grep -E "SuccessfulRescale|Scaled up replica set|Scaled down replica set" +``` + +### Why it didn’t work (initially) and why it worked after + +* **No metrics = no autoscaling.** HPA needs resource metrics (CPU/memory) to calculate utilization. Before metrics-server was enabled, the HPA showed CPU as unknown in `kubectl get hpa -w`, so it couldn’t decide to scale. + +* **Once metrics-server was enabled, HPA behaved as designed.** After metrics became available, `kubectl get hpa -w` showed real utilization (e.g., `cpu: 240%/10%`), and the event log confirmed rescaling decisions like `SuccessfulRescale ... cpu resource utilization (percentage of request) above target`. + +* **My target was intentionally low (10%), so scaling was aggressive.** With `targetCPUUtilizationPercentage: 10`, even a modest CPU spike can trigger scaling quickly, which explains why it ramped up to `New size: 10` (and stopped there because `maxReplicas: 10`). + +* **Scale-down takes time.** Even after CPU fell back to `cpu: 2%/10%`–`3%/10%`, replicas stayed high for a while before scaling down (`reason: All metrics below target`). + + +2. How HPA works + +* **HPA watches a metric** for a workload (usually average CPU or memory across the pods in a Deployment). +* It needs a **metrics source** (typically `metrics-server`). If metrics aren’t available, the HPA shows CPU as *unknown* and can’t scale. +* For CPU scaling, it compares **actual CPU usage** to the pod’s **CPU request** and computes “% of request”. +* If usage is **above the target** (e.g., `cpu: 240%/10%`), it **increases** the Deployment replicas (up to `maxReplicas`). +* If usage stays **below the target** (e.g., `cpu: 2–3%/10%`), it **scales down**, but usually **more slowly** to avoid bouncing up/down (“thrashing”). diff --git a/README.md b/README.md index 79bb814..057f54d 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ GitOps with FastAPI ***University of Amsterdam*** -# 1. Introduction +# 1. Introduction In this tutorial, we use GitOps practices with FastAPI, including CI/CD pipelines, code quality tools, and automated testing. @@ -35,7 +35,7 @@ In this tutorial, we use GitOps practices with FastAPI, including CI/CD pipeline -# 2. Tutorial +# 2. Tutorial The steps of this tutorial are as follows: - [Building REST APIs with FastAPI](#21-setting-up-the-project) @@ -60,17 +60,17 @@ Prerequisites: ``` * Set Up the Python Environmentt: - + ```bash # Create a virtual environment python -m venv venv - + # Activate the virtual environment # On Linux/MacOS: source venv/bin/activate # On Windows: venv\Scripts\activate - + # Install dependencies pip install -r requirements.txt ``` @@ -110,7 +110,7 @@ Prerequisites: ```bash # Check for issues ruff check app/ tests/ - + # Fix auto-fixable issues ruff check app/ tests/ --fix ``` @@ -120,7 +120,7 @@ Prerequisites: ```bash # Check formatting black --check app/ tests/ - + # Format code black app/ tests/ ``` @@ -134,7 +134,7 @@ Pre-commit hooks automatically run checks before each commit to ensure consisten ```bash # Install pre-commit pip install pre-commit - + # Install the git hooks pre-commit install ``` @@ -142,11 +142,11 @@ Pre-commit hooks automatically run checks before each commit to ensure consisten * Using Pre-commit: Pre-commit will now run automatically on `git commit`. You can also run it manually: - + ```bash # Run on all files pre-commit run --all-files - + # Run on staged files pre-commit run ``` @@ -200,14 +200,14 @@ This repository includes a Helm chart for deploying the application to Kubernete - Kubernetes 1.19+ - Helm 3.0+ -* Install the Helm Chart: +* Install the Helm Chart: ```bash helm install my-release ./helm/fastapi-gitops-starter ``` -* Uninstall the Helm Chart: - +* Uninstall the Helm Chart: + ```bash helm uninstall my-release ``` @@ -239,7 +239,7 @@ including host and paths. * To make sure we do not commit secrets * To check code style - + ## 3.2 Add a New Endpoint 1. Open `app/main.py` diff --git a/app/main.py b/app/main.py index ae8adc6..5abaa1f 100644 --- a/app/main.py +++ b/app/main.py @@ -3,6 +3,14 @@ import uvicorn from fastapi import FastAPI from fastapi.responses import JSONResponse +from pydantic import BaseModel + + +# used: https://fastapi.tiangolo.com/tutorial/body/ +class ItemCreate(BaseModel): + name: str + description: str + app = FastAPI( title="FastAPI GitOps Starter", @@ -49,5 +57,16 @@ async def get_item(item_id: int): } +@app.post("/api/items") +async def create_item(item: ItemCreate): + """Create a new item.""" + return { + "id": 999, + "name": item.name, + "description": item.description, + "created": True, + } + + if __name__ == "__main__": uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/helm/fastapi-gitops-starter/custom-values.yaml b/helm/fastapi-gitops-starter/custom-values.yaml new file mode 100644 index 0000000..8cc051a --- /dev/null +++ b/helm/fastapi-gitops-starter/custom-values.yaml @@ -0,0 +1,37 @@ +# Enable HPA +autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 10 + targetCPUUtilizationPercentage: 10 # Low threshold for testing + +# Set resource requests (required for HPA) +resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m # HPA uses this as baseline + memory: 128Mi + +# Standard deployment settings +replicaCount: 1 # Ignored when HPA enabled +image: + repository: fastapi-gitops-starter + tag: latest + pullPolicy: Never # Use local image from minikube + +service: + type: NodePort + port: 80 + targetPort: 8000 + +# Enable ingress for testing +ingress: + enabled: true + className: nginx + hosts: + - host: minikube.test + paths: + - path: /GitOps-Starter + pathType: Prefix diff --git a/helm/fastapi-gitops-starter/values.yaml b/helm/fastapi-gitops-starter/values.yaml index 20c08be..30cb88c 100644 --- a/helm/fastapi-gitops-starter/values.yaml +++ b/helm/fastapi-gitops-starter/values.yaml @@ -5,10 +5,10 @@ replicaCount: 1 image: - repository: ghcr.io/qcdis/fastapi-gitops-starter - pullPolicy: IfNotPresent + repository: fastapi-gitops-starter + pullPolicy: Never # Overrides the image tag whose default is the chart appVersion. - tag: "v0.4" + tag: latest registry: createImagePullSecret: true diff --git a/tests/test_main.py b/tests/test_main.py index db89e2f..4228f12 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -41,3 +41,16 @@ def test_get_item(): assert data["id"] == 5 assert data["name"] == "Item 5" assert "item number 5" in data["description"] + + +def test_create_item(): + """Test the create item endpoint.""" + response = client.post( + url="/api/items", json={"name": "test item", "description": "test description"} + ) + assert response.status_code == 200 + data = response.json() + assert data["id"] == 999 + assert data["name"] == "test item" + assert data["description"] == "test description" + assert data["created"] is True