diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c6d9063..f71973d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,36 +1,75 @@ -# This is a basic workflow to help you get started with Actions +name: quick-actions-aws-lambda-template-ci -name: CI - -# Controls when the workflow will run on: - # Triggers the workflow on push or pull request events but only for the "main" branch push: branches: [main] pull_request: branches: [main] - - # Allows you to run this workflow manually from the Actions tab workflow_dispatch: -# A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - # This workflow contains a single job called "build" - build: - # The type of runner that the job will run on + code-quality: + name: Code Quality runs-on: ubuntu-latest - - # Steps represent a sequence of tasks that will be executed as part of the job steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v3 - # Runs a single command using the runners shell - - name: Run a one-line script - run: echo Hello, world! + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install Python dependencies + run: pip install ruff ruff-lsp + + - name: Lint Python with Ruff + run: ruff check python/ + + - name: Check Python formatting with Ruff Format + run: ruff format python/ --check + + - name: Set up Terraform + uses: hashicorp/setup-terraform@v3 + + - name: Check Terraform formatting + run: terraform fmt terraform/ -check + + - name: Install yamllint and toml + run: pip install yamllint toml - # Runs a set of commands using the runners shell - - name: Run a multi-line script + - name: Lint TOML files run: | - echo Add other actions to build, - echo test, and deploy your project. + for file in $(find . -name "*.toml"); do + python -c "import toml; toml.load(open('$file'))" || exit 1 + done + + - name: Lint YAML files + run: yamllint . -c=./config/.yamllint.yaml + + - name: Format YAML files (check only) + run: echo "No standard YAML formatter in check mode; skipping." + + - name: Format Markdown files (check only) + run: | + pip install mdformat + mdformat --check . + + # terraform: + # name: Terraform + # runs-on: ubuntu-latest + # needs: code-quality + # steps: + # - uses: actions/checkout@v3 + + # - name: Set up Terraform + # uses: hashicorp/setup-terraform@v3 + + # - name: Terraform Init + # run: terraform -chdir=terraform/ init + + # - name: Terraform Plan + # run: terraform -chdir=terraform/ plan + + # - name: Terraform Apply + # if: github.ref == 'refs/heads/main' + # run: terraform -chdir=terraform/ apply -auto-approve diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0a7d93d --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +node_modules +/.nova +.DS_Store +package-lock.json +.venv +.pytest_cache +.ruff_cache +.vscode +# Ignore Python bytecode files +__pycache__/ \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..ca73e53 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,7 @@ +{ + "python.testing.pytestArgs": [ + "python" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} \ No newline at end of file diff --git a/README.md b/README.md index f27a261..a40404a 100644 --- a/README.md +++ b/README.md @@ -1 +1,158 @@ # quick-actions-aws-lambda-template + +A template project for deploying AWS Lambda functions with best practices, infrastructure-as-code (Terraform), and automated code quality checks using GitHub Actions. + +______________________________________________________________________ + +## Features + +- **AWS Lambda Function**: Python 3.12 Lambda with event source mapping (SQS trigger). +- **Infrastructure as Code**: All AWS resources (Lambda, IAM, SQS, CloudWatch Logs) managed via Terraform. +- **CI/CD**: GitHub Actions pipeline for linting, formatting, and Terraform plan/apply. +- **Code Quality**: Python linting and formatting (ruff), TOML and YAML linting, Markdown formatting. +- **Best Practices**: Environment variables, tagging, log retention, and least-privilege IAM. + +______________________________________________________________________ + +## Project Structure + +``` +. +├── python/ +│ └── src/ +│ └── base_lambda.py # Lambda function source code +├── terraform/ +│ └── lambda.tf # Terraform for Lambda, IAM, SQS, etc. +├── .github/ +│ └── workflows/ +│ └── ci.yml # GitHub Actions pipeline +├── README.md +``` + +______________________________________________________________________ + +## Lambda Function + +The Lambda function (`base_lambda.py`) provides utility actions via AWS APIs: + +- Delete another Lambda function +- Redrive messages from an SQS DLQ +- Get ECR login and repository URI + +The handler expects an event with a key indicating the action to perform. + +______________________________________________________________________ + +## Infrastructure + +Terraform provisions: + +- **Lambda Function** (`example_lambda`) +- **IAM Role** for Lambda execution +- **CloudWatch Log Group** for Lambda logs +- **SQS Queue** as event source +- **Event Source Mapping** between SQS and Lambda + +All resources are tagged for project and environment tracking. + +______________________________________________________________________ + +## CI/CD Pipeline + +GitHub Actions workflow (`ci.yml`) includes: + +### Code Quality Stage + +- Lint Python with Ruff +- Check Python formatting with Ruff Format +- Check Terraform formatting +- Lint TOML files +- Lint YAML files +- Check Markdown formatting + +### Terraform Stage + +- Terraform init, plan, and apply (on main branch) + +______________________________________________________________________ + +## Getting Started + +### Prerequisites + +- [Python 3.12+](https://www.python.org/) +- [Terraform](https://www.terraform.io/) +- [AWS CLI](https://aws.amazon.com/cli/) +- Docker (for packaging Lambda, if needed) + +### Setup + +1. **Clone the repository** + + ```sh + git clone https://github.com/your-org/quick-actions-aws-lambda-template.git + cd quick-actions-aws-lambda-template + ``` + +1. **Install Python dependencies** + + ```sh + pip install -r requirements.txt + ``` + +1. **Build Lambda deployment package** + + ```sh + mkdir -p build + cd python/src + zip -r ../../build/example_lambda.zip . + cd ../../ + ``` + +1. **Initialize and apply Terraform** + + ```sh + cd terraform + terraform init + terraform apply + ``` + +______________________________________________________________________ + +## Usage + +The Lambda function expects an event with a key indicating the action, for example: + +```json +{ + "body": "{\"event\": \"delete_lambda_function\", \"function_name\": \"target_lambda\"}" +} +``` + +Supported events: + +- `delete_lambda_function` +- `redrive_sqs_dlq` +- `get_ecr_login_and_repo_uri` + +Update the Lambda code to handle your specific use cases. + +______________________________________________________________________ + +## Customization + +- **Add more Lambda actions** in `base_lambda.py` +- **Adjust Terraform** for additional AWS resources or permissions +- **Modify CI pipeline** in `.github/workflows/ci.yml` for your team's standards + +______________________________________________________________________ + +## License + +MIT License + +______________________________________________________________________ + +## Authors + +- [Riyan Imam](https://github.com/riyanimam) diff --git a/config/.yamllint.yaml b/config/.yamllint.yaml index 7712f7a..16addfa 100644 --- a/config/.yamllint.yaml +++ b/config/.yamllint.yaml @@ -4,4 +4,4 @@ rules: line-length: max: 500 level: warning - new-lines: disable \ No newline at end of file + new-lines: disable diff --git a/python/src/base_lambda.py b/python/src/base_lambda.py index d34c941..ef33bfd 100644 --- a/python/src/base_lambda.py +++ b/python/src/base_lambda.py @@ -1,5 +1,7 @@ from aws_lambda_powertools.shared.types import JSONType from aws_lambda_powertools.utilities.typing import LambdaContext +import json +import boto3 from botocore.config import Config @@ -10,5 +12,46 @@ ) +def delete_lambda_function(function_name: str): + client = boto3.client("lambda", config=aws_client_config) + response = client.delete_function(FunctionName=function_name) + return response + + +def redrive_sqs_dlq(source_queue_url: str, dlq_url: str, max_messages: int = 10): + sqs = boto3.client("sqs", config=aws_client_config) + messages = sqs.receive_message( + QueueUrl=dlq_url, MaxNumberOfMessages=max_messages, WaitTimeSeconds=2 + ).get("Messages", []) + + for msg in messages: + sqs.send_message(QueueUrl=source_queue_url, MessageBody=msg["Body"]) + sqs.delete_message(QueueUrl=dlq_url, ReceiptHandle=msg["ReceiptHandle"]) + return {"redriven": len(messages)} + + +def get_ecr_login_and_repo_uri(repository_name: str): + ecr = boto3.client("ecr", config=aws_client_config) + auth = ecr.get_authorization_token() + repo = ecr.describe_repositories(repositoryNames=[repository_name]) + repo_uri = repo["repositories"][0]["repositoryUri"] + token = auth["authorizationData"][0]["authorizationToken"] + proxy_endpoint = auth["authorizationData"][0]["proxyEndpoint"] + return { + "repository_uri": repo_uri, + "auth_token": token, + "proxy_endpoint": proxy_endpoint, + } + + def lambda_handler(payload: JSONType, context: LambdaContext): - return + event = json.loads(payload["body"]) + + if event == "delete_lambda_function": + delete_lambda_function("function_name_here") + elif event == "redrive_sqs_dlq": + redrive_sqs_dlq("source_queue_url_here", "dlq_url_here") + elif event == "get_ecr_login_and_repo_uri": + get_ecr_login_and_repo_uri("repository_name_here") + else: + return "Invalid or no event received" diff --git a/python/src/requirements.txt b/python/src/requirements.txt index e69de29..abc7d7b 100644 --- a/python/src/requirements.txt +++ b/python/src/requirements.txt @@ -0,0 +1,2 @@ +boto3 +json \ No newline at end of file diff --git a/python/test/__pycache__/test_base_lambda.cpython-311-pytest-8.1.1.pyc b/python/test/__pycache__/test_base_lambda.cpython-311-pytest-8.1.1.pyc new file mode 100644 index 0000000..dc6212b Binary files /dev/null and b/python/test/__pycache__/test_base_lambda.cpython-311-pytest-8.1.1.pyc differ diff --git a/python/test/requirements.txt b/python/test/requirements.txt index 18ed95d..5173710 100644 --- a/python/test/requirements.txt +++ b/python/test/requirements.txt @@ -4,6 +4,7 @@ aws-xray-sdk boto3 boto3-stubs dataclasses_json +json mdformat mdformat-gfm mdformat-black diff --git a/python/test/test_base_lambda.py b/python/test/test_base_lambda.py new file mode 100644 index 0000000..f7bdf6b --- /dev/null +++ b/python/test/test_base_lambda.py @@ -0,0 +1,121 @@ +from unittest import mock +import json + +import src.base_lambda as base_lambda + + +@mock.patch("src.base_lambda.boto3.client") +def test_delete_lambda_function_calls_boto3(mock_boto_client): + mock_lambda = mock.Mock() + mock_boto_client.return_value = mock_lambda + mock_lambda.delete_function.return_value = { + "ResponseMetadata": {"HTTPStatusCode": 204} + } + + resp = base_lambda.delete_lambda_function("my-func") + mock_boto_client.assert_called_once_with( + "lambda", config=base_lambda.aws_client_config + ) + mock_lambda.delete_function.assert_called_once_with(FunctionName="my-func") + assert resp == {"ResponseMetadata": {"HTTPStatusCode": 204}} + + +@mock.patch("src.base_lambda.boto3.client") +def test_redrive_sqs_dlq_moves_messages(mock_boto_client): + mock_sqs = mock.Mock() + mock_boto_client.return_value = mock_sqs + messages = [ + {"Body": "msg1", "ReceiptHandle": "rh1"}, + {"Body": "msg2", "ReceiptHandle": "rh2"}, + ] + mock_sqs.receive_message.return_value = {"Messages": messages} + + result = base_lambda.redrive_sqs_dlq("source_url", "dlq_url", max_messages=2) + + mock_boto_client.assert_called_once_with( + "sqs", config=base_lambda.aws_client_config + ) + mock_sqs.receive_message.assert_called_once_with( + QueueUrl="dlq_url", MaxNumberOfMessages=2, WaitTimeSeconds=2 + ) + assert mock_sqs.send_message.call_count == 2 + assert mock_sqs.delete_message.call_count == 2 + mock_sqs.send_message.assert_any_call(QueueUrl="source_url", MessageBody="msg1") + mock_sqs.send_message.assert_any_call(QueueUrl="source_url", MessageBody="msg2") + mock_sqs.delete_message.assert_any_call(QueueUrl="dlq_url", ReceiptHandle="rh1") + mock_sqs.delete_message.assert_any_call(QueueUrl="dlq_url", ReceiptHandle="rh2") + assert result == {"redriven": 2} + + +@mock.patch("src.base_lambda.boto3.client") +def test_redrive_sqs_dlq_no_messages(mock_boto_client): + mock_sqs = mock.Mock() + mock_boto_client.return_value = mock_sqs + mock_sqs.receive_message.return_value = {} + + result = base_lambda.redrive_sqs_dlq("source_url", "dlq_url") + assert result == {"redriven": 0} + mock_sqs.send_message.assert_not_called() + mock_sqs.delete_message.assert_not_called() + + +@mock.patch("src.base_lambda.boto3.client") +def test_get_ecr_login_and_repo_uri(mock_boto_client): + mock_ecr = mock.Mock() + mock_boto_client.return_value = mock_ecr + mock_ecr.get_authorization_token.return_value = { + "authorizationData": [ + {"authorizationToken": "token123", "proxyEndpoint": "https://ecr.aws"} + ] + } + mock_ecr.describe_repositories.return_value = { + "repositories": [ + {"repositoryUri": "123456789012.dkr.ecr.us-east-1.amazonaws.com/my-repo"} + ] + } + + result = base_lambda.get_ecr_login_and_repo_uri("my-repo") + assert result == { + "repository_uri": "123456789012.dkr.ecr.us-east-1.amazonaws.com/my-repo", + "auth_token": "token123", + "proxy_endpoint": "https://ecr.aws", + } + + +@mock.patch("src.base_lambda.delete_lambda_function") +@mock.patch("src.base_lambda.json") +def test_lambda_handler_delete_lambda_function(mock_json, mock_delete): + payload = {"body": json.dumps("delete_lambda_function")} + mock_json.loads.return_value = "delete_lambda_function" + context = mock.Mock() + base_lambda.lambda_handler(payload, context) + mock_delete.assert_called_once_with("function_name_here") + + +@mock.patch("src.base_lambda.redrive_sqs_dlq") +@mock.patch("src.base_lambda.json") +def test_lambda_handler_redrive_sqs_dlq(mock_json, mock_redrive): + payload = {"body": json.dumps("redrive_sqs_dlq")} + mock_json.loads.return_value = "redrive_sqs_dlq" + context = mock.Mock() + base_lambda.lambda_handler(payload, context) + mock_redrive.assert_called_once_with("source_queue_url_here", "dlq_url_here") + + +@mock.patch("src.base_lambda.get_ecr_login_and_repo_uri") +@mock.patch("src.base_lambda.json") +def test_lambda_handler_get_ecr_login_and_repo_uri(mock_json, mock_get_ecr): + payload = {"body": json.dumps("get_ecr_login_and_repo_uri")} + mock_json.loads.return_value = "get_ecr_login_and_repo_uri" + context = mock.Mock() + base_lambda.lambda_handler(payload, context) + mock_get_ecr.assert_called_once_with("repository_name_here") + + +@mock.patch("src.base_lambda.json") +def test_lambda_handler_invalid_event(mock_json): + payload = {"body": json.dumps("unknown_event")} + mock_json.loads.return_value = "unknown_event" + context = mock.Mock() + result = base_lambda.lambda_handler(payload, context) + assert result == "Invalid or no event received" diff --git a/terraform/iam_lambda.tf b/terraform/iam_lambda.tf index e69de29..0ba5b0f 100644 --- a/terraform/iam_lambda.tf +++ b/terraform/iam_lambda.tf @@ -0,0 +1,25 @@ +resource "aws_iam_role" "lambda_exec" { + name = "example_lambda_exec_role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + Service = "lambda.amazonaws.com" + } + }] + }) + + tags = { + Project = "QuickActions" + Environment = "Production" + Owner = "DevOps" + } +} + +resource "aws_iam_role_policy_attachment" "lambda_logs" { + role = aws_iam_role.lambda_exec.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" +} \ No newline at end of file diff --git a/terraform/lambda.tf b/terraform/lambda.tf index e69de29..4c99594 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -0,0 +1,48 @@ +resource "aws_lambda_function" "example_lambda" { + function_name = "example_lambda" + filename = "build/example_lambda.zip" + handler = "base_lambda.lambda_handler" + runtime = "python3.12" + role = aws_iam_role.lambda_exec.arn + + source_code_hash = filebase64sha256("build/example_lambda.zip") + + environment { + variables = { + ENV = "production" + } + } + + tags = { + Project = "QuickActions" + Environment = "Production" + } +} + +resource "aws_cloudwatch_log_group" "example_lambda" { + name = "/aws/lambda/${aws_lambda_function.example_lambda.function_name}" + retention_in_days = 14 + + tags = { + Project = "QuickActions" + Environment = "Production" + Owner = "DevOps" + } +} + +resource "aws_lambda_event_source_mapping" "example_lambda_event" { + event_source_arn = aws_sqs_queue.example_queue.arn + function_name = aws_lambda_function.example_lambda.arn + enabled = true + batch_size = 10 + starting_position = "LATEST" +} + +resource "aws_sqs_queue" "example_queue" { + name = "example-lambda-event-queue" + + tags = { + Project = "QuickActions" + Environment = "Production" + } +}