diff --git a/deployment/live/cloudbuild/dev/terragrunt.hcl b/deployment/live/cloudbuild/dev/terragrunt.hcl index 42236c3..fb7688c 100644 --- a/deployment/live/cloudbuild/dev/terragrunt.hcl +++ b/deployment/live/cloudbuild/dev/terragrunt.hcl @@ -8,6 +8,7 @@ inputs = merge( { distributor_cloud_run_service = "distributor-service-dev" witness_cloud_run_service = "witness-service-dev" + feeder_cloud_run_service = "feeder-service-dev" slack_template_json = file("slack.json") } ) diff --git a/deployment/live/feeder/dev/terragrunt.hcl b/deployment/live/feeder/dev/terragrunt.hcl new file mode 100644 index 0000000..3870435 --- /dev/null +++ b/deployment/live/feeder/dev/terragrunt.hcl @@ -0,0 +1,16 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +inputs = merge( + include.root.locals, + { + feeder_docker_image = "us-central1-docker.pkg.dev/checkpoint-distributor/distributor-docker-dev/feeder:latest" + extra_args = [ + "--witness_url=https://api.transparency.dev/dev/witness/little-garden", + ] + ephemeral = true + } +) + diff --git a/deployment/live/feeder/root.hcl b/deployment/live/feeder/root.hcl new file mode 100644 index 0000000..9473667 --- /dev/null +++ b/deployment/live/feeder/root.hcl @@ -0,0 +1,24 @@ +terraform { + source = "${get_repo_root()}/deployment/modules/feeder" +} + +locals { + project_id = "checkpoint-distributor" + region = "us-central1" + env = path_relative_to_include() +} + +remote_state { + backend = "gcs" + + config = { + project = local.project_id + location = local.region + bucket = "${local.project_id}-feeder-${local.env}-terraform-state" + prefix = "${path_relative_to_include()}/terraform.tfstate" + + gcs_bucket_labels = { + name = "terraform_state_storage" + } + } +} diff --git a/deployment/modules/cloudbuild/main.tf b/deployment/modules/cloudbuild/main.tf index ef8c07a..ee2ec86 100644 --- a/deployment/modules/cloudbuild/main.tf +++ b/deployment/modules/cloudbuild/main.tf @@ -32,6 +32,7 @@ locals { artifact_repo = "${var.region}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.distributor_docker.name}" distributor_docker_image = "${local.artifact_repo}/distributor" witness_docker_image = "${local.artifact_repo}/witness" + feeder_docker_image = "${local.artifact_repo}/feeder" } resource "google_cloudbuild_trigger" "distributor_docker" { @@ -234,6 +235,106 @@ resource "google_cloudbuild_trigger" "witness_docker_tag" { } } +resource "google_cloudbuild_trigger" "feeder_docker" { + name = "build-feeder-docker-${var.env}" + service_account = google_service_account.cloudbuild_service_account.id + location = var.region + + github { + owner = "transparency-dev" + name = "witness" + push { + branch = "^main$" + } + } + + build { + step { + name = "gcr.io/cloud-builders/docker" + args = [ + "build", + "-t", "${local.feeder_docker_image}:$SHORT_SHA", + "-t", "${local.feeder_docker_image}:latest", + "-f", "./cmd/feedwitness/Dockerfile", + "." + ] + } + step { + name = "gcr.io/cloud-builders/docker" + args = [ + "push", + "--all-tags", + local.feeder_docker_image + ] + } + # Deploy container image to Cloud Run + step { + name = "gcr.io/google.com/cloudsdktool/cloud-sdk" + entrypoint = "gcloud" + args = [ + "run", + "deploy", + var.feeder_cloud_run_service, + "--image", + "${local.feeder_docker_image}:$SHORT_SHA", + "--region", + var.region + ] + } + options { + logging = "CLOUD_LOGGING_ONLY" + } + } +} + +# When a new tag is pushed to GitHub, add that tag to the docker +# image that was already pushed to the repo for the corresponding +# commit hash. +# This requires that the above step has already completed, but that +# seems like a fair assumption given that we'd have deployed it in ci +# before tagging it. +resource "google_cloudbuild_trigger" "feeder_docker_tag" { + name = "tag-feeder-docker-${var.env}" + service_account = google_service_account.cloudbuild_service_account.id + location = var.region + + github { + owner = "transparency-dev" + name = "witness" + push { + tag = ".*" + } + } + + build { + step { + name = "gcr.io/cloud-builders/docker" + args = [ + "pull", + "${local.feeder_docker_image}:$SHORT_SHA", + ] + } + step { + name = "gcr.io/cloud-builders/docker" + args = [ + "tag", + "${local.feeder_docker_image}:$SHORT_SHA", + "${local.feeder_docker_image}:$TAG_NAME", + ] + } + step { + name = "gcr.io/cloud-builders/docker" + args = [ + "push", + "${local.feeder_docker_image}:$TAG_NAME", + ] + } + options { + logging = "CLOUD_LOGGING_ONLY" + } + } +} + resource "google_service_account" "cloudbuild_service_account" { account_id = "cloudbuild-${var.env}-sa" diff --git a/deployment/modules/cloudbuild/outputs.tf b/deployment/modules/cloudbuild/outputs.tf index 831a0d3..153f9cd 100644 --- a/deployment/modules/cloudbuild/outputs.tf +++ b/deployment/modules/cloudbuild/outputs.tf @@ -38,3 +38,8 @@ output "witness_docker_image" { description = "The address of the witness docker image that will be built" value = local.witness_docker_image } + +output "feeder_docker_image" { + description = "The address of the feeder docker image that will be built" + value = local.feeder_docker_image +} diff --git a/deployment/modules/cloudbuild/variables.tf b/deployment/modules/cloudbuild/variables.tf index be2adf7..b2167c7 100644 --- a/deployment/modules/cloudbuild/variables.tf +++ b/deployment/modules/cloudbuild/variables.tf @@ -39,6 +39,11 @@ variable "witness_cloud_run_service" { type = string } +variable "feeder_cloud_run_service" { + description = "The name of the cloud run service running the feeder that new feeder images should be pushed to" + type = string +} + variable "slack_template_json" { description = "Contents of the Slack template (https://cloud.google.com/build/docs/configuring-notifications/configure-slack#configuring_slack_notifications)" type = string diff --git a/deployment/modules/feeder/main.tf b/deployment/modules/feeder/main.tf new file mode 100644 index 0000000..2ed753b --- /dev/null +++ b/deployment/modules/feeder/main.tf @@ -0,0 +1,122 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# Project data +provider "google" { + project = var.project_id +} + +data "google_project" "project" { + project_id = var.project_id +} + +# This will be configured by terragrunt when deploying +terraform { + backend "gcs" {} + required_providers { + google = { + source = "hashicorp/google" + version = "6.0.1" + } + google-beta = { + source = "hashicorp/google-beta" + version = "6.0.1" + } + } +} + +# Enable Cloud Run API +resource "google_project_service" "cloudrun_api" { + service = "run.googleapis.com" + disable_on_destroy = false +} + +### +### Set up Cloud Run service +### +resource "google_service_account" "cloudrun_service_account" { + account_id = "cloudrun-feeder-${var.env}-sa" + display_name = "Service Account for feeder Cloud Run (${var.env})" +} + +resource "google_project_iam_member" "iam_act_as" { + project = var.project_id + role = "roles/iam.serviceAccountUser" + member = "serviceAccount:${google_service_account.cloudrun_service_account.email}" +} +resource "google_project_iam_member" "iam_metrics_writer" { + project = var.project_id + role = "roles/monitoring.metricWriter" + member = "serviceAccount:${google_service_account.cloudrun_service_account.email}" +} +resource "google_project_iam_member" "iam_service_agent" { + project = var.project_id + role = "roles/run.serviceAgent" + member = "serviceAccount:${google_service_account.cloudrun_service_account.email}" +} + +resource "google_cloud_run_v2_service" "default" { + name = "feeder-service-${var.env}" + location = var.region + launch_stage = "GA" + + + template { + service_account = google_service_account.cloudrun_service_account.email + scaling { + min_instance_count = 1 + max_instance_count = 1 + } + containers { + image = var.feeder_docker_image + name = "feeder" + args = concat([ + "--logtostderr", + "--v=1", + "--metrics_listen=:8081", + "--max_qps=${var.max_qps}", + ], var.extra_args) + ports { + container_port = 8081 + } + + startup_probe { + initial_delay_seconds = 1 + timeout_seconds = 1 + period_seconds = 10 + failure_threshold = 3 + tcp_socket { + port = 8081 + } + } + + } + containers { + image = "us-docker.pkg.dev/cloud-ops-agents-artifacts/cloud-run-gmp-sidecar/cloud-run-gmp-sidecar:1.3.0" + name = "collector" + depends_on = ["feeder"] + } + } + client = "terraform" + depends_on = [ + google_project_service.cloudrun_api, + google_project_iam_member.iam_act_as, + google_project_iam_member.iam_metrics_writer, + google_project_iam_member.iam_service_agent, + ] + + deletion_protection = !var.ephemeral +} diff --git a/deployment/modules/feeder/variables.tf b/deployment/modules/feeder/variables.tf new file mode 100644 index 0000000..15b74a7 --- /dev/null +++ b/deployment/modules/feeder/variables.tf @@ -0,0 +1,54 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" + type = string +} + +variable "region" { + description = "The region to host the cluster in" + type = string +} + +variable "env" { + description = "Unique identifier for the env, e.g. ci or prod" + type = string +} + +variable "feeder_docker_image" { + description = "The full image URL (path & tag) for the feeder docker image to deploy" + type = string +} + +variable "extra_args" { + description = "Extra arguments to be provided to the feeder invoked in cloud run" + type = list(string) + default = [] +} + +variable "ephemeral" { + description = "Set to true if this is a CI/temporary deploy" + type = bool + default = false +} + +variable "max_qps" { + description = "Max qps to send to witnesses" + type = number + default = 2.0 +} +