Distributor feeder

deployment/live/cloudbuild/dev/terragrunt.hcl

@@ -8,6 +8,7 @@ inputs = merge(
   {
     distributor_cloud_run_service = "distributor-service-dev"
     witness_cloud_run_service     = "witness-service-dev"
+    feeder_cloud_run_service     = "feeder-service-dev"
     slack_template_json           = file("slack.json")
   }
 )

deployment/live/feeder/dev/terragrunt.hcl

@@ -0,0 +1,16 @@
+include "root" {
+  path   = find_in_parent_folders("root.hcl")
+  expose = true
+}
+
+inputs = merge(
+  include.root.locals,
+  {
+    feeder_docker_image = "us-central1-docker.pkg.dev/checkpoint-distributor/distributor-docker-dev/feeder:latest"
+    extra_args = [
+      "--witness_url=https://api.transparency.dev/dev/witness/little-garden",
+    ]
+    ephemeral = true
+  }
+)
+

deployment/live/feeder/root.hcl

@@ -0,0 +1,24 @@
+terraform {
+  source = "${get_repo_root()}/deployment/modules/feeder"
+}
+
+locals {
+  project_id = "checkpoint-distributor"
+  region     = "us-central1"
+  env        = path_relative_to_include()
+}
+
+remote_state {
+  backend = "gcs"
+
+  config = {
+    project  = local.project_id
+    location = local.region
+    bucket   = "${local.project_id}-feeder-${local.env}-terraform-state"
+    prefix   = "${path_relative_to_include()}/terraform.tfstate"
+
+    gcs_bucket_labels = {
+      name = "terraform_state_storage"
+    }
+  }
+}

deployment/modules/cloudbuild/main.tf

@@ -32,6 +32,7 @@ locals {
   artifact_repo = "${var.region}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.distributor_docker.name}"
   distributor_docker_image  = "${local.artifact_repo}/distributor"
   witness_docker_image  = "${local.artifact_repo}/witness"
+  feeder_docker_image  = "${local.artifact_repo}/feeder"
 }
 
 resource "google_cloudbuild_trigger" "distributor_docker" {
@@ -234,6 +235,106 @@ resource "google_cloudbuild_trigger" "witness_docker_tag" {
   }
 }
 
+resource "google_cloudbuild_trigger" "feeder_docker" {
+  name            = "build-feeder-docker-${var.env}"
+  service_account = google_service_account.cloudbuild_service_account.id
+  location        = var.region
+
+  github {
+    owner = "transparency-dev"
+    name  = "witness"
+    push {
+      branch = "^main$"
+    }
+  }
+
+  build {
+    step {
+      name = "gcr.io/cloud-builders/docker"
+      args = [
+        "build",
+        "-t", "${local.feeder_docker_image}:$SHORT_SHA",
+        "-t", "${local.feeder_docker_image}:latest",
+        "-f", "./cmd/feedwitness/Dockerfile",
+        "."
+      ]
+    }
+    step {
+      name = "gcr.io/cloud-builders/docker"
+      args = [
+        "push",
+        "--all-tags",
+        local.feeder_docker_image
+      ]
+    }
+    # Deploy container image to Cloud Run
+    step {
+      name       = "gcr.io/google.com/cloudsdktool/cloud-sdk"
+      entrypoint = "gcloud"
+      args = [
+        "run",
+        "deploy",
+        var.feeder_cloud_run_service,
+        "--image",
+        "${local.feeder_docker_image}:$SHORT_SHA",
+        "--region",
+        var.region
+      ]
+    }
+    options {
+      logging = "CLOUD_LOGGING_ONLY"
+    }
+  }
+}
+
+# When a new tag is pushed to GitHub, add that tag to the docker
+# image that was already pushed to the repo for the corresponding
+# commit hash.
+# This requires that the above step has already completed, but that
+# seems like a fair assumption given that we'd have deployed it in ci
+# before tagging it.
+resource "google_cloudbuild_trigger" "feeder_docker_tag" {
+  name            = "tag-feeder-docker-${var.env}"
+  service_account = google_service_account.cloudbuild_service_account.id
+  location        = var.region
+
+  github {
+    owner = "transparency-dev"
+    name  = "witness"
+    push {
+      tag = ".*"
+    }
+  }
+
+  build {
+    step {
+      name = "gcr.io/cloud-builders/docker"
+      args = [
+        "pull",
+        "${local.feeder_docker_image}:$SHORT_SHA",
+      ]
+    }
+    step {
+      name = "gcr.io/cloud-builders/docker"
+      args = [
+        "tag",
+        "${local.feeder_docker_image}:$SHORT_SHA",
+        "${local.feeder_docker_image}:$TAG_NAME",
+      ]
+    }
+    step {
+      name = "gcr.io/cloud-builders/docker"
+      args = [
+        "push",
+        "${local.feeder_docker_image}:$TAG_NAME",
+      ]
+    }
+    options {
+      logging = "CLOUD_LOGGING_ONLY"
+    }
+  }
+}
+
 
 resource "google_service_account" "cloudbuild_service_account" {
   account_id   = "cloudbuild-${var.env}-sa"

deployment/modules/cloudbuild/outputs.tf

@@ -38,3 +38,8 @@ output "witness_docker_image" {
   description = "The address of the witness docker image that will be built"
   value       = local.witness_docker_image
 }
+
+output "feeder_docker_image" {
+  description = "The address of the feeder docker image that will be built"
+  value       = local.feeder_docker_image
+}

deployment/modules/cloudbuild/variables.tf

@@ -39,6 +39,11 @@ variable "witness_cloud_run_service" {
   type        = string
 }
 
+variable "feeder_cloud_run_service" {
+  description = "The name of the cloud run service running the feeder that new feeder images should be pushed to"
+  type        = string
+}
+
 variable "slack_template_json" {
   description = "Contents of the Slack template (https://cloud.google.com/build/docs/configuring-notifications/configure-slack#configuring_slack_notifications)"
   type        = string

deployment/modules/feeder/main.tf

@@ -0,0 +1,122 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# Project data
+provider "google" {
+  project = var.project_id
+}
+
+data "google_project" "project" {
+  project_id = var.project_id
+}
+
+# This will be configured by terragrunt when deploying
+terraform {
+  backend "gcs" {}
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "6.0.1"
+    }
+    google-beta = {
+      source  = "hashicorp/google-beta"
+      version = "6.0.1"
+    }
+  }
+}
+
+# Enable Cloud Run API
+resource "google_project_service" "cloudrun_api" {
+  service            = "run.googleapis.com"
+  disable_on_destroy = false
+}
+
+###
+### Set up Cloud Run service
+###
+resource "google_service_account" "cloudrun_service_account" {
+  account_id   = "cloudrun-feeder-${var.env}-sa"
+  display_name = "Service Account for feeder Cloud Run (${var.env})"
+}
+
+resource "google_project_iam_member" "iam_act_as" {
+  project = var.project_id
+  role    = "roles/iam.serviceAccountUser"
+  member  = "serviceAccount:${google_service_account.cloudrun_service_account.email}"
+}
+resource "google_project_iam_member" "iam_metrics_writer" {
+  project = var.project_id
+  role    = "roles/monitoring.metricWriter"
+  member  = "serviceAccount:${google_service_account.cloudrun_service_account.email}"
+}
+resource "google_project_iam_member" "iam_service_agent" {
+  project = var.project_id
+  role    = "roles/run.serviceAgent"
+  member  = "serviceAccount:${google_service_account.cloudrun_service_account.email}"
+}
+
+resource "google_cloud_run_v2_service" "default" {
+  name         = "feeder-service-${var.env}"
+  location     = var.region
+  launch_stage = "GA"
+
+
+  template {
+    service_account = google_service_account.cloudrun_service_account.email
+    scaling {
+      min_instance_count = 1
+      max_instance_count = 1
+    }
+    containers {
+      image = var.feeder_docker_image
+      name  = "feeder"
+      args = concat([
+        "--logtostderr",
+        "--v=1",
+        "--metrics_listen=:8081",
+        "--max_qps=${var.max_qps}",
+      ], var.extra_args)
+     ports {
+        container_port = 8081
+      }
+
+      startup_probe {
+        initial_delay_seconds = 1
+        timeout_seconds       = 1
+        period_seconds        = 10
+        failure_threshold     = 3
+        tcp_socket {
+          port = 8081
+        }
+      }
+
+    }
+    containers {
+      image      = "us-docker.pkg.dev/cloud-ops-agents-artifacts/cloud-run-gmp-sidecar/cloud-run-gmp-sidecar:1.3.0"
+      name       = "collector"
+      depends_on = ["feeder"]
+    }
+  }
+  client = "terraform"
+  depends_on = [
+    google_project_service.cloudrun_api,
+    google_project_iam_member.iam_act_as,
+    google_project_iam_member.iam_metrics_writer,
+    google_project_iam_member.iam_service_agent,
+  ]
+
+  deletion_protection = !var.ephemeral
+}

deployment/modules/feeder/variables.tf

@@ -0,0 +1,54 @@
+/**
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+  description = "The project ID to host the cluster in"
+  type        = string
+}
+
+variable "region" {
+  description = "The region to host the cluster in"
+  type        = string
+}
+
+variable "env" {
+  description = "Unique identifier for the env, e.g. ci or prod"
+  type        = string
+}
+
+variable "feeder_docker_image" {
+  description = "The full image URL (path & tag) for the feeder docker image to deploy"
+  type        = string
+}
+
+variable "extra_args" {
+  description = "Extra arguments to be provided to the feeder invoked in cloud run"
+  type        = list(string)
+  default     = []
+}
+
+variable "ephemeral" {
+  description = "Set to true if this is a CI/temporary deploy"
+  type        = bool
+  default     = false
+}
+
+variable "max_qps" {
+  description = "Max qps to send to witnesses"
+  type = number
+  default = 2.0
+}
+