From 5fbe95d0bd1e870b8263bd5cbf6f6fd06157b693 Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Wed, 26 Aug 2026 11:43:38 +0000 Subject: [PATCH 01/14] Auto-generate terraform module for gcs-spanner-dv --- .../dataflow_job.tf | 292 ++++++++++++++++++ 1 file changed, 292 insertions(+) create mode 100644 v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf diff --git a/v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf b/v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf new file mode 100644 index 0000000000..de3fe72443 --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf @@ -0,0 +1,292 @@ + + +# Autogenerated file. DO NOT EDIT. +# +# Copyright (C) 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + + +variable "on_delete" { + type = string + description = "One of \"drain\" or \"cancel\". Specifies behavior of deletion during terraform destroy." +} + +variable "project" { + type = string + description = "The Google Cloud Project ID within which this module provisions resources." +} + +variable "region" { + type = string + description = "The region in which the created job should run." +} + +variable "gcsInputDirectory" { + type = string + description = "This directory is used to read the AVRO files of the records read from source. For example, `gs://your-bucket/your-path`" + default = null +} + +variable "projectId" { + type = string + description = "This is the name of the Cloud Spanner project." + default = null +} + +variable "spannerHost" { + type = string + description = "The Cloud Spanner endpoint to call in the template. For example, `https://batch-spanner.googleapis.com`. Defaults to: https://batch-spanner.googleapis.com." + default = null +} + +variable "instanceId" { + type = string + description = "The destination Cloud Spanner instance." + +} + +variable "databaseId" { + type = string + description = "The destination Cloud Spanner database." + +} + +variable "spannerPriority" { + type = string + description = "The request priority for Cloud Spanner calls. The value must be one of: [`HIGH`,`MEDIUM`,`LOW`]. Defaults to `HIGH`." + default = null +} + +variable "sessionFilePath" { + type = string + description = "Session file path in Cloud Storage that contains mapping information from Spanner Migration Tool. Defaults to empty." + default = null +} + +variable "schemaOverridesFilePath" { + type = string + description = "A file which specifies the table and the column name overrides from source to spanner. Defaults to empty." + default = null +} + +variable "tableOverrides" { + type = string + description = "These are the table name overrides from source to spanner. They are written in the following format: [{SourceTableName1, SpannerTableName1}, {SourceTableName2, SpannerTableName2}] This example shows mapping Singers table to Vocalists and Albums table to Records. For example, `[{Singers, Vocalists}, {Albums, Records}]`. Defaults to empty." + default = null +} + +variable "columnOverrides" { + type = string + description = "These are the column name overrides from source to spanner. They are written in the following format: [{SourceTableName1.SourceColumnName1, SourceTableName1.SpannerColumnName1}, {SourceTableName2.SourceColumnName1, SourceTableName2.SpannerColumnName1}]Note that the SourceTableName should remain the same in both the source and spanner pair. To override table names, use tableOverrides.The example shows mapping SingerName to TalentName and AlbumName to RecordName in Singers and Albums table respectively. For example, `[{Singers.SingerName, Singers.TalentName}, {Albums.AlbumName, Albums.RecordName}]`. Defaults to empty." + default = null +} + +variable "bigQueryDataset" { + type = string + description = "The BigQuery dataset ID where the validation results will be stored. For example, `validation_report_dataset`" + +} + +variable "runId" { + type = string + description = "A unique identifier for the validation run. If not provided, the Dataflow Job Name will be used. For example, `run_20230101_120000`" + default = null +} + +variable "transformationJarPath" { + type = string + description = "Custom jar location in Cloud Storage that contains the custom transformation logic for processing records. Defaults to empty." + default = null +} + +variable "transformationClassName" { + type = string + description = "Fully qualified class name having the custom transformation logic. It is a mandatory field in case transformationJarPath is specified. Defaults to empty." + default = null +} + +variable "transformationCustomParameters" { + type = string + description = "String containing any custom parameters to be passed to the custom transformation class. Defaults to empty." + default = null +} + + +provider "google" { + project = var.project +} + +provider "google-beta" { + project = var.project +} + +variable "additional_experiments" { + type = set(string) + description = "List of experiments that should be used by the job. An example value is 'enable_stackdriver_agent_metrics'." + default = null +} + +variable "autoscaling_algorithm" { + type = string + description = "The algorithm to use for autoscaling" + default = null +} + +variable "enable_streaming_engine" { + type = bool + description = "Indicates if the job should use the streaming engine feature." + default = null +} + +variable "ip_configuration" { + type = string + description = "The configuration for VM IPs. Options are 'WORKER_IP_PUBLIC' or 'WORKER_IP_PRIVATE'." + default = null +} + +variable "kms_key_name" { + type = string + description = "The name for the Cloud KMS key for the job. Key format is: projects/PROJECT_ID/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY" + default = null +} + +variable "labels" { + type = map(string) + description = "User labels to be specified for the job. Keys and values should follow the restrictions specified in the labeling restrictions page. NOTE: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field 'effective_labels' for all of the labels present on the resource." + default = null +} + +variable "launcher_machine_type" { + type = string + description = "The machine type to use for launching the job. The default is n1-standard-1." + default = null +} + +variable "machine_type" { + type = string + description = "The machine type to use for the job." + default = null +} + +variable "max_workers" { + type = number + description = "The maximum number of Google Compute Engine instances to be made available to your pipeline during execution, from 1 to 1000." + default = null +} + +variable "name" { + type = string +} + +variable "network" { + type = string + description = "The network to which VMs will be assigned. If it is not provided, 'default' will be used." + default = null +} + +variable "num_workers" { + type = number + description = "The initial number of Google Compute Engine instances for the job." + default = null +} + +variable "sdk_container_image" { + type = string + description = "Docker registry location of container image to use for the 'worker harness. Default is the container for the version of the SDK. Note this field is only valid for portable pipelines." + default = null +} + +variable "service_account_email" { + type = string + description = "The Service Account email used to create the job." + default = null +} + +variable "skip_wait_on_job_termination" { + type = bool + description = "If true, treat DRAINING and CANCELLING as terminal job states and do not wait for further changes before removing from terraform state and moving on. WARNING: this will lead to job name conflicts if you do not ensure that the job names are different, e.g. by embedding a release ID or by using a random_id." + default = null +} + +variable "staging_location" { + type = string + description = "The Cloud Storage path to use for staging files. Must be a valid Cloud Storage URL, beginning with gs://." + default = null +} + +variable "subnetwork" { + type = string + description = "The subnetwork to which VMs will be assigned. Should be of the form 'regions/REGION/subnetworks/SUBNETWORK'." + default = null +} + +variable "temp_location" { + type = string + description = "The Cloud Storage path to use for temporary files. Must be a valid Cloud Storage URL, beginning with gs://." + default = null +} + +resource "google_project_service" "required" { + service = "dataflow.googleapis.com" + disable_on_destroy = false +} + +resource "google_dataflow_flex_template_job" "generated" { + depends_on = [google_project_service.required] + provider = google-beta + container_spec_gcs_path = "gs://dataflow-templates-${var.region}/latest/flex/GCS_Spanner_Data_Validator" + parameters = { + gcsInputDirectory = var.gcsInputDirectory + projectId = var.projectId + spannerHost = var.spannerHost + instanceId = var.instanceId + databaseId = var.databaseId + spannerPriority = var.spannerPriority + sessionFilePath = var.sessionFilePath + schemaOverridesFilePath = var.schemaOverridesFilePath + tableOverrides = var.tableOverrides + columnOverrides = var.columnOverrides + bigQueryDataset = var.bigQueryDataset + runId = var.runId + transformationJarPath = var.transformationJarPath + transformationClassName = var.transformationClassName + transformationCustomParameters = var.transformationCustomParameters + } + + additional_experiments = var.additional_experiments + autoscaling_algorithm = var.autoscaling_algorithm + enable_streaming_engine = var.enable_streaming_engine + ip_configuration = var.ip_configuration + kms_key_name = var.kms_key_name + labels = var.labels + launcher_machine_type = var.launcher_machine_type + machine_type = var.machine_type + max_workers = var.max_workers + name = var.name + network = var.network + num_workers = var.num_workers + sdk_container_image = var.sdk_container_image + service_account_email = var.service_account_email + skip_wait_on_job_termination = var.skip_wait_on_job_termination + staging_location = var.staging_location + subnetwork = var.subnetwork + temp_location = var.temp_location + region = var.region +} + +output "dataflow_job_url" { + value = "https://console.cloud.google.com/dataflow/jobs/${var.region}/${google_dataflow_flex_template_job.generated.job_id}" +} + From 1ccd1849678606326da0374370f09e3b093744b8 Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Wed, 26 Aug 2026 12:02:26 +0000 Subject: [PATCH 02/14] Add terraform deployment sample for gcs-spanner-dv --- .../terraform/samples/gcs-spanner-dv/main.tf | 62 ++++++++ .../samples/gcs-spanner-dv/outputs.tf | 4 + .../samples/gcs-spanner-dv/variables.tf | 142 ++++++++++++++++++ 3 files changed, 208 insertions(+) create mode 100644 v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/main.tf create mode 100644 v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/outputs.tf create mode 100644 v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/variables.tf diff --git a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/main.tf b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/main.tf new file mode 100644 index 0000000000..2ced85e813 --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/main.tf @@ -0,0 +1,62 @@ +provider "google" { + project = var.project +} + +provider "google-beta" { + project = var.project +} + +data "google_project" "project" { + project_id = var.project +} + +data "google_compute_default_service_account" "gce_account" { + project = var.project +} + +resource "google_project_iam_member" "dataflow_roles" { + for_each = var.add_policies_to_service_account ? toset([ + "roles/dataflow.worker", + "roles/spanner.databaseAdmin", + "roles/storage.objectViewer", + "roles/bigquery.dataEditor" + ]) : toset([]) + project = data.google_project.project.id + role = each.key + member = var.service_account_email != null ? "serviceAccount:${var.service_account_email}" : "serviceAccount:${data.google_compute_default_service_account.gce_account.email}" +} + +resource "google_dataflow_flex_template_job" "gcs_spanner_dv_job" { + provider = google-beta + name = var.job_name + project = var.project + region = var.region + container_spec_gcs_path = "gs://dataflow-templates-${var.region}/latest/flex/GCS_Spanner_Data_Validator" + + parameters = { + gcsInputDirectory = var.gcs_input_directory + projectId = var.spanner_project_id != null ? var.spanner_project_id : var.project + instanceId = var.instance_id + databaseId = var.database_id + bigQueryDataset = var.bigquery_dataset + spannerHost = var.spanner_host + spannerPriority = var.spanner_priority + sessionFilePath = var.session_file_path + schemaOverridesFilePath = var.schema_overrides_file_path + tableOverrides = var.table_overrides + columnOverrides = var.column_overrides + runId = var.run_id + transformationJarPath = var.transformation_jar_path + transformationClassName = var.transformation_class_name + transformationCustomParameters = var.transformation_custom_parameters + } + + service_account_email = var.service_account_email + network = var.network != null ? "projects/${var.project}/global/networks/${var.network}" : null + subnetwork = var.subnetwork != null ? "https://www.googleapis.com/compute/v1/projects/${var.project}/regions/${var.region}/subnetworks/${var.subnetwork}" : null + machine_type = var.machine_type + max_workers = var.max_workers + additional_experiments = var.additional_experiments + + depends_on = [google_project_iam_member.dataflow_roles] +} diff --git a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/outputs.tf b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/outputs.tf new file mode 100644 index 0000000000..1a446716c0 --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/outputs.tf @@ -0,0 +1,4 @@ +output "dataflow_job_url" { + description = "The URL of the created Dataflow job in the Google Cloud Console." + value = "https://console.cloud.google.com/dataflow/jobs/${var.region}/${google_dataflow_flex_template_job.gcs_spanner_dv_job.job_id}?project=${var.project}" +} diff --git a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/variables.tf b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/variables.tf new file mode 100644 index 0000000000..f66b62eb39 --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/variables.tf @@ -0,0 +1,142 @@ +variable "project" { + type = string + description = "Google Cloud Project ID where Dataflow will run." +} + +variable "region" { + type = string + description = "Google Cloud region to run Dataflow in." +} + +variable "job_name" { + type = string + description = "Dataflow job name." +} + +variable "add_policies_to_service_account" { + type = bool + description = "Terraform will add the required permission to the dataflow service account." + default = true +} + +variable "service_account_email" { + type = string + description = "Service account email for Dataflow workers. If not set, the default Compute Engine service account is used." + default = null +} + +variable "network" { + type = string + description = "Network for Dataflow workers." + default = null +} + +variable "subnetwork" { + type = string + description = "Subnetwork for Dataflow workers." + default = null +} + +variable "machine_type" { + type = string + description = "Machine type for Dataflow worker VMs." + default = "n1-standard-4" +} + +variable "max_workers" { + type = number + description = "Maximum number of Dataflow worker VMs." + default = 10 +} + +variable "gcs_input_directory" { + type = string + description = "The GCS directory containing the data for validation. Example: gs://my-bucket/path/" +} + +variable "instance_id" { + type = string + description = "Cloud Spanner instance ID." +} + +variable "database_id" { + type = string + description = "Cloud Spanner database ID." +} + +variable "spanner_project_id" { + type = string + description = "Google Cloud Project ID containing the Spanner instance. Defaults to the Dataflow project if not provided." + default = null +} + +variable "bigquery_dataset" { + type = string + description = "The BigQuery dataset ID where the validation results will be stored. Example: validation_report_dataset" +} + +variable "spanner_host" { + type = string + description = "Custom Spanner host endpoint." + default = null +} + +variable "spanner_priority" { + type = string + description = "Priority for Spanner RPCs (e.g. HIGH, MEDIUM, LOW)." + default = null +} + +variable "session_file_path" { + type = string + description = "GCS path to the session file." + default = null +} + +variable "schema_overrides_file_path" { + type = string + description = "GCS path to the schema overrides file." + default = null +} + +variable "table_overrides" { + type = string + description = "Table name overrides from source to Spanner." + default = null +} + +variable "column_overrides" { + type = string + description = "Column name overrides from source to Spanner." + default = null +} + +variable "run_id" { + type = string + description = "A unique identifier for the validation run." + default = null +} + +variable "transformation_jar_path" { + type = string + description = "GCS path to the transformation JAR file." + default = null +} + +variable "transformation_class_name" { + type = string + description = "Fully qualified transformation class name." + default = null +} + +variable "transformation_custom_parameters" { + type = string + description = "Custom parameters for the transformation." + default = null +} + +variable "additional_experiments" { + type = list(string) + description = "Additional Dataflow experiments." + default = [] +} From 2c8832b49300bae3e760c2f5cbb80ca0fa35e24a Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Wed, 26 Aug 2026 12:03:03 +0000 Subject: [PATCH 03/14] Add terraform_simple.tfvars placeholder --- .../samples/gcs-spanner-dv/terraform_simple.tfvars | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform_simple.tfvars diff --git a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform_simple.tfvars b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform_simple.tfvars new file mode 100644 index 0000000000..45d129fb52 --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform_simple.tfvars @@ -0,0 +1,7 @@ +project = "my-gcp-project" +region = "us-central1" +job_name = "gcs-spanner-dv-sample" +gcs_input_directory = "gs://my-bucket/validation-data/" +instance_id = "my-spanner-instance" +database_id = "my-spanner-database" +bigquery_dataset = "validation_report_dataset" From f7ab988149fadb2afcf6efb9c8683f3ac952344d Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Wed, 26 Aug 2026 12:09:37 +0000 Subject: [PATCH 04/14] Update docs with Terraform samples for GCS Spanner DV --- .../README_GCS_Spanner_Data_Validator.md | 2 + v2/gcs-spanner-dv/terraform/samples/README.md | 57 +++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 v2/gcs-spanner-dv/terraform/samples/README.md diff --git a/v2/gcs-spanner-dv/README_GCS_Spanner_Data_Validator.md b/v2/gcs-spanner-dv/README_GCS_Spanner_Data_Validator.md index 8c34a9a472..86419932dd 100644 --- a/v2/gcs-spanner-dv/README_GCS_Spanner_Data_Validator.md +++ b/v2/gcs-spanner-dv/README_GCS_Spanner_Data_Validator.md @@ -270,3 +270,5 @@ resource "google_dataflow_flex_template_job" "gcs_spanner_data_validator" { } } ``` + +For more advanced end-to-end deployment examples using Terraform, please refer to the [Terraform samples directory](terraform/samples/). diff --git a/v2/gcs-spanner-dv/terraform/samples/README.md b/v2/gcs-spanner-dv/terraform/samples/README.md new file mode 100644 index 0000000000..c36e411f30 --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/README.md @@ -0,0 +1,57 @@ +# Terraform Samples for GCS to Spanner Data Validation + +This directory contains Terraform deployment samples to run the [GCS to Spanner Data Validation](../../README_GCS_Spanner_Data_Validator.md) Dataflow pipeline. + +The provided sample in [`gcs-spanner-dv`](./gcs-spanner-dv/) demonstrates how to easily launch the Dataflow job while automatically attaching all necessary IAM roles. + +## What this sample does + +The Terraform module will create the following Google Cloud resources: +1. **Dataflow Flex Template Job:** Uses `google_dataflow_flex_template_job` to launch the data validation pipeline. +2. **IAM Role Bindings:** Grants the Dataflow worker service account the required roles to run the validation: + * `roles/dataflow.worker` + * `roles/spanner.databaseAdmin` + * `roles/storage.objectViewer` + * `roles/bigquery.dataEditor` (required to write validation reports to BigQuery) + +## Prerequisites + +Before executing the sample, ensure you meet the following requirements: + +1. **APIs Enabled**: The following APIs must be enabled in your Google Cloud Project: + * Dataflow API (`dataflow.googleapis.com`) + * Cloud Spanner API (`spanner.googleapis.com`) + * Cloud Storage API (`storage-component.googleapis.com`) + * BigQuery API (`bigquery.googleapis.com`) + +2. **Terraform**: Make sure Terraform is installed locally and you are authenticated using `gcloud auth application-default login`. + +## Step-by-Step Usage Instructions + +1. **Navigate to the sample directory** + ```shell + cd gcs-spanner-dv/ + ``` + +2. **Initialize Terraform** + Download the required providers and initialize the state. + ```shell + terraform init + ``` + +3. **Configure the variables** + Open the `terraform_simple.tfvars` file and modify the placeholder values (such as `project`, `instance_id`, `database_id`, and `gcs_input_directory`) to match your environment. + +4. **Plan the deployment** + Review the resources that Terraform will create: + ```shell + terraform plan -var-file=terraform_simple.tfvars + ``` + +5. **Apply the deployment** + Execute the creation of the resources: + ```shell + terraform apply -var-file=terraform_simple.tfvars + ``` + + Once the command completes, Terraform will output the `dataflow_job_url`, which you can click to view the running job in the Google Cloud Console. From 6e2f6f07ae775e8a9373a34779d2158baa2793d7 Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Wed, 26 Aug 2026 12:49:12 +0000 Subject: [PATCH 05/14] Address code review feedback for gcs-spanner-dv terraform --- .../terraform/samples/gcs-spanner-dv/main.tf | 4 +++- .../samples/gcs-spanner-dv/terraform_simple.tfvars | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/main.tf b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/main.tf index 2ced85e813..4143595f06 100644 --- a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/main.tf +++ b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/main.tf @@ -14,10 +14,11 @@ data "google_compute_default_service_account" "gce_account" { project = var.project } +# Add roles to the service account that will run Dataflow for data validation resource "google_project_iam_member" "dataflow_roles" { for_each = var.add_policies_to_service_account ? toset([ "roles/dataflow.worker", - "roles/spanner.databaseAdmin", + "roles/spanner.databaseReader", "roles/storage.objectViewer", "roles/bigquery.dataEditor" ]) : toset([]) @@ -26,6 +27,7 @@ resource "google_project_iam_member" "dataflow_roles" { member = var.service_account_email != null ? "serviceAccount:${var.service_account_email}" : "serviceAccount:${data.google_compute_default_service_account.gce_account.email}" } +# Define the Dataflow Flex Template job for GCS to Spanner Data Validation resource "google_dataflow_flex_template_job" "gcs_spanner_dv_job" { provider = google-beta name = var.job_name diff --git a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform_simple.tfvars b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform_simple.tfvars index 45d129fb52..2b7b4f08c3 100644 --- a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform_simple.tfvars +++ b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform_simple.tfvars @@ -1,7 +1,21 @@ +# Below is a simplified version of the configuration for GCS to Spanner Data Validation +# Provide the Google Cloud project ID project = "my-gcp-project" + +# The region to deploy the Dataflow job region = "us-central1" + +# The name of the Dataflow job job_name = "gcs-spanner-dv-sample" + +# The Cloud Storage directory containing validation data (should end with a '/') gcs_input_directory = "gs://my-bucket/validation-data/" + +# The Spanner instance ID to validate against instance_id = "my-spanner-instance" + +# The Spanner database ID to validate against database_id = "my-spanner-database" + +# The BigQuery dataset to store validation reports bigquery_dataset = "validation_report_dataset" From 0923f147b97876766637a0a2201ed9d76edca609 Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Thu, 27 Aug 2026 10:05:07 +0000 Subject: [PATCH 06/14] improvements --- .../main/resources/terraform-flex-template.tf | 1 + .../dataflow_job.tf | 256 +++++++++--------- v2/gcs-spanner-dv/terraform/samples/README.md | 69 ++--- .../samples/gcs-spanner-dv/README.md | 54 ++++ .../terraform/samples/gcs-spanner-dv/main.tf | 81 +++--- .../samples/gcs-spanner-dv/outputs.tf | 9 +- .../samples/gcs-spanner-dv/terraform.tf | 37 +++ .../gcs-spanner-dv/terraform_simple.tfvars | 35 ++- .../samples/gcs-spanner-dv/variables.tf | 79 ++++-- 9 files changed, 383 insertions(+), 238 deletions(-) create mode 100644 v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/README.md create mode 100644 v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform.tf diff --git a/plugins/core-plugin/src/main/resources/terraform-flex-template.tf b/plugins/core-plugin/src/main/resources/terraform-flex-template.tf index d9234d7ef7..0847194279 100644 --- a/plugins/core-plugin/src/main/resources/terraform-flex-template.tf +++ b/plugins/core-plugin/src/main/resources/terraform-flex-template.tf @@ -190,6 +190,7 @@ resource "google_dataflow_flex_template_job" "generated" { subnetwork = var.subnetwork temp_location = var.temp_location region = var.region + on_delete = var.on_delete } output "dataflow_job_url" { diff --git a/v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf b/v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf index de3fe72443..b52990eff1 100644 --- a/v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf +++ b/v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf @@ -34,259 +34,261 @@ variable "region" { } variable "gcsInputDirectory" { - type = string + type = string description = "This directory is used to read the AVRO files of the records read from source. For example, `gs://your-bucket/your-path`" - default = null + default = null } variable "projectId" { - type = string + type = string description = "This is the name of the Cloud Spanner project." - default = null + default = null } variable "spannerHost" { - type = string + type = string description = "The Cloud Spanner endpoint to call in the template. For example, `https://batch-spanner.googleapis.com`. Defaults to: https://batch-spanner.googleapis.com." - default = null + default = null } variable "instanceId" { - type = string + type = string description = "The destination Cloud Spanner instance." - + } variable "databaseId" { - type = string + type = string description = "The destination Cloud Spanner database." - + } variable "spannerPriority" { - type = string + type = string description = "The request priority for Cloud Spanner calls. The value must be one of: [`HIGH`,`MEDIUM`,`LOW`]. Defaults to `HIGH`." - default = null + default = null } variable "sessionFilePath" { - type = string + type = string description = "Session file path in Cloud Storage that contains mapping information from Spanner Migration Tool. Defaults to empty." - default = null + default = null } variable "schemaOverridesFilePath" { - type = string + type = string description = "A file which specifies the table and the column name overrides from source to spanner. Defaults to empty." - default = null + default = null } variable "tableOverrides" { - type = string + type = string description = "These are the table name overrides from source to spanner. They are written in the following format: [{SourceTableName1, SpannerTableName1}, {SourceTableName2, SpannerTableName2}] This example shows mapping Singers table to Vocalists and Albums table to Records. For example, `[{Singers, Vocalists}, {Albums, Records}]`. Defaults to empty." - default = null + default = null } variable "columnOverrides" { - type = string + type = string description = "These are the column name overrides from source to spanner. They are written in the following format: [{SourceTableName1.SourceColumnName1, SourceTableName1.SpannerColumnName1}, {SourceTableName2.SourceColumnName1, SourceTableName2.SpannerColumnName1}]Note that the SourceTableName should remain the same in both the source and spanner pair. To override table names, use tableOverrides.The example shows mapping SingerName to TalentName and AlbumName to RecordName in Singers and Albums table respectively. For example, `[{Singers.SingerName, Singers.TalentName}, {Albums.AlbumName, Albums.RecordName}]`. Defaults to empty." - default = null + default = null } variable "bigQueryDataset" { - type = string + type = string description = "The BigQuery dataset ID where the validation results will be stored. For example, `validation_report_dataset`" - + } variable "runId" { - type = string + type = string description = "A unique identifier for the validation run. If not provided, the Dataflow Job Name will be used. For example, `run_20230101_120000`" - default = null + default = null } variable "transformationJarPath" { - type = string + type = string description = "Custom jar location in Cloud Storage that contains the custom transformation logic for processing records. Defaults to empty." - default = null + default = null } variable "transformationClassName" { - type = string + type = string description = "Fully qualified class name having the custom transformation logic. It is a mandatory field in case transformationJarPath is specified. Defaults to empty." - default = null + default = null } variable "transformationCustomParameters" { - type = string + type = string description = "String containing any custom parameters to be passed to the custom transformation class. Defaults to empty." - default = null + default = null } provider "google" { - project = var.project + project = var.project } provider "google-beta" { - project = var.project + project = var.project } variable "additional_experiments" { - type = set(string) - description = "List of experiments that should be used by the job. An example value is 'enable_stackdriver_agent_metrics'." - default = null + type = set(string) + description = "List of experiments that should be used by the job. An example value is 'enable_stackdriver_agent_metrics'." + default = null } variable "autoscaling_algorithm" { - type = string - description = "The algorithm to use for autoscaling" - default = null + type = string + description = "The algorithm to use for autoscaling" + default = null } variable "enable_streaming_engine" { - type = bool - description = "Indicates if the job should use the streaming engine feature." - default = null + type = bool + description = "Indicates if the job should use the streaming engine feature." + default = null } variable "ip_configuration" { - type = string - description = "The configuration for VM IPs. Options are 'WORKER_IP_PUBLIC' or 'WORKER_IP_PRIVATE'." - default = null + type = string + description = "The configuration for VM IPs. Options are 'WORKER_IP_PUBLIC' or 'WORKER_IP_PRIVATE'." + default = null } variable "kms_key_name" { - type = string - description = "The name for the Cloud KMS key for the job. Key format is: projects/PROJECT_ID/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY" - default = null + type = string + description = "The name for the Cloud KMS key for the job. Key format is: projects/PROJECT_ID/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY" + default = null } variable "labels" { - type = map(string) - description = "User labels to be specified for the job. Keys and values should follow the restrictions specified in the labeling restrictions page. NOTE: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field 'effective_labels' for all of the labels present on the resource." - default = null + type = map(string) + description = "User labels to be specified for the job. Keys and values should follow the restrictions specified in the labeling restrictions page. NOTE: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field 'effective_labels' for all of the labels present on the resource." + default = null } variable "launcher_machine_type" { - type = string - description = "The machine type to use for launching the job. The default is n1-standard-1." - default = null + type = string + description = "The machine type to use for launching the job. The default is n1-standard-1." + default = null } variable "machine_type" { - type = string - description = "The machine type to use for the job." - default = null + type = string + description = "The machine type to use for the job." + default = null } variable "max_workers" { - type = number - description = "The maximum number of Google Compute Engine instances to be made available to your pipeline during execution, from 1 to 1000." - default = null + type = number + description = "The maximum number of Google Compute Engine instances to be made available to your pipeline during execution, from 1 to 1000." + default = null } variable "name" { - type = string + description = "The unique name of the Dataflow job." + type = string } variable "network" { - type = string - description = "The network to which VMs will be assigned. If it is not provided, 'default' will be used." - default = null + type = string + description = "The network to which VMs will be assigned. If it is not provided, 'default' will be used." + default = null } variable "num_workers" { - type = number - description = "The initial number of Google Compute Engine instances for the job." - default = null + type = number + description = "The initial number of Google Compute Engine instances for the job." + default = null } variable "sdk_container_image" { - type = string - description = "Docker registry location of container image to use for the 'worker harness. Default is the container for the version of the SDK. Note this field is only valid for portable pipelines." - default = null + type = string + description = "Docker registry location of container image to use for the 'worker harness. Default is the container for the version of the SDK. Note this field is only valid for portable pipelines." + default = null } variable "service_account_email" { - type = string - description = "The Service Account email used to create the job." - default = null + type = string + description = "The Service Account email used to create the job." + default = null } variable "skip_wait_on_job_termination" { - type = bool - description = "If true, treat DRAINING and CANCELLING as terminal job states and do not wait for further changes before removing from terraform state and moving on. WARNING: this will lead to job name conflicts if you do not ensure that the job names are different, e.g. by embedding a release ID or by using a random_id." - default = null + type = bool + description = "If true, treat DRAINING and CANCELLING as terminal job states and do not wait for further changes before removing from terraform state and moving on. WARNING: this will lead to job name conflicts if you do not ensure that the job names are different, e.g. by embedding a release ID or by using a random_id." + default = null } variable "staging_location" { - type = string - description = "The Cloud Storage path to use for staging files. Must be a valid Cloud Storage URL, beginning with gs://." - default = null + type = string + description = "The Cloud Storage path to use for staging files. Must be a valid Cloud Storage URL, beginning with gs://." + default = null } variable "subnetwork" { - type = string - description = "The subnetwork to which VMs will be assigned. Should be of the form 'regions/REGION/subnetworks/SUBNETWORK'." - default = null + type = string + description = "The subnetwork to which VMs will be assigned. Should be of the form 'regions/REGION/subnetworks/SUBNETWORK'." + default = null } variable "temp_location" { - type = string - description = "The Cloud Storage path to use for temporary files. Must be a valid Cloud Storage URL, beginning with gs://." - default = null + type = string + description = "The Cloud Storage path to use for temporary files. Must be a valid Cloud Storage URL, beginning with gs://." + default = null } resource "google_project_service" "required" { - service = "dataflow.googleapis.com" - disable_on_destroy = false + service = "dataflow.googleapis.com" + disable_on_destroy = false } resource "google_dataflow_flex_template_job" "generated" { - depends_on = [google_project_service.required] - provider = google-beta - container_spec_gcs_path = "gs://dataflow-templates-${var.region}/latest/flex/GCS_Spanner_Data_Validator" - parameters = { - gcsInputDirectory = var.gcsInputDirectory - projectId = var.projectId - spannerHost = var.spannerHost - instanceId = var.instanceId - databaseId = var.databaseId - spannerPriority = var.spannerPriority - sessionFilePath = var.sessionFilePath - schemaOverridesFilePath = var.schemaOverridesFilePath - tableOverrides = var.tableOverrides - columnOverrides = var.columnOverrides - bigQueryDataset = var.bigQueryDataset - runId = var.runId - transformationJarPath = var.transformationJarPath - transformationClassName = var.transformationClassName - transformationCustomParameters = var.transformationCustomParameters - } - - additional_experiments = var.additional_experiments - autoscaling_algorithm = var.autoscaling_algorithm - enable_streaming_engine = var.enable_streaming_engine - ip_configuration = var.ip_configuration - kms_key_name = var.kms_key_name - labels = var.labels - launcher_machine_type = var.launcher_machine_type - machine_type = var.machine_type - max_workers = var.max_workers - name = var.name - network = var.network - num_workers = var.num_workers - sdk_container_image = var.sdk_container_image - service_account_email = var.service_account_email - skip_wait_on_job_termination = var.skip_wait_on_job_termination - staging_location = var.staging_location - subnetwork = var.subnetwork - temp_location = var.temp_location - region = var.region + depends_on = [google_project_service.required] + provider = google-beta + container_spec_gcs_path = "gs://dataflow-templates-${var.region}/latest/flex/GCS_Spanner_Data_Validator" + parameters = { + gcsInputDirectory = var.gcsInputDirectory + projectId = var.projectId + spannerHost = var.spannerHost + instanceId = var.instanceId + databaseId = var.databaseId + spannerPriority = var.spannerPriority + sessionFilePath = var.sessionFilePath + schemaOverridesFilePath = var.schemaOverridesFilePath + tableOverrides = var.tableOverrides + columnOverrides = var.columnOverrides + bigQueryDataset = var.bigQueryDataset + runId = var.runId + transformationJarPath = var.transformationJarPath + transformationClassName = var.transformationClassName + transformationCustomParameters = var.transformationCustomParameters + } + + additional_experiments = var.additional_experiments + autoscaling_algorithm = var.autoscaling_algorithm + enable_streaming_engine = var.enable_streaming_engine + ip_configuration = var.ip_configuration + kms_key_name = var.kms_key_name + labels = var.labels + launcher_machine_type = var.launcher_machine_type + machine_type = var.machine_type + max_workers = var.max_workers + name = var.name + network = var.network + num_workers = var.num_workers + sdk_container_image = var.sdk_container_image + service_account_email = var.service_account_email + skip_wait_on_job_termination = var.skip_wait_on_job_termination + staging_location = var.staging_location + subnetwork = var.subnetwork + temp_location = var.temp_location + region = var.region + on_delete = var.on_delete } output "dataflow_job_url" { - value = "https://console.cloud.google.com/dataflow/jobs/${var.region}/${google_dataflow_flex_template_job.generated.job_id}" + value = "https://console.cloud.google.com/dataflow/jobs/${var.region}/${google_dataflow_flex_template_job.generated.job_id}" } diff --git a/v2/gcs-spanner-dv/terraform/samples/README.md b/v2/gcs-spanner-dv/terraform/samples/README.md index c36e411f30..3791c70cb2 100644 --- a/v2/gcs-spanner-dv/terraform/samples/README.md +++ b/v2/gcs-spanner-dv/terraform/samples/README.md @@ -1,57 +1,26 @@ # Terraform Samples for GCS to Spanner Data Validation -This directory contains Terraform deployment samples to run the [GCS to Spanner Data Validation](../../README_GCS_Spanner_Data_Validator.md) Dataflow pipeline. +This directory provides samples for common scenarios users might have while trying to run a data validation job. -The provided sample in [`gcs-spanner-dv`](./gcs-spanner-dv/) demonstrates how to easily launch the Dataflow job while automatically attaching all necessary IAM roles. +Pick a sample that is closest to your use-case, and use it as a starting point, tailoring it to your own specific needs. -## What this sample does +## Prerequisites -The Terraform module will create the following Google Cloud resources: -1. **Dataflow Flex Template Job:** Uses `google_dataflow_flex_template_job` to launch the data validation pipeline. -2. **IAM Role Bindings:** Grants the Dataflow worker service account the required roles to run the validation: - * `roles/dataflow.worker` - * `roles/spanner.databaseAdmin` - * `roles/storage.objectViewer` - * `roles/bigquery.dataEditor` (required to write validation reports to BigQuery) +Before using these samples, ensure you have: +- [Terraform](https://developer.hashicorp.com/terraform/downloads) installed. +- Authenticated with Google Cloud (e.g., using `gcloud auth application-default login`). +- A Google Cloud project with the necessary permissions. -## Prerequisites +## List of examples + +1. [Launching a Data Validation Job](gcs-spanner-dv/README.md) - A basic example that configures the necessary variables and provisions the Dataflow pipeline for GCS to Spanner Data Validation. + +## How to add a new sample + +We strongly recommend copying an existing sample and modifying it for your scenario. This ensures a consistent style across all Terraform samples. -Before executing the sample, ensure you meet the following requirements: - -1. **APIs Enabled**: The following APIs must be enabled in your Google Cloud Project: - * Dataflow API (`dataflow.googleapis.com`) - * Cloud Spanner API (`spanner.googleapis.com`) - * Cloud Storage API (`storage-component.googleapis.com`) - * BigQuery API (`bigquery.googleapis.com`) - -2. **Terraform**: Make sure Terraform is installed locally and you are authenticated using `gcloud auth application-default login`. - -## Step-by-Step Usage Instructions - -1. **Navigate to the sample directory** - ```shell - cd gcs-spanner-dv/ - ``` - -2. **Initialize Terraform** - Download the required providers and initialize the state. - ```shell - terraform init - ``` - -3. **Configure the variables** - Open the `terraform_simple.tfvars` file and modify the placeholder values (such as `project`, `instance_id`, `database_id`, and `gcs_input_directory`) to match your environment. - -4. **Plan the deployment** - Review the resources that Terraform will create: - ```shell - terraform plan -var-file=terraform_simple.tfvars - ``` - -5. **Apply the deployment** - Execute the creation of the resources: - ```shell - terraform apply -var-file=terraform_simple.tfvars - ``` - - Once the command completes, Terraform will output the `dataflow_job_url`, which you can click to view the running job in the Google Cloud Console. +```shell +mkdir my-new-sample +cp -r gcs-spanner-dv/ my-new-sample/ +cd my-new-sample/ +``` diff --git a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/README.md b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/README.md new file mode 100644 index 0000000000..3bd4ccc611 --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/README.md @@ -0,0 +1,54 @@ +# GCS to Spanner Data Validation + +The provided sample in [`gcs-spanner-dv`](./) demonstrates how to easily launch the Dataflow job while automatically attaching all necessary IAM roles. + +## What this sample does + +The Terraform module will create the following Google Cloud resources: +1. **Dataflow Flex Template Job:** Uses `google_dataflow_flex_template_job` to launch the data validation pipeline. +2. **IAM Role Bindings:** Grants the Dataflow worker service account the required roles to run the validation: + * `roles/dataflow.worker` (required to execute Dataflow jobs) + * `roles/spanner.databaseReader` (required to read records from Spanner) + * `roles/storage.objectAdmin` (required to read/write objects in Cloud Storage) + * `roles/bigquery.dataEditor` (required to write validation reports to BigQuery) + * `roles/bigquery.jobUser` (required to execute BigQuery load jobs) + * `roles/monitoring.metricWriter` (required to write Dataflow metrics) + * `roles/cloudprofiler.agent` (required for Cloud Profiler) + +## Prerequisites + +Before executing the sample, ensure you meet the following requirements: + +1. **APIs Enabled**: The following APIs must be enabled in your Google Cloud Project: + * Dataflow API (`dataflow.googleapis.com`) + * Cloud Spanner API (`spanner.googleapis.com`) + * Cloud Storage API (`storage.googleapis.com`) + * BigQuery API (`bigquery.googleapis.com`) + +2. **Terraform**: Make sure Terraform is installed locally and you are authenticated using `gcloud auth application-default login`. + +## Usage + +1. **Clone the repository** + ```shell + git clone https://github.com/GoogleCloudPlatform/DataflowTemplates.git + cd DataflowTemplates/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv + ``` + +2. **Initialize Terraform** + ```shell + terraform init + ``` + +3. **Configure the variables** + Open the `terraform_simple.tfvars` file and modify the placeholder values (such as `project`, `instance_id`, `database_id`, `gcs_input_directory`, and `bigquery_dataset`) to match your environment. + +4. **Review the execution plan** + ```shell + terraform plan -var-file=terraform_simple.tfvars + ``` + +5. **Apply the configuration** + ```shell + terraform apply -var-file=terraform_simple.tfvars + ``` diff --git a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/main.tf b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/main.tf index 4143595f06..919d79d44b 100644 --- a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/main.tf +++ b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/main.tf @@ -1,30 +1,38 @@ -provider "google" { - project = var.project -} - -provider "google-beta" { - project = var.project -} - data "google_project" "project" { project_id = var.project + depends_on = [google_project_service.enabled_apis] } data "google_compute_default_service_account" "gce_account" { - project = var.project + project = var.project + depends_on = [google_project_service.enabled_apis] +} + +locals { + effective_sa_email = (var.service_account_email != null && var.service_account_email != "") ? var.service_account_email : data.google_compute_default_service_account.gce_account.email } # Add roles to the service account that will run Dataflow for data validation resource "google_project_iam_member" "dataflow_roles" { for_each = var.add_policies_to_service_account ? toset([ "roles/dataflow.worker", - "roles/spanner.databaseReader", - "roles/storage.objectViewer", - "roles/bigquery.dataEditor" + "roles/storage.objectAdmin", + "roles/bigquery.dataEditor", + "roles/bigquery.jobUser", + "roles/monitoring.metricWriter", + "roles/cloudprofiler.agent" ]) : toset([]) - project = data.google_project.project.id + + project = var.project role = each.key - member = var.service_account_email != null ? "serviceAccount:${var.service_account_email}" : "serviceAccount:${data.google_compute_default_service_account.gce_account.email}" + member = "serviceAccount:${local.effective_sa_email}" +} + +resource "google_project_iam_member" "spanner_reader_role" { + count = var.add_policies_to_service_account ? 1 : 0 + project = var.spanner_project_id != null && var.spanner_project_id != "" ? var.spanner_project_id : var.project + role = "roles/spanner.databaseReader" + member = "serviceAccount:${local.effective_sa_email}" } # Define the Dataflow Flex Template job for GCS to Spanner Data Validation @@ -34,31 +42,36 @@ resource "google_dataflow_flex_template_job" "gcs_spanner_dv_job" { project = var.project region = var.region container_spec_gcs_path = "gs://dataflow-templates-${var.region}/latest/flex/GCS_Spanner_Data_Validator" - + parameters = { - gcsInputDirectory = var.gcs_input_directory - projectId = var.spanner_project_id != null ? var.spanner_project_id : var.project - instanceId = var.instance_id - databaseId = var.database_id - bigQueryDataset = var.bigquery_dataset - spannerHost = var.spanner_host - spannerPriority = var.spanner_priority - sessionFilePath = var.session_file_path - schemaOverridesFilePath = var.schema_overrides_file_path - tableOverrides = var.table_overrides - columnOverrides = var.column_overrides - runId = var.run_id - transformationJarPath = var.transformation_jar_path - transformationClassName = var.transformation_class_name - transformationCustomParameters = var.transformation_custom_parameters + for k, v in { + gcsInputDirectory = var.gcs_input_directory + projectId = var.spanner_project_id != null && var.spanner_project_id != "" ? var.spanner_project_id : var.project + instanceId = var.instance_id + databaseId = var.database_id + bigQueryDataset = var.bigquery_dataset + spannerHost = var.spanner_host + spannerPriority = var.spanner_priority + sessionFilePath = var.session_file_path + schemaOverridesFilePath = var.schema_overrides_file_path + tableOverrides = var.table_overrides + columnOverrides = var.column_overrides + runId = var.run_id + transformationJarPath = var.transformation_jar_path + transformationClassName = var.transformation_class_name + transformationCustomParameters = var.transformation_custom_parameters + } : k => v if v != null && v != "" } - service_account_email = var.service_account_email - network = var.network != null ? "projects/${var.project}/global/networks/${var.network}" : null - subnetwork = var.subnetwork != null ? "https://www.googleapis.com/compute/v1/projects/${var.project}/regions/${var.region}/subnetworks/${var.subnetwork}" : null + service_account_email = local.effective_sa_email + network = (var.network != null && var.network != "") ? (var.host_project != null && var.host_project != "") ? "projects/${var.host_project}/global/networks/${var.network}" : "projects/${var.project}/global/networks/${var.network}" : null + subnetwork = (var.subnetwork != null && var.subnetwork != "") ? (var.host_project != null && var.host_project != "") ? "https://www.googleapis.com/compute/v1/projects/${var.host_project}/regions/${var.region}/subnetworks/${var.subnetwork}" : "https://www.googleapis.com/compute/v1/projects/${var.project}/regions/${var.region}/subnetworks/${var.subnetwork}" : null machine_type = var.machine_type max_workers = var.max_workers additional_experiments = var.additional_experiments - depends_on = [google_project_iam_member.dataflow_roles] + depends_on = [ + google_project_iam_member.dataflow_roles, + google_project_iam_member.spanner_reader_role + ] } diff --git a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/outputs.tf b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/outputs.tf index 1a446716c0..bf02f5ff19 100644 --- a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/outputs.tf +++ b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/outputs.tf @@ -1,4 +1,9 @@ output "dataflow_job_url" { - description = "The URL of the created Dataflow job in the Google Cloud Console." - value = "https://console.cloud.google.com/dataflow/jobs/${var.region}/${google_dataflow_flex_template_job.gcs_spanner_dv_job.job_id}?project=${var.project}" + description = "URL for the created Dataflow Flex Template job." + value = "https://console.cloud.google.com/dataflow/jobs/${var.region}/${google_dataflow_flex_template_job.gcs_spanner_dv_job.job_id}?project=${google_dataflow_flex_template_job.gcs_spanner_dv_job.project}" +} + +output "dataflow_job_id" { + description = "The unique ID of the created Dataflow Flex Template job." + value = google_dataflow_flex_template_job.gcs_spanner_dv_job.job_id } diff --git a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform.tf b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform.tf new file mode 100644 index 0000000000..7e38536772 --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform.tf @@ -0,0 +1,37 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "~> 4.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = "~> 4.0" + } + } + required_version = "~> 1.2" +} + +provider "google" { + project = var.project + region = var.region +} + +provider "google-beta" { + project = var.project + region = var.region +} + +# Enable the APIs +resource "google_project_service" "enabled_apis" { + for_each = toset([ + "iam.googleapis.com", + "dataflow.googleapis.com", + "storage.googleapis.com", + "spanner.googleapis.com", + "bigquery.googleapis.com" + ]) + service = each.key + project = var.project + disable_on_destroy = false +} diff --git a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform_simple.tfvars b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform_simple.tfvars index 2b7b4f08c3..a7c72871c0 100644 --- a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform_simple.tfvars +++ b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform_simple.tfvars @@ -1,21 +1,38 @@ -# Below is a simplified version of the configuration for GCS to Spanner Data Validation +# ========================================== +# Global Project & Provider Configuration +# ========================================== # Provide the Google Cloud project ID -project = "my-gcp-project" +project = "" # The region to deploy the Dataflow job -region = "us-central1" +region = "" +# ========================================== +# Dataflow Job Configuration +# ========================================== # The name of the Dataflow job -job_name = "gcs-spanner-dv-sample" +job_name = "" -# The Cloud Storage directory containing validation data (should end with a '/') -gcs_input_directory = "gs://my-bucket/validation-data/" +# ========================================== +# Source (GCS) Configuration +# ========================================== +# The Cloud Storage directory containing validation data +gcs_input_directory = "gs:////" +# ========================================== +# Destination (Spanner) Configuration +# ========================================== # The Spanner instance ID to validate against -instance_id = "my-spanner-instance" +instance_id = "" # The Spanner database ID to validate against -database_id = "my-spanner-database" +database_id = "" +# The Google Cloud project ID where the Spanner instance is located +spanner_project_id = "" + +# ========================================== +# Reporting (BigQuery) Configuration +# ========================================== # The BigQuery dataset to store validation reports -bigquery_dataset = "validation_report_dataset" +bigquery_dataset = "" diff --git a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/variables.tf b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/variables.tf index f66b62eb39..461f316f11 100644 --- a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/variables.tf +++ b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/variables.tf @@ -3,6 +3,12 @@ variable "project" { description = "Google Cloud Project ID where Dataflow will run." } +variable "host_project" { + type = string + description = "Project id hosting the network in case of a shared vpc setup." + default = "" +} + variable "region" { type = string description = "Google Cloud region to run Dataflow in." @@ -22,19 +28,19 @@ variable "add_policies_to_service_account" { variable "service_account_email" { type = string description = "Service account email for Dataflow workers. If not set, the default Compute Engine service account is used." - default = null + default = "" } variable "network" { type = string description = "Network for Dataflow workers." - default = null + default = "" } variable "subnetwork" { type = string description = "Subnetwork for Dataflow workers." - default = null + default = "" } variable "machine_type" { @@ -46,7 +52,7 @@ variable "machine_type" { variable "max_workers" { type = number description = "Maximum number of Dataflow worker VMs." - default = 10 + default = null } variable "gcs_input_directory" { @@ -67,7 +73,7 @@ variable "database_id" { variable "spanner_project_id" { type = string description = "Google Cloud Project ID containing the Spanner instance. Defaults to the Dataflow project if not provided." - default = null + default = "" } variable "bigquery_dataset" { @@ -78,61 +84,66 @@ variable "bigquery_dataset" { variable "spanner_host" { type = string description = "Custom Spanner host endpoint." - default = null + default = "" } variable "spanner_priority" { type = string - description = "Priority for Spanner RPCs (e.g. HIGH, MEDIUM, LOW)." - default = null + description = "Priority for Spanner RPC invocations" + default = "" + + validation { + condition = var.spanner_priority == null ? true : contains(["HIGH", "MEDIUM", "LOW"], var.spanner_priority) + error_message = "spanner_priority must be one of 'HIGH', 'MEDIUM', 'LOW', or null." + } } variable "session_file_path" { type = string description = "GCS path to the session file." - default = null + default = "" } variable "schema_overrides_file_path" { type = string description = "GCS path to the schema overrides file." - default = null + default = "" } variable "table_overrides" { type = string description = "Table name overrides from source to Spanner." - default = null + default = "" } variable "column_overrides" { type = string description = "Column name overrides from source to Spanner." - default = null + default = "" } variable "run_id" { type = string description = "A unique identifier for the validation run." - default = null + default = "" } variable "transformation_jar_path" { type = string description = "GCS path to the transformation JAR file." - default = null + default = "" } variable "transformation_class_name" { type = string description = "Fully qualified transformation class name." - default = null + default = "" } variable "transformation_custom_parameters" { type = string description = "Custom parameters for the transformation." - default = null + default = "" } variable "additional_experiments" { @@ -140,3 +151,39 @@ variable "additional_experiments" { description = "Additional Dataflow experiments." default = [] } + +variable "ip_configuration" { + type = string + description = "IP configuration for Dataflow workers (e.g. 'WORKER_IP_PRIVATE')." + default = "" +} + +variable "launcher_machine_type" { + type = string + description = "Machine type for the Dataflow launcher VM." + default = "" +} + +variable "num_workers" { + type = number + description = "Initial number of Dataflow worker VMs." + default = null +} + +variable "additional_pipeline_options" { + type = map(string) + description = "Additional Dataflow pipeline options." + default = {} +} + +variable "labels" { + type = map(string) + description = "Labels to apply to the Dataflow job." + default = {} +} + +variable "kms_key_name" { + type = string + description = "Cloud KMS key name for data encryption." + default = "" +} From 1c3cbd9ba077dacb21d86275db3e214b1d3c6f19 Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Thu, 27 Aug 2026 12:39:14 +0000 Subject: [PATCH 07/14] Refactor gcs-spanner-dv terraform sample to simple-validation-job --- v2/gcs-spanner-dv/terraform/samples/README.md | 4 +- .../samples/gcs-spanner-dv/README.md | 54 --- .../terraform/samples/gcs-spanner-dv/main.tf | 77 ---- .../gcs-spanner-dv/terraform_simple.tfvars | 38 -- .../samples/gcs-spanner-dv/variables.tf | 189 --------- .../samples/simple-validation-job/README.md | 332 +++++++++++++++ .../samples/simple-validation-job/main.tf | 85 ++++ .../outputs.tf | 2 +- .../terraform.tf | 22 +- .../terraform.tfstate.1787826398.backup | 391 ++++++++++++++++++ .../simple-validation-job/variables.tf | 50 +++ 11 files changed, 878 insertions(+), 366 deletions(-) delete mode 100644 v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/README.md delete mode 100644 v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/main.tf delete mode 100644 v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform_simple.tfvars delete mode 100644 v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/variables.tf create mode 100644 v2/gcs-spanner-dv/terraform/samples/simple-validation-job/README.md create mode 100644 v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf rename v2/gcs-spanner-dv/terraform/samples/{gcs-spanner-dv => simple-validation-job}/outputs.tf (67%) rename v2/gcs-spanner-dv/terraform/samples/{gcs-spanner-dv => simple-validation-job}/terraform.tf (50%) create mode 100644 v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfstate.1787826398.backup create mode 100644 v2/gcs-spanner-dv/terraform/samples/simple-validation-job/variables.tf diff --git a/v2/gcs-spanner-dv/terraform/samples/README.md b/v2/gcs-spanner-dv/terraform/samples/README.md index 3791c70cb2..99c511903b 100644 --- a/v2/gcs-spanner-dv/terraform/samples/README.md +++ b/v2/gcs-spanner-dv/terraform/samples/README.md @@ -13,7 +13,7 @@ Before using these samples, ensure you have: ## List of examples -1. [Launching a Data Validation Job](gcs-spanner-dv/README.md) - A basic example that configures the necessary variables and provisions the Dataflow pipeline for GCS to Spanner Data Validation. +1. [Launching a single Data Validation Job](simple-validation-job/README.md) - A basic example that configures the necessary variables and provisions the Dataflow pipeline for GCS to Spanner Data Validation. ## How to add a new sample @@ -21,6 +21,6 @@ We strongly recommend copying an existing sample and modifying it for your scena ```shell mkdir my-new-sample -cp -r gcs-spanner-dv/ my-new-sample/ +cp -r simple-validation-job/ my-new-sample/ cd my-new-sample/ ``` diff --git a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/README.md b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/README.md deleted file mode 100644 index 3bd4ccc611..0000000000 --- a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# GCS to Spanner Data Validation - -The provided sample in [`gcs-spanner-dv`](./) demonstrates how to easily launch the Dataflow job while automatically attaching all necessary IAM roles. - -## What this sample does - -The Terraform module will create the following Google Cloud resources: -1. **Dataflow Flex Template Job:** Uses `google_dataflow_flex_template_job` to launch the data validation pipeline. -2. **IAM Role Bindings:** Grants the Dataflow worker service account the required roles to run the validation: - * `roles/dataflow.worker` (required to execute Dataflow jobs) - * `roles/spanner.databaseReader` (required to read records from Spanner) - * `roles/storage.objectAdmin` (required to read/write objects in Cloud Storage) - * `roles/bigquery.dataEditor` (required to write validation reports to BigQuery) - * `roles/bigquery.jobUser` (required to execute BigQuery load jobs) - * `roles/monitoring.metricWriter` (required to write Dataflow metrics) - * `roles/cloudprofiler.agent` (required for Cloud Profiler) - -## Prerequisites - -Before executing the sample, ensure you meet the following requirements: - -1. **APIs Enabled**: The following APIs must be enabled in your Google Cloud Project: - * Dataflow API (`dataflow.googleapis.com`) - * Cloud Spanner API (`spanner.googleapis.com`) - * Cloud Storage API (`storage.googleapis.com`) - * BigQuery API (`bigquery.googleapis.com`) - -2. **Terraform**: Make sure Terraform is installed locally and you are authenticated using `gcloud auth application-default login`. - -## Usage - -1. **Clone the repository** - ```shell - git clone https://github.com/GoogleCloudPlatform/DataflowTemplates.git - cd DataflowTemplates/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv - ``` - -2. **Initialize Terraform** - ```shell - terraform init - ``` - -3. **Configure the variables** - Open the `terraform_simple.tfvars` file and modify the placeholder values (such as `project`, `instance_id`, `database_id`, `gcs_input_directory`, and `bigquery_dataset`) to match your environment. - -4. **Review the execution plan** - ```shell - terraform plan -var-file=terraform_simple.tfvars - ``` - -5. **Apply the configuration** - ```shell - terraform apply -var-file=terraform_simple.tfvars - ``` diff --git a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/main.tf b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/main.tf deleted file mode 100644 index 919d79d44b..0000000000 --- a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/main.tf +++ /dev/null @@ -1,77 +0,0 @@ -data "google_project" "project" { - project_id = var.project - depends_on = [google_project_service.enabled_apis] -} - -data "google_compute_default_service_account" "gce_account" { - project = var.project - depends_on = [google_project_service.enabled_apis] -} - -locals { - effective_sa_email = (var.service_account_email != null && var.service_account_email != "") ? var.service_account_email : data.google_compute_default_service_account.gce_account.email -} - -# Add roles to the service account that will run Dataflow for data validation -resource "google_project_iam_member" "dataflow_roles" { - for_each = var.add_policies_to_service_account ? toset([ - "roles/dataflow.worker", - "roles/storage.objectAdmin", - "roles/bigquery.dataEditor", - "roles/bigquery.jobUser", - "roles/monitoring.metricWriter", - "roles/cloudprofiler.agent" - ]) : toset([]) - - project = var.project - role = each.key - member = "serviceAccount:${local.effective_sa_email}" -} - -resource "google_project_iam_member" "spanner_reader_role" { - count = var.add_policies_to_service_account ? 1 : 0 - project = var.spanner_project_id != null && var.spanner_project_id != "" ? var.spanner_project_id : var.project - role = "roles/spanner.databaseReader" - member = "serviceAccount:${local.effective_sa_email}" -} - -# Define the Dataflow Flex Template job for GCS to Spanner Data Validation -resource "google_dataflow_flex_template_job" "gcs_spanner_dv_job" { - provider = google-beta - name = var.job_name - project = var.project - region = var.region - container_spec_gcs_path = "gs://dataflow-templates-${var.region}/latest/flex/GCS_Spanner_Data_Validator" - - parameters = { - for k, v in { - gcsInputDirectory = var.gcs_input_directory - projectId = var.spanner_project_id != null && var.spanner_project_id != "" ? var.spanner_project_id : var.project - instanceId = var.instance_id - databaseId = var.database_id - bigQueryDataset = var.bigquery_dataset - spannerHost = var.spanner_host - spannerPriority = var.spanner_priority - sessionFilePath = var.session_file_path - schemaOverridesFilePath = var.schema_overrides_file_path - tableOverrides = var.table_overrides - columnOverrides = var.column_overrides - runId = var.run_id - transformationJarPath = var.transformation_jar_path - transformationClassName = var.transformation_class_name - transformationCustomParameters = var.transformation_custom_parameters - } : k => v if v != null && v != "" - } - - service_account_email = local.effective_sa_email - network = (var.network != null && var.network != "") ? (var.host_project != null && var.host_project != "") ? "projects/${var.host_project}/global/networks/${var.network}" : "projects/${var.project}/global/networks/${var.network}" : null - subnetwork = (var.subnetwork != null && var.subnetwork != "") ? (var.host_project != null && var.host_project != "") ? "https://www.googleapis.com/compute/v1/projects/${var.host_project}/regions/${var.region}/subnetworks/${var.subnetwork}" : "https://www.googleapis.com/compute/v1/projects/${var.project}/regions/${var.region}/subnetworks/${var.subnetwork}" : null - machine_type = var.machine_type - max_workers = var.max_workers - additional_experiments = var.additional_experiments - - depends_on = [ - google_project_iam_member.dataflow_roles, - google_project_iam_member.spanner_reader_role - ] -} diff --git a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform_simple.tfvars b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform_simple.tfvars deleted file mode 100644 index a7c72871c0..0000000000 --- a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform_simple.tfvars +++ /dev/null @@ -1,38 +0,0 @@ -# ========================================== -# Global Project & Provider Configuration -# ========================================== -# Provide the Google Cloud project ID -project = "" - -# The region to deploy the Dataflow job -region = "" - -# ========================================== -# Dataflow Job Configuration -# ========================================== -# The name of the Dataflow job -job_name = "" - -# ========================================== -# Source (GCS) Configuration -# ========================================== -# The Cloud Storage directory containing validation data -gcs_input_directory = "gs:////" - -# ========================================== -# Destination (Spanner) Configuration -# ========================================== -# The Spanner instance ID to validate against -instance_id = "" - -# The Spanner database ID to validate against -database_id = "" - -# The Google Cloud project ID where the Spanner instance is located -spanner_project_id = "" - -# ========================================== -# Reporting (BigQuery) Configuration -# ========================================== -# The BigQuery dataset to store validation reports -bigquery_dataset = "" diff --git a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/variables.tf b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/variables.tf deleted file mode 100644 index 461f316f11..0000000000 --- a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/variables.tf +++ /dev/null @@ -1,189 +0,0 @@ -variable "project" { - type = string - description = "Google Cloud Project ID where Dataflow will run." -} - -variable "host_project" { - type = string - description = "Project id hosting the network in case of a shared vpc setup." - default = "" -} - -variable "region" { - type = string - description = "Google Cloud region to run Dataflow in." -} - -variable "job_name" { - type = string - description = "Dataflow job name." -} - -variable "add_policies_to_service_account" { - type = bool - description = "Terraform will add the required permission to the dataflow service account." - default = true -} - -variable "service_account_email" { - type = string - description = "Service account email for Dataflow workers. If not set, the default Compute Engine service account is used." - default = "" -} - -variable "network" { - type = string - description = "Network for Dataflow workers." - default = "" -} - -variable "subnetwork" { - type = string - description = "Subnetwork for Dataflow workers." - default = "" -} - -variable "machine_type" { - type = string - description = "Machine type for Dataflow worker VMs." - default = "n1-standard-4" -} - -variable "max_workers" { - type = number - description = "Maximum number of Dataflow worker VMs." - default = null -} - -variable "gcs_input_directory" { - type = string - description = "The GCS directory containing the data for validation. Example: gs://my-bucket/path/" -} - -variable "instance_id" { - type = string - description = "Cloud Spanner instance ID." -} - -variable "database_id" { - type = string - description = "Cloud Spanner database ID." -} - -variable "spanner_project_id" { - type = string - description = "Google Cloud Project ID containing the Spanner instance. Defaults to the Dataflow project if not provided." - default = "" -} - -variable "bigquery_dataset" { - type = string - description = "The BigQuery dataset ID where the validation results will be stored. Example: validation_report_dataset" -} - -variable "spanner_host" { - type = string - description = "Custom Spanner host endpoint." - default = "" -} - -variable "spanner_priority" { - type = string - description = "Priority for Spanner RPC invocations" - default = "" - - validation { - condition = var.spanner_priority == null ? true : contains(["HIGH", "MEDIUM", "LOW"], var.spanner_priority) - error_message = "spanner_priority must be one of 'HIGH', 'MEDIUM', 'LOW', or null." - } -} - -variable "session_file_path" { - type = string - description = "GCS path to the session file." - default = "" -} - -variable "schema_overrides_file_path" { - type = string - description = "GCS path to the schema overrides file." - default = "" -} - -variable "table_overrides" { - type = string - description = "Table name overrides from source to Spanner." - default = "" -} - -variable "column_overrides" { - type = string - description = "Column name overrides from source to Spanner." - default = "" -} - -variable "run_id" { - type = string - description = "A unique identifier for the validation run." - default = "" -} - -variable "transformation_jar_path" { - type = string - description = "GCS path to the transformation JAR file." - default = "" -} - -variable "transformation_class_name" { - type = string - description = "Fully qualified transformation class name." - default = "" -} - -variable "transformation_custom_parameters" { - type = string - description = "Custom parameters for the transformation." - default = "" -} - -variable "additional_experiments" { - type = list(string) - description = "Additional Dataflow experiments." - default = [] -} - -variable "ip_configuration" { - type = string - description = "IP configuration for Dataflow workers (e.g. 'WORKER_IP_PRIVATE')." - default = "" -} - -variable "launcher_machine_type" { - type = string - description = "Machine type for the Dataflow launcher VM." - default = "" -} - -variable "num_workers" { - type = number - description = "Initial number of Dataflow worker VMs." - default = null -} - -variable "additional_pipeline_options" { - type = map(string) - description = "Additional Dataflow pipeline options." - default = {} -} - -variable "labels" { - type = map(string) - description = "Labels to apply to the Dataflow job." - default = {} -} - -variable "kms_key_name" { - type = string - description = "Cloud KMS key name for data encryption." - default = "" -} diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/README.md b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/README.md new file mode 100644 index 0000000000..1da56aa615 --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/README.md @@ -0,0 +1,332 @@ +# GCS to Spanner Data Validation + +This sample demonstrates how to easily launch the Dataflow job while automatically attaching all necessary IAM roles. + +## What this sample does + +The Terraform module will create the following Google Cloud resources: +1. **Dataflow Flex Template Job:** Uses `google_dataflow_flex_template_job` to launch the data validation pipeline. +2. **GCS Session File:** The GCS object created by uploading the local session file (if specified). +3. **IAM Role Bindings:** Grants the Dataflow worker service account the required roles to run the validation: + * `roles/dataflow.worker` (required to execute Dataflow jobs) + * `roles/spanner.databaseReader` (required to read records from Spanner) + * `roles/storage.objectAdmin` (required to read/write objects in Cloud Storage) + * `roles/bigquery.dataEditor` (required to write validation reports to BigQuery) + * `roles/bigquery.jobUser` (required to execute BigQuery load jobs) + * `roles/monitoring.metricWriter` (required to write Dataflow metrics) + * `roles/cloudprofiler.agent` (required for Cloud Profiler) + +## Terraform permissions + +In order to create the resources in this sample, the `Service account`/`User account` being used to run Terraform should have the required permissions. +There are two ways to add permissions - + +1. Adding pre-defined roles to the service account running Terraform. +2. Creating a custom role with the granular permissions and attaching it to the service account running Terraform. + +### Using custom role and granular permissions (recommended) + +Following permissions are required - + +```shell +- dataflow.jobs.cancel +- dataflow.jobs.create +- dataflow.jobs.updateContents +- iam.roles.get +- iam.serviceAccounts.actAs +- resourcemanager.projects.setIamPolicy +- storage.objects.create +- storage.objects.delete +- serviceusage.services.use +- serviceusage.services.enable +``` + +**Note**: Add the `roles/viewer` role as well to the service account. + +### Using pre-defined roles + +Following roles are required - + +```shell +roles/dataflow.admin +roles/iam.securityAdmin +roles/iam.serviceAccountUser +roles/storage.admin +roles/viewer +``` + +## Dataflow permissions + +The Dataflow service account needs to be provided with the required roles. This sample will attempt to automatically bind the following roles to the specified service account: `roles/dataflow.worker`, `roles/spanner.databaseReader`, `roles/storage.objectAdmin`, `roles/bigquery.dataEditor`, `roles/bigquery.jobUser`, `roles/monitoring.metricWriter`, and `roles/cloudprofiler.agent`. + +## Assumptions + +It makes the following assumptions - + +1. Appropriate permissions are added to the service account running Terraform to allow resource creation. +2. The BigQuery dataset to store validation reports is already created and correctly named in `var.bigquery_dataset`. +3. The Source AVRO records in GCS exist in `var.gcs_input_directory`. +4. A Spanner instance with database containing the destination records is created and accessible. +5. If the source and Spanner schema is not like-to-like (e.g., column/table renames), an SMT generated session file, or an overrides file needs to be provided containing the schema mapping information. + +Given these assumptions, the job compares the source AVRO records against the Spanner database and writes mismatch reports and validation statistics to the specified BigQuery dataset. + +## Description + +This sample contains the following files - + +1. `main.tf` - This contains the Terraform resources which will be created. +2. `outputs.tf` - This declares the outputs that will be output as part of running this terraform example. +3. `variables.tf` - This declares the input variables that are required to configure the resources. +4. `terraform.tf` - This contains the required providers and APIs/project configurations for this sample. +5. `terraform.tfvars` - This contains the minimal list of dummy inputs that need to be populated to run this example. + +## Prerequisites + +Before executing the sample, ensure you meet the following requirements: + +1. **APIs Enabled**: The following APIs must be enabled in your Google Cloud Project: + * Dataflow API (`dataflow.googleapis.com`) + * Cloud Spanner API (`spanner.googleapis.com`) + * Cloud Storage API (`storage.googleapis.com`) + * BigQuery API (`bigquery.googleapis.com`) + +2. **Terraform**: Make sure Terraform is installed locally and you are authenticated using `gcloud auth application-default login`. + +## Usage + +1. **Clone the repository** + ```shell + git clone https://github.com/GoogleCloudPlatform/DataflowTemplates.git + cd DataflowTemplates/v2/gcs-spanner-dv/terraform/samples/simple-validation-job + ``` + +2. **Initialize Terraform** + ```shell + terraform init + ``` + +3. **Configure the variables** + Open the `terraform.tfvars` file and modify the placeholder values (such as `project`, `instance_id`, `database_id`, `gcs_input_directory`, and `bigquery_dataset`) to match your environment. + +4. **Review the execution plan** + ```shell + terraform plan -var-file=terraform.tfvars + ``` + +5. **Apply the configuration** + ```shell + terraform apply -var-file=terraform.tfvars + ``` + +This will launch the configured jobs and produce an output like below - + +```shell +Apply complete! Resources: 1 added, 0 changed, 0 destroyed. + +Outputs: + +dataflow_job_id = [ + "2024-06-05_00_41_11-4759981257849547781", +] +dataflow_job_url = [ + "https://console.cloud.google.com/dataflow/jobs/us-central1/2024-06-05_00_41_11-4759981257849547781", +] +``` + +### Cleanup + +Once the jobs have finished running, you can cleanup by running - + +```shell +terraform destroy +``` + +## Observability + +To monitor the data validation job, you can view the Dataflow job in the Google Cloud Console. The resulting metrics from the validation job such as matched records, mismatched records, and missing records are available in the BigQuery dataset specified in `var.bigquery_dataset`. + +## FAQ + +### Dataflow job is failing with "Timeout in polling result file" + +Dataflow has a 10-minute timeout within which the launcher VM logic should complete. There could be multiple reasons for it to take over 10 mins: + +- **Job logs not present after the log "launcher VM started":** A sign would be there are only 3-4 log statements in the job logs. This is likely due to private Google access not being enabled for the subnetwork. Please enable private Google access in your network. + +### Job graph is not loading/Custom counters not visible on Dataflow panel + +For very large graphs, this can happen. The graph section would be empty and the counters won't load. But worry not, the validation should progress nonetheless. In such cases, the Dataflow custom metrics can be directly viewed on [Cloud monitoring](https://cloud.google.com/dataflow/docs/guides/using-monitoring-intf). + +### Data Validation is taking too long + +There can be multiple reasons for this: + +- **Check Spanner metrics:** Are memory/CPU limits being hit during reads? Consider increasing the number of nodes if Spanner is struggling to serve the reads. +- **Check Dataflow metrics:** Are memory/CPU limits being hit? Dataflow should autoscale to the required number of nodes. + - CPU/memory limits being hit means the `max_workers` parameter might be too low. It is recommended to use smaller machines (`Ex: n1-standard-4`) for most workloads. However, if you have very large datasets, we recommend scaling up the number of workers. + +### Configuring to run using a VPC + +#### Dataflow + +1. Set the `network` and the `subnetwork` parameters to run the Dataflow job inside a VPC. Specify [network](https://cloud.google.com/dataflow/docs/guides/specifying-networks#network_parameter) and [subnetwork](https://cloud.google.com/dataflow/docs/guides/specifying-networks#subnetwork_parameter) according to the linked guidelines. +2. Set the `ip_configuration` to `WORKER_IP_PRIVATE` to disable public IP addresses for the worker VMs. +3. If only certain network tags are allowlisted via a firewall, specify the network tags via the [additional-experiments flag](https://cloud.google.com/dataflow/docs/guides/routes-firewall#network-tags-flex) (e.g. `use_network_tags=allow-dataflow`). Dataflow automatically assigns + the `dataflow` network tag if any network tag is additionally specified. You need + specify the tag for both worker VMs and launcher VMs. + + +> **_NOTE:_** You can use a shared VPC by specifying the `host_project` in the subnet path. +> This will result in the Dataflow jobs being launched inside the shared VPC. +> Usage of shared VPC requires cross-project permissions. They +> are available as a Terraform +> template [here](../../../../spanner-common/terraform/samples/configure-shared-vpc/README.md). +> Dataflow service account permissions are +> documented [here](https://cloud.google.com/dataflow/docs/guides/specifying-networks#shared). + + +If you are facing issue with VPC connectivity, check the following Dataflow +[guide](https://cloud.google.com/dataflow/docs/guides/troubleshoot-networking) +to debug common networking issues. + +### Updating workers of a Dataflow job + +Currently, the Terraform `google_dataflow_flex_template_job` resource does not support updating the workers of a Dataflow job. +If the worker counts are changed in `tfvars` and a Terraform apply is run, Terraform will attempt to cancel the existing Dataflow job and replace it with a new one. +**This is not recommended**. Instead, use the `gcloud` CLI to update the worker counts of a launched Dataflow job: + +```shell +gcloud dataflow jobs update-options \ + --region=us-central1 \ + --min-num-workers=5 \ + --max-num-workers=20 \ + +``` + +### Specifying schema changes + +By default, the validation job performs a like-like schema mapping between the source AVRO records and Spanner. Any schema changes between the source and Spanner can be specified using a `session file` or `overrides` parameters. + +**We highly recommend using the schema overrides parameters (`table_overrides` and `column_overrides`) instead of a session file** when dealing with schema differences. + +#### Using Schema Overrides (Recommended) + +When passing schema overrides to the job, you must strictly follow the required `[{}]` bracket-brace format. If the format is not matched exactly, Dataflow will reject the configuration with a regex error. + +* **For `table_overrides`**: Use the format `[{OldTableName,NewTableName}]`. + * *Example:* `[{Singers, Vocalists}]` +* **For `column_overrides`**: You **MUST** include the table name alongside the column names. Use the format `[{TableName.OldColumnName,TableName.NewColumnName}]`. Missing the table name will cause the pipeline to crash. + * *Example:* `[{Singers.SingerId, Singers.VocalistId}]` + +You can pass these overrides directly to your Terraform configuration using `var.table_overrides` and `var.column_overrides`. + +#### Using a Session File + +If you prefer or need to use a session file, you can generate one using the Spanner Migration Tool (SMT): + +1. Setup SMT and [launch the UI](https://googlecloudplatform.github.io/spanner-migration-tool/ui#launching-the-web-ui-for-spanner-migration-tool). +2. Perform a [schema conversion](https://googlecloudplatform.github.io/spanner-migration-tool/ui/schema-conv) and download the session file locally. + +To provide this session file to Terraform: + +1. Copy the SMT generated `session file` to the Terraform working directory and name it `session.json`. +2. Set the `var.local_session_file_path` variable to `"session.json"` (or the relative/absolute path to your file). +3. Ensure `var.working_directory_bucket` and `var.working_directory_prefix` are configured so Terraform knows where to upload the file in GCS. + +This will automatically upload the local file to GCS and configure it in the Dataflow job. + +### Adding access to Terraform service account + +#### Using custom role and granular permissions (recommended) + +You can run the following gcloud command to create a custom role in your GCP project. + +```shell +gcloud iam roles create dv_terraform_role --project= --file=perms.yaml --quiet +``` + +The `YAML` file required for the above will be like so - + +```shell +title: "Data Validation Terraform Role" +description: "Custom role for running Spanner Data Validation via Terraform." +stage: "GA" +includedPermissions: +- iam.roles.get +- iam.serviceAccounts.actAs +# ....add all permissions from the list defined in the 'Terraform permissions' section above. +``` + +Then attach the role to the service account - + +```shell +gcloud iam service-accounts add-iam-policy-binding @.iam.gserviceaccount.com \ + --member="serviceAccount:@.iam.gserviceaccount.com" \ + --role="projects//roles/dv_terraform_role" +``` + +#### Using pre-defined roles + +You can run the following shell script to add roles to the service account being used to run Terraform. This will have to done by a user which has the authority to grant the specified roles to a service account - + +```shell +#!/bin/bash + +# Service account to be granted roles +SERVICE_ACCOUNT="@.iam.gserviceaccount.com" + +# Project ID where roles will be granted +PROJECT_ID="" + +# Array of roles to grant +ROLES=( + "roles/dataflow.admin" + "roles/iam.securityAdmin" + "roles/iam.serviceAccountUser" + "roles/storage.admin" + "roles/viewer" +) + +# Loop through each role and grant it to the service account +for ROLE in "${ROLES[@]}" +do + gcloud projects add-iam-policy-binding "$PROJECT_ID" \ + --member="serviceAccount:${SERVICE_ACCOUNT}" \ + --role="$ROLE" +done +``` + +### Verifying access in the Terraform service account + +Verify that the custom role is attached to the service account - + +```shell +gcloud projects get-iam-policy \ +--flatten="bindings[].members" \ +--format='table(bindings.role)' \ +--filter="bindings.members:@.iam.gserviceaccount.com" +``` + +Verify that the role has the correct set of permissions + +```shell +gcloud iam roles describe bulk_migrations_role --project= +``` + +### Impersonating the Terraform service account + +#### Using GCE VM instance (recommended) + +A GCE VM created using the service account setup above will automatically use the service account for all API requests triggered by Terraform. Running terraform from such a GCE VM does not require downloading service keys and is the recommended approach. + +#### Using key file + +1. Activate the service account - + ```shell + gcloud auth activate-service-account @.iam.gserviceaccount.com --key-file=path/to/key_file --project=project_id + ``` +2. Impersonate service account while fetching the ADC credentials - + ```shell + gcloud auth application-default login --impersonate-service-account @.iam.gserviceaccount.com + ``` diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf new file mode 100644 index 0000000000..5c45142397 --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf @@ -0,0 +1,85 @@ +locals { + effective_sa_email = (var.dataflow_params.runner_params.service_account_email != null && var.dataflow_params.runner_params.service_account_email != "") ? var.dataflow_params.runner_params.service_account_email : data.google_compute_default_service_account.gce_account.email +} + +# upload local session file to the working GCS bucket +resource "google_storage_bucket_object" "session_file_object" { + count = var.dataflow_params.template_params.local_session_file_path != null ? 1 : 0 + depends_on = [google_project_service.enabled_apis] + name = "${var.dataflow_params.template_params.working_directory_prefix}/session.json" + source = var.dataflow_params.template_params.local_session_file_path + content_type = "application/json" + bucket = var.dataflow_params.template_params.working_directory_bucket +} + +# Add roles to the service account that will run Dataflow for data validation +resource "google_project_iam_member" "dataflow_roles" { + for_each = var.common_params.add_policies_to_service_account ? toset([ + "roles/dataflow.worker", + "roles/storage.objectAdmin", + "roles/bigquery.dataEditor", + "roles/bigquery.jobUser", + "roles/monitoring.metricWriter", + "roles/cloudprofiler.agent" + ]) : toset([]) + + project = var.common_params.project + role = each.key + member = "serviceAccount:${local.effective_sa_email}" +} + +resource "google_project_iam_member" "spanner_reader_role" { + count = var.common_params.add_policies_to_service_account ? 1 : 0 + project = var.dataflow_params.template_params.spanner_project_id != null && var.dataflow_params.template_params.spanner_project_id != "" ? var.dataflow_params.template_params.spanner_project_id : var.common_params.project + role = "roles/spanner.databaseReader" + member = "serviceAccount:${local.effective_sa_email}" +} + +# Define the Dataflow Flex Template job for GCS to Spanner Data Validation +resource "google_dataflow_flex_template_job" "gcs_spanner_dv_job" { + provider = google-beta + name = var.dataflow_params.runner_params.job_name + project = var.common_params.project + region = var.common_params.region + container_spec_gcs_path = "gs://dataflow-templates-${var.common_params.region}/latest/flex/GCS_Spanner_Data_Validator" + + parameters = { + for k, v in { + gcsInputDirectory = var.dataflow_params.template_params.gcs_input_directory + projectId = var.dataflow_params.template_params.spanner_project_id != null && var.dataflow_params.template_params.spanner_project_id != "" ? var.dataflow_params.template_params.spanner_project_id : var.common_params.project + instanceId = var.dataflow_params.template_params.instance_id + databaseId = var.dataflow_params.template_params.database_id + bigQueryDataset = var.dataflow_params.template_params.bigquery_dataset + spannerHost = var.dataflow_params.template_params.spanner_host + spannerPriority = var.dataflow_params.template_params.spanner_priority + sessionFilePath = var.dataflow_params.template_params.local_session_file_path != null ? "gs://${var.dataflow_params.template_params.working_directory_bucket}/${var.dataflow_params.template_params.working_directory_prefix}/session.json" : var.dataflow_params.template_params.session_file_path + schemaOverridesFilePath = var.dataflow_params.template_params.schema_overrides_file_path + tableOverrides = var.dataflow_params.template_params.table_overrides + columnOverrides = var.dataflow_params.template_params.column_overrides + runId = var.dataflow_params.template_params.run_id + transformationJarPath = var.dataflow_params.template_params.transformation_jar_path + transformationClassName = var.dataflow_params.template_params.transformation_class_name + transformationCustomParameters = var.dataflow_params.template_params.transformation_custom_parameters + } : k => v if v != null && v != "" + } + + service_account_email = local.effective_sa_email + network = (var.dataflow_params.runner_params.network != null && var.dataflow_params.runner_params.network != "") ? (var.common_params.host_project != null && var.common_params.host_project != "") ? "projects/${var.common_params.host_project}/global/networks/${var.dataflow_params.runner_params.network}" : "projects/${var.common_params.project}/global/networks/${var.dataflow_params.runner_params.network}" : null + subnetwork = (var.dataflow_params.runner_params.subnetwork != null && var.dataflow_params.runner_params.subnetwork != "") ? (var.common_params.host_project != null && var.common_params.host_project != "") ? "https://www.googleapis.com/compute/v1/projects/${var.common_params.host_project}/regions/${var.common_params.region}/subnetworks/${var.dataflow_params.runner_params.subnetwork}" : "https://www.googleapis.com/compute/v1/projects/${var.common_params.project}/regions/${var.common_params.region}/subnetworks/${var.dataflow_params.runner_params.subnetwork}" : null + machine_type = var.dataflow_params.runner_params.machine_type + max_workers = var.dataflow_params.runner_params.max_workers + additional_experiments = var.dataflow_params.runner_params.additional_experiments + additional_pipeline_options = var.dataflow_params.runner_params.additional_pipeline_options + launcher_machine_type = var.dataflow_params.runner_params.launcher_machine_type + ip_configuration = var.dataflow_params.runner_params.ip_configuration + num_workers = var.dataflow_params.runner_params.num_workers + labels = merge(var.dataflow_params.runner_params.labels, { + "migration_id" = var.dataflow_params.runner_params.job_name + }) + + depends_on = [ + google_project_iam_member.dataflow_roles, + google_project_iam_member.spanner_reader_role, + google_storage_bucket_object.session_file_object + ] +} diff --git a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/outputs.tf b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/outputs.tf similarity index 67% rename from v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/outputs.tf rename to v2/gcs-spanner-dv/terraform/samples/simple-validation-job/outputs.tf index bf02f5ff19..d8cb916c14 100644 --- a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/outputs.tf +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/outputs.tf @@ -1,6 +1,6 @@ output "dataflow_job_url" { description = "URL for the created Dataflow Flex Template job." - value = "https://console.cloud.google.com/dataflow/jobs/${var.region}/${google_dataflow_flex_template_job.gcs_spanner_dv_job.job_id}?project=${google_dataflow_flex_template_job.gcs_spanner_dv_job.project}" + value = "https://console.cloud.google.com/dataflow/jobs/${var.common_params.region}/${google_dataflow_flex_template_job.gcs_spanner_dv_job.job_id}?project=${google_dataflow_flex_template_job.gcs_spanner_dv_job.project}" } output "dataflow_job_id" { diff --git a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform.tf b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tf similarity index 50% rename from v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform.tf rename to v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tf index 7e38536772..ba4bd1db2b 100644 --- a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform.tf +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tf @@ -13,13 +13,13 @@ terraform { } provider "google" { - project = var.project - region = var.region + project = var.common_params.project + region = var.common_params.region } provider "google-beta" { - project = var.project - region = var.region + project = var.common_params.project + region = var.common_params.region } # Enable the APIs @@ -32,6 +32,18 @@ resource "google_project_service" "enabled_apis" { "bigquery.googleapis.com" ]) service = each.key - project = var.project + project = var.common_params.project disable_on_destroy = false } + +# To fetch project number +data "google_project" "project" { + project_id = var.common_params.project + depends_on = [google_project_service.enabled_apis] +} + +# Fetch the default service account for Compute Engine (used by Dataflow) +data "google_compute_default_service_account" "gce_account" { + project = var.common_params.project + depends_on = [google_project_service.enabled_apis] +} diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfstate.1787826398.backup b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfstate.1787826398.backup new file mode 100644 index 0000000000..ce82049198 --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfstate.1787826398.backup @@ -0,0 +1,391 @@ +{ + "version": 4, + "terraform_version": "1.8.5", + "serial": 16, + "lineage": "d4268df6-92b7-745e-d8df-45f7ea9b0a96", + "outputs": { + "dataflow_job_id": { + "value": "2026-06-03_03_28_29-6712826463759274662", + "type": "string" + }, + "dataflow_job_url": { + "value": "https://console.cloud.google.com/dataflow/jobs/us-central1/2026-06-03_03_28_29-6712826463759274662", + "type": "string" + } + }, + "resources": [ + { + "mode": "data", + "type": "google_compute_default_service_account", + "name": "gce_account", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "display_name": "Compute Engine default service account", + "email": "443856856628-compute@developer.gserviceaccount.com", + "id": "projects/span-cloud-ck-testing-external/serviceAccounts/443856856628-compute@developer.gserviceaccount.com", + "member": "serviceAccount:443856856628-compute@developer.gserviceaccount.com", + "name": "projects/span-cloud-ck-testing-external/serviceAccounts/443856856628-compute@developer.gserviceaccount.com", + "project": "span-cloud-ck-testing-external", + "unique_id": "112173336591113827983" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "google_project", + "name": "project", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "auto_create_network": null, + "billing_account": "0161AC-3A2851-1A98A1", + "folder_id": "", + "id": "projects/span-cloud-ck-testing-external", + "labels": {}, + "name": "span-cloud-ck-testing-external", + "number": "443856856628", + "org_id": "673587647347", + "project_id": "span-cloud-ck-testing-external", + "skip_delete": null + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "managed", + "type": "google_dataflow_flex_template_job", + "name": "generated", + "provider": "provider[\"registry.terraform.io/hashicorp/google-beta\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "additional_experiments": [ + "disable_runner_v2", + "use_network_tags=allow-dataflow", + "use_network_tags_for_flex_templates=allow-dataflow" + ], + "additional_pipeline_options": null, + "autoscaling_algorithm": null, + "container_spec_gcs_path": "gs://dataflow-templates-us-central1/latest/flex/GCS_Spanner_Data_Validator", + "create_ignore_already_exists": false, + "deletion_policy": "DELETE", + "effective_labels": { + "goog-dataflow-provided-template-name": "gcs_spanner_data_validator", + "goog-dataflow-provided-template-type": "flex", + "goog-dataflow-provided-template-version": "2026-05-26-00_rc00", + "goog-terraform-provisioned": "true" + }, + "enable_streaming_engine": null, + "id": "2026-06-03_03_28_29-6712826463759274662", + "ip_configuration": null, + "job_id": "2026-06-03_03_28_29-6712826463759274662", + "kms_key_name": "", + "labels": null, + "launcher_machine_type": null, + "machine_type": "n1-standard-1", + "max_workers": 0, + "name": "validation-terraform-test", + "network": "default", + "num_workers": 0, + "on_delete": "cancel", + "parameters": { + "bigQueryDataset": "validation_report_dataset", + "databaseId": "bulk_dlq_format_test", + "gcsInputDirectory": "gs://ea-functional-tests/bulk_dlq_format_test/source-avro", + "instanceId": "ea-functional-tests", + "projectId": "span-cloud-ck-testing-external", + "runId": "bulk_validation_run_test17", + "schemaOverridesFilePath": "gs://ea-functional-tests/bulk_dlq_format_test/bulk_dlq_format_test_overrides.json" + }, + "project": "span-cloud-ck-testing-external", + "region": "us-central1", + "sdk_container_image": "gcr.io/cloud-dataflow/v1beta3/beam-java17-batch:2.73.0", + "service_account_email": "443856856628-compute@developer.gserviceaccount.com", + "skip_wait_on_job_termination": false, + "staging_location": "gs://dataflow-staging-us-central1-443856856628/staging", + "state": "JOB_STATE_RUNNING", + "subnetwork": "", + "temp_location": "gs://dataflow-staging-us-central1-443856856628/tmp", + "terraform_labels": { + "goog-terraform-provisioned": "true" + }, + "transform_name_mapping": null, + "type": "JOB_TYPE_BATCH" + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "google_project_service.enabled_apis" + ] + } + ] + }, + { + "mode": "managed", + "type": "google_project_iam_member", + "name": "validation_roles", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "index_key": "roles/bigquery.dataEditor", + "schema_version": 0, + "attributes": { + "condition": [], + "etag": "BwZTVuPbDfs=", + "id": "projects/span-cloud-ck-testing-external/roles/bigquery.dataEditor/serviceAccount:443856856628-compute@developer.gserviceaccount.com", + "member": "serviceAccount:443856856628-compute@developer.gserviceaccount.com", + "project": "projects/span-cloud-ck-testing-external", + "role": "roles/bigquery.dataEditor" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.google_compute_default_service_account.gce_account", + "data.google_project.project", + "google_project_service.enabled_apis" + ] + }, + { + "index_key": "roles/bigquery.jobUser", + "schema_version": 0, + "attributes": { + "condition": [], + "etag": "BwZTVuPbDfs=", + "id": "projects/span-cloud-ck-testing-external/roles/bigquery.jobUser/serviceAccount:443856856628-compute@developer.gserviceaccount.com", + "member": "serviceAccount:443856856628-compute@developer.gserviceaccount.com", + "project": "projects/span-cloud-ck-testing-external", + "role": "roles/bigquery.jobUser" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.google_compute_default_service_account.gce_account", + "data.google_project.project", + "google_project_service.enabled_apis" + ] + }, + { + "index_key": "roles/cloudprofiler.agent", + "schema_version": 0, + "attributes": { + "condition": [], + "etag": "BwZTVuPbDfs=", + "id": "projects/span-cloud-ck-testing-external/roles/cloudprofiler.agent/serviceAccount:443856856628-compute@developer.gserviceaccount.com", + "member": "serviceAccount:443856856628-compute@developer.gserviceaccount.com", + "project": "projects/span-cloud-ck-testing-external", + "role": "roles/cloudprofiler.agent" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.google_compute_default_service_account.gce_account", + "data.google_project.project", + "google_project_service.enabled_apis" + ] + }, + { + "index_key": "roles/dataflow.admin", + "schema_version": 0, + "attributes": { + "condition": [], + "etag": "BwZTVuPbDfs=", + "id": "projects/span-cloud-ck-testing-external/roles/dataflow.admin/serviceAccount:443856856628-compute@developer.gserviceaccount.com", + "member": "serviceAccount:443856856628-compute@developer.gserviceaccount.com", + "project": "projects/span-cloud-ck-testing-external", + "role": "roles/dataflow.admin" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.google_compute_default_service_account.gce_account", + "data.google_project.project", + "google_project_service.enabled_apis" + ] + }, + { + "index_key": "roles/dataflow.worker", + "schema_version": 0, + "attributes": { + "condition": [], + "etag": "BwZTVuPbDfs=", + "id": "projects/span-cloud-ck-testing-external/roles/dataflow.worker/serviceAccount:443856856628-compute@developer.gserviceaccount.com", + "member": "serviceAccount:443856856628-compute@developer.gserviceaccount.com", + "project": "projects/span-cloud-ck-testing-external", + "role": "roles/dataflow.worker" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.google_compute_default_service_account.gce_account", + "data.google_project.project", + "google_project_service.enabled_apis" + ] + }, + { + "index_key": "roles/monitoring.metricWriter", + "schema_version": 0, + "attributes": { + "condition": [], + "etag": "BwZTVuPbDfs=", + "id": "projects/span-cloud-ck-testing-external/roles/monitoring.metricWriter/serviceAccount:443856856628-compute@developer.gserviceaccount.com", + "member": "serviceAccount:443856856628-compute@developer.gserviceaccount.com", + "project": "projects/span-cloud-ck-testing-external", + "role": "roles/monitoring.metricWriter" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.google_compute_default_service_account.gce_account", + "data.google_project.project", + "google_project_service.enabled_apis" + ] + }, + { + "index_key": "roles/spanner.databaseReader", + "schema_version": 0, + "attributes": { + "condition": [], + "etag": "BwZTVuPbDfs=", + "id": "projects/span-cloud-ck-testing-external/roles/spanner.databaseReader/serviceAccount:443856856628-compute@developer.gserviceaccount.com", + "member": "serviceAccount:443856856628-compute@developer.gserviceaccount.com", + "project": "projects/span-cloud-ck-testing-external", + "role": "roles/spanner.databaseReader" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.google_compute_default_service_account.gce_account", + "data.google_project.project", + "google_project_service.enabled_apis" + ] + }, + { + "index_key": "roles/storage.objectViewer", + "schema_version": 0, + "attributes": { + "condition": [], + "etag": "BwZTVuPbDfs=", + "id": "projects/span-cloud-ck-testing-external/roles/storage.objectViewer/serviceAccount:443856856628-compute@developer.gserviceaccount.com", + "member": "serviceAccount:443856856628-compute@developer.gserviceaccount.com", + "project": "projects/span-cloud-ck-testing-external", + "role": "roles/storage.objectViewer" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.google_compute_default_service_account.gce_account", + "data.google_project.project", + "google_project_service.enabled_apis" + ] + }, + { + "index_key": "roles/viewer", + "schema_version": 0, + "attributes": { + "condition": [], + "etag": "BwZTVuPbDfs=", + "id": "projects/span-cloud-ck-testing-external/roles/viewer/serviceAccount:443856856628-compute@developer.gserviceaccount.com", + "member": "serviceAccount:443856856628-compute@developer.gserviceaccount.com", + "project": "projects/span-cloud-ck-testing-external", + "role": "roles/viewer" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.google_compute_default_service_account.gce_account", + "data.google_project.project", + "google_project_service.enabled_apis" + ] + } + ] + }, + { + "mode": "managed", + "type": "google_project_service", + "name": "enabled_apis", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "index_key": "bigquery.googleapis.com", + "schema_version": 0, + "attributes": { + "disable_dependent_services": null, + "disable_on_destroy": false, + "id": "span-cloud-ck-testing-external/bigquery.googleapis.com", + "project": "span-cloud-ck-testing-external", + "service": "bigquery.googleapis.com", + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjoxMjAwMDAwMDAwMDAwfX0=" + }, + { + "index_key": "dataflow.googleapis.com", + "schema_version": 0, + "attributes": { + "disable_dependent_services": null, + "disable_on_destroy": false, + "id": "span-cloud-ck-testing-external/dataflow.googleapis.com", + "project": "span-cloud-ck-testing-external", + "service": "dataflow.googleapis.com", + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjoxMjAwMDAwMDAwMDAwfX0=" + }, + { + "index_key": "iam.googleapis.com", + "schema_version": 0, + "attributes": { + "disable_dependent_services": null, + "disable_on_destroy": false, + "id": "span-cloud-ck-testing-external/iam.googleapis.com", + "project": "span-cloud-ck-testing-external", + "service": "iam.googleapis.com", + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjoxMjAwMDAwMDAwMDAwfX0=" + }, + { + "index_key": "spanner.googleapis.com", + "schema_version": 0, + "attributes": { + "disable_dependent_services": null, + "disable_on_destroy": false, + "id": "span-cloud-ck-testing-external/spanner.googleapis.com", + "project": "span-cloud-ck-testing-external", + "service": "spanner.googleapis.com", + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjoxMjAwMDAwMDAwMDAwfX0=" + }, + { + "index_key": "storage.googleapis.com", + "schema_version": 0, + "attributes": { + "disable_dependent_services": null, + "disable_on_destroy": false, + "id": "span-cloud-ck-testing-external/storage.googleapis.com", + "project": "span-cloud-ck-testing-external", + "service": "storage.googleapis.com", + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjoxMjAwMDAwMDAwMDAwfX0=" + } + ] + } + ], + "check_results": null +} diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/variables.tf b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/variables.tf new file mode 100644 index 0000000000..238b8c668b --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/variables.tf @@ -0,0 +1,50 @@ +variable "common_params" { + description = "Parameters that are common to multiple resources" + type = object({ + project = string + host_project = optional(string, "") + region = string + add_policies_to_service_account = optional(bool, true) + }) +} + +variable "dataflow_params" { + description = "Parameters for the Dataflow job." + type = object({ + template_params = object({ + gcs_input_directory = string + instance_id = string + database_id = string + spanner_project_id = optional(string, "") + bigquery_dataset = string + spanner_host = optional(string, "") + spanner_priority = optional(string, null) + session_file_path = optional(string, "") + local_session_file_path = optional(string, null) + schema_overrides_file_path = optional(string, "") + table_overrides = optional(string, "") + column_overrides = optional(string, "") + run_id = optional(string, "") + transformation_jar_path = optional(string, "") + transformation_class_name = optional(string, "") + transformation_custom_parameters = optional(string, "") + working_directory_bucket = optional(string, null) + working_directory_prefix = optional(string, null) + }) + runner_params = object({ + job_name = string + service_account_email = optional(string, "") + network = optional(string, "") + subnetwork = optional(string, "") + machine_type = optional(string, "n1-standard-4") + max_workers = optional(number, null) + num_workers = optional(number, null) + additional_experiments = optional(list(string), []) + ip_configuration = optional(string, "") + launcher_machine_type = optional(string, "") + additional_pipeline_options = optional(map(string), {}) + labels = optional(map(string), {}) + kms_key_name = optional(string, "") + }) + }) +} From 82c49e7ce377cd380c4fe88a23c80c7c1fa61ad3 Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Thu, 27 Aug 2026 12:49:11 +0000 Subject: [PATCH 08/14] Fix terraform validation errors and update simple tfvars placeholders --- .../terraform/samples/simple-validation-job/main.tf | 1 - .../terraform/samples/simple-validation-job/variables.tf | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf index 5c45142397..6fd41764e0 100644 --- a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf @@ -69,7 +69,6 @@ resource "google_dataflow_flex_template_job" "gcs_spanner_dv_job" { machine_type = var.dataflow_params.runner_params.machine_type max_workers = var.dataflow_params.runner_params.max_workers additional_experiments = var.dataflow_params.runner_params.additional_experiments - additional_pipeline_options = var.dataflow_params.runner_params.additional_pipeline_options launcher_machine_type = var.dataflow_params.runner_params.launcher_machine_type ip_configuration = var.dataflow_params.runner_params.ip_configuration num_workers = var.dataflow_params.runner_params.num_workers diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/variables.tf b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/variables.tf index 238b8c668b..8ca2045f3c 100644 --- a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/variables.tf +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/variables.tf @@ -40,8 +40,8 @@ variable "dataflow_params" { max_workers = optional(number, null) num_workers = optional(number, null) additional_experiments = optional(list(string), []) - ip_configuration = optional(string, "") - launcher_machine_type = optional(string, "") + ip_configuration = optional(string, null) + launcher_machine_type = optional(string, null) additional_pipeline_options = optional(map(string), {}) labels = optional(map(string), {}) kms_key_name = optional(string, "") From 8cf739ec7d4fdb840fcfad7a983c414b803c0123 Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Thu, 27 Aug 2026 12:51:23 +0000 Subject: [PATCH 09/14] Add ignored tfvars files, revert flex template plugin change, and remove tfstate backup --- .../main/resources/terraform-flex-template.tf | 1 - .../terraform.tfstate.1787826398.backup | 391 ------------------ .../simple-validation-job/terraform.tfvars | 45 ++ .../terraform_simple.tfvars | 20 + 4 files changed, 65 insertions(+), 392 deletions(-) delete mode 100644 v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfstate.1787826398.backup create mode 100644 v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfvars create mode 100644 v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform_simple.tfvars diff --git a/plugins/core-plugin/src/main/resources/terraform-flex-template.tf b/plugins/core-plugin/src/main/resources/terraform-flex-template.tf index 0847194279..d9234d7ef7 100644 --- a/plugins/core-plugin/src/main/resources/terraform-flex-template.tf +++ b/plugins/core-plugin/src/main/resources/terraform-flex-template.tf @@ -190,7 +190,6 @@ resource "google_dataflow_flex_template_job" "generated" { subnetwork = var.subnetwork temp_location = var.temp_location region = var.region - on_delete = var.on_delete } output "dataflow_job_url" { diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfstate.1787826398.backup b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfstate.1787826398.backup deleted file mode 100644 index ce82049198..0000000000 --- a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfstate.1787826398.backup +++ /dev/null @@ -1,391 +0,0 @@ -{ - "version": 4, - "terraform_version": "1.8.5", - "serial": 16, - "lineage": "d4268df6-92b7-745e-d8df-45f7ea9b0a96", - "outputs": { - "dataflow_job_id": { - "value": "2026-06-03_03_28_29-6712826463759274662", - "type": "string" - }, - "dataflow_job_url": { - "value": "https://console.cloud.google.com/dataflow/jobs/us-central1/2026-06-03_03_28_29-6712826463759274662", - "type": "string" - } - }, - "resources": [ - { - "mode": "data", - "type": "google_compute_default_service_account", - "name": "gce_account", - "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "display_name": "Compute Engine default service account", - "email": "443856856628-compute@developer.gserviceaccount.com", - "id": "projects/span-cloud-ck-testing-external/serviceAccounts/443856856628-compute@developer.gserviceaccount.com", - "member": "serviceAccount:443856856628-compute@developer.gserviceaccount.com", - "name": "projects/span-cloud-ck-testing-external/serviceAccounts/443856856628-compute@developer.gserviceaccount.com", - "project": "span-cloud-ck-testing-external", - "unique_id": "112173336591113827983" - }, - "sensitive_attributes": [] - } - ] - }, - { - "mode": "data", - "type": "google_project", - "name": "project", - "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "auto_create_network": null, - "billing_account": "0161AC-3A2851-1A98A1", - "folder_id": "", - "id": "projects/span-cloud-ck-testing-external", - "labels": {}, - "name": "span-cloud-ck-testing-external", - "number": "443856856628", - "org_id": "673587647347", - "project_id": "span-cloud-ck-testing-external", - "skip_delete": null - }, - "sensitive_attributes": [] - } - ] - }, - { - "mode": "managed", - "type": "google_dataflow_flex_template_job", - "name": "generated", - "provider": "provider[\"registry.terraform.io/hashicorp/google-beta\"]", - "instances": [ - { - "schema_version": 1, - "attributes": { - "additional_experiments": [ - "disable_runner_v2", - "use_network_tags=allow-dataflow", - "use_network_tags_for_flex_templates=allow-dataflow" - ], - "additional_pipeline_options": null, - "autoscaling_algorithm": null, - "container_spec_gcs_path": "gs://dataflow-templates-us-central1/latest/flex/GCS_Spanner_Data_Validator", - "create_ignore_already_exists": false, - "deletion_policy": "DELETE", - "effective_labels": { - "goog-dataflow-provided-template-name": "gcs_spanner_data_validator", - "goog-dataflow-provided-template-type": "flex", - "goog-dataflow-provided-template-version": "2026-05-26-00_rc00", - "goog-terraform-provisioned": "true" - }, - "enable_streaming_engine": null, - "id": "2026-06-03_03_28_29-6712826463759274662", - "ip_configuration": null, - "job_id": "2026-06-03_03_28_29-6712826463759274662", - "kms_key_name": "", - "labels": null, - "launcher_machine_type": null, - "machine_type": "n1-standard-1", - "max_workers": 0, - "name": "validation-terraform-test", - "network": "default", - "num_workers": 0, - "on_delete": "cancel", - "parameters": { - "bigQueryDataset": "validation_report_dataset", - "databaseId": "bulk_dlq_format_test", - "gcsInputDirectory": "gs://ea-functional-tests/bulk_dlq_format_test/source-avro", - "instanceId": "ea-functional-tests", - "projectId": "span-cloud-ck-testing-external", - "runId": "bulk_validation_run_test17", - "schemaOverridesFilePath": "gs://ea-functional-tests/bulk_dlq_format_test/bulk_dlq_format_test_overrides.json" - }, - "project": "span-cloud-ck-testing-external", - "region": "us-central1", - "sdk_container_image": "gcr.io/cloud-dataflow/v1beta3/beam-java17-batch:2.73.0", - "service_account_email": "443856856628-compute@developer.gserviceaccount.com", - "skip_wait_on_job_termination": false, - "staging_location": "gs://dataflow-staging-us-central1-443856856628/staging", - "state": "JOB_STATE_RUNNING", - "subnetwork": "", - "temp_location": "gs://dataflow-staging-us-central1-443856856628/tmp", - "terraform_labels": { - "goog-terraform-provisioned": "true" - }, - "transform_name_mapping": null, - "type": "JOB_TYPE_BATCH" - }, - "sensitive_attributes": [], - "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", - "dependencies": [ - "google_project_service.enabled_apis" - ] - } - ] - }, - { - "mode": "managed", - "type": "google_project_iam_member", - "name": "validation_roles", - "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", - "instances": [ - { - "index_key": "roles/bigquery.dataEditor", - "schema_version": 0, - "attributes": { - "condition": [], - "etag": "BwZTVuPbDfs=", - "id": "projects/span-cloud-ck-testing-external/roles/bigquery.dataEditor/serviceAccount:443856856628-compute@developer.gserviceaccount.com", - "member": "serviceAccount:443856856628-compute@developer.gserviceaccount.com", - "project": "projects/span-cloud-ck-testing-external", - "role": "roles/bigquery.dataEditor" - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "data.google_compute_default_service_account.gce_account", - "data.google_project.project", - "google_project_service.enabled_apis" - ] - }, - { - "index_key": "roles/bigquery.jobUser", - "schema_version": 0, - "attributes": { - "condition": [], - "etag": "BwZTVuPbDfs=", - "id": "projects/span-cloud-ck-testing-external/roles/bigquery.jobUser/serviceAccount:443856856628-compute@developer.gserviceaccount.com", - "member": "serviceAccount:443856856628-compute@developer.gserviceaccount.com", - "project": "projects/span-cloud-ck-testing-external", - "role": "roles/bigquery.jobUser" - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "data.google_compute_default_service_account.gce_account", - "data.google_project.project", - "google_project_service.enabled_apis" - ] - }, - { - "index_key": "roles/cloudprofiler.agent", - "schema_version": 0, - "attributes": { - "condition": [], - "etag": "BwZTVuPbDfs=", - "id": "projects/span-cloud-ck-testing-external/roles/cloudprofiler.agent/serviceAccount:443856856628-compute@developer.gserviceaccount.com", - "member": "serviceAccount:443856856628-compute@developer.gserviceaccount.com", - "project": "projects/span-cloud-ck-testing-external", - "role": "roles/cloudprofiler.agent" - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "data.google_compute_default_service_account.gce_account", - "data.google_project.project", - "google_project_service.enabled_apis" - ] - }, - { - "index_key": "roles/dataflow.admin", - "schema_version": 0, - "attributes": { - "condition": [], - "etag": "BwZTVuPbDfs=", - "id": "projects/span-cloud-ck-testing-external/roles/dataflow.admin/serviceAccount:443856856628-compute@developer.gserviceaccount.com", - "member": "serviceAccount:443856856628-compute@developer.gserviceaccount.com", - "project": "projects/span-cloud-ck-testing-external", - "role": "roles/dataflow.admin" - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "data.google_compute_default_service_account.gce_account", - "data.google_project.project", - "google_project_service.enabled_apis" - ] - }, - { - "index_key": "roles/dataflow.worker", - "schema_version": 0, - "attributes": { - "condition": [], - "etag": "BwZTVuPbDfs=", - "id": "projects/span-cloud-ck-testing-external/roles/dataflow.worker/serviceAccount:443856856628-compute@developer.gserviceaccount.com", - "member": "serviceAccount:443856856628-compute@developer.gserviceaccount.com", - "project": "projects/span-cloud-ck-testing-external", - "role": "roles/dataflow.worker" - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "data.google_compute_default_service_account.gce_account", - "data.google_project.project", - "google_project_service.enabled_apis" - ] - }, - { - "index_key": "roles/monitoring.metricWriter", - "schema_version": 0, - "attributes": { - "condition": [], - "etag": "BwZTVuPbDfs=", - "id": "projects/span-cloud-ck-testing-external/roles/monitoring.metricWriter/serviceAccount:443856856628-compute@developer.gserviceaccount.com", - "member": "serviceAccount:443856856628-compute@developer.gserviceaccount.com", - "project": "projects/span-cloud-ck-testing-external", - "role": "roles/monitoring.metricWriter" - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "data.google_compute_default_service_account.gce_account", - "data.google_project.project", - "google_project_service.enabled_apis" - ] - }, - { - "index_key": "roles/spanner.databaseReader", - "schema_version": 0, - "attributes": { - "condition": [], - "etag": "BwZTVuPbDfs=", - "id": "projects/span-cloud-ck-testing-external/roles/spanner.databaseReader/serviceAccount:443856856628-compute@developer.gserviceaccount.com", - "member": "serviceAccount:443856856628-compute@developer.gserviceaccount.com", - "project": "projects/span-cloud-ck-testing-external", - "role": "roles/spanner.databaseReader" - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "data.google_compute_default_service_account.gce_account", - "data.google_project.project", - "google_project_service.enabled_apis" - ] - }, - { - "index_key": "roles/storage.objectViewer", - "schema_version": 0, - "attributes": { - "condition": [], - "etag": "BwZTVuPbDfs=", - "id": "projects/span-cloud-ck-testing-external/roles/storage.objectViewer/serviceAccount:443856856628-compute@developer.gserviceaccount.com", - "member": "serviceAccount:443856856628-compute@developer.gserviceaccount.com", - "project": "projects/span-cloud-ck-testing-external", - "role": "roles/storage.objectViewer" - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "data.google_compute_default_service_account.gce_account", - "data.google_project.project", - "google_project_service.enabled_apis" - ] - }, - { - "index_key": "roles/viewer", - "schema_version": 0, - "attributes": { - "condition": [], - "etag": "BwZTVuPbDfs=", - "id": "projects/span-cloud-ck-testing-external/roles/viewer/serviceAccount:443856856628-compute@developer.gserviceaccount.com", - "member": "serviceAccount:443856856628-compute@developer.gserviceaccount.com", - "project": "projects/span-cloud-ck-testing-external", - "role": "roles/viewer" - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "data.google_compute_default_service_account.gce_account", - "data.google_project.project", - "google_project_service.enabled_apis" - ] - } - ] - }, - { - "mode": "managed", - "type": "google_project_service", - "name": "enabled_apis", - "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", - "instances": [ - { - "index_key": "bigquery.googleapis.com", - "schema_version": 0, - "attributes": { - "disable_dependent_services": null, - "disable_on_destroy": false, - "id": "span-cloud-ck-testing-external/bigquery.googleapis.com", - "project": "span-cloud-ck-testing-external", - "service": "bigquery.googleapis.com", - "timeouts": null - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjoxMjAwMDAwMDAwMDAwfX0=" - }, - { - "index_key": "dataflow.googleapis.com", - "schema_version": 0, - "attributes": { - "disable_dependent_services": null, - "disable_on_destroy": false, - "id": "span-cloud-ck-testing-external/dataflow.googleapis.com", - "project": "span-cloud-ck-testing-external", - "service": "dataflow.googleapis.com", - "timeouts": null - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjoxMjAwMDAwMDAwMDAwfX0=" - }, - { - "index_key": "iam.googleapis.com", - "schema_version": 0, - "attributes": { - "disable_dependent_services": null, - "disable_on_destroy": false, - "id": "span-cloud-ck-testing-external/iam.googleapis.com", - "project": "span-cloud-ck-testing-external", - "service": "iam.googleapis.com", - "timeouts": null - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjoxMjAwMDAwMDAwMDAwfX0=" - }, - { - "index_key": "spanner.googleapis.com", - "schema_version": 0, - "attributes": { - "disable_dependent_services": null, - "disable_on_destroy": false, - "id": "span-cloud-ck-testing-external/spanner.googleapis.com", - "project": "span-cloud-ck-testing-external", - "service": "spanner.googleapis.com", - "timeouts": null - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjoxMjAwMDAwMDAwMDAwfX0=" - }, - { - "index_key": "storage.googleapis.com", - "schema_version": 0, - "attributes": { - "disable_dependent_services": null, - "disable_on_destroy": false, - "id": "span-cloud-ck-testing-external/storage.googleapis.com", - "project": "span-cloud-ck-testing-external", - "service": "storage.googleapis.com", - "timeouts": null - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjoxMjAwMDAwMDAwMDAwfX0=" - } - ] - } - ], - "check_results": null -} diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfvars b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfvars new file mode 100644 index 0000000000..2c1829819b --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfvars @@ -0,0 +1,45 @@ +common_params = { + project = "" # Replace with your GCP project ID + host_project = "" # Optional: Project ID hosting the network if using a shared VPC + region = "" # Replace with your desired GCP region + add_policies_to_service_account = true # Optional: Whether Terraform should add required permissions to the service account +} + +dataflow_params = { + template_params = { + gcs_input_directory = "gs:///source-avro" # The Cloud Storage directory containing validation data + instance_id = "" # The Spanner instance ID to validate against + database_id = "" # The Spanner database ID to validate against + spanner_project_id = "" # Optional: Project ID where the Spanner instance is located (defaults to Dataflow project) + bigquery_dataset = "" # The BigQuery dataset to store validation reports (e.g., validation_report_dataset) + spanner_host = "" # Optional: Custom Spanner host endpoint + spanner_priority = "HIGH" # Optional: Priority for Spanner RPC invocations (HIGH, MEDIUM, LOW) + local_session_file_path = "" # Optional: Local path to the session file (will be uploaded to working directory) + session_file_path = "" # Optional: Existing GCS path to the session file + schema_overrides_file_path = "" # Optional: GCS path to your overrides file + table_overrides = "" # Optional: Table name overrides (e.g., "[{OldTableName,NewTableName}]") + column_overrides = "" # Optional: Column name overrides (e.g., "[{TableName.OldColumnName,TableName.NewColumnName}]") + transformation_jar_path = "" # Optional: GCS path to the transformation JAR file + transformation_class_name = "" # Optional: Fully qualified transformation class name + transformation_custom_parameters = "" # Optional: Custom parameters for the transformation + working_directory_bucket = "" # Optional: Bucket for uploading session file and creating output directory + working_directory_prefix = "" # Optional: Prefix within the GCS bucket for working directory + run_id = "" # Optional: Custom run identifier + } + + runner_params = { + job_name = "data-validation-job" # Or your custom job name + service_account_email = "" # Optional: Service account email for Dataflow workers + network = "" # Optional: Network for Dataflow workers + subnetwork = "" # Optional: Subnetwork for Dataflow workers + machine_type = "n1-standard-4" # Optional: Machine type for Dataflow worker VMs + max_workers = 10 # Optional: Maximum number of Dataflow worker VMs + num_workers = 4 # Optional: Initial number of Dataflow worker VMs + additional_experiments = [""] # Optional: Additional Dataflow experiments (list of strings) + ip_configuration = "WORKER_IP_PRIVATE" # Optional: IP configuration for Dataflow workers + launcher_machine_type = "n1-standard-1" # Optional: Machine type for the Dataflow launcher VM + additional_pipeline_options = { "" = "" } # Optional: Additional Dataflow pipeline options (map of strings) + labels = { "env" = "test" } # Optional: Labels to apply to the Dataflow job (map of strings) + kms_key_name = "" # Optional: Cloud KMS key name for data encryption + } +} diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform_simple.tfvars b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform_simple.tfvars new file mode 100644 index 0000000000..380fab4609 --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform_simple.tfvars @@ -0,0 +1,20 @@ +common_params = { + project = "" # Replace with your GCP project ID + region = "" # Replace with your desired GCP region +} + +dataflow_params = { + template_params = { + instance_id = "" # The Spanner instance ID to validate against + database_id = "" # The Spanner database ID to validate against + spanner_project_id = "" # Optional: Project ID where the Spanner instance is located + gcs_input_directory = "gs:///source-avro" # The Cloud Storage directory containing validation data + bigquery_dataset = "" # The BigQuery dataset to store validation reports + schema_overrides_file_path = "" # Optional: GCS path to your overrides file + run_id = "" # Optional: Custom run identifier + } + + runner_params = { + job_name = "" # Custom job name for the Dataflow pipeline + } +} From 2e7c27589fe6e12ff8869160d8cac42a3d7c98a5 Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Thu, 27 Aug 2026 13:09:40 +0000 Subject: [PATCH 10/14] final changes --- .../dataflow_job.tf | 7 ----- .../samples/simple-validation-job/README.md | 17 ++++++------ .../samples/simple-validation-job/main.tf | 26 ++++++++++++++++--- .../simple-validation-job/terraform.tf | 2 +- .../simple-validation-job/terraform.tfvars | 2 +- 5 files changed, 33 insertions(+), 21 deletions(-) diff --git a/v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf b/v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf index b52990eff1..9220ff40c5 100644 --- a/v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf +++ b/v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf @@ -124,13 +124,6 @@ variable "transformationCustomParameters" { } -provider "google" { - project = var.project -} - -provider "google-beta" { - project = var.project -} variable "additional_experiments" { type = set(string) diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/README.md b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/README.md index 1da56aa615..3cc1f5977e 100644 --- a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/README.md +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/README.md @@ -261,7 +261,7 @@ includedPermissions: Then attach the role to the service account - ```shell -gcloud iam service-accounts add-iam-policy-binding @.iam.gserviceaccount.com \ +gcloud projects add-iam-policy-binding \ --member="serviceAccount:@.iam.gserviceaccount.com" \ --role="projects//roles/dv_terraform_role" ``` @@ -281,11 +281,12 @@ PROJECT_ID="" # Array of roles to grant ROLES=( - "roles/dataflow.admin" - "roles/iam.securityAdmin" - "roles/iam.serviceAccountUser" - "roles/storage.admin" - "roles/viewer" + "roles/dataflow.admin" # Required to create and manage Dataflow jobs + "roles/iam.securityAdmin" # Required to bind roles to the Dataflow worker SA + "roles/iam.serviceAccountUser" # Required to impersonate the Dataflow worker SA + "roles/storage.admin" # Required to manage GCS staging objects + "roles/viewer" # Required to fetch current project states + "roles/serviceusage.serviceUsageAdmin" # Required to enable required APIs ) # Loop through each role and grant it to the service account @@ -305,13 +306,13 @@ Verify that the custom role is attached to the service account - gcloud projects get-iam-policy \ --flatten="bindings[].members" \ --format='table(bindings.role)' \ ---filter="bindings.members:@.iam.gserviceaccount.com" +--filter="bindings.members:serviceAccount:@.iam.gserviceaccount.com" ``` Verify that the role has the correct set of permissions ```shell -gcloud iam roles describe bulk_migrations_role --project= +gcloud iam roles describe dv_terraform_role --project= ``` ### Impersonating the Terraform service account diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf index 6fd41764e0..76a0e26d94 100644 --- a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf @@ -1,11 +1,27 @@ locals { effective_sa_email = (var.dataflow_params.runner_params.service_account_email != null && var.dataflow_params.runner_params.service_account_email != "") ? var.dataflow_params.runner_params.service_account_email : data.google_compute_default_service_account.gce_account.email + + # Network resolution (handling Shared VPC structures) + network_project = (var.common_params.host_project != null && var.common_params.host_project != "") ? var.common_params.host_project : var.common_params.project + network_uri = (var.dataflow_params.runner_params.network != null && var.dataflow_params.runner_params.network != "") ? "projects/${local.network_project}/global/networks/${var.dataflow_params.runner_params.network}" : null + subnetwork_uri = (var.dataflow_params.runner_params.subnetwork != null && var.dataflow_params.runner_params.subnetwork != "") ? "https://www.googleapis.com/compute/v1/projects/${local.network_project}/regions/${var.common_params.region}/subnetworks/${var.dataflow_params.runner_params.subnetwork}" : null + + # Spanner project resolution + spanner_project_id = (var.dataflow_params.template_params.spanner_project_id != null && var.dataflow_params.template_params.spanner_project_id != "") ? var.dataflow_params.template_params.spanner_project_id : var.common_params.project } # upload local session file to the working GCS bucket resource "google_storage_bucket_object" "session_file_object" { count = var.dataflow_params.template_params.local_session_file_path != null ? 1 : 0 depends_on = [google_project_service.enabled_apis] + + lifecycle { + precondition { + condition = var.dataflow_params.template_params.working_directory_bucket != null + error_message = "You must provide a working_directory_bucket in template_params when uploading a local_session_file_path." + } + } + name = "${var.dataflow_params.template_params.working_directory_prefix}/session.json" source = var.dataflow_params.template_params.local_session_file_path content_type = "application/json" @@ -30,7 +46,7 @@ resource "google_project_iam_member" "dataflow_roles" { resource "google_project_iam_member" "spanner_reader_role" { count = var.common_params.add_policies_to_service_account ? 1 : 0 - project = var.dataflow_params.template_params.spanner_project_id != null && var.dataflow_params.template_params.spanner_project_id != "" ? var.dataflow_params.template_params.spanner_project_id : var.common_params.project + project = local.spanner_project_id role = "roles/spanner.databaseReader" member = "serviceAccount:${local.effective_sa_email}" } @@ -46,7 +62,7 @@ resource "google_dataflow_flex_template_job" "gcs_spanner_dv_job" { parameters = { for k, v in { gcsInputDirectory = var.dataflow_params.template_params.gcs_input_directory - projectId = var.dataflow_params.template_params.spanner_project_id != null && var.dataflow_params.template_params.spanner_project_id != "" ? var.dataflow_params.template_params.spanner_project_id : var.common_params.project + projectId = local.spanner_project_id instanceId = var.dataflow_params.template_params.instance_id databaseId = var.dataflow_params.template_params.database_id bigQueryDataset = var.dataflow_params.template_params.bigquery_dataset @@ -64,8 +80,10 @@ resource "google_dataflow_flex_template_job" "gcs_spanner_dv_job" { } service_account_email = local.effective_sa_email - network = (var.dataflow_params.runner_params.network != null && var.dataflow_params.runner_params.network != "") ? (var.common_params.host_project != null && var.common_params.host_project != "") ? "projects/${var.common_params.host_project}/global/networks/${var.dataflow_params.runner_params.network}" : "projects/${var.common_params.project}/global/networks/${var.dataflow_params.runner_params.network}" : null - subnetwork = (var.dataflow_params.runner_params.subnetwork != null && var.dataflow_params.runner_params.subnetwork != "") ? (var.common_params.host_project != null && var.common_params.host_project != "") ? "https://www.googleapis.com/compute/v1/projects/${var.common_params.host_project}/regions/${var.common_params.region}/subnetworks/${var.dataflow_params.runner_params.subnetwork}" : "https://www.googleapis.com/compute/v1/projects/${var.common_params.project}/regions/${var.common_params.region}/subnetworks/${var.dataflow_params.runner_params.subnetwork}" : null + network = local.network_uri + subnetwork = local.subnetwork_uri + kms_key_name = var.dataflow_params.runner_params.kms_key_name != "" ? var.dataflow_params.runner_params.kms_key_name : null + additional_pipeline_options = var.dataflow_params.runner_params.additional_pipeline_options machine_type = var.dataflow_params.runner_params.machine_type max_workers = var.dataflow_params.runner_params.max_workers additional_experiments = var.dataflow_params.runner_params.additional_experiments diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tf b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tf index ba4bd1db2b..d94bfd4f92 100644 --- a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tf +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tf @@ -27,6 +27,7 @@ resource "google_project_service" "enabled_apis" { for_each = toset([ "iam.googleapis.com", "dataflow.googleapis.com", + "compute.googleapis.com", "storage.googleapis.com", "spanner.googleapis.com", "bigquery.googleapis.com" @@ -39,7 +40,6 @@ resource "google_project_service" "enabled_apis" { # To fetch project number data "google_project" "project" { project_id = var.common_params.project - depends_on = [google_project_service.enabled_apis] } # Fetch the default service account for Compute Engine (used by Dataflow) diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfvars b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfvars index 2c1829819b..934137b95c 100644 --- a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfvars +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfvars @@ -32,7 +32,7 @@ dataflow_params = { service_account_email = "" # Optional: Service account email for Dataflow workers network = "" # Optional: Network for Dataflow workers subnetwork = "" # Optional: Subnetwork for Dataflow workers - machine_type = "n1-standard-4" # Optional: Machine type for Dataflow worker VMs + machine_type = "n2-standard-4" # Optional: Machine type for Dataflow worker VMs max_workers = 10 # Optional: Maximum number of Dataflow worker VMs num_workers = 4 # Optional: Initial number of Dataflow worker VMs additional_experiments = [""] # Optional: Additional Dataflow experiments (list of strings) From 7b010aeb4b9faa489401741148d6eed60d05ac95 Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Fri, 28 Aug 2026 05:22:30 +0000 Subject: [PATCH 11/14] gemini-review --- .../terraform/samples/simple-validation-job/main.tf | 10 +++++----- .../samples/simple-validation-job/terraform.tfvars | 1 - .../samples/simple-validation-job/variables.tf | 1 - 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf index 76a0e26d94..76fa9e7521 100644 --- a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf @@ -12,17 +12,17 @@ locals { # upload local session file to the working GCS bucket resource "google_storage_bucket_object" "session_file_object" { - count = var.dataflow_params.template_params.local_session_file_path != null ? 1 : 0 + count = (var.dataflow_params.template_params.local_session_file_path != null && var.dataflow_params.template_params.local_session_file_path != "") ? 1 : 0 depends_on = [google_project_service.enabled_apis] lifecycle { precondition { - condition = var.dataflow_params.template_params.working_directory_bucket != null + condition = var.dataflow_params.template_params.working_directory_bucket != null && var.dataflow_params.template_params.working_directory_bucket != "" error_message = "You must provide a working_directory_bucket in template_params when uploading a local_session_file_path." } } - name = "${var.dataflow_params.template_params.working_directory_prefix}/session.json" + name = (var.dataflow_params.template_params.working_directory_prefix != null && var.dataflow_params.template_params.working_directory_prefix != "") ? "${var.dataflow_params.template_params.working_directory_prefix}/session.json" : "session.json" source = var.dataflow_params.template_params.local_session_file_path content_type = "application/json" bucket = var.dataflow_params.template_params.working_directory_bucket @@ -68,7 +68,7 @@ resource "google_dataflow_flex_template_job" "gcs_spanner_dv_job" { bigQueryDataset = var.dataflow_params.template_params.bigquery_dataset spannerHost = var.dataflow_params.template_params.spanner_host spannerPriority = var.dataflow_params.template_params.spanner_priority - sessionFilePath = var.dataflow_params.template_params.local_session_file_path != null ? "gs://${var.dataflow_params.template_params.working_directory_bucket}/${var.dataflow_params.template_params.working_directory_prefix}/session.json" : var.dataflow_params.template_params.session_file_path + sessionFilePath = (var.dataflow_params.template_params.local_session_file_path != null && var.dataflow_params.template_params.local_session_file_path != "") ? ((var.dataflow_params.template_params.working_directory_prefix != null && var.dataflow_params.template_params.working_directory_prefix != "") ? "gs://${var.dataflow_params.template_params.working_directory_bucket}/${var.dataflow_params.template_params.working_directory_prefix}/session.json" : "gs://${var.dataflow_params.template_params.working_directory_bucket}/session.json") : var.dataflow_params.template_params.session_file_path schemaOverridesFilePath = var.dataflow_params.template_params.schema_overrides_file_path tableOverrides = var.dataflow_params.template_params.table_overrides columnOverrides = var.dataflow_params.template_params.column_overrides @@ -83,7 +83,7 @@ resource "google_dataflow_flex_template_job" "gcs_spanner_dv_job" { network = local.network_uri subnetwork = local.subnetwork_uri kms_key_name = var.dataflow_params.runner_params.kms_key_name != "" ? var.dataflow_params.runner_params.kms_key_name : null - additional_pipeline_options = var.dataflow_params.runner_params.additional_pipeline_options + machine_type = var.dataflow_params.runner_params.machine_type max_workers = var.dataflow_params.runner_params.max_workers additional_experiments = var.dataflow_params.runner_params.additional_experiments diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfvars b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfvars index 934137b95c..2ddecac9d4 100644 --- a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfvars +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfvars @@ -38,7 +38,6 @@ dataflow_params = { additional_experiments = [""] # Optional: Additional Dataflow experiments (list of strings) ip_configuration = "WORKER_IP_PRIVATE" # Optional: IP configuration for Dataflow workers launcher_machine_type = "n1-standard-1" # Optional: Machine type for the Dataflow launcher VM - additional_pipeline_options = { "" = "" } # Optional: Additional Dataflow pipeline options (map of strings) labels = { "env" = "test" } # Optional: Labels to apply to the Dataflow job (map of strings) kms_key_name = "" # Optional: Cloud KMS key name for data encryption } diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/variables.tf b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/variables.tf index 8ca2045f3c..c381196211 100644 --- a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/variables.tf +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/variables.tf @@ -42,7 +42,6 @@ variable "dataflow_params" { additional_experiments = optional(list(string), []) ip_configuration = optional(string, null) launcher_machine_type = optional(string, null) - additional_pipeline_options = optional(map(string), {}) labels = optional(map(string), {}) kms_key_name = optional(string, "") }) From c3a1283f6f11d0434c20bbc5430562bf909d4543 Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Fri, 28 Aug 2026 06:00:02 +0000 Subject: [PATCH 12/14] review --- .../dataflow_job.tf | 2 +- .../samples/simple-validation-job/README.md | 10 ++---- .../samples/simple-validation-job/main.tf | 28 +++------------- .../simple-validation-job/terraform.tf | 2 +- .../simple-validation-job/terraform.tfvars | 3 -- .../simple-validation-job/variables.tf | 33 +++++++++---------- 6 files changed, 25 insertions(+), 53 deletions(-) diff --git a/v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf b/v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf index 9220ff40c5..4a94cc19cd 100644 --- a/v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf +++ b/v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf @@ -2,7 +2,7 @@ # Autogenerated file. DO NOT EDIT. # -# Copyright (C) 2024 Google LLC +# Copyright (C) 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may not # use this file except in compliance with the License. You may obtain a copy of diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/README.md b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/README.md index 3cc1f5977e..331364e043 100644 --- a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/README.md +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/README.md @@ -6,8 +6,7 @@ This sample demonstrates how to easily launch the Dataflow job while automatical The Terraform module will create the following Google Cloud resources: 1. **Dataflow Flex Template Job:** Uses `google_dataflow_flex_template_job` to launch the data validation pipeline. -2. **GCS Session File:** The GCS object created by uploading the local session file (if specified). -3. **IAM Role Bindings:** Grants the Dataflow worker service account the required roles to run the validation: +2. **IAM Role Bindings:** Grants the Dataflow worker service account the required roles to run the validation: * `roles/dataflow.worker` (required to execute Dataflow jobs) * `roles/spanner.databaseReader` (required to read records from Spanner) * `roles/storage.objectAdmin` (required to read/write objects in Cloud Storage) @@ -230,11 +229,8 @@ If you prefer or need to use a session file, you can generate one using the Span To provide this session file to Terraform: -1. Copy the SMT generated `session file` to the Terraform working directory and name it `session.json`. -2. Set the `var.local_session_file_path` variable to `"session.json"` (or the relative/absolute path to your file). -3. Ensure `var.working_directory_bucket` and `var.working_directory_prefix` are configured so Terraform knows where to upload the file in GCS. - -This will automatically upload the local file to GCS and configure it in the Dataflow job. +1. Upload the SMT generated `session file` to a Cloud Storage bucket. +2. Set the `var.session_file_path` variable to the GCS path of your uploaded file (e.g. `gs://my-bucket/path/to/session.json`). ### Adding access to Terraform service account diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf index 76fa9e7521..63ad44c8a7 100644 --- a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf @@ -3,30 +3,13 @@ locals { # Network resolution (handling Shared VPC structures) network_project = (var.common_params.host_project != null && var.common_params.host_project != "") ? var.common_params.host_project : var.common_params.project - network_uri = (var.dataflow_params.runner_params.network != null && var.dataflow_params.runner_params.network != "") ? "projects/${local.network_project}/global/networks/${var.dataflow_params.runner_params.network}" : null - subnetwork_uri = (var.dataflow_params.runner_params.subnetwork != null && var.dataflow_params.runner_params.subnetwork != "") ? "https://www.googleapis.com/compute/v1/projects/${local.network_project}/regions/${var.common_params.region}/subnetworks/${var.dataflow_params.runner_params.subnetwork}" : null + network_uri = (var.dataflow_params.runner_params.network != null && var.dataflow_params.runner_params.network != "") ? (can(regex("/", var.dataflow_params.runner_params.network)) ? var.dataflow_params.runner_params.network : "projects/${local.network_project}/global/networks/${var.dataflow_params.runner_params.network}") : null + subnetwork_uri = (var.dataflow_params.runner_params.subnetwork != null && var.dataflow_params.runner_params.subnetwork != "") ? (can(regex("/", var.dataflow_params.runner_params.subnetwork)) ? var.dataflow_params.runner_params.subnetwork : "https://www.googleapis.com/compute/v1/projects/${local.network_project}/regions/${var.common_params.region}/subnetworks/${var.dataflow_params.runner_params.subnetwork}") : null # Spanner project resolution spanner_project_id = (var.dataflow_params.template_params.spanner_project_id != null && var.dataflow_params.template_params.spanner_project_id != "") ? var.dataflow_params.template_params.spanner_project_id : var.common_params.project } -# upload local session file to the working GCS bucket -resource "google_storage_bucket_object" "session_file_object" { - count = (var.dataflow_params.template_params.local_session_file_path != null && var.dataflow_params.template_params.local_session_file_path != "") ? 1 : 0 - depends_on = [google_project_service.enabled_apis] - - lifecycle { - precondition { - condition = var.dataflow_params.template_params.working_directory_bucket != null && var.dataflow_params.template_params.working_directory_bucket != "" - error_message = "You must provide a working_directory_bucket in template_params when uploading a local_session_file_path." - } - } - - name = (var.dataflow_params.template_params.working_directory_prefix != null && var.dataflow_params.template_params.working_directory_prefix != "") ? "${var.dataflow_params.template_params.working_directory_prefix}/session.json" : "session.json" - source = var.dataflow_params.template_params.local_session_file_path - content_type = "application/json" - bucket = var.dataflow_params.template_params.working_directory_bucket -} # Add roles to the service account that will run Dataflow for data validation resource "google_project_iam_member" "dataflow_roles" { @@ -68,7 +51,7 @@ resource "google_dataflow_flex_template_job" "gcs_spanner_dv_job" { bigQueryDataset = var.dataflow_params.template_params.bigquery_dataset spannerHost = var.dataflow_params.template_params.spanner_host spannerPriority = var.dataflow_params.template_params.spanner_priority - sessionFilePath = (var.dataflow_params.template_params.local_session_file_path != null && var.dataflow_params.template_params.local_session_file_path != "") ? ((var.dataflow_params.template_params.working_directory_prefix != null && var.dataflow_params.template_params.working_directory_prefix != "") ? "gs://${var.dataflow_params.template_params.working_directory_bucket}/${var.dataflow_params.template_params.working_directory_prefix}/session.json" : "gs://${var.dataflow_params.template_params.working_directory_bucket}/session.json") : var.dataflow_params.template_params.session_file_path + sessionFilePath = var.dataflow_params.template_params.session_file_path schemaOverridesFilePath = var.dataflow_params.template_params.schema_overrides_file_path tableOverrides = var.dataflow_params.template_params.table_overrides columnOverrides = var.dataflow_params.template_params.column_overrides @@ -82,7 +65,7 @@ resource "google_dataflow_flex_template_job" "gcs_spanner_dv_job" { service_account_email = local.effective_sa_email network = local.network_uri subnetwork = local.subnetwork_uri - kms_key_name = var.dataflow_params.runner_params.kms_key_name != "" ? var.dataflow_params.runner_params.kms_key_name : null + kms_key_name = (var.dataflow_params.runner_params.kms_key_name != null && var.dataflow_params.runner_params.kms_key_name != "") ? var.dataflow_params.runner_params.kms_key_name : null machine_type = var.dataflow_params.runner_params.machine_type max_workers = var.dataflow_params.runner_params.max_workers @@ -96,7 +79,6 @@ resource "google_dataflow_flex_template_job" "gcs_spanner_dv_job" { depends_on = [ google_project_iam_member.dataflow_roles, - google_project_iam_member.spanner_reader_role, - google_storage_bucket_object.session_file_object + google_project_iam_member.spanner_reader_role ] } diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tf b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tf index d94bfd4f92..dfc622a7d1 100644 --- a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tf +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tf @@ -9,7 +9,7 @@ terraform { version = "~> 4.0" } } - required_version = "~> 1.2" + required_version = "~> 1.3" } provider "google" { diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfvars b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfvars index 2ddecac9d4..98db61b706 100644 --- a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfvars +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfvars @@ -14,7 +14,6 @@ dataflow_params = { bigquery_dataset = "" # The BigQuery dataset to store validation reports (e.g., validation_report_dataset) spanner_host = "" # Optional: Custom Spanner host endpoint spanner_priority = "HIGH" # Optional: Priority for Spanner RPC invocations (HIGH, MEDIUM, LOW) - local_session_file_path = "" # Optional: Local path to the session file (will be uploaded to working directory) session_file_path = "" # Optional: Existing GCS path to the session file schema_overrides_file_path = "" # Optional: GCS path to your overrides file table_overrides = "" # Optional: Table name overrides (e.g., "[{OldTableName,NewTableName}]") @@ -22,8 +21,6 @@ dataflow_params = { transformation_jar_path = "" # Optional: GCS path to the transformation JAR file transformation_class_name = "" # Optional: Fully qualified transformation class name transformation_custom_parameters = "" # Optional: Custom parameters for the transformation - working_directory_bucket = "" # Optional: Bucket for uploading session file and creating output directory - working_directory_prefix = "" # Optional: Prefix within the GCS bucket for working directory run_id = "" # Optional: Custom run identifier } diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/variables.tf b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/variables.tf index c381196211..ebab17b030 100644 --- a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/variables.tf +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/variables.tf @@ -2,7 +2,7 @@ variable "common_params" { description = "Parameters that are common to multiple resources" type = object({ project = string - host_project = optional(string, "") + host_project = optional(string, null) region = string add_policies_to_service_account = optional(bool, true) }) @@ -15,27 +15,24 @@ variable "dataflow_params" { gcs_input_directory = string instance_id = string database_id = string - spanner_project_id = optional(string, "") + spanner_project_id = optional(string, null) bigquery_dataset = string - spanner_host = optional(string, "") + spanner_host = optional(string, null) spanner_priority = optional(string, null) - session_file_path = optional(string, "") - local_session_file_path = optional(string, null) - schema_overrides_file_path = optional(string, "") - table_overrides = optional(string, "") - column_overrides = optional(string, "") - run_id = optional(string, "") - transformation_jar_path = optional(string, "") - transformation_class_name = optional(string, "") - transformation_custom_parameters = optional(string, "") - working_directory_bucket = optional(string, null) - working_directory_prefix = optional(string, null) + session_file_path = optional(string, null) + schema_overrides_file_path = optional(string, null) + table_overrides = optional(string, null) + column_overrides = optional(string, null) + run_id = optional(string, null) + transformation_jar_path = optional(string, null) + transformation_class_name = optional(string, null) + transformation_custom_parameters = optional(string, null) }) runner_params = object({ job_name = string - service_account_email = optional(string, "") - network = optional(string, "") - subnetwork = optional(string, "") + service_account_email = optional(string, null) + network = optional(string, null) + subnetwork = optional(string, null) machine_type = optional(string, "n1-standard-4") max_workers = optional(number, null) num_workers = optional(number, null) @@ -43,7 +40,7 @@ variable "dataflow_params" { ip_configuration = optional(string, null) launcher_machine_type = optional(string, null) labels = optional(map(string), {}) - kms_key_name = optional(string, "") + kms_key_name = optional(string, null) }) }) } From b60677728b64fff4f1e8bbc109530ffa771f7098 Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Fri, 28 Aug 2026 06:05:05 +0000 Subject: [PATCH 13/14] bulk readme changes --- .../samples/sharded-bulk-migration/README.md | 24 ++++++++++++++----- .../README.md | 2 +- .../single-job-bulk-migration/README.md | 23 ++++++++++++++---- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/README.md b/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/README.md index 72104a5412..8ca3134406 100644 --- a/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/README.md +++ b/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/README.md @@ -299,14 +299,26 @@ gcloud dataflow jobs update-options \ 2024-06-17_01_21_44-12198433486526363702 ``` -### Specifying schema overrides +### Specifying schema changes -By default, the bulk job performs a like-like mapping between -source and Spanner. However, for a sharded migration, a session file is mandatory for migration. -Any schema changes between source and Spanner can be -specified using the `session file`. +By default, the validation job performs a like-like schema mapping between the source AVRO records and Spanner. Any schema changes between the source and Spanner can be specified using a `session file` or `overrides` parameters. -To generate a session file: +**We highly recommend using the schema overrides parameters (`table_overrides` and `column_overrides`) instead of a session file** when dealing with schema differences. + +#### Using Schema Overrides (Recommended) + +When passing schema overrides to the job, you must strictly follow the required `[{}]` bracket-brace format. If the format is not matched exactly, Dataflow will reject the configuration with a regex error. + +* **For `table_overrides`**: Use the format `[{OldTableName,NewTableName}]`. + * *Example:* `[{Singers, Vocalists}]` +* **For `column_overrides`**: You **MUST** include the table name alongside the column names. Use the format `[{TableName.OldColumnName,TableName.NewColumnName}]`. Missing the table name will cause the pipeline to crash. + * *Example:* `[{Singers.SingerId, Singers.VocalistId}]` + +You can pass these overrides directly to your Terraform configuration using `var.table_overrides` and `var.column_overrides`. + +#### Using a Session File + +If you prefer or need to use a session file, you can generate one using the Spanner Migration Tool (SMT): 1. Setup SMT and [launch the UI](https://googlecloudplatform.github.io/spanner-migration-tool/ui#launching-the-web-ui-for-spanner-migration-tool). diff --git a/v2/sourcedb-to-spanner/terraform/samples/single-job-bulk-migration-postgres/README.md b/v2/sourcedb-to-spanner/terraform/samples/single-job-bulk-migration-postgres/README.md index 1c9b110635..1e93efc526 100644 --- a/v2/sourcedb-to-spanner/terraform/samples/single-job-bulk-migration-postgres/README.md +++ b/v2/sourcedb-to-spanner/terraform/samples/single-job-bulk-migration-postgres/README.md @@ -409,7 +409,7 @@ Once the roles are added, run the following command to verify them - gcloud projects get-iam-policy \ --flatten="bindings[].members" \ --format='table(bindings.role)' \ ---filter="bindings.members:@.iam.gserviceaccount.com" +--filter="bindings.members:serviceAccount:@.iam.gserviceaccount.com" ``` Sample output - diff --git a/v2/sourcedb-to-spanner/terraform/samples/single-job-bulk-migration/README.md b/v2/sourcedb-to-spanner/terraform/samples/single-job-bulk-migration/README.md index 0eff2392d1..d784639ae3 100644 --- a/v2/sourcedb-to-spanner/terraform/samples/single-job-bulk-migration/README.md +++ b/v2/sourcedb-to-spanner/terraform/samples/single-job-bulk-migration/README.md @@ -285,13 +285,26 @@ gcloud dataflow jobs update-options \ 2024-06-17_01_21_44-12198433486526363702 ``` -### Specifying schema overrides +### Specifying schema changes -By default, the bulk job performs a like-like mapping between -source and Spanner. Any schema changes between source and Spanner can be -specified using the `session file`. +By default, the validation job performs a like-like schema mapping between the source AVRO records and Spanner. Any schema changes between the source and Spanner can be specified using a `session file` or `overrides` parameters. -To generate a session file: +**We highly recommend using the schema overrides parameters (`table_overrides` and `column_overrides`) instead of a session file** when dealing with schema differences. + +#### Using Schema Overrides (Recommended) + +When passing schema overrides to the job, you must strictly follow the required `[{}]` bracket-brace format. If the format is not matched exactly, Dataflow will reject the configuration with a regex error. + +* **For `table_overrides`**: Use the format `[{OldTableName,NewTableName}]`. + * *Example:* `[{Singers, Vocalists}]` +* **For `column_overrides`**: You **MUST** include the table name alongside the column names. Use the format `[{TableName.OldColumnName,TableName.NewColumnName}]`. Missing the table name will cause the pipeline to crash. + * *Example:* `[{Singers.SingerId, Singers.VocalistId}]` + +You can pass these overrides directly to your Terraform configuration using `var.table_overrides` and `var.column_overrides`. + +#### Using a Session File + +If you prefer or need to use a session file, you can generate one using the Spanner Migration Tool (SMT): 1. Setup SMT and [launch the UI](https://googlecloudplatform.github.io/spanner-migration-tool/ui#launching-the-web-ui-for-spanner-migration-tool). From 0af93bdae9b66650604fe8ba7d5c711e9e21d295 Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Fri, 28 Aug 2026 06:19:27 +0000 Subject: [PATCH 14/14] gemini-review --- .../terraform/samples/simple-validation-job/main.tf | 2 +- .../terraform/samples/simple-validation-job/terraform.tf | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf index 63ad44c8a7..b25fdabe8f 100644 --- a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf @@ -1,5 +1,5 @@ locals { - effective_sa_email = (var.dataflow_params.runner_params.service_account_email != null && var.dataflow_params.runner_params.service_account_email != "") ? var.dataflow_params.runner_params.service_account_email : data.google_compute_default_service_account.gce_account.email + effective_sa_email = (var.dataflow_params.runner_params.service_account_email != null && var.dataflow_params.runner_params.service_account_email != "") ? var.dataflow_params.runner_params.service_account_email : "${data.google_project.project.number}-compute@developer.gserviceaccount.com" # Network resolution (handling Shared VPC structures) network_project = (var.common_params.host_project != null && var.common_params.host_project != "") ? var.common_params.host_project : var.common_params.project diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tf b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tf index dfc622a7d1..976b0c59fc 100644 --- a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tf +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tf @@ -42,8 +42,3 @@ data "google_project" "project" { project_id = var.common_params.project } -# Fetch the default service account for Compute Engine (used by Dataflow) -data "google_compute_default_service_account" "gce_account" { - project = var.common_params.project - depends_on = [google_project_service.enabled_apis] -}