diff --git a/v2/gcs-spanner-dv/README_GCS_Spanner_Data_Validator.md b/v2/gcs-spanner-dv/README_GCS_Spanner_Data_Validator.md index 8c34a9a472..86419932dd 100644 --- a/v2/gcs-spanner-dv/README_GCS_Spanner_Data_Validator.md +++ b/v2/gcs-spanner-dv/README_GCS_Spanner_Data_Validator.md @@ -270,3 +270,5 @@ resource "google_dataflow_flex_template_job" "gcs_spanner_data_validator" { } } ``` + +For more advanced end-to-end deployment examples using Terraform, please refer to the [Terraform samples directory](terraform/samples/). diff --git a/v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf b/v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf new file mode 100644 index 0000000000..4a94cc19cd --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf @@ -0,0 +1,287 @@ + + +# Autogenerated file. DO NOT EDIT. +# +# Copyright (C) 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + + +variable "on_delete" { + type = string + description = "One of \"drain\" or \"cancel\". Specifies behavior of deletion during terraform destroy." +} + +variable "project" { + type = string + description = "The Google Cloud Project ID within which this module provisions resources." +} + +variable "region" { + type = string + description = "The region in which the created job should run." +} + +variable "gcsInputDirectory" { + type = string + description = "This directory is used to read the AVRO files of the records read from source. For example, `gs://your-bucket/your-path`" + default = null +} + +variable "projectId" { + type = string + description = "This is the name of the Cloud Spanner project." + default = null +} + +variable "spannerHost" { + type = string + description = "The Cloud Spanner endpoint to call in the template. For example, `https://batch-spanner.googleapis.com`. Defaults to: https://batch-spanner.googleapis.com." + default = null +} + +variable "instanceId" { + type = string + description = "The destination Cloud Spanner instance." + +} + +variable "databaseId" { + type = string + description = "The destination Cloud Spanner database." + +} + +variable "spannerPriority" { + type = string + description = "The request priority for Cloud Spanner calls. The value must be one of: [`HIGH`,`MEDIUM`,`LOW`]. Defaults to `HIGH`." + default = null +} + +variable "sessionFilePath" { + type = string + description = "Session file path in Cloud Storage that contains mapping information from Spanner Migration Tool. Defaults to empty." + default = null +} + +variable "schemaOverridesFilePath" { + type = string + description = "A file which specifies the table and the column name overrides from source to spanner. Defaults to empty." + default = null +} + +variable "tableOverrides" { + type = string + description = "These are the table name overrides from source to spanner. They are written in the following format: [{SourceTableName1, SpannerTableName1}, {SourceTableName2, SpannerTableName2}] This example shows mapping Singers table to Vocalists and Albums table to Records. For example, `[{Singers, Vocalists}, {Albums, Records}]`. Defaults to empty." + default = null +} + +variable "columnOverrides" { + type = string + description = "These are the column name overrides from source to spanner. They are written in the following format: [{SourceTableName1.SourceColumnName1, SourceTableName1.SpannerColumnName1}, {SourceTableName2.SourceColumnName1, SourceTableName2.SpannerColumnName1}]Note that the SourceTableName should remain the same in both the source and spanner pair. To override table names, use tableOverrides.The example shows mapping SingerName to TalentName and AlbumName to RecordName in Singers and Albums table respectively. For example, `[{Singers.SingerName, Singers.TalentName}, {Albums.AlbumName, Albums.RecordName}]`. Defaults to empty." + default = null +} + +variable "bigQueryDataset" { + type = string + description = "The BigQuery dataset ID where the validation results will be stored. For example, `validation_report_dataset`" + +} + +variable "runId" { + type = string + description = "A unique identifier for the validation run. If not provided, the Dataflow Job Name will be used. For example, `run_20230101_120000`" + default = null +} + +variable "transformationJarPath" { + type = string + description = "Custom jar location in Cloud Storage that contains the custom transformation logic for processing records. Defaults to empty." + default = null +} + +variable "transformationClassName" { + type = string + description = "Fully qualified class name having the custom transformation logic. It is a mandatory field in case transformationJarPath is specified. Defaults to empty." + default = null +} + +variable "transformationCustomParameters" { + type = string + description = "String containing any custom parameters to be passed to the custom transformation class. Defaults to empty." + default = null +} + + + +variable "additional_experiments" { + type = set(string) + description = "List of experiments that should be used by the job. An example value is 'enable_stackdriver_agent_metrics'." + default = null +} + +variable "autoscaling_algorithm" { + type = string + description = "The algorithm to use for autoscaling" + default = null +} + +variable "enable_streaming_engine" { + type = bool + description = "Indicates if the job should use the streaming engine feature." + default = null +} + +variable "ip_configuration" { + type = string + description = "The configuration for VM IPs. Options are 'WORKER_IP_PUBLIC' or 'WORKER_IP_PRIVATE'." + default = null +} + +variable "kms_key_name" { + type = string + description = "The name for the Cloud KMS key for the job. Key format is: projects/PROJECT_ID/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY" + default = null +} + +variable "labels" { + type = map(string) + description = "User labels to be specified for the job. Keys and values should follow the restrictions specified in the labeling restrictions page. NOTE: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field 'effective_labels' for all of the labels present on the resource." + default = null +} + +variable "launcher_machine_type" { + type = string + description = "The machine type to use for launching the job. The default is n1-standard-1." + default = null +} + +variable "machine_type" { + type = string + description = "The machine type to use for the job." + default = null +} + +variable "max_workers" { + type = number + description = "The maximum number of Google Compute Engine instances to be made available to your pipeline during execution, from 1 to 1000." + default = null +} + +variable "name" { + description = "The unique name of the Dataflow job." + type = string +} + +variable "network" { + type = string + description = "The network to which VMs will be assigned. If it is not provided, 'default' will be used." + default = null +} + +variable "num_workers" { + type = number + description = "The initial number of Google Compute Engine instances for the job." + default = null +} + +variable "sdk_container_image" { + type = string + description = "Docker registry location of container image to use for the 'worker harness. Default is the container for the version of the SDK. Note this field is only valid for portable pipelines." + default = null +} + +variable "service_account_email" { + type = string + description = "The Service Account email used to create the job." + default = null +} + +variable "skip_wait_on_job_termination" { + type = bool + description = "If true, treat DRAINING and CANCELLING as terminal job states and do not wait for further changes before removing from terraform state and moving on. WARNING: this will lead to job name conflicts if you do not ensure that the job names are different, e.g. by embedding a release ID or by using a random_id." + default = null +} + +variable "staging_location" { + type = string + description = "The Cloud Storage path to use for staging files. Must be a valid Cloud Storage URL, beginning with gs://." + default = null +} + +variable "subnetwork" { + type = string + description = "The subnetwork to which VMs will be assigned. Should be of the form 'regions/REGION/subnetworks/SUBNETWORK'." + default = null +} + +variable "temp_location" { + type = string + description = "The Cloud Storage path to use for temporary files. Must be a valid Cloud Storage URL, beginning with gs://." + default = null +} + +resource "google_project_service" "required" { + service = "dataflow.googleapis.com" + disable_on_destroy = false +} + +resource "google_dataflow_flex_template_job" "generated" { + depends_on = [google_project_service.required] + provider = google-beta + container_spec_gcs_path = "gs://dataflow-templates-${var.region}/latest/flex/GCS_Spanner_Data_Validator" + parameters = { + gcsInputDirectory = var.gcsInputDirectory + projectId = var.projectId + spannerHost = var.spannerHost + instanceId = var.instanceId + databaseId = var.databaseId + spannerPriority = var.spannerPriority + sessionFilePath = var.sessionFilePath + schemaOverridesFilePath = var.schemaOverridesFilePath + tableOverrides = var.tableOverrides + columnOverrides = var.columnOverrides + bigQueryDataset = var.bigQueryDataset + runId = var.runId + transformationJarPath = var.transformationJarPath + transformationClassName = var.transformationClassName + transformationCustomParameters = var.transformationCustomParameters + } + + additional_experiments = var.additional_experiments + autoscaling_algorithm = var.autoscaling_algorithm + enable_streaming_engine = var.enable_streaming_engine + ip_configuration = var.ip_configuration + kms_key_name = var.kms_key_name + labels = var.labels + launcher_machine_type = var.launcher_machine_type + machine_type = var.machine_type + max_workers = var.max_workers + name = var.name + network = var.network + num_workers = var.num_workers + sdk_container_image = var.sdk_container_image + service_account_email = var.service_account_email + skip_wait_on_job_termination = var.skip_wait_on_job_termination + staging_location = var.staging_location + subnetwork = var.subnetwork + temp_location = var.temp_location + region = var.region + on_delete = var.on_delete +} + +output "dataflow_job_url" { + value = "https://console.cloud.google.com/dataflow/jobs/${var.region}/${google_dataflow_flex_template_job.generated.job_id}" +} + diff --git a/v2/gcs-spanner-dv/terraform/samples/README.md b/v2/gcs-spanner-dv/terraform/samples/README.md new file mode 100644 index 0000000000..99c511903b --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/README.md @@ -0,0 +1,26 @@ +# Terraform Samples for GCS to Spanner Data Validation + +This directory provides samples for common scenarios users might have while trying to run a data validation job. + +Pick a sample that is closest to your use-case, and use it as a starting point, tailoring it to your own specific needs. + +## Prerequisites + +Before using these samples, ensure you have: +- [Terraform](https://developer.hashicorp.com/terraform/downloads) installed. +- Authenticated with Google Cloud (e.g., using `gcloud auth application-default login`). +- A Google Cloud project with the necessary permissions. + +## List of examples + +1. [Launching a single Data Validation Job](simple-validation-job/README.md) - A basic example that configures the necessary variables and provisions the Dataflow pipeline for GCS to Spanner Data Validation. + +## How to add a new sample + +We strongly recommend copying an existing sample and modifying it for your scenario. This ensures a consistent style across all Terraform samples. + +```shell +mkdir my-new-sample +cp -r simple-validation-job/ my-new-sample/ +cd my-new-sample/ +``` diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/README.md b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/README.md new file mode 100644 index 0000000000..331364e043 --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/README.md @@ -0,0 +1,329 @@ +# GCS to Spanner Data Validation + +This sample demonstrates how to easily launch the Dataflow job while automatically attaching all necessary IAM roles. + +## What this sample does + +The Terraform module will create the following Google Cloud resources: +1. **Dataflow Flex Template Job:** Uses `google_dataflow_flex_template_job` to launch the data validation pipeline. +2. **IAM Role Bindings:** Grants the Dataflow worker service account the required roles to run the validation: + * `roles/dataflow.worker` (required to execute Dataflow jobs) + * `roles/spanner.databaseReader` (required to read records from Spanner) + * `roles/storage.objectAdmin` (required to read/write objects in Cloud Storage) + * `roles/bigquery.dataEditor` (required to write validation reports to BigQuery) + * `roles/bigquery.jobUser` (required to execute BigQuery load jobs) + * `roles/monitoring.metricWriter` (required to write Dataflow metrics) + * `roles/cloudprofiler.agent` (required for Cloud Profiler) + +## Terraform permissions + +In order to create the resources in this sample, the `Service account`/`User account` being used to run Terraform should have the required permissions. +There are two ways to add permissions - + +1. Adding pre-defined roles to the service account running Terraform. +2. Creating a custom role with the granular permissions and attaching it to the service account running Terraform. + +### Using custom role and granular permissions (recommended) + +Following permissions are required - + +```shell +- dataflow.jobs.cancel +- dataflow.jobs.create +- dataflow.jobs.updateContents +- iam.roles.get +- iam.serviceAccounts.actAs +- resourcemanager.projects.setIamPolicy +- storage.objects.create +- storage.objects.delete +- serviceusage.services.use +- serviceusage.services.enable +``` + +**Note**: Add the `roles/viewer` role as well to the service account. + +### Using pre-defined roles + +Following roles are required - + +```shell +roles/dataflow.admin +roles/iam.securityAdmin +roles/iam.serviceAccountUser +roles/storage.admin +roles/viewer +``` + +## Dataflow permissions + +The Dataflow service account needs to be provided with the required roles. This sample will attempt to automatically bind the following roles to the specified service account: `roles/dataflow.worker`, `roles/spanner.databaseReader`, `roles/storage.objectAdmin`, `roles/bigquery.dataEditor`, `roles/bigquery.jobUser`, `roles/monitoring.metricWriter`, and `roles/cloudprofiler.agent`. + +## Assumptions + +It makes the following assumptions - + +1. Appropriate permissions are added to the service account running Terraform to allow resource creation. +2. The BigQuery dataset to store validation reports is already created and correctly named in `var.bigquery_dataset`. +3. The Source AVRO records in GCS exist in `var.gcs_input_directory`. +4. A Spanner instance with database containing the destination records is created and accessible. +5. If the source and Spanner schema is not like-to-like (e.g., column/table renames), an SMT generated session file, or an overrides file needs to be provided containing the schema mapping information. + +Given these assumptions, the job compares the source AVRO records against the Spanner database and writes mismatch reports and validation statistics to the specified BigQuery dataset. + +## Description + +This sample contains the following files - + +1. `main.tf` - This contains the Terraform resources which will be created. +2. `outputs.tf` - This declares the outputs that will be output as part of running this terraform example. +3. `variables.tf` - This declares the input variables that are required to configure the resources. +4. `terraform.tf` - This contains the required providers and APIs/project configurations for this sample. +5. `terraform.tfvars` - This contains the minimal list of dummy inputs that need to be populated to run this example. + +## Prerequisites + +Before executing the sample, ensure you meet the following requirements: + +1. **APIs Enabled**: The following APIs must be enabled in your Google Cloud Project: + * Dataflow API (`dataflow.googleapis.com`) + * Cloud Spanner API (`spanner.googleapis.com`) + * Cloud Storage API (`storage.googleapis.com`) + * BigQuery API (`bigquery.googleapis.com`) + +2. **Terraform**: Make sure Terraform is installed locally and you are authenticated using `gcloud auth application-default login`. + +## Usage + +1. **Clone the repository** + ```shell + git clone https://github.com/GoogleCloudPlatform/DataflowTemplates.git + cd DataflowTemplates/v2/gcs-spanner-dv/terraform/samples/simple-validation-job + ``` + +2. **Initialize Terraform** + ```shell + terraform init + ``` + +3. **Configure the variables** + Open the `terraform.tfvars` file and modify the placeholder values (such as `project`, `instance_id`, `database_id`, `gcs_input_directory`, and `bigquery_dataset`) to match your environment. + +4. **Review the execution plan** + ```shell + terraform plan -var-file=terraform.tfvars + ``` + +5. **Apply the configuration** + ```shell + terraform apply -var-file=terraform.tfvars + ``` + +This will launch the configured jobs and produce an output like below - + +```shell +Apply complete! Resources: 1 added, 0 changed, 0 destroyed. + +Outputs: + +dataflow_job_id = [ + "2024-06-05_00_41_11-4759981257849547781", +] +dataflow_job_url = [ + "https://console.cloud.google.com/dataflow/jobs/us-central1/2024-06-05_00_41_11-4759981257849547781", +] +``` + +### Cleanup + +Once the jobs have finished running, you can cleanup by running - + +```shell +terraform destroy +``` + +## Observability + +To monitor the data validation job, you can view the Dataflow job in the Google Cloud Console. The resulting metrics from the validation job such as matched records, mismatched records, and missing records are available in the BigQuery dataset specified in `var.bigquery_dataset`. + +## FAQ + +### Dataflow job is failing with "Timeout in polling result file" + +Dataflow has a 10-minute timeout within which the launcher VM logic should complete. There could be multiple reasons for it to take over 10 mins: + +- **Job logs not present after the log "launcher VM started":** A sign would be there are only 3-4 log statements in the job logs. This is likely due to private Google access not being enabled for the subnetwork. Please enable private Google access in your network. + +### Job graph is not loading/Custom counters not visible on Dataflow panel + +For very large graphs, this can happen. The graph section would be empty and the counters won't load. But worry not, the validation should progress nonetheless. In such cases, the Dataflow custom metrics can be directly viewed on [Cloud monitoring](https://cloud.google.com/dataflow/docs/guides/using-monitoring-intf). + +### Data Validation is taking too long + +There can be multiple reasons for this: + +- **Check Spanner metrics:** Are memory/CPU limits being hit during reads? Consider increasing the number of nodes if Spanner is struggling to serve the reads. +- **Check Dataflow metrics:** Are memory/CPU limits being hit? Dataflow should autoscale to the required number of nodes. + - CPU/memory limits being hit means the `max_workers` parameter might be too low. It is recommended to use smaller machines (`Ex: n1-standard-4`) for most workloads. However, if you have very large datasets, we recommend scaling up the number of workers. + +### Configuring to run using a VPC + +#### Dataflow + +1. Set the `network` and the `subnetwork` parameters to run the Dataflow job inside a VPC. Specify [network](https://cloud.google.com/dataflow/docs/guides/specifying-networks#network_parameter) and [subnetwork](https://cloud.google.com/dataflow/docs/guides/specifying-networks#subnetwork_parameter) according to the linked guidelines. +2. Set the `ip_configuration` to `WORKER_IP_PRIVATE` to disable public IP addresses for the worker VMs. +3. If only certain network tags are allowlisted via a firewall, specify the network tags via the [additional-experiments flag](https://cloud.google.com/dataflow/docs/guides/routes-firewall#network-tags-flex) (e.g. `use_network_tags=allow-dataflow`). Dataflow automatically assigns + the `dataflow` network tag if any network tag is additionally specified. You need + specify the tag for both worker VMs and launcher VMs. + + +> **_NOTE:_** You can use a shared VPC by specifying the `host_project` in the subnet path. +> This will result in the Dataflow jobs being launched inside the shared VPC. +> Usage of shared VPC requires cross-project permissions. They +> are available as a Terraform +> template [here](../../../../spanner-common/terraform/samples/configure-shared-vpc/README.md). +> Dataflow service account permissions are +> documented [here](https://cloud.google.com/dataflow/docs/guides/specifying-networks#shared). + + +If you are facing issue with VPC connectivity, check the following Dataflow +[guide](https://cloud.google.com/dataflow/docs/guides/troubleshoot-networking) +to debug common networking issues. + +### Updating workers of a Dataflow job + +Currently, the Terraform `google_dataflow_flex_template_job` resource does not support updating the workers of a Dataflow job. +If the worker counts are changed in `tfvars` and a Terraform apply is run, Terraform will attempt to cancel the existing Dataflow job and replace it with a new one. +**This is not recommended**. Instead, use the `gcloud` CLI to update the worker counts of a launched Dataflow job: + +```shell +gcloud dataflow jobs update-options \ + --region=us-central1 \ + --min-num-workers=5 \ + --max-num-workers=20 \ + +``` + +### Specifying schema changes + +By default, the validation job performs a like-like schema mapping between the source AVRO records and Spanner. Any schema changes between the source and Spanner can be specified using a `session file` or `overrides` parameters. + +**We highly recommend using the schema overrides parameters (`table_overrides` and `column_overrides`) instead of a session file** when dealing with schema differences. + +#### Using Schema Overrides (Recommended) + +When passing schema overrides to the job, you must strictly follow the required `[{}]` bracket-brace format. If the format is not matched exactly, Dataflow will reject the configuration with a regex error. + +* **For `table_overrides`**: Use the format `[{OldTableName,NewTableName}]`. + * *Example:* `[{Singers, Vocalists}]` +* **For `column_overrides`**: You **MUST** include the table name alongside the column names. Use the format `[{TableName.OldColumnName,TableName.NewColumnName}]`. Missing the table name will cause the pipeline to crash. + * *Example:* `[{Singers.SingerId, Singers.VocalistId}]` + +You can pass these overrides directly to your Terraform configuration using `var.table_overrides` and `var.column_overrides`. + +#### Using a Session File + +If you prefer or need to use a session file, you can generate one using the Spanner Migration Tool (SMT): + +1. Setup SMT and [launch the UI](https://googlecloudplatform.github.io/spanner-migration-tool/ui#launching-the-web-ui-for-spanner-migration-tool). +2. Perform a [schema conversion](https://googlecloudplatform.github.io/spanner-migration-tool/ui/schema-conv) and download the session file locally. + +To provide this session file to Terraform: + +1. Upload the SMT generated `session file` to a Cloud Storage bucket. +2. Set the `var.session_file_path` variable to the GCS path of your uploaded file (e.g. `gs://my-bucket/path/to/session.json`). + +### Adding access to Terraform service account + +#### Using custom role and granular permissions (recommended) + +You can run the following gcloud command to create a custom role in your GCP project. + +```shell +gcloud iam roles create dv_terraform_role --project= --file=perms.yaml --quiet +``` + +The `YAML` file required for the above will be like so - + +```shell +title: "Data Validation Terraform Role" +description: "Custom role for running Spanner Data Validation via Terraform." +stage: "GA" +includedPermissions: +- iam.roles.get +- iam.serviceAccounts.actAs +# ....add all permissions from the list defined in the 'Terraform permissions' section above. +``` + +Then attach the role to the service account - + +```shell +gcloud projects add-iam-policy-binding \ + --member="serviceAccount:@.iam.gserviceaccount.com" \ + --role="projects//roles/dv_terraform_role" +``` + +#### Using pre-defined roles + +You can run the following shell script to add roles to the service account being used to run Terraform. This will have to done by a user which has the authority to grant the specified roles to a service account - + +```shell +#!/bin/bash + +# Service account to be granted roles +SERVICE_ACCOUNT="@.iam.gserviceaccount.com" + +# Project ID where roles will be granted +PROJECT_ID="" + +# Array of roles to grant +ROLES=( + "roles/dataflow.admin" # Required to create and manage Dataflow jobs + "roles/iam.securityAdmin" # Required to bind roles to the Dataflow worker SA + "roles/iam.serviceAccountUser" # Required to impersonate the Dataflow worker SA + "roles/storage.admin" # Required to manage GCS staging objects + "roles/viewer" # Required to fetch current project states + "roles/serviceusage.serviceUsageAdmin" # Required to enable required APIs +) + +# Loop through each role and grant it to the service account +for ROLE in "${ROLES[@]}" +do + gcloud projects add-iam-policy-binding "$PROJECT_ID" \ + --member="serviceAccount:${SERVICE_ACCOUNT}" \ + --role="$ROLE" +done +``` + +### Verifying access in the Terraform service account + +Verify that the custom role is attached to the service account - + +```shell +gcloud projects get-iam-policy \ +--flatten="bindings[].members" \ +--format='table(bindings.role)' \ +--filter="bindings.members:serviceAccount:@.iam.gserviceaccount.com" +``` + +Verify that the role has the correct set of permissions + +```shell +gcloud iam roles describe dv_terraform_role --project= +``` + +### Impersonating the Terraform service account + +#### Using GCE VM instance (recommended) + +A GCE VM created using the service account setup above will automatically use the service account for all API requests triggered by Terraform. Running terraform from such a GCE VM does not require downloading service keys and is the recommended approach. + +#### Using key file + +1. Activate the service account - + ```shell + gcloud auth activate-service-account @.iam.gserviceaccount.com --key-file=path/to/key_file --project=project_id + ``` +2. Impersonate service account while fetching the ADC credentials - + ```shell + gcloud auth application-default login --impersonate-service-account @.iam.gserviceaccount.com + ``` diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf new file mode 100644 index 0000000000..b25fdabe8f --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/main.tf @@ -0,0 +1,84 @@ +locals { + effective_sa_email = (var.dataflow_params.runner_params.service_account_email != null && var.dataflow_params.runner_params.service_account_email != "") ? var.dataflow_params.runner_params.service_account_email : "${data.google_project.project.number}-compute@developer.gserviceaccount.com" + + # Network resolution (handling Shared VPC structures) + network_project = (var.common_params.host_project != null && var.common_params.host_project != "") ? var.common_params.host_project : var.common_params.project + network_uri = (var.dataflow_params.runner_params.network != null && var.dataflow_params.runner_params.network != "") ? (can(regex("/", var.dataflow_params.runner_params.network)) ? var.dataflow_params.runner_params.network : "projects/${local.network_project}/global/networks/${var.dataflow_params.runner_params.network}") : null + subnetwork_uri = (var.dataflow_params.runner_params.subnetwork != null && var.dataflow_params.runner_params.subnetwork != "") ? (can(regex("/", var.dataflow_params.runner_params.subnetwork)) ? var.dataflow_params.runner_params.subnetwork : "https://www.googleapis.com/compute/v1/projects/${local.network_project}/regions/${var.common_params.region}/subnetworks/${var.dataflow_params.runner_params.subnetwork}") : null + + # Spanner project resolution + spanner_project_id = (var.dataflow_params.template_params.spanner_project_id != null && var.dataflow_params.template_params.spanner_project_id != "") ? var.dataflow_params.template_params.spanner_project_id : var.common_params.project +} + + +# Add roles to the service account that will run Dataflow for data validation +resource "google_project_iam_member" "dataflow_roles" { + for_each = var.common_params.add_policies_to_service_account ? toset([ + "roles/dataflow.worker", + "roles/storage.objectAdmin", + "roles/bigquery.dataEditor", + "roles/bigquery.jobUser", + "roles/monitoring.metricWriter", + "roles/cloudprofiler.agent" + ]) : toset([]) + + project = var.common_params.project + role = each.key + member = "serviceAccount:${local.effective_sa_email}" +} + +resource "google_project_iam_member" "spanner_reader_role" { + count = var.common_params.add_policies_to_service_account ? 1 : 0 + project = local.spanner_project_id + role = "roles/spanner.databaseReader" + member = "serviceAccount:${local.effective_sa_email}" +} + +# Define the Dataflow Flex Template job for GCS to Spanner Data Validation +resource "google_dataflow_flex_template_job" "gcs_spanner_dv_job" { + provider = google-beta + name = var.dataflow_params.runner_params.job_name + project = var.common_params.project + region = var.common_params.region + container_spec_gcs_path = "gs://dataflow-templates-${var.common_params.region}/latest/flex/GCS_Spanner_Data_Validator" + + parameters = { + for k, v in { + gcsInputDirectory = var.dataflow_params.template_params.gcs_input_directory + projectId = local.spanner_project_id + instanceId = var.dataflow_params.template_params.instance_id + databaseId = var.dataflow_params.template_params.database_id + bigQueryDataset = var.dataflow_params.template_params.bigquery_dataset + spannerHost = var.dataflow_params.template_params.spanner_host + spannerPriority = var.dataflow_params.template_params.spanner_priority + sessionFilePath = var.dataflow_params.template_params.session_file_path + schemaOverridesFilePath = var.dataflow_params.template_params.schema_overrides_file_path + tableOverrides = var.dataflow_params.template_params.table_overrides + columnOverrides = var.dataflow_params.template_params.column_overrides + runId = var.dataflow_params.template_params.run_id + transformationJarPath = var.dataflow_params.template_params.transformation_jar_path + transformationClassName = var.dataflow_params.template_params.transformation_class_name + transformationCustomParameters = var.dataflow_params.template_params.transformation_custom_parameters + } : k => v if v != null && v != "" + } + + service_account_email = local.effective_sa_email + network = local.network_uri + subnetwork = local.subnetwork_uri + kms_key_name = (var.dataflow_params.runner_params.kms_key_name != null && var.dataflow_params.runner_params.kms_key_name != "") ? var.dataflow_params.runner_params.kms_key_name : null + + machine_type = var.dataflow_params.runner_params.machine_type + max_workers = var.dataflow_params.runner_params.max_workers + additional_experiments = var.dataflow_params.runner_params.additional_experiments + launcher_machine_type = var.dataflow_params.runner_params.launcher_machine_type + ip_configuration = var.dataflow_params.runner_params.ip_configuration + num_workers = var.dataflow_params.runner_params.num_workers + labels = merge(var.dataflow_params.runner_params.labels, { + "migration_id" = var.dataflow_params.runner_params.job_name + }) + + depends_on = [ + google_project_iam_member.dataflow_roles, + google_project_iam_member.spanner_reader_role + ] +} diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/outputs.tf b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/outputs.tf new file mode 100644 index 0000000000..d8cb916c14 --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/outputs.tf @@ -0,0 +1,9 @@ +output "dataflow_job_url" { + description = "URL for the created Dataflow Flex Template job." + value = "https://console.cloud.google.com/dataflow/jobs/${var.common_params.region}/${google_dataflow_flex_template_job.gcs_spanner_dv_job.job_id}?project=${google_dataflow_flex_template_job.gcs_spanner_dv_job.project}" +} + +output "dataflow_job_id" { + description = "The unique ID of the created Dataflow Flex Template job." + value = google_dataflow_flex_template_job.gcs_spanner_dv_job.job_id +} diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tf b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tf new file mode 100644 index 0000000000..976b0c59fc --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tf @@ -0,0 +1,44 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "~> 4.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = "~> 4.0" + } + } + required_version = "~> 1.3" +} + +provider "google" { + project = var.common_params.project + region = var.common_params.region +} + +provider "google-beta" { + project = var.common_params.project + region = var.common_params.region +} + +# Enable the APIs +resource "google_project_service" "enabled_apis" { + for_each = toset([ + "iam.googleapis.com", + "dataflow.googleapis.com", + "compute.googleapis.com", + "storage.googleapis.com", + "spanner.googleapis.com", + "bigquery.googleapis.com" + ]) + service = each.key + project = var.common_params.project + disable_on_destroy = false +} + +# To fetch project number +data "google_project" "project" { + project_id = var.common_params.project +} + diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfvars b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfvars new file mode 100644 index 0000000000..98db61b706 --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform.tfvars @@ -0,0 +1,41 @@ +common_params = { + project = "" # Replace with your GCP project ID + host_project = "" # Optional: Project ID hosting the network if using a shared VPC + region = "" # Replace with your desired GCP region + add_policies_to_service_account = true # Optional: Whether Terraform should add required permissions to the service account +} + +dataflow_params = { + template_params = { + gcs_input_directory = "gs:///source-avro" # The Cloud Storage directory containing validation data + instance_id = "" # The Spanner instance ID to validate against + database_id = "" # The Spanner database ID to validate against + spanner_project_id = "" # Optional: Project ID where the Spanner instance is located (defaults to Dataflow project) + bigquery_dataset = "" # The BigQuery dataset to store validation reports (e.g., validation_report_dataset) + spanner_host = "" # Optional: Custom Spanner host endpoint + spanner_priority = "HIGH" # Optional: Priority for Spanner RPC invocations (HIGH, MEDIUM, LOW) + session_file_path = "" # Optional: Existing GCS path to the session file + schema_overrides_file_path = "" # Optional: GCS path to your overrides file + table_overrides = "" # Optional: Table name overrides (e.g., "[{OldTableName,NewTableName}]") + column_overrides = "" # Optional: Column name overrides (e.g., "[{TableName.OldColumnName,TableName.NewColumnName}]") + transformation_jar_path = "" # Optional: GCS path to the transformation JAR file + transformation_class_name = "" # Optional: Fully qualified transformation class name + transformation_custom_parameters = "" # Optional: Custom parameters for the transformation + run_id = "" # Optional: Custom run identifier + } + + runner_params = { + job_name = "data-validation-job" # Or your custom job name + service_account_email = "" # Optional: Service account email for Dataflow workers + network = "" # Optional: Network for Dataflow workers + subnetwork = "" # Optional: Subnetwork for Dataflow workers + machine_type = "n2-standard-4" # Optional: Machine type for Dataflow worker VMs + max_workers = 10 # Optional: Maximum number of Dataflow worker VMs + num_workers = 4 # Optional: Initial number of Dataflow worker VMs + additional_experiments = [""] # Optional: Additional Dataflow experiments (list of strings) + ip_configuration = "WORKER_IP_PRIVATE" # Optional: IP configuration for Dataflow workers + launcher_machine_type = "n1-standard-1" # Optional: Machine type for the Dataflow launcher VM + labels = { "env" = "test" } # Optional: Labels to apply to the Dataflow job (map of strings) + kms_key_name = "" # Optional: Cloud KMS key name for data encryption + } +} diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform_simple.tfvars b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform_simple.tfvars new file mode 100644 index 0000000000..380fab4609 --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/terraform_simple.tfvars @@ -0,0 +1,20 @@ +common_params = { + project = "" # Replace with your GCP project ID + region = "" # Replace with your desired GCP region +} + +dataflow_params = { + template_params = { + instance_id = "" # The Spanner instance ID to validate against + database_id = "" # The Spanner database ID to validate against + spanner_project_id = "" # Optional: Project ID where the Spanner instance is located + gcs_input_directory = "gs:///source-avro" # The Cloud Storage directory containing validation data + bigquery_dataset = "" # The BigQuery dataset to store validation reports + schema_overrides_file_path = "" # Optional: GCS path to your overrides file + run_id = "" # Optional: Custom run identifier + } + + runner_params = { + job_name = "" # Custom job name for the Dataflow pipeline + } +} diff --git a/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/variables.tf b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/variables.tf new file mode 100644 index 0000000000..ebab17b030 --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/simple-validation-job/variables.tf @@ -0,0 +1,46 @@ +variable "common_params" { + description = "Parameters that are common to multiple resources" + type = object({ + project = string + host_project = optional(string, null) + region = string + add_policies_to_service_account = optional(bool, true) + }) +} + +variable "dataflow_params" { + description = "Parameters for the Dataflow job." + type = object({ + template_params = object({ + gcs_input_directory = string + instance_id = string + database_id = string + spanner_project_id = optional(string, null) + bigquery_dataset = string + spanner_host = optional(string, null) + spanner_priority = optional(string, null) + session_file_path = optional(string, null) + schema_overrides_file_path = optional(string, null) + table_overrides = optional(string, null) + column_overrides = optional(string, null) + run_id = optional(string, null) + transformation_jar_path = optional(string, null) + transformation_class_name = optional(string, null) + transformation_custom_parameters = optional(string, null) + }) + runner_params = object({ + job_name = string + service_account_email = optional(string, null) + network = optional(string, null) + subnetwork = optional(string, null) + machine_type = optional(string, "n1-standard-4") + max_workers = optional(number, null) + num_workers = optional(number, null) + additional_experiments = optional(list(string), []) + ip_configuration = optional(string, null) + launcher_machine_type = optional(string, null) + labels = optional(map(string), {}) + kms_key_name = optional(string, null) + }) + }) +} diff --git a/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/README.md b/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/README.md index 72104a5412..8ca3134406 100644 --- a/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/README.md +++ b/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/README.md @@ -299,14 +299,26 @@ gcloud dataflow jobs update-options \ 2024-06-17_01_21_44-12198433486526363702 ``` -### Specifying schema overrides +### Specifying schema changes -By default, the bulk job performs a like-like mapping between -source and Spanner. However, for a sharded migration, a session file is mandatory for migration. -Any schema changes between source and Spanner can be -specified using the `session file`. +By default, the validation job performs a like-like schema mapping between the source AVRO records and Spanner. Any schema changes between the source and Spanner can be specified using a `session file` or `overrides` parameters. -To generate a session file: +**We highly recommend using the schema overrides parameters (`table_overrides` and `column_overrides`) instead of a session file** when dealing with schema differences. + +#### Using Schema Overrides (Recommended) + +When passing schema overrides to the job, you must strictly follow the required `[{}]` bracket-brace format. If the format is not matched exactly, Dataflow will reject the configuration with a regex error. + +* **For `table_overrides`**: Use the format `[{OldTableName,NewTableName}]`. + * *Example:* `[{Singers, Vocalists}]` +* **For `column_overrides`**: You **MUST** include the table name alongside the column names. Use the format `[{TableName.OldColumnName,TableName.NewColumnName}]`. Missing the table name will cause the pipeline to crash. + * *Example:* `[{Singers.SingerId, Singers.VocalistId}]` + +You can pass these overrides directly to your Terraform configuration using `var.table_overrides` and `var.column_overrides`. + +#### Using a Session File + +If you prefer or need to use a session file, you can generate one using the Spanner Migration Tool (SMT): 1. Setup SMT and [launch the UI](https://googlecloudplatform.github.io/spanner-migration-tool/ui#launching-the-web-ui-for-spanner-migration-tool). diff --git a/v2/sourcedb-to-spanner/terraform/samples/single-job-bulk-migration-postgres/README.md b/v2/sourcedb-to-spanner/terraform/samples/single-job-bulk-migration-postgres/README.md index 1c9b110635..1e93efc526 100644 --- a/v2/sourcedb-to-spanner/terraform/samples/single-job-bulk-migration-postgres/README.md +++ b/v2/sourcedb-to-spanner/terraform/samples/single-job-bulk-migration-postgres/README.md @@ -409,7 +409,7 @@ Once the roles are added, run the following command to verify them - gcloud projects get-iam-policy \ --flatten="bindings[].members" \ --format='table(bindings.role)' \ ---filter="bindings.members:@.iam.gserviceaccount.com" +--filter="bindings.members:serviceAccount:@.iam.gserviceaccount.com" ``` Sample output - diff --git a/v2/sourcedb-to-spanner/terraform/samples/single-job-bulk-migration/README.md b/v2/sourcedb-to-spanner/terraform/samples/single-job-bulk-migration/README.md index 0eff2392d1..d784639ae3 100644 --- a/v2/sourcedb-to-spanner/terraform/samples/single-job-bulk-migration/README.md +++ b/v2/sourcedb-to-spanner/terraform/samples/single-job-bulk-migration/README.md @@ -285,13 +285,26 @@ gcloud dataflow jobs update-options \ 2024-06-17_01_21_44-12198433486526363702 ``` -### Specifying schema overrides +### Specifying schema changes -By default, the bulk job performs a like-like mapping between -source and Spanner. Any schema changes between source and Spanner can be -specified using the `session file`. +By default, the validation job performs a like-like schema mapping between the source AVRO records and Spanner. Any schema changes between the source and Spanner can be specified using a `session file` or `overrides` parameters. -To generate a session file: +**We highly recommend using the schema overrides parameters (`table_overrides` and `column_overrides`) instead of a session file** when dealing with schema differences. + +#### Using Schema Overrides (Recommended) + +When passing schema overrides to the job, you must strictly follow the required `[{}]` bracket-brace format. If the format is not matched exactly, Dataflow will reject the configuration with a regex error. + +* **For `table_overrides`**: Use the format `[{OldTableName,NewTableName}]`. + * *Example:* `[{Singers, Vocalists}]` +* **For `column_overrides`**: You **MUST** include the table name alongside the column names. Use the format `[{TableName.OldColumnName,TableName.NewColumnName}]`. Missing the table name will cause the pipeline to crash. + * *Example:* `[{Singers.SingerId, Singers.VocalistId}]` + +You can pass these overrides directly to your Terraform configuration using `var.table_overrides` and `var.column_overrides`. + +#### Using a Session File + +If you prefer or need to use a session file, you can generate one using the Spanner Migration Tool (SMT): 1. Setup SMT and [launch the UI](https://googlecloudplatform.github.io/spanner-migration-tool/ui#launching-the-web-ui-for-spanner-migration-tool).