From 7cb1a0770afe674a51897e3d02e94b61315b5275 Mon Sep 17 00:00:00 2001 From: Leopoldo Araujo <68853890+poloaraujo@users.noreply.github.com> Date: Thu, 15 Jun 2023 14:21:48 +0200 Subject: [PATCH 01/31] on-run-end hook correction (#357) README.md - Removed backticks and one parenthesis --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9a055e8c..94853de2 100644 --- a/README.md +++ b/README.md @@ -55,15 +55,15 @@ packages: 3. Add an on-run-end hook to your `dbt_project.yml` ```yml - `on-run-end: - - "{{ dbt_artifacts.upload_results(results) }}"` + on-run-end: + - "{{ dbt_artifacts.upload_results(results) }}" ``` We recommend adding a conditional here so that the upload only occurs in your production environment, such as: ```yml on-run-end: - - "{% if target.name == 'prod' %}{{ dbt_artifacts.upload_results(results) }}{% endif %}"`) + - "{% if target.name == 'prod' %}{{ dbt_artifacts.upload_results(results) }}{% endif %}" ``` 4. Run the tables! From 358b3d0b2df29ef88e6a3ab2e0d03a8ba9992a78 Mon Sep 17 00:00:00 2001 From: calkan-cue <116581096+calkan-cue@users.noreply.github.com> Date: Thu, 15 Jun 2023 17:04:47 +0300 Subject: [PATCH 02/31] parse_json({{ field }}, wide_number_mode=>'round') was added for the bigquery --- macros/parse_json.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/parse_json.sql b/macros/parse_json.sql index 1184b542..4930c088 100644 --- a/macros/parse_json.sql +++ b/macros/parse_json.sql @@ -11,6 +11,6 @@ {%- endmacro %} {% macro bigquery__parse_json(field) -%} - parse_json({{ field }}) + parse_json({{ field }}, wide_number_mode=>'round') {%- endmacro %} From 5e20a38a44b07b773a6bfa275147eb7a82f6036c Mon Sep 17 00:00:00 2001 From: "Jun Han (Johnson) Ooi" Date: Thu, 15 Jun 2023 11:10:23 -0400 Subject: [PATCH 03/31] Add wide_number_mode to remaining BigQuery upload macros (#356) --- macros/upload_test_executions.sql | 2 +- macros/upload_tests.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/macros/upload_test_executions.sql b/macros/upload_test_executions.sql index 198fc95f..45a31221 100644 --- a/macros/upload_test_executions.sql +++ b/macros/upload_test_executions.sql @@ -120,7 +120,7 @@ null, {# rows_affected not available in Databricks #} {{ 'null' if test.failures is none else test.failures }}, {# failures #} '{{ test.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}', {# message #} - parse_json('{{ tojson(test.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# adapter_response #} + parse_json('{{ tojson(test.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}', wide_number_mode=>'round') {# adapter_response #} ) {%- if not loop.last %},{%- endif %} diff --git a/macros/upload_tests.sql b/macros/upload_tests.sql index ab4f2a1b..5d7c268d 100644 --- a/macros/upload_tests.sql +++ b/macros/upload_tests.sql @@ -51,7 +51,7 @@ '{{ test.package_name }}', {# package_name #} '{{ test.original_file_path | replace('\\', '\\\\') }}', {# test_path #} {{ tojson(test.tags) }}, {# tags #} - parse_json('{{ tojson(test) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}') {# all_fields #} + parse_json('{{ tojson(test) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}', wide_number_mode=>'round') {# all_fields #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} From 247615d0715d1b5a9be84ef871c1ac5344e75eda Mon Sep 17 00:00:00 2001 From: Michael Manganiello Date: Thu, 15 Jun 2023 12:19:34 -0300 Subject: [PATCH 04/31] fix: Escape single quotes in invocation_args (#355) Related to #316, this change fixes another occurrence, this time for `invocations.invocation_args`. As DBT variables can contain single quotes, they must be escaped. --- macros/upload_invocations.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/macros/upload_invocations.sql b/macros/upload_invocations.sql index 0f891ce1..b1d6e7bc 100644 --- a/macros/upload_invocations.sql +++ b/macros/upload_invocations.sql @@ -83,7 +83,7 @@ null, {# dbt_vars #} {% endif %} - '{{ tojson(invocation_args_dict) | replace('\\', '\\\\') }}', {# invocation_args #} + '{{ tojson(invocation_args_dict) | replace('\\', '\\\\') | replace("'", "\\'") }}', {# invocation_args #} {% set metadata_env = {} %} {% for key, value in dbt_metadata_envs.items() %} @@ -146,7 +146,7 @@ {% endif %} {% endif %} - safe.parse_json('''{{ tojson(invocation_args_dict) }}'''), {# invocation_args #} + safe.parse_json('''{{ tojson(invocation_args_dict) | replace("'", "\\'") }}'''), {# invocation_args #} {% set metadata_env = {} %} {% for key, value in dbt_metadata_envs.items() %} From 8ad9142fd84017ff4e97164de58ee14dff5f1534 Mon Sep 17 00:00:00 2001 From: calkan-cue <116581096+calkan-cue@users.noreply.github.com> Date: Tue, 20 Jun 2023 20:08:11 +0300 Subject: [PATCH 05/31] removed the wide_number_mode from bigquery parts --- macros/upload_exposures.sql | 2 +- macros/upload_models.sql | 2 +- macros/upload_seeds.sql | 2 +- macros/upload_snapshots.sql | 2 +- macros/upload_sources.sql | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/macros/upload_exposures.sql b/macros/upload_exposures.sql index 3aa74a29..daa86fea 100644 --- a/macros/upload_exposures.sql +++ b/macros/upload_exposures.sql @@ -70,7 +70,7 @@ '{{ exposure.package_name }}', {# package_name #} {{ tojson(exposure.depends_on.nodes) }}, {# depends_on_nodes #} {{ tojson(exposure.tags) }}, {# tags #} - parse_json('{{ tojson(exposure) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}', wide_number_mode=>'round') {# all_results #} + parse_json('{{ tojson(exposure) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# all_results #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_models.sql b/macros/upload_models.sql index 7673f3f4..b8fce69e 100644 --- a/macros/upload_models.sql +++ b/macros/upload_models.sql @@ -71,7 +71,7 @@ {{ tojson(model.tags) }}, {# tags #} parse_json('''{{ tojson(model.config.meta) }}'''), {# meta #} '{{ model.alias }}', {# alias #} - parse_json('{{ tojson(model) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}', wide_number_mode=>'round') {# all_results #} + parse_json('{{ tojson(model) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}') {# all_results #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_seeds.sql b/macros/upload_seeds.sql index f256c4a1..61683f75 100644 --- a/macros/upload_seeds.sql +++ b/macros/upload_seeds.sql @@ -64,7 +64,7 @@ '{{ seed.checksum.checksum }}', {# checksum #} parse_json('''{{ tojson(seed.config.meta) }}'''), {# meta #} '{{ seed.alias }}', {# alias #} - parse_json('{{ tojson(seed) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}', wide_number_mode=>'round') {# all_results #} + parse_json('{{ tojson(seed) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}') {# all_results #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_snapshots.sql b/macros/upload_snapshots.sql index 711d3a00..cac08abc 100644 --- a/macros/upload_snapshots.sql +++ b/macros/upload_snapshots.sql @@ -71,7 +71,7 @@ '{{ snapshot.config.strategy }}', {# strategy #} parse_json('''{{ tojson(snapshot.config.meta) }}'''), {# meta #} '{{ snapshot.alias }}', {# alias #} - parse_json('{{ tojson(snapshot) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}', wide_number_mode=>'round') {# all_results #} + parse_json('{{ tojson(snapshot) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}') {# all_results #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_sources.sql b/macros/upload_sources.sql index ffe0e2b3..a624ea4b 100644 --- a/macros/upload_sources.sql +++ b/macros/upload_sources.sql @@ -60,7 +60,7 @@ '{{ source.identifier }}', {# identifier #} '{{ source.loaded_at_field | replace("'","\\'") }}', {# loaded_at_field #} parse_json('{{ tojson(source.freshness) | replace("'","\\'") }}'), {# freshness #} - parse_json('{{ tojson(source) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}', wide_number_mode=>'round') {# all_results #} + parse_json('{{ tojson(source) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# all_results #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} From 2367b47e0008bd11f74fabe7ab46eab1020d0f3a Mon Sep 17 00:00:00 2001 From: Baiyue Cao Date: Tue, 20 Jun 2023 16:05:31 -0400 Subject: [PATCH 06/31] adding escape newline for bigquery executions --- macros/upload_model_executions.sql | 2 +- macros/upload_seed_executions.sql | 2 +- macros/upload_snapshot_executions.sql | 2 +- macros/upload_test_executions.sql | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/macros/upload_model_executions.sql b/macros/upload_model_executions.sql index 476c7ba7..6b7828a8 100644 --- a/macros/upload_model_executions.sql +++ b/macros/upload_model_executions.sql @@ -130,7 +130,7 @@ '{{ model.node.schema }}', {# schema #} '{{ model.node.name }}', {# name #} '{{ model.node.alias }}', {# alias #} - '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}', {# message #} + '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') | | replace("\n", "\\n") }}', {# message #} parse_json('{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# adapter_response #} ) {%- if not loop.last %},{%- endif %} diff --git a/macros/upload_seed_executions.sql b/macros/upload_seed_executions.sql index a3a9d521..54e161c7 100644 --- a/macros/upload_seed_executions.sql +++ b/macros/upload_seed_executions.sql @@ -128,7 +128,7 @@ '{{ model.node.schema }}', {# schema #} '{{ model.node.name }}', {# name #} '{{ model.node.alias }}', {# alias #} - '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}', {# message #} + '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') | | replace("\n", "\\n") }}', {# message #} parse_json('{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# adapter_response #} ) {%- if not loop.last %},{%- endif %} diff --git a/macros/upload_snapshot_executions.sql b/macros/upload_snapshot_executions.sql index e0bf6dac..4903f48e 100644 --- a/macros/upload_snapshot_executions.sql +++ b/macros/upload_snapshot_executions.sql @@ -128,7 +128,7 @@ '{{ model.node.schema }}', {# schema #} '{{ model.node.name }}', {# name #} '{{ model.node.alias }}', {# alias #} - '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}', {# message #} + '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') | replace("\n", "\\n") }}', {# message #} parse_json('{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# adapter_response #} ) {%- if not loop.last %},{%- endif %} diff --git a/macros/upload_test_executions.sql b/macros/upload_test_executions.sql index 198fc95f..55112f4a 100644 --- a/macros/upload_test_executions.sql +++ b/macros/upload_test_executions.sql @@ -119,7 +119,7 @@ {{ test.execution_time }}, {# total_node_runtime #} null, {# rows_affected not available in Databricks #} {{ 'null' if test.failures is none else test.failures }}, {# failures #} - '{{ test.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}', {# message #} + '{{ test.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') | replace("\n", "\\n") }}', {# message #} parse_json('{{ tojson(test.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# adapter_response #} ) From b464e60e551949282d972e5fd2fc81e2b8773032 Mon Sep 17 00:00:00 2001 From: Baiyue Cao Date: Tue, 20 Jun 2023 16:06:12 -0400 Subject: [PATCH 07/31] fix typo --- macros/upload_model_executions.sql | 2 +- macros/upload_seed_executions.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/macros/upload_model_executions.sql b/macros/upload_model_executions.sql index 6b7828a8..14917d1a 100644 --- a/macros/upload_model_executions.sql +++ b/macros/upload_model_executions.sql @@ -130,7 +130,7 @@ '{{ model.node.schema }}', {# schema #} '{{ model.node.name }}', {# name #} '{{ model.node.alias }}', {# alias #} - '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') | | replace("\n", "\\n") }}', {# message #} + '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') | replace("\n", "\\n") }}', {# message #} parse_json('{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# adapter_response #} ) {%- if not loop.last %},{%- endif %} diff --git a/macros/upload_seed_executions.sql b/macros/upload_seed_executions.sql index 54e161c7..a7ada664 100644 --- a/macros/upload_seed_executions.sql +++ b/macros/upload_seed_executions.sql @@ -128,7 +128,7 @@ '{{ model.node.schema }}', {# schema #} '{{ model.node.name }}', {# name #} '{{ model.node.alias }}', {# alias #} - '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') | | replace("\n", "\\n") }}', {# message #} + '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') | replace("\n", "\\n") }}', {# message #} parse_json('{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# adapter_response #} ) {%- if not loop.last %},{%- endif %} From ee9156263057b5ede1ba666484c6c75819532c38 Mon Sep 17 00:00:00 2001 From: Gemma Down <52132406+glsdown@users.noreply.github.com> Date: Fri, 21 Jul 2023 16:52:53 +0100 Subject: [PATCH 08/31] Fix BigQuery parse_json macro usage (#371) * Use safe.parse_json instead of parse_json * Replace hard-coded parse_json with macro --- macros/parse_json.sql | 2 +- macros/upload_exposures.sql | 4 ++-- macros/upload_invocations.sql | 8 ++++---- macros/upload_model_executions.sql | 2 +- macros/upload_models.sql | 4 ++-- macros/upload_seed_executions.sql | 2 +- macros/upload_seeds.sql | 4 ++-- macros/upload_snapshot_executions.sql | 2 +- macros/upload_snapshots.sql | 4 ++-- macros/upload_sources.sql | 4 ++-- macros/upload_test_executions.sql | 2 +- macros/upload_tests.sql | 2 +- 12 files changed, 20 insertions(+), 20 deletions(-) diff --git a/macros/parse_json.sql b/macros/parse_json.sql index 4930c088..39ea5d8f 100644 --- a/macros/parse_json.sql +++ b/macros/parse_json.sql @@ -11,6 +11,6 @@ {%- endmacro %} {% macro bigquery__parse_json(field) -%} - parse_json({{ field }}, wide_number_mode=>'round') + safe.parse_json("""{{ field }}""", wide_number_mode=>'round') {%- endmacro %} diff --git a/macros/upload_exposures.sql b/macros/upload_exposures.sql index daa86fea..d8653154 100644 --- a/macros/upload_exposures.sql +++ b/macros/upload_exposures.sql @@ -62,7 +62,7 @@ '{{ run_started_at }}', {# run_started_at #} '{{ exposure.name | replace("'","\\'") }}', {# name #} '{{ exposure.type }}', {# type #} - parse_json('{{ tojson(exposure.owner) | replace("'","\\'") }}'), {# owner #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(exposure.owner) | replace("'","\\'")) }}, {# owner #} '{{ exposure.maturity }}', {# maturity #} '{{ exposure.original_file_path | replace('\\', '\\\\') }}', {# path #} """{{ exposure.description | replace("'","\\'") }}""", {# description #} @@ -70,7 +70,7 @@ '{{ exposure.package_name }}', {# package_name #} {{ tojson(exposure.depends_on.nodes) }}, {# depends_on_nodes #} {{ tojson(exposure.tags) }}, {# tags #} - parse_json('{{ tojson(exposure) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# all_results #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(exposure) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"')) }} {# all_results #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_invocations.sql b/macros/upload_invocations.sql index b1d6e7bc..f3dcdd6b 100644 --- a/macros/upload_invocations.sql +++ b/macros/upload_invocations.sql @@ -122,7 +122,7 @@ {% for env_variable in var('env_vars') %} {% do env_vars_dict.update({env_variable: (env_var(env_variable, ''))}) %} {% endfor %} - parse_json('''{{ tojson(env_vars_dict) }}'''), {# env_vars #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(env_vars_dict)) }}, {# env_vars #} {% else %} null, {# env_vars #} {% endif %} @@ -132,7 +132,7 @@ {% for dbt_var in var('dbt_vars') %} {% do dbt_vars_dict.update({dbt_var: (var(dbt_var, ''))}) %} {% endfor %} - parse_json('''{{ tojson(dbt_vars_dict) }}'''), {# dbt_vars #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(dbt_vars_dict)) }}, {# dbt_vars #} {% else %} null, {# dbt_vars #} {% endif %} @@ -146,13 +146,13 @@ {% endif %} {% endif %} - safe.parse_json('''{{ tojson(invocation_args_dict) | replace("'", "\\'") }}'''), {# invocation_args #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(invocation_args_dict) | replace("'", "\\'")) }}, {# invocation_args #} {% set metadata_env = {} %} {% for key, value in dbt_metadata_envs.items() %} {% do metadata_env.update({key: value}) %} {% endfor %} - parse_json('''{{ tojson(metadata_env) | replace('\\', '\\\\') }}''') {# dbt_custom_envs #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(metadata_env) | replace('\\', '\\\\')) }} {# dbt_custom_envs #} ) {% endset %} diff --git a/macros/upload_model_executions.sql b/macros/upload_model_executions.sql index 14917d1a..a8ebbe89 100644 --- a/macros/upload_model_executions.sql +++ b/macros/upload_model_executions.sql @@ -131,7 +131,7 @@ '{{ model.node.name }}', {# name #} '{{ model.node.alias }}', {# alias #} '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') | replace("\n", "\\n") }}', {# message #} - parse_json('{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# adapter_response #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"')) }} {# adapter_response #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_models.sql b/macros/upload_models.sql index b8fce69e..bc640f52 100644 --- a/macros/upload_models.sql +++ b/macros/upload_models.sql @@ -69,9 +69,9 @@ '{{ model.checksum.checksum }}', {# checksum #} '{{ model.config.materialized }}', {# materialization #} {{ tojson(model.tags) }}, {# tags #} - parse_json('''{{ tojson(model.config.meta) }}'''), {# meta #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model.config.meta)) }}, {# meta #} '{{ model.alias }}', {# alias #} - parse_json('{{ tojson(model) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}') {# all_results #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_results #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_seed_executions.sql b/macros/upload_seed_executions.sql index a7ada664..4be58b09 100644 --- a/macros/upload_seed_executions.sql +++ b/macros/upload_seed_executions.sql @@ -129,7 +129,7 @@ '{{ model.node.name }}', {# name #} '{{ model.node.alias }}', {# alias #} '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') | replace("\n", "\\n") }}', {# message #} - parse_json('{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# adapter_response #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"')) }} {# adapter_response #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_seeds.sql b/macros/upload_seeds.sql index 61683f75..8158bf84 100644 --- a/macros/upload_seeds.sql +++ b/macros/upload_seeds.sql @@ -62,9 +62,9 @@ '{{ seed.package_name }}', {# package_name #} '{{ seed.original_file_path | replace('\\', '\\\\') }}', {# path #} '{{ seed.checksum.checksum }}', {# checksum #} - parse_json('''{{ tojson(seed.config.meta) }}'''), {# meta #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(seed.config.meta)) }}, {# meta #} '{{ seed.alias }}', {# alias #} - parse_json('{{ tojson(seed) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}') {# all_results #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(seed) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_results #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_snapshot_executions.sql b/macros/upload_snapshot_executions.sql index 4903f48e..3e8c03d0 100644 --- a/macros/upload_snapshot_executions.sql +++ b/macros/upload_snapshot_executions.sql @@ -129,7 +129,7 @@ '{{ model.node.name }}', {# name #} '{{ model.node.alias }}', {# alias #} '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') | replace("\n", "\\n") }}', {# message #} - parse_json('{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# adapter_response #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"')) }} {# adapter_response #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_snapshots.sql b/macros/upload_snapshots.sql index cac08abc..477f46a6 100644 --- a/macros/upload_snapshots.sql +++ b/macros/upload_snapshots.sql @@ -69,9 +69,9 @@ '{{ snapshot.original_file_path | replace('\\', '\\\\') }}', {# path #} '{{ snapshot.checksum.checksum }}', {# checksum #} '{{ snapshot.config.strategy }}', {# strategy #} - parse_json('''{{ tojson(snapshot.config.meta) }}'''), {# meta #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(snapshot.config.meta)) }}, {# meta #} '{{ snapshot.alias }}', {# alias #} - parse_json('{{ tojson(snapshot) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}') {# all_results #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(snapshot) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_results #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_sources.sql b/macros/upload_sources.sql index a624ea4b..38ab6ec5 100644 --- a/macros/upload_sources.sql +++ b/macros/upload_sources.sql @@ -59,8 +59,8 @@ '{{ source.name }}', {# name #} '{{ source.identifier }}', {# identifier #} '{{ source.loaded_at_field | replace("'","\\'") }}', {# loaded_at_field #} - parse_json('{{ tojson(source.freshness) | replace("'","\\'") }}'), {# freshness #} - parse_json('{{ tojson(source) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# all_results #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(source.freshness) | replace("'","\\'")) }}, {# freshness #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(source) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"')) }} {# all_results #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_test_executions.sql b/macros/upload_test_executions.sql index 1f0f43d7..a8747099 100644 --- a/macros/upload_test_executions.sql +++ b/macros/upload_test_executions.sql @@ -120,7 +120,7 @@ null, {# rows_affected not available in Databricks #} {{ 'null' if test.failures is none else test.failures }}, {# failures #} '{{ test.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') | replace("\n", "\\n") }}', {# message #} - parse_json('{{ tojson(test.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}', wide_number_mode=>'round') {# adapter_response #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(test.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"')) }} {# adapter_response #} ) {%- if not loop.last %},{%- endif %} diff --git a/macros/upload_tests.sql b/macros/upload_tests.sql index 5d7c268d..9d6a02a6 100644 --- a/macros/upload_tests.sql +++ b/macros/upload_tests.sql @@ -51,7 +51,7 @@ '{{ test.package_name }}', {# package_name #} '{{ test.original_file_path | replace('\\', '\\\\') }}', {# test_path #} {{ tojson(test.tags) }}, {# tags #} - parse_json('{{ tojson(test) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}', wide_number_mode=>'round') {# all_fields #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(test) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_fields #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} From fbe99f82ac19fdc9c460e682cf7c322c6c336840 Mon Sep 17 00:00:00 2001 From: Gemma Down <52132406+glsdown@users.noreply.github.com> Date: Mon, 24 Jul 2023 09:57:11 +0100 Subject: [PATCH 09/31] [RELEASE] Version 2.4.3 (#370) * Update version numbers * Update PR template --- .github/pull_request_template.md | 1 + README.md | 2 +- dbt_project.yml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1888f576..3ce860d8 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -13,6 +13,7 @@ - [ ] New features (breaking change) - [ ] Other (non-breaking change) - [ ] Other (breaking change) +- [ ] Release preparation ## What does this solve? diff --git a/README.md b/README.md index 94853de2..45800893 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ See the generated [dbt docs site](https://brooklyn-data.github.io/dbt_artifacts/ ``` packages: - package: brooklyn-data/dbt_artifacts - version: 2.4.2 + version: 2.4.3 ``` :construction_worker: Make sure to fix at least the **minor** version, to avoid issues when a new release is open. See the notes on upgrading below for more detail. diff --git a/dbt_project.yml b/dbt_project.yml index 24da069d..5e723cac 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: "dbt_artifacts" -version: "2.4.2" +version: "2.4.3" config-version: 2 require-dbt-version: [">=1.3.0", "<1.6.0"] profile: "dbt_artifacts" From 8c8fbed751b0352cf6a74c54034778614ab94794 Mon Sep 17 00:00:00 2001 From: Kushagra Maurya <98528579+sfc-gh-kumaurya@users.noreply.github.com> Date: Mon, 11 Sep 2023 17:48:12 +0530 Subject: [PATCH 10/31] adding support for dbt v1.6 (#374) --- .github/workflows/ci_lint_package.yml | 2 +- .github/workflows/ci_test_package.yml | 4 ++-- .github/workflows/main_lint_package.yml | 2 +- .github/workflows/main_test_package.yml | 2 +- .github/workflows/publish_docs_on_release.yml | 2 +- dbt_project.yml | 2 +- tox.ini | 6 +++--- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci_lint_package.yml b/.github/workflows/ci_lint_package.yml index bbd3264c..b7d43a01 100644 --- a/.github/workflows/ci_lint_package.yml +++ b/.github/workflows/ci_lint_package.yml @@ -44,7 +44,7 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR - name: Install Python packages - run: python -m pip install dbt-snowflake~=1.5.0 sqlfluff-templater-dbt + run: python -m pip install dbt-snowflake~=1.6.0 sqlfluff-templater-dbt - name: Test database connection run: dbt debug diff --git a/.github/workflows/ci_test_package.yml b/.github/workflows/ci_test_package.yml index 34c47163..edd8cdc9 100644 --- a/.github/workflows/ci_test_package.yml +++ b/.github/workflows/ci_test_package.yml @@ -21,7 +21,7 @@ env: DBT_ENV_SECRET_DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DBT_ENV_SECRET_GCP_PROJECT: ${{ secrets.GCP_PROJECT }} # Env var to test version - LAST_RELEASE_SUPPORTED_DBT_VERSION: 1_4_0 # A dbt version supported by both the last release and this one + LAST_RELEASE_SUPPORTED_DBT_VERSION: 1_6_0 # A dbt version supported by both the last release and this one # Env vars to test invocations model DBT_CLOUD_PROJECT_ID: 123 DBT_CLOUD_JOB_ID: ABC @@ -102,7 +102,7 @@ jobs: matrix: warehouse: ["snowflake", "bigquery"] # When supporting a new version, update the list here - version: ["1_3_0", "1_4_0", "1_5_0"] + version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0"] runs-on: ubuntu-latest environment: name: Approve Integration Tests diff --git a/.github/workflows/main_lint_package.yml b/.github/workflows/main_lint_package.yml index e3d25dd0..2feaaba1 100644 --- a/.github/workflows/main_lint_package.yml +++ b/.github/workflows/main_lint_package.yml @@ -40,7 +40,7 @@ jobs: uses: actions/checkout@v3 - name: Install Python packages - run: python -m pip install dbt-snowflake~=1.5.0 sqlfluff-templater-dbt + run: python -m pip install dbt-snowflake~=1.6.0 sqlfluff-templater-dbt - name: Test database connection run: dbt debug diff --git a/.github/workflows/main_test_package.yml b/.github/workflows/main_test_package.yml index 2202c05c..9aa70ae3 100644 --- a/.github/workflows/main_test_package.yml +++ b/.github/workflows/main_test_package.yml @@ -35,7 +35,7 @@ jobs: strategy: matrix: warehouse: ["snowflake", "bigquery"] - version: ["1_3_0", "1_4_0", "1_5_0"] + version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0"] runs-on: ubuntu-latest permissions: contents: "read" diff --git a/.github/workflows/publish_docs_on_release.yml b/.github/workflows/publish_docs_on_release.yml index cb588373..d558a464 100644 --- a/.github/workflows/publish_docs_on_release.yml +++ b/.github/workflows/publish_docs_on_release.yml @@ -39,7 +39,7 @@ jobs: uses: actions/checkout@v3 - name: Install Python packages - run: python -m pip install dbt-snowflake~=1.3.0 + run: python -m pip install dbt-snowflake~=1.6.0 - name: Test database connection run: dbt debug diff --git a/dbt_project.yml b/dbt_project.yml index 5e723cac..977f219e 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,7 +1,7 @@ name: "dbt_artifacts" version: "2.4.3" config-version: 2 -require-dbt-version: [">=1.3.0", "<1.6.0"] +require-dbt-version: [">=1.3.0", "<1.7.0"] profile: "dbt_artifacts" clean-targets: # folders to be removed by `dbt clean` diff --git a/tox.ini b/tox.ini index f311d76d..064d234f 100644 --- a/tox.ini +++ b/tox.ini @@ -36,7 +36,7 @@ rules = LT01,LT02,LT03,CP01,AL01,AL02,CP02,ST08,LT06,LT07,AM01,LT08,AL05,RF02,RF deps = sqlfluff-templater-dbt~=2.0.2 - dbt-snowflake~=1.3.0 + dbt-snowflake~=1.6.0 [sqlfluff:indentation] indent_unit = space @@ -114,13 +114,13 @@ commands = sqlfluff fix models --ignore parsing # Generate docs [testenv:generate_docs] -deps = dbt-snowflake~=1.5.0 +deps = dbt-snowflake~=1.6.0 commands = dbt docs generate --profiles-dir integration_test_project # Snowflake integration tests [testenv:integration_snowflake] changedir = integration_test_project -deps = dbt-snowflake~=1.4.0 +deps = dbt-snowflake~=1.6.0 commands = dbt clean dbt deps From 8910738702ea9f2a65ddbd0e33200151b03f1e18 Mon Sep 17 00:00:00 2001 From: Gemma Down <52132406+glsdown@users.noreply.github.com> Date: Mon, 11 Sep 2023 13:44:44 +0100 Subject: [PATCH 11/31] Add CI checks for 1.6 (#378) * Add CI checks for 1.6 * Add databricks tests * Specify linter version * Remove databricks --- .github/workflows/ci_lint_package.yml | 2 +- tox.ini | 28 +++++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_lint_package.yml b/.github/workflows/ci_lint_package.yml index b7d43a01..111f6357 100644 --- a/.github/workflows/ci_lint_package.yml +++ b/.github/workflows/ci_lint_package.yml @@ -44,7 +44,7 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR - name: Install Python packages - run: python -m pip install dbt-snowflake~=1.6.0 sqlfluff-templater-dbt + run: python -m pip install dbt-snowflake~=1.6.0 sqlfluff-templater-dbt~=2.0.2 - name: Test database connection run: dbt debug diff --git a/tox.ini b/tox.ini index 064d234f..2070a7eb 100644 --- a/tox.ini +++ b/tox.ini @@ -151,10 +151,18 @@ commands = dbt deps dbt build --target snowflake +[testenv:integration_snowflake_1_6_0] +changedir = integration_test_project +deps = dbt-snowflake~=1.6.0 +commands = + dbt clean + dbt deps + dbt build --target snowflake + # Databricks integration tests [testenv:integration_databricks] changedir = integration_test_project -deps = dbt-databricks~=1.4.0 +deps = dbt-databricks~=1.6.0 commands = dbt clean dbt deps @@ -184,10 +192,18 @@ commands = dbt deps dbt build --target databricks +[testenv:integration_databricks_1_6_0] +changedir = integration_test_project +deps = dbt-databricks~=1.6.0 +commands = + dbt clean + dbt deps + dbt build --target databricks + # Bigquery integration tests [testenv:integration_bigquery] changedir = integration_test_project -deps = dbt-bigquery~=1.4.0 +deps = dbt-bigquery~=1.6.0 commands = dbt clean dbt deps @@ -217,6 +233,14 @@ commands = dbt deps dbt build --target bigquery --vars '"my_var": "my value"' +[testenv:integration_bigquery_1_6_0] +changedir = integration_test_project +deps = dbt-bigquery~=1.6.0 +commands = + dbt clean + dbt deps + dbt build --target bigquery --vars '"my_var": "my value"' + # Spark integration test (disabled) [testenv:integration_spark] changedir = integration_test_project From db1b9b1a4ea08d463bd780fa6c13dcd905dd5c0a Mon Sep 17 00:00:00 2001 From: Gemma Down <52132406+glsdown@users.noreply.github.com> Date: Mon, 11 Sep 2023 13:52:58 +0100 Subject: [PATCH 12/31] Fix SQLFluff check on push to main (#379) * Add CI checks for 1.6 * Add databricks tests * Specify linter version * Remove databricks * Fix main lint --- .github/workflows/main_lint_package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main_lint_package.yml b/.github/workflows/main_lint_package.yml index 2feaaba1..45985ec2 100644 --- a/.github/workflows/main_lint_package.yml +++ b/.github/workflows/main_lint_package.yml @@ -40,7 +40,7 @@ jobs: uses: actions/checkout@v3 - name: Install Python packages - run: python -m pip install dbt-snowflake~=1.6.0 sqlfluff-templater-dbt + run: python -m pip install dbt-snowflake~=1.6.0 sqlfluff-templater-dbt~=2.0.2 - name: Test database connection run: dbt debug From 9fbca4febb08bed14bddaa70cc7c1679a1392849 Mon Sep 17 00:00:00 2001 From: Gemma Down <52132406+glsdown@users.noreply.github.com> Date: Mon, 11 Sep 2023 13:56:11 +0100 Subject: [PATCH 13/31] Bump version number (#377) --- README.md | 2 +- dbt_project.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 45800893..b546e98c 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ See the generated [dbt docs site](https://brooklyn-data.github.io/dbt_artifacts/ ``` packages: - package: brooklyn-data/dbt_artifacts - version: 2.4.3 + version: 2.5.0 ``` :construction_worker: Make sure to fix at least the **minor** version, to avoid issues when a new release is open. See the notes on upgrading below for more detail. diff --git a/dbt_project.yml b/dbt_project.yml index 977f219e..83a4ccad 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: "dbt_artifacts" -version: "2.4.3" +version: "2.5.0" config-version: 2 require-dbt-version: [">=1.3.0", "<1.7.0"] profile: "dbt_artifacts" From 62554f36eb85297c5238b0561374096094045a53 Mon Sep 17 00:00:00 2001 From: Gemma Down <52132406+glsdown@users.noreply.github.com> Date: Mon, 11 Sep 2023 14:40:52 +0100 Subject: [PATCH 14/31] Add Python setup (#380) --- .github/workflows/ci_lint_package.yml | 8 +++++++- .github/workflows/main_lint_package.yml | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_lint_package.yml b/.github/workflows/ci_lint_package.yml index 111f6357..9f1508f3 100644 --- a/.github/workflows/ci_lint_package.yml +++ b/.github/workflows/ci_lint_package.yml @@ -43,8 +43,14 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: "3.9.x" + architecture: "x64" + - name: Install Python packages - run: python -m pip install dbt-snowflake~=1.6.0 sqlfluff-templater-dbt~=2.0.2 + run: python -m pip install dbt-snowflake~=1.6.0 sqlfluff-templater-dbt~=2.3.2 - name: Test database connection run: dbt debug diff --git a/.github/workflows/main_lint_package.yml b/.github/workflows/main_lint_package.yml index 45985ec2..7747f565 100644 --- a/.github/workflows/main_lint_package.yml +++ b/.github/workflows/main_lint_package.yml @@ -39,8 +39,14 @@ jobs: - name: Checkout branch uses: actions/checkout@v3 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: "3.9.x" + architecture: "x64" + - name: Install Python packages - run: python -m pip install dbt-snowflake~=1.6.0 sqlfluff-templater-dbt~=2.0.2 + run: python -m pip install dbt-snowflake~=1.6.0 sqlfluff-templater-dbt~=2.3.2 - name: Test database connection run: dbt debug From cef1cec7aab2226bea49c611bf8b25c0172fefbc Mon Sep 17 00:00:00 2001 From: Jared Rimmer <100997264+jared-rimmer@users.noreply.github.com> Date: Wed, 13 Sep 2023 16:43:01 +0100 Subject: [PATCH 15/31] Add escape for windows backslash on checksum columns (#353) Co-authored-by: Gemma Down <52132406+glsdown@users.noreply.github.com> --- macros/upload_models.sql | 4 ++-- macros/upload_seeds.sql | 4 ++-- macros/upload_snapshots.sql | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/macros/upload_models.sql b/macros/upload_models.sql index bc640f52..cb396a05 100644 --- a/macros/upload_models.sql +++ b/macros/upload_models.sql @@ -35,7 +35,7 @@ '{{ tojson(model.depends_on.nodes) | replace('\\', '\\\\') }}', {# depends_on_nodes #} '{{ model.package_name }}', {# package_name #} '{{ model.original_file_path | replace('\\', '\\\\') }}', {# path #} - '{{ model.checksum.checksum }}', {# checksum #} + '{{ model.checksum.checksum | replace('\\', '\\\\') }}', {# checksum #} '{{ model.config.materialized }}', {# materialization #} '{{ tojson(model.tags) }}', {# tags #} '{{ tojson(model.config.meta) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}', {# meta #} @@ -66,7 +66,7 @@ {{ tojson(model.depends_on.nodes) }}, {# depends_on_nodes #} '{{ model.package_name }}', {# package_name #} '{{ model.original_file_path | replace('\\', '\\\\') }}', {# path #} - '{{ model.checksum.checksum }}', {# checksum #} + '{{ model.checksum.checksum | replace('\\', '\\\\') }}', {# checksum #} '{{ model.config.materialized }}', {# materialization #} {{ tojson(model.tags) }}, {# tags #} {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model.config.meta)) }}, {# meta #} diff --git a/macros/upload_seeds.sql b/macros/upload_seeds.sql index 8158bf84..0a766bcb 100644 --- a/macros/upload_seeds.sql +++ b/macros/upload_seeds.sql @@ -34,7 +34,7 @@ '{{ seed.name }}', {# name #} '{{ seed.package_name }}', {# package_name #} '{{ seed.original_file_path | replace('\\', '\\\\') }}', {# path #} - '{{ seed.checksum.checksum }}', {# checksum #} + '{{ seed.checksum.checksum | replace('\\', '\\\\') }}', {# checksum #} '{{ tojson(seed.config.meta) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}', {# meta #} '{{ seed.alias }}', {# alias #} '{{ tojson(seed) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}' {# all_results #} @@ -61,7 +61,7 @@ '{{ seed.name }}', {# name #} '{{ seed.package_name }}', {# package_name #} '{{ seed.original_file_path | replace('\\', '\\\\') }}', {# path #} - '{{ seed.checksum.checksum }}', {# checksum #} + '{{ seed.checksum.checksum | replace('\\', '\\\\')}}', {# checksum #} {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(seed.config.meta)) }}, {# meta #} '{{ seed.alias }}', {# alias #} {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(seed) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_results #} diff --git a/macros/upload_snapshots.sql b/macros/upload_snapshots.sql index 477f46a6..05dad1be 100644 --- a/macros/upload_snapshots.sql +++ b/macros/upload_snapshots.sql @@ -38,7 +38,7 @@ '{{ tojson(snapshot.depends_on.nodes) }}', {# depends_on_nodes #} '{{ snapshot.package_name }}', {# package_name #} '{{ snapshot.original_file_path | replace('\\', '\\\\') }}', {# path #} - '{{ snapshot.checksum.checksum }}', {# checksum #} + '{{ snapshot.checksum.checksum | replace('\\', '\\\\') }}', {# checksum #} '{{ snapshot.config.strategy }}', {# strategy #} '{{ tojson(snapshot.config.meta) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}', {# meta #} '{{ snapshot.alias }}', {# alias #} @@ -67,7 +67,7 @@ {{ tojson(snapshot.depends_on.nodes) }}, {# depends_on_nodes #} '{{ snapshot.package_name }}', {# package_name #} '{{ snapshot.original_file_path | replace('\\', '\\\\') }}', {# path #} - '{{ snapshot.checksum.checksum }}', {# checksum #} + '{{ snapshot.checksum.checksum | replace('\\', '\\\\') }}', {# checksum #} '{{ snapshot.config.strategy }}', {# strategy #} {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(snapshot.config.meta)) }}, {# meta #} '{{ snapshot.alias }}', {# alias #} From 532edbb9c8343c1643fe390e4f4bdc280f7a197b Mon Sep 17 00:00:00 2001 From: Gemma Down <52132406+glsdown@users.noreply.github.com> Date: Mon, 18 Sep 2023 14:02:50 +0100 Subject: [PATCH 16/31] Add variable to allow users to exclude all results columns (#382) * Add ability to exclude all results column * Add documentation about the variable --- README.md | 54 ++++++++++++++---------- integration_test_project/dbt_project.yml | 1 + macros/upload_exposures.sql | 12 +++++- macros/upload_models.sql | 12 +++++- macros/upload_seeds.sql | 12 +++++- macros/upload_snapshots.sql | 12 +++++- macros/upload_sources.sql | 12 +++++- macros/upload_tests.sql | 12 +++++- 8 files changed, 92 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index b546e98c..a63bba3c 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,37 @@ vars: ] ``` +## Creating custom marts tables + +Multiple modelled `dim` and `fct` models have been provided for ease of use, but we recognise that some use cases may require custom ones. To this end, you can disable all but the raw sources tables using the following in your `dbt_project.yml` file: + +```yml +# dbt_project.yml + +models: + dbt_artifacts: + +enabled: false + sources: + +enabled: true +``` + +In these sources tables, you will find a JSON column `all_results` which contains a JSON blob of the results object used, which you can use in your own analysis: + +- exposures +- models +- seeds +- snapshots +- sources +- tests + +This column can cause queries to become too long - particularly in BigQuery. Therefore, if you want to disable this column, you can make use of the `dbt_artifacts_exclude_all_results` variable, and set this to `true` in your `dbt_project.yml` file. + +``` +# dbt_project.yml +vars: + dbt_artifacts_exclude_all_results: true +``` + ## Upgrading from 1.x to >=2.0.0 If you were using the following variables: @@ -189,29 +220,6 @@ An example operation is as follows: dbt run-operation migrate_from_v0_to_v1 --args '{old_database: analytics, old_schema: dbt_artifacts, new_database: analytics, new_schema: artifact_sources}' ``` -## Creating custom marts tables - -Multiple modelled `dim` and `fct` models have been provided for ease of use, but we recognise that some use cases may require custom ones. To this end, you can disable all but the raw sources tables using the following in your `dbt_project.yml` file: - -```yml -# dbt_project.yml - -models: - dbt_artifacts: - +enabled: false - sources: - +enabled: true -``` - -In these sources tables, you will find a JSON column `all_results` which contains a JSON blob of the results object used, which you can use in your own analysis: - -- exposures -- models -- seeds -- snapshots -- sources -- tests - ## Acknowledgements Thank you to [Tails.com](https://tails.com/gb/careers/) for initial development and maintenance of this package. On 2021/12/20, the repository was transferred from the Tails.com GitHub organization to Brooklyn Data Co. diff --git a/integration_test_project/dbt_project.yml b/integration_test_project/dbt_project.yml index a6550009..dba89a50 100644 --- a/integration_test_project/dbt_project.yml +++ b/integration_test_project/dbt_project.yml @@ -21,6 +21,7 @@ vars: env_vars: ["TEST_ENV_VAR_NUMBER", "TEST_ENV_VAR_EMPTY", "TEST_ENV_VAR_WITH_QUOTE"] dbt_vars: ["test_dbt_vars_1", "test_dbt_vars_2", "test_dbt_vars_3"] + dbt_artifacts_exclude_all_results: true models: +persist_docs: diff --git a/macros/upload_exposures.sql b/macros/upload_exposures.sql index d8653154..8ef51ce2 100644 --- a/macros/upload_exposures.sql +++ b/macros/upload_exposures.sql @@ -41,7 +41,11 @@ '{{ exposure.package_name }}', {# package_name #} '{{ tojson(exposure.depends_on.nodes) }}', {# depends_on_nodes #} '{{ tojson(exposure.tags) }}', {# tags #} - '{{ tojson(exposure) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(exposure) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} @@ -70,7 +74,11 @@ '{{ exposure.package_name }}', {# package_name #} {{ tojson(exposure.depends_on.nodes) }}, {# depends_on_nodes #} {{ tojson(exposure.tags) }}, {# tags #} - {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(exposure) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"')) }} {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(exposure) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"')) }} {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_models.sql b/macros/upload_models.sql index cb396a05..f0748876 100644 --- a/macros/upload_models.sql +++ b/macros/upload_models.sql @@ -40,7 +40,11 @@ '{{ tojson(model.tags) }}', {# tags #} '{{ tojson(model.config.meta) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}', {# meta #} '{{ model.alias }}', {# alias #} - '{{ tojson(model) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}' {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(model) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}' {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} @@ -71,7 +75,11 @@ {{ tojson(model.tags) }}, {# tags #} {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model.config.meta)) }}, {# meta #} '{{ model.alias }}', {# alias #} - {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_seeds.sql b/macros/upload_seeds.sql index 0a766bcb..ea288e21 100644 --- a/macros/upload_seeds.sql +++ b/macros/upload_seeds.sql @@ -37,7 +37,11 @@ '{{ seed.checksum.checksum | replace('\\', '\\\\') }}', {# checksum #} '{{ tojson(seed.config.meta) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}', {# meta #} '{{ seed.alias }}', {# alias #} - '{{ tojson(seed) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}' {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(seed) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}' {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} @@ -64,7 +68,11 @@ '{{ seed.checksum.checksum | replace('\\', '\\\\')}}', {# checksum #} {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(seed.config.meta)) }}, {# meta #} '{{ seed.alias }}', {# alias #} - {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(seed) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(seed) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_snapshots.sql b/macros/upload_snapshots.sql index 05dad1be..3c7a7cbf 100644 --- a/macros/upload_snapshots.sql +++ b/macros/upload_snapshots.sql @@ -42,7 +42,11 @@ '{{ snapshot.config.strategy }}', {# strategy #} '{{ tojson(snapshot.config.meta) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}', {# meta #} '{{ snapshot.alias }}', {# alias #} - '{{ tojson(snapshot) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}' {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(snapshot) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}' {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} @@ -71,7 +75,11 @@ '{{ snapshot.config.strategy }}', {# strategy #} {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(snapshot.config.meta)) }}, {# meta #} '{{ snapshot.alias }}', {# alias #} - {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(snapshot) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(snapshot) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_sources.sql b/macros/upload_sources.sql index 38ab6ec5..23208d6d 100644 --- a/macros/upload_sources.sql +++ b/macros/upload_sources.sql @@ -33,7 +33,11 @@ '{{ source.identifier }}', {# identifier #} '{{ source.loaded_at_field | replace("'","\\'") }}', {# loaded_at_field #} '{{ tojson(source.freshness) | replace("'","\\'") }}', {# freshness #} - '{{ tojson(source) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(source) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} @@ -60,7 +64,11 @@ '{{ source.identifier }}', {# identifier #} '{{ source.loaded_at_field | replace("'","\\'") }}', {# loaded_at_field #} {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(source.freshness) | replace("'","\\'")) }}, {# freshness #} - {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(source) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"')) }} {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(source) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"')) }} {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_tests.sql b/macros/upload_tests.sql index 9d6a02a6..d29341ff 100644 --- a/macros/upload_tests.sql +++ b/macros/upload_tests.sql @@ -27,7 +27,11 @@ '{{ test.package_name }}', {# package_name #} '{{ test.original_file_path | replace('\\', '\\\\') }}', {# test_path #} '{{ tojson(test.tags) }}', {# tags #} - '{{ tojson(test) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}' {# all_fields #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(test) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}' {# all_fields #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} @@ -51,7 +55,11 @@ '{{ test.package_name }}', {# package_name #} '{{ test.original_file_path | replace('\\', '\\\\') }}', {# test_path #} {{ tojson(test.tags) }}, {# tags #} - {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(test) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_fields #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(test) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_fields #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} From 7a33172ae51221a7a9c5a0a91ed1b3cb4388447d Mon Sep 17 00:00:00 2001 From: Gemma Down <52132406+glsdown@users.noreply.github.com> Date: Mon, 18 Sep 2023 14:39:50 +0100 Subject: [PATCH 17/31] Improve insert method used (#381) * Add named columns * Remove unused `rows_affected` column in `test_executions` #195 * Reorganise macro folder * Add documentation for macros * Consolidate logic in upload results * Simplify how nodes are extracted * Generalise identifying objects to load further * Have all results included in the loop * Split out get_dataset_content * Move the get table content into its own macro * Bug fixing * Move relation definition to insert macro to avoid quoting issues * Revert "Remove unused `rows_affected` column in `test_executions` #195" This reverts commit 9182fdcb281362d04831371636757846cef2c577. * Remove ---debug left in databricks tox * Apply suggestions from code review Co-authored-by: Jared Rimmer <100997264+jared-rimmer@users.noreply.github.com> --------- Co-authored-by: Jared Rimmer <100997264+jared-rimmer@users.noreply.github.com> --- macros/_macros.yml | 246 ++++++++++++++++++ .../column_identifier.sql | 0 .../generate_surrogate_key.sql | 0 .../get_relation.sql | 14 + .../parse_json.sql | 0 .../type_helpers.sql | 8 +- macros/insert_into_metadata_table.sql | 38 --- .../{ => migration}/migrate_from_v0_to_v1.sql | 0 .../upload_exposures.sql | 6 +- .../upload_invocations.sql | 0 .../upload_model_executions.sql | 8 +- .../upload_models.sql | 0 .../upload_seed_executions.sql | 8 +- .../upload_seeds.sql | 6 +- .../upload_snapshot_executions.sql | 8 +- .../upload_snapshots.sql | 7 +- .../upload_sources.sql | 0 .../upload_test_executions.sql | 8 +- .../upload_tests.sql | 0 macros/upload_results.sql | 169 ------------ .../upload_results/get_column_name_lists.sql | 230 ++++++++++++++++ macros/upload_results/get_dataset_content.sql | 22 ++ .../get_table_content_values.sql | 32 +++ .../insert_into_metadata_table.sql | 49 ++++ macros/upload_results/upload_results.sql | 50 ++++ tox.ini | 2 +- 26 files changed, 656 insertions(+), 255 deletions(-) create mode 100644 macros/_macros.yml rename macros/{ => database_specific_helpers}/column_identifier.sql (100%) rename macros/{ => database_specific_helpers}/generate_surrogate_key.sql (100%) create mode 100644 macros/database_specific_helpers/get_relation.sql rename macros/{ => database_specific_helpers}/parse_json.sql (100%) rename macros/{ => database_specific_helpers}/type_helpers.sql (95%) delete mode 100644 macros/insert_into_metadata_table.sql rename macros/{ => migration}/migrate_from_v0_to_v1.sql (100%) rename macros/{ => upload_individual_datasets}/upload_exposures.sql (96%) rename macros/{ => upload_individual_datasets}/upload_invocations.sql (100%) rename macros/{ => upload_individual_datasets}/upload_model_executions.sql (97%) rename macros/{ => upload_individual_datasets}/upload_models.sql (100%) rename macros/{ => upload_individual_datasets}/upload_seed_executions.sql (97%) rename macros/{ => upload_individual_datasets}/upload_seeds.sql (95%) rename macros/{ => upload_individual_datasets}/upload_snapshot_executions.sql (97%) rename macros/{ => upload_individual_datasets}/upload_snapshots.sql (95%) rename macros/{ => upload_individual_datasets}/upload_sources.sql (100%) rename macros/{ => upload_individual_datasets}/upload_test_executions.sql (96%) rename macros/{ => upload_individual_datasets}/upload_tests.sql (100%) delete mode 100644 macros/upload_results.sql create mode 100644 macros/upload_results/get_column_name_lists.sql create mode 100644 macros/upload_results/get_dataset_content.sql create mode 100644 macros/upload_results/get_table_content_values.sql create mode 100644 macros/upload_results/insert_into_metadata_table.sql create mode 100644 macros/upload_results/upload_results.sql diff --git a/macros/_macros.yml b/macros/_macros.yml new file mode 100644 index 00000000..7b798447 --- /dev/null +++ b/macros/_macros.yml @@ -0,0 +1,246 @@ +version: 2 + +macros: + ## DATABASE SPECIFIC HELPERS ## + - name: column_identifier + description: | + Dependent on the adapter type, return the identifier for a column using a numerical index. + arguments: + - name: column_index + type: integer + description: | + The index of the column to return the identifier for + + - name: generate_surrogate_key + description: | + Since folks commonly install dbt_artifacts alongside a myriad of other packages, + we copy the dbt_utils implementation of the surrogate_key macro so we don't have + any dependencies to make conflicts worse! + + This version is: + URL: https://github.com/dbt-labs/dbt-utils/blob/main/macros/sql/generate_surrogate_key.sql + Commit SHA: eaa0e41b033bdf252eff0ae014ec11888f37ebff + Date: 2023-04-28 + arguments: + - name: field_list + type: list + description: | + A list of fields to concatenate together to form the surrogate key + + - name: get_relation + description: | + Identify a relation in the graph from a relation name + arguments: + - name: get_relation_name + type: string + description: | + The name of the relation to return from the graph + + - name: parse_json + description: | + Dependent on the adapter type, return a column which parses the JSON field. + arguments: + - name: field + type: string + description: | + The name of the field to parse + + - name: type_array + description: | + Dependent on the adapter type, returns the native type for storing an array. + + - name: type_boolean + description: | + Dependent on the adapter type, returns the native boolean type. + + - name: type_json + description: | + Dependent on the adapter type, returns the native type for storing JSON. + + ## MIGRATION ## + - name: migrate_from_v0_to_v1 + description: | + A macro to assist with migrating from v0 to v1 of dbt_artifacts. See + https://github.com/brooklyn-data/dbt_artifacts/blob/main/README.md#migrating-from-100-to-100 + for details on the usage. + arguments: + - name: old_database + type: string + description: | + The database of the <1.0.0 output (fct_/dim_) models - does not have to be different to `new_database` + - name: old_schema + type: string + description: | + The schema of the <1.0.0 output (fct_/dim_) models - does not have to be different to `new_schema` + - name: new_database + type: string + description: | + The target database that the v1 artifact sources are in - does not have to be different to `old_database` + - name: new_schema + type: string + description: | + The target schema that the v1 artifact sources are in - does not have to be different to `old_schema` + + ## UPLOAD INDIVIDUAL DATASETS ## + - name: upload_exposures + description: | + The macro to support upload of the data to the exposures table. + arguments: + - name: exposures + type: list + description: | + A list of exposure objects extracted from the dbt graph + + - name: upload_invocations + description: | + The macro to support upload of the data to the invocations table. + + - name: upload_model_executions + description: | + The macro to support upload of the data to the model_executions table. + arguments: + - name: models + type: list + description: | + A list of model execution results objects extracted from the dbt result object + + - name: upload_models + description: | + The macro to support upload of the data to the models table. + arguments: + - name: models + type: list + description: | + A list of test objects extracted from the dbt graph + + - name: upload_seed_executions + description: | + The macro to support upload of the data to the seed_executions table. + arguments: + - name: seeds + type: list + description: | + A list of seed execution results objects extracted from the dbt result object + + - name: upload_seeds + description: | + The macro to support upload of the data to the seeds table. + arguments: + - name: seeds + type: list + description: | + A list of seeds objects extracted from the dbt graph + + - name: upload_snapshot_executions + description: | + The macro to support upload of the data to the snapshot_executions table. + arguments: + - name: snapshots + type: list + description: | + A list of snapshot execution results objects extracted from the dbt result object + + - name: upload_snapshots + description: | + The macro to support upload of the data to the snapshots table. + arguments: + - name: snapshots + type: list + description: | + A list of snapshots objects extracted from the dbt graph + + - name: upload_sources + description: | + The macro to support upload of the data to the sources table. + arguments: + - name: sources + type: list + description: | + A list of sources objects extracted from the dbt graph + + - name: upload_test_executions + description: | + The macro to support upload of the data to the test_executions table. + arguments: + - name: tests + type: list + description: | + A list of test execution results objects extracted from the dbt result object + + - name: upload_tests + description: | + The macro to support upload of the data to the tests table. + arguments: + - name: tests + type: list + description: | + A list of test objects extracted from the dbt graph + + ## UPLOAD RESULTS ## + - name: get_column_name_list + description: | + A macro to return the list of column names for a particular dataset. Returns a comment if the dataset is not + valid. + arguments: + - name: dataset + type: string + description: | + The name of the dataset to return the column names for e.g. `models` + + - name: get_dataset_content + description: | + A macro to extract the data to be uploaded from either the results or the graph object. + arguments: + - name: dataset + type: string + description: | + The name of the dataset to return the data for e.g. `models` + + - name: get_table_content_values + description: | + A macro to create the insert statement values required to be uploaded to the table. + arguments: + - name: dataset + type: string + description: | + The name of the dataset to return the column names for e.g. `models` + - name: objects_to_upload + type: list + description: | + The objects to be used to generate the insert statement values - extracted from `get_dataset_content` + + - name: insert_into_metadata_table + description: | + Dependent on the adapter type, the wrapper to insert the data into a table from a list of values. Used in the + `upload_results` macro, alongside the `get_column_lists` macro to generate the column names and the + `upload_dataset` macros to generate the data to be inserted. + arguments: + - name: database_name + type: string + description: | + The database name for the relation that the data is to be inserted into + - name: schema_name + type: string + description: | + The schema name for the relation that the data is to be inserted into + - name: table_name + type: string + description: | + The table name for the relation that the data is to be inserted into + - name: fields + type: string + description: | + The list of fields for the relation that the data is to be inserted into + - name: content + type: string + description: | + The data content to insert into the relation + + - name: upload_results + description: | + The main macro called to upload the metadata into each of the source tables. + arguments: + - name: results + type: list + description: | + The results object from dbt. diff --git a/macros/column_identifier.sql b/macros/database_specific_helpers/column_identifier.sql similarity index 100% rename from macros/column_identifier.sql rename to macros/database_specific_helpers/column_identifier.sql diff --git a/macros/generate_surrogate_key.sql b/macros/database_specific_helpers/generate_surrogate_key.sql similarity index 100% rename from macros/generate_surrogate_key.sql rename to macros/database_specific_helpers/generate_surrogate_key.sql diff --git a/macros/database_specific_helpers/get_relation.sql b/macros/database_specific_helpers/get_relation.sql new file mode 100644 index 00000000..ed7d02e5 --- /dev/null +++ b/macros/database_specific_helpers/get_relation.sql @@ -0,0 +1,14 @@ +{% macro get_relation(relation_name) %} + {% if execute %} + {% set model_get_relation_node = graph.nodes.values() | selectattr('name', 'equalto', relation_name) | first %} + {% set relation = api.Relation.create( + database = model_get_relation_node.database, + schema = model_get_relation_node.schema, + identifier = model_get_relation_node.alias + ) + %} + {% do return(relation) %} + {% else %} + {% do return(api.Relation.create()) %} + {% endif %} +{% endmacro %} diff --git a/macros/parse_json.sql b/macros/database_specific_helpers/parse_json.sql similarity index 100% rename from macros/parse_json.sql rename to macros/database_specific_helpers/parse_json.sql diff --git a/macros/type_helpers.sql b/macros/database_specific_helpers/type_helpers.sql similarity index 95% rename from macros/type_helpers.sql rename to macros/database_specific_helpers/type_helpers.sql index 19c3a718..4064ad46 100644 --- a/macros/type_helpers.sql +++ b/macros/database_specific_helpers/type_helpers.sql @@ -19,11 +19,11 @@ {% endmacro %} {% macro snowflake__type_json() %} - OBJECT + object {% endmacro %} {% macro bigquery__type_json() %} - JSON + json {% endmacro %} {#- ARRAY -#} @@ -37,9 +37,9 @@ {% endmacro %} {% macro snowflake__type_array() %} - ARRAY + array {% endmacro %} {% macro bigquery__type_array() %} - ARRAY + array {% endmacro %} diff --git a/macros/insert_into_metadata_table.sql b/macros/insert_into_metadata_table.sql deleted file mode 100644 index 235d2326..00000000 --- a/macros/insert_into_metadata_table.sql +++ /dev/null @@ -1,38 +0,0 @@ -{% macro insert_into_metadata_table(database_name, schema_name, table_name, content) -%} - {% if content != "" %} - {{ return(adapter.dispatch('insert_into_metadata_table', 'dbt_artifacts')(database_name, schema_name, table_name, content)) }} - {% endif %} -{%- endmacro %} - -{% macro spark__insert_into_metadata_table(database_name, schema_name, table_name, content) -%} - {% set insert_into_table_query %} - insert into {% if database_name %}{{ database_name }}.{% endif %}{{ schema_name }}.{{ table_name }} - {{ content }} - {% endset %} - - {% do run_query(insert_into_table_query) %} -{%- endmacro %} - -{% macro snowflake__insert_into_metadata_table(database_name, schema_name, table_name, content) -%} - {% set insert_into_table_query %} - insert into {{database_name}}.{{ schema_name }}.{{ table_name }} - {{ content }} - {% endset %} - - {% do run_query(insert_into_table_query) %} -{%- endmacro %} - -{% macro bigquery__insert_into_metadata_table(database_name, schema_name, table_name, content) -%} - - {% set insert_into_table_query %} - insert into `{{database_name}}.{{ schema_name }}.{{ table_name }}` - VALUES - {{ content }} - {% endset %} - - {% do run_query(insert_into_table_query) %} - -{%- endmacro %} - -{% macro default__insert_into_metadata_table(database_name, schema_name, table_name, content) -%} -{%- endmacro %} diff --git a/macros/migrate_from_v0_to_v1.sql b/macros/migration/migrate_from_v0_to_v1.sql similarity index 100% rename from macros/migrate_from_v0_to_v1.sql rename to macros/migration/migrate_from_v0_to_v1.sql diff --git a/macros/upload_exposures.sql b/macros/upload_individual_datasets/upload_exposures.sql similarity index 96% rename from macros/upload_exposures.sql rename to macros/upload_individual_datasets/upload_exposures.sql index 8ef51ce2..488f9ffd 100644 --- a/macros/upload_exposures.sql +++ b/macros/upload_individual_datasets/upload_exposures.sql @@ -1,8 +1,4 @@ -{% macro upload_exposures(graph) -%} - {% set exposures = [] %} - {% for node in graph.exposures.values() %} - {% do exposures.append(node) %} - {% endfor %} +{% macro upload_exposures(exposures) -%} {{ return(adapter.dispatch('get_exposures_dml_sql', 'dbt_artifacts')(exposures)) }} {%- endmacro %} diff --git a/macros/upload_invocations.sql b/macros/upload_individual_datasets/upload_invocations.sql similarity index 100% rename from macros/upload_invocations.sql rename to macros/upload_individual_datasets/upload_invocations.sql diff --git a/macros/upload_model_executions.sql b/macros/upload_individual_datasets/upload_model_executions.sql similarity index 97% rename from macros/upload_model_executions.sql rename to macros/upload_individual_datasets/upload_model_executions.sql index a8ebbe89..3ff87cc5 100644 --- a/macros/upload_model_executions.sql +++ b/macros/upload_individual_datasets/upload_model_executions.sql @@ -1,10 +1,4 @@ -{% macro upload_model_executions(results) -%} - {% set models = [] %} - {% for result in results %} - {% if result.node.resource_type == "model" %} - {% do models.append(result) %} - {% endif %} - {% endfor %} +{% macro upload_model_executions(models) -%} {{ return(adapter.dispatch('get_model_executions_dml_sql', 'dbt_artifacts')(models)) }} {%- endmacro %} diff --git a/macros/upload_models.sql b/macros/upload_individual_datasets/upload_models.sql similarity index 100% rename from macros/upload_models.sql rename to macros/upload_individual_datasets/upload_models.sql diff --git a/macros/upload_seed_executions.sql b/macros/upload_individual_datasets/upload_seed_executions.sql similarity index 97% rename from macros/upload_seed_executions.sql rename to macros/upload_individual_datasets/upload_seed_executions.sql index 4be58b09..69c4540c 100644 --- a/macros/upload_seed_executions.sql +++ b/macros/upload_individual_datasets/upload_seed_executions.sql @@ -1,10 +1,4 @@ -{% macro upload_seed_executions(results) -%} - {% set seeds = [] %} - {% for result in results %} - {% if result.node.resource_type == "seed" %} - {% do seeds.append(result) %} - {% endif %} - {% endfor %} +{% macro upload_seed_executions(seeds) -%} {{ return(adapter.dispatch('get_seed_executions_dml_sql', 'dbt_artifacts')(seeds)) }} {%- endmacro %} diff --git a/macros/upload_seeds.sql b/macros/upload_individual_datasets/upload_seeds.sql similarity index 95% rename from macros/upload_seeds.sql rename to macros/upload_individual_datasets/upload_seeds.sql index ea288e21..2f12507e 100644 --- a/macros/upload_seeds.sql +++ b/macros/upload_individual_datasets/upload_seeds.sql @@ -1,8 +1,4 @@ -{% macro upload_seeds(graph) -%} - {% set seeds = [] %} - {% for node in graph.nodes.values() | selectattr("resource_type", "equalto", "seed") %} - {% do seeds.append(node) %} - {% endfor %} +{% macro upload_seeds(seeds) -%} {{ return(adapter.dispatch('get_seeds_dml_sql', 'dbt_artifacts')(seeds)) }} {%- endmacro %} diff --git a/macros/upload_snapshot_executions.sql b/macros/upload_individual_datasets/upload_snapshot_executions.sql similarity index 97% rename from macros/upload_snapshot_executions.sql rename to macros/upload_individual_datasets/upload_snapshot_executions.sql index 3e8c03d0..a9980b18 100644 --- a/macros/upload_snapshot_executions.sql +++ b/macros/upload_individual_datasets/upload_snapshot_executions.sql @@ -1,10 +1,4 @@ -{% macro upload_snapshot_executions(results) -%} - {% set snapshots = [] %} - {% for result in results %} - {% if result.node.resource_type == "snapshot" %} - {% do snapshots.append(result) %} - {% endif %} - {% endfor %} +{% macro upload_snapshot_executions(snapshots) -%} {{ return(adapter.dispatch('get_snapshot_executions_dml_sql', 'dbt_artifacts')(snapshots)) }} {%- endmacro %} diff --git a/macros/upload_snapshots.sql b/macros/upload_individual_datasets/upload_snapshots.sql similarity index 95% rename from macros/upload_snapshots.sql rename to macros/upload_individual_datasets/upload_snapshots.sql index 3c7a7cbf..82789e3a 100644 --- a/macros/upload_snapshots.sql +++ b/macros/upload_individual_datasets/upload_snapshots.sql @@ -1,8 +1,5 @@ -{% macro upload_snapshots(graph) -%} - {% set snapshots = [] %} - {% for node in graph.nodes.values() | selectattr("resource_type", "equalto", "snapshot") %} - {% do snapshots.append(node) %} - {% endfor %} +{% macro upload_snapshots(snapshots) -%} + {{ return(adapter.dispatch('get_snapshots_dml_sql', 'dbt_artifacts')(snapshots)) }} {%- endmacro %} diff --git a/macros/upload_sources.sql b/macros/upload_individual_datasets/upload_sources.sql similarity index 100% rename from macros/upload_sources.sql rename to macros/upload_individual_datasets/upload_sources.sql diff --git a/macros/upload_test_executions.sql b/macros/upload_individual_datasets/upload_test_executions.sql similarity index 96% rename from macros/upload_test_executions.sql rename to macros/upload_individual_datasets/upload_test_executions.sql index a8747099..60986658 100644 --- a/macros/upload_test_executions.sql +++ b/macros/upload_individual_datasets/upload_test_executions.sql @@ -1,10 +1,4 @@ -{% macro upload_test_executions(results) -%} - {% set tests = [] %} - {% for result in results %} - {% if result.node.resource_type == "test" %} - {% do tests.append(result) %} - {% endif %} - {% endfor %} +{% macro upload_test_executions(tests) -%} {{ return(adapter.dispatch('get_test_executions_dml_sql', 'dbt_artifacts')(tests)) }} {%- endmacro %} diff --git a/macros/upload_tests.sql b/macros/upload_individual_datasets/upload_tests.sql similarity index 100% rename from macros/upload_tests.sql rename to macros/upload_individual_datasets/upload_tests.sql diff --git a/macros/upload_results.sql b/macros/upload_results.sql deleted file mode 100644 index dbfb8051..00000000 --- a/macros/upload_results.sql +++ /dev/null @@ -1,169 +0,0 @@ -{# dbt doesn't like us ref'ing in an operation so we fetch the info from the graph #} -{% macro get_relation(get_relation_name) %} - {% if execute %} - {% set model_get_relation_node = graph.nodes.values() | selectattr('name', 'equalto', get_relation_name) | first %} - {% set relation = api.Relation.create( - database = model_get_relation_node.database, - schema = model_get_relation_node.schema, - identifier = model_get_relation_node.alias - ) - %} - {% do return(relation) %} - {% else %} - {% do return(api.Relation.create()) %} - {% endif %} -{% endmacro %} - -{% macro upload_results(results) -%} - - {% if execute %} - - {% if results != [] %} - {% do log("Uploading model executions", true) %} - {% set model_executions = dbt_artifacts.get_relation('model_executions') %} - {% set content_model_executions = dbt_artifacts.upload_model_executions(results) %} - {{ dbt_artifacts.insert_into_metadata_table( - database_name=model_executions.database, - schema_name=model_executions.schema, - table_name=model_executions.identifier, - content=content_model_executions - ) - }} - - {% do log("Uploading seed executions", true) %} - {% set seed_executions = dbt_artifacts.get_relation('seed_executions') %} - {% set content_seed_executions = dbt_artifacts.upload_seed_executions(results) %} - {{ dbt_artifacts.insert_into_metadata_table( - database_name=seed_executions.database, - schema_name=seed_executions.schema, - table_name=seed_executions.identifier, - content=content_seed_executions - ) - }} - - {% do log("Uploading snapshot executions", true) %} - {% set snapshot_executions = dbt_artifacts.get_relation('snapshot_executions') %} - {% set content_snapshot_executions = dbt_artifacts.upload_snapshot_executions(results) %} - {{ dbt_artifacts.insert_into_metadata_table( - database_name=snapshot_executions.database, - schema_name=snapshot_executions.schema, - table_name=snapshot_executions.identifier, - content=content_snapshot_executions - ) - }} - - {% do log("Uploading test executions", true) %} - {% set test_executions = dbt_artifacts.get_relation('test_executions') %} - {% set content_test_executions = dbt_artifacts.upload_test_executions(results) %} - {{ dbt_artifacts.insert_into_metadata_table( - database_name=test_executions.database, - schema_name=test_executions.schema, - table_name=test_executions.identifier, - content=content_test_executions - ) - }} - - {% endif %} - - {% do log("Uploading exposures", true) %} - {% set exposures = dbt_artifacts.get_relation('exposures') %} - {% set content_exposures = dbt_artifacts.upload_exposures(graph) %} - {{ dbt_artifacts.insert_into_metadata_table( - database_name=exposures.database, - schema_name=exposures.schema, - table_name=exposures.identifier, - content=content_exposures - ) - }} - - {% do log("Uploading tests", true) %} - {% set tests = dbt_artifacts.get_relation('tests') %} - {% set tests_set = [] %} - {% for node in graph.nodes.values() | selectattr("resource_type", "equalto", "test") %} - {% do tests_set.append(node) %} - {% endfor %} - {# upload tests in chunks of 5000 tests (300 for BigQuery), or less #} - {% set upload_limit = 300 if target.type == 'bigquery' else 5000 %} - {% for i in range(0, tests_set | length, upload_limit) -%} - {% set content_tests = dbt_artifacts.upload_tests(tests_set[i: i + upload_limit]) %} - {{ dbt_artifacts.insert_into_metadata_table( - database_name=tests.database, - schema_name=tests.schema, - table_name=tests.identifier, - content=content_tests - ) - }} - {%- endfor %} - - {% do log("Uploading seeds", true) %} - {% set seeds = dbt_artifacts.get_relation('seeds') %} - {% set content_seeds = dbt_artifacts.upload_seeds(graph) %} - {{ dbt_artifacts.insert_into_metadata_table( - database_name=seeds.database, - schema_name=seeds.schema, - table_name=seeds.identifier, - content=content_seeds - ) - }} - - {% do log("Uploading models", true) %} - {% set models = dbt_artifacts.get_relation('models') %} - {% set models_set = [] %} - {% for node in graph.nodes.values() | selectattr("resource_type", "equalto", "model") %} - {% do models_set.append(node) %} - {% endfor %} - {% set upload_limit = 50 if target.type == 'bigquery' else 100 %} - {% for i in range(0, models_set | length, upload_limit) -%} - {% set content_models = dbt_artifacts.upload_models(models_set[i: i + upload_limit]) %} - {{ dbt_artifacts.insert_into_metadata_table( - database_name=models.database, - schema_name=models.schema, - table_name=models.identifier, - content=content_models - ) - }} - {%- endfor %} - - {% do log("Uploading sources", true) %} - {% set sources = dbt_artifacts.get_relation('sources') %} - {% set sources_set = [] %} - {% for node in graph.sources.values() %} - {% do sources_set.append(node) %} - {% endfor %} - {# upload sources in chunks of 5000 sources (300 for BigQuery), or less #} - {% set upload_limit = 300 if target.type == 'bigquery' else 5000 %} - {% for i in range(0, sources_set | length, upload_limit) -%} - {% set content_sources = dbt_artifacts.upload_sources(sources_set[i: i + upload_limit]) %} - {{ dbt_artifacts.insert_into_metadata_table( - database_name=sources.database, - schema_name=sources.schema, - table_name=sources.identifier, - content=content_sources - ) - }} - {%- endfor %} - - {% do log("Uploading snapshots", true) %} - {% set snapshots = dbt_artifacts.get_relation('snapshots') %} - {% set content_snapshots = dbt_artifacts.upload_snapshots(graph) %} - {{ dbt_artifacts.insert_into_metadata_table( - database_name=snapshots.database, - schema_name=snapshots.schema, - table_name=snapshots.identifier, - content=content_snapshots - ) - }} - - {% do log("Uploading invocations", true) %} - {% set invocations = dbt_artifacts.get_relation('invocations') %} - {% set content_invocations = dbt_artifacts.upload_invocations() %} - {{ dbt_artifacts.insert_into_metadata_table( - database_name=invocations.database, - schema_name=invocations.schema, - table_name=invocations.identifier, - content=content_invocations - ) - }} - - {% endif %} -{%- endmacro %} diff --git a/macros/upload_results/get_column_name_lists.sql b/macros/upload_results/get_column_name_lists.sql new file mode 100644 index 00000000..ea708467 --- /dev/null +++ b/macros/upload_results/get_column_name_lists.sql @@ -0,0 +1,230 @@ + +{# + These are the column lists used as part of the upload macros - the order here should be the same + as the order in each individual `upload_dataset` macro. +#} + +{% macro get_column_name_list(dataset) -%} + + {% if dataset == 'exposures' %} + + ( + command_invocation_id, + node_id, + run_started_at, + name, + type, + owner, + maturity, + path, + description, + url, + package_name, + depends_on_nodes, + tags, + all_results + ) + + {% elif dataset == 'invocations' %} + + ( + command_invocation_id, + dbt_version, + project_name, + run_started_at, + dbt_command, + full_refresh_flag, + target_profile_name, + target_name, + target_schema, + target_threads, + dbt_cloud_project_id, + dbt_cloud_job_id, + dbt_cloud_run_id, + dbt_cloud_run_reason_category, + dbt_cloud_run_reason, + env_vars, + dbt_vars, + invocation_args, + dbt_custom_envs + ) + + {% elif dataset == 'model_executions' %} + + ( + node_id, + command_invocation_id, + run_started_at, + was_full_refresh, + thread_id, + status, + compile_started_at, + query_completed_at, + total_node_runtime, + rows_affected, + {% if target.type == 'bigquery' %} + bytes_processed, + {% endif %} + materialization, + schema, + name, + alias, + message, + adapter_response + ) + + {% elif dataset == 'models' %} + + ( + node_id, + command_invocation_id, + run_started_at, + database, + schema, + name, + depends_on_nodes, + package_name, + path, + checksum, + materialization, + tags, + meta, + alias, + all_results + ) + + + {% elif dataset == 'seed_executions' %} + + ( + command_invocation_id, + node_id, + run_started_at, + was_full_refresh, + thread_id, + status, + compile_started_at, + query_completed_at, + total_node_runtime, + rows_affected, + materialization, + schema, + name, + alias, + message, + adapter_response + ) + + {% elif dataset == 'seeds' %} + + ( + command_invocation_id, + node_id, + run_started_at, + database, + schema, + name, + package_name, + path, + checksum, + meta, + alias, + all_results + ) + + {% elif dataset == 'snapshot_executions' %} + + ( + command_invocation_id, + node_id, + run_started_at, + was_full_refresh, + thread_id, + status, + compile_started_at, + query_completed_at, + total_node_runtime, + rows_affected, + materialization, + schema, + name, + alias, + message, + adapter_response + ) + + {% elif dataset == 'snapshots' %} + + ( + command_invocation_id, + node_id, + run_started_at, + database, + schema, + name, + depends_on_nodes, + package_name, + path, + checksum, + strategy, + meta, + alias, + all_results + ) + + {% elif dataset == 'sources' %} + + ( + command_invocation_id, + node_id, + run_started_at, + database, + schema, + source_name, + loader, + name, + identifier, + loaded_at_field, + freshness, + all_results + ) + + {% elif dataset == 'test_executions' %} + + ( + command_invocation_id, + node_id, + run_started_at, + was_full_refresh, + thread_id, + status, + compile_started_at, + query_completed_at, + total_node_runtime, + rows_affected, + failures, + message, + adapter_response + ) + + {% elif dataset == 'tests' %} + + ( + command_invocation_id, + node_id, + run_started_at, + name, + depends_on_nodes, + package_name, + test_path, + tags, + all_results + ) + + {% else %} + + /* No column list available */ + + {% endif %} + +{%- endmacro %} diff --git a/macros/upload_results/get_dataset_content.sql b/macros/upload_results/get_dataset_content.sql new file mode 100644 index 00000000..b34e39cd --- /dev/null +++ b/macros/upload_results/get_dataset_content.sql @@ -0,0 +1,22 @@ +{% macro get_dataset_content(dataset) %} + + {% if dataset in ['model_executions', 'seed_executions', 'test_executions', 'snapshot_executions'] %} + {# Executions make use of the results object #} + {% set objects = results | selectattr("node.resource_type", "equalto", dataset.split("_")[0]) | list %} + {% elif dataset in ['seeds', 'snapshots', 'tests', 'models'] %} + {# Use the nodes in the [graph](https://docs.getdbt.com/reference/dbt-jinja-functions/graph) to extract details #} + {% set objects = graph.nodes.values() | selectattr("resource_type", "equalto", dataset[:-1]) | list %} + {% elif dataset in ['exposures', 'sources'] %} + {# Use the [graph](https://docs.getdbt.com/reference/dbt-jinja-functions/graph) to extract details #} + {% set objects = graph.get(dataset).values() | list %} + {% elif dataset == 'invocations' %} + {# + Invocations doesn't need anything input, but we include this so that it will still be picked up + as part of the loop below - the length must be >0 to allow for an upload, hence the empty string + #} + {% set objects = [''] %} + {% endif %} + + {{ return(objects) }} + +{% endmacro %} diff --git a/macros/upload_results/get_table_content_values.sql b/macros/upload_results/get_table_content_values.sql new file mode 100644 index 00000000..277899c5 --- /dev/null +++ b/macros/upload_results/get_table_content_values.sql @@ -0,0 +1,32 @@ +{% macro get_table_content_values(dataset, objects_to_upload) %} + + {# Convert the results to data to be imported #} + + {% if dataset == 'model_executions' %} + {% set content = dbt_artifacts.upload_model_executions(objects_to_upload) %} + {% elif dataset == 'seed_executions' %} + {% set content = dbt_artifacts.upload_seed_executions(objects_to_upload) %} + {% elif dataset == 'test_executions' %} + {% set content = dbt_artifacts.upload_test_executions(objects_to_upload) %} + {% elif dataset == 'snapshot_executions' %} + {% set content = dbt_artifacts.upload_snapshot_executions(objects_to_upload) %} + {% elif dataset == 'exposures' %} + {% set content = dbt_artifacts.upload_exposures(objects_to_upload) %} + {% elif dataset == 'models' %} + {% set content = dbt_artifacts.upload_models(objects_to_upload) %} + {% elif dataset == 'seeds' %} + {% set content = dbt_artifacts.upload_seeds(objects_to_upload) %} + {% elif dataset == 'snapshots' %} + {% set content = dbt_artifacts.upload_snapshots(objects_to_upload) %} + {% elif dataset == 'sources' %} + {% set content = dbt_artifacts.upload_sources(objects_to_upload) %} + {% elif dataset == 'tests' %} + {% set content = dbt_artifacts.upload_tests(objects_to_upload) %} + {# Invocations only requires data from variables available in the macro #} + {% elif dataset == 'invocations' %} + {% set content = dbt_artifacts.upload_invocations() %} + {% endif %} + + {{ return(content) }} + +{% endmacro %} diff --git a/macros/upload_results/insert_into_metadata_table.sql b/macros/upload_results/insert_into_metadata_table.sql new file mode 100644 index 00000000..2ec4d5d5 --- /dev/null +++ b/macros/upload_results/insert_into_metadata_table.sql @@ -0,0 +1,49 @@ +{% macro insert_into_metadata_table(dataset, fields, content) -%} + + {% if content != "" %} + + {# Get the relation that the results will be uploaded to #} + {% set dataset_relation = dbt_artifacts.get_relation(dataset) %} + {# Insert the data into the table #} + {{ return(adapter.dispatch('insert_into_metadata_table', 'dbt_artifacts')(dataset_relation, fields, content)) }} + + {% endif %} + +{%- endmacro %} + +{% macro spark__insert_into_metadata_table(relation, fields, content) -%} + + {% set insert_into_table_query %} + insert into {{ relation }} {{ fields }} + {{ content }} + {% endset %} + + {% do run_query(insert_into_table_query) %} + +{%- endmacro %} + +{% macro snowflake__insert_into_metadata_table(relation, fields, content) -%} + + {% set insert_into_table_query %} + insert into {{ relation }} {{ fields }} + {{ content }} + {% endset %} + + {% do run_query(insert_into_table_query) %} + +{%- endmacro %} + +{% macro bigquery__insert_into_metadata_table(relation, fields, content) -%} + + {% set insert_into_table_query %} + insert into {{ relation }} {{ fields }} + values + {{ content }} + {% endset %} + + {% do run_query(insert_into_table_query) %} + +{%- endmacro %} + +{% macro default__insert_into_metadata_table(relation, fields, content) -%} +{%- endmacro %} diff --git a/macros/upload_results/upload_results.sql b/macros/upload_results/upload_results.sql new file mode 100644 index 00000000..fcadc199 --- /dev/null +++ b/macros/upload_results/upload_results.sql @@ -0,0 +1,50 @@ +{# dbt doesn't like us ref'ing in an operation so we fetch the info from the graph #} + +{% macro upload_results(results) -%} + + {% if execute %} + + {% set datasets_to_load = ['exposures', 'seeds', 'snapshots', 'invocations', 'sources', 'tests', 'models'] %} + {% if results != [] %} + {# When executing, and results are available, then upload the results #} + {% set datasets_to_load = ['model_executions', 'seed_executions', 'test_executions', 'snapshot_executions'] + datasets_to_load %} + {% endif %} + + {# Upload each data set in turn #} + {% for dataset in datasets_to_load %} + + {% do log("Uploading " ~ dataset.replace("_", " "), true) %} + + {# Get the results that need to be uploaded #} + {% set objects = dbt_artifacts.get_dataset_content(dataset) %} + + {# Upload in chunks to reduce query size #} + {% if dataset == 'model' %} + {% set upload_limit = 50 if target.type == 'bigquery' else 100 %} + {% else %} + {% set upload_limit = 300 if target.type == 'bigquery' else 5000 %} + {% endif %} + + {# Loop through each chunk in turn #} + {% for i in range(0, objects | length, upload_limit) -%} + + {# Get just the objects to load on this loop #} + {% set content = dbt_artifacts.get_table_content_values(dataset, objects[i: i + upload_limit]) %} + + {# Insert the content into the metadata table #} + {{ dbt_artifacts.insert_into_metadata_table( + dataset=dataset, + fields=dbt_artifacts.get_column_name_list(dataset), + content=content + ) + }} + + {# Loop the next 'chunk' #} + {% endfor %} + + {# Loop the next 'dataset' #} + {% endfor %} + + {% endif %} + +{%- endmacro %} diff --git a/tox.ini b/tox.ini index 2070a7eb..fbff0176 100644 --- a/tox.ini +++ b/tox.ini @@ -166,7 +166,7 @@ deps = dbt-databricks~=1.6.0 commands = dbt clean dbt deps - dbt --debug build --target databricks + dbt build --target databricks [testenv:integration_databricks_1_3_0] changedir = integration_test_project From 3bde04677b9bf8d41b1c9c63bf05d670fc2130b4 Mon Sep 17 00:00:00 2001 From: Gemma Down <52132406+glsdown@users.noreply.github.com> Date: Thu, 21 Sep 2023 09:34:09 +0100 Subject: [PATCH 18/31] Fix bug with calculation of last_run stats for incremental models (#384) --- models/dim_dbt__current_models.sql | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/models/dim_dbt__current_models.sql b/models/dim_dbt__current_models.sql index c78a063b..4d7a3c46 100644 --- a/models/dim_dbt__current_models.sql +++ b/models/dim_dbt__current_models.sql @@ -27,10 +27,16 @@ latest_models_runs as ( {% if target.type == 'bigquery' %} , model_executions.bytes_processed {% endif %} + /* Row number by refresh and node ID */ , row_number() over ( partition by latest_models.node_id, model_executions.was_full_refresh order by model_executions.query_completed_at desc /* most recent ranked first */ ) as run_idx + /* Row number by node ID */ + , row_number() over ( + partition by latest_models.node_id + order by model_executions.query_completed_at desc /* most recent ranked first */ + ) as run_idx_id_only from model_executions inner join latest_models on model_executions.node_id = latest_models.node_id where model_executions.status = 'success' @@ -45,11 +51,17 @@ latest_model_stats as ( {% if target.type == 'bigquery' %} , max(case when was_full_refresh then bytes_processed end) as last_full_refresh_run_bytes_processed {% endif %} - , max(query_completed_at) as last_run_completed_at - , max(total_node_runtime) as last_run_total_runtime - , max(rows_affected) as last_run_rows_affected + , max(case when run_idx_id_only = 1 then query_completed_at end) as last_run_completed_at + , max(case when run_idx_id_only = 1 then total_node_runtime end) as last_run_total_runtime + , max(case when run_idx_id_only = 1 then rows_affected end) as last_run_rows_affected + {% if target.type == 'bigquery' %} + , max(case when run_idx_id_only = 1 then bytes_processed end) as last_run_bytes_processed + {% endif %} + , max(case when not was_full_refresh then query_completed_at end) as last_incremental_run_completed_at + , max(case when not was_full_refresh then total_node_runtime end) as last_incremental_run_total_runtime + , max(case when not was_full_refresh then rows_affected end) as last_incremental_run_rows_affected {% if target.type == 'bigquery' %} - , max(bytes_processed) as last_run_bytes_processed + , max(case when not was_full_refresh then bytes_processed end) as last_incremental_run_bytes_processed {% endif %} from latest_models_runs where run_idx = 1 @@ -71,6 +83,12 @@ final as ( {% if target.type == 'bigquery' %} , latest_model_stats.last_run_bytes_processed {% endif %} + , latest_model_stats.last_incremental_run_completed_at + , latest_model_stats.last_incremental_run_total_runtime + , latest_model_stats.last_incremental_run_rows_affected + {% if target.type == 'bigquery' %} + , latest_model_stats.last_incremental_run_bytes_processed + {% endif %} from latest_models left join latest_model_stats on latest_models.node_id = latest_model_stats.node_id From 3cf1473a55748e9537ff6c65cc3e58e0a54a9ccd Mon Sep 17 00:00:00 2001 From: Jono <89880566+kenpokiwi@users.noreply.github.com> Date: Fri, 29 Sep 2023 02:27:29 +1300 Subject: [PATCH 19/31] Fix missing value in upload model executions - add else clause to for if loops (#386) * add else clause to for if loops * Refactor logic for timestamp values Refactor logic to evaluate compile_started_at and query_completed_at values. * Refactor logic for timestamp values for all macros * Update macros/upload_individual_datasets/upload_model_executions.sql * Fix issue with `.get` usage --------- Co-authored-by: Jonathan Sellar Co-authored-by: Gemma Down <52132406+glsdown@users.noreply.github.com> Co-authored-by: Gemma Down --- .../upload_model_executions.sql | 72 ++++--------------- 1 file changed, 12 insertions(+), 60 deletions(-) diff --git a/macros/upload_individual_datasets/upload_model_executions.sql b/macros/upload_individual_datasets/upload_model_executions.sql index 3ff87cc5..fca386b2 100644 --- a/macros/upload_individual_datasets/upload_model_executions.sql +++ b/macros/upload_individual_datasets/upload_model_executions.sql @@ -39,26 +39,10 @@ '{{ model.thread_id }}', {# thread_id #} '{{ model.status }}', {# status #} - {% if model.timing != [] %} - {% for stage in model.timing if stage.name == "compile" %} - {% if loop.length == 0 %} - null, {# compile_started_at #} - {% else %} - '{{ stage.started_at }}', {# compile_started_at #} - {% endif %} - {% endfor %} - - {% for stage in model.timing if stage.name == "execute" %} - {% if loop.length == 0 %} - null, {# query_completed_at #} - {% else %} - '{{ stage.completed_at }}', {# query_completed_at #} - {% endif %} - {% endfor %} - {% else %} - null, {# compile_started_at #} - null, {# query_completed_at #} - {% endif %} + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} {{ model.execution_time }}, {# total_node_runtime #} null, -- rows_affected not available {# Only available in Snowflake & BigQuery #} @@ -96,26 +80,10 @@ '{{ model.thread_id }}', {# thread_id #} '{{ model.status }}', {# status #} - {% if model.timing != [] %} - {% for stage in model.timing if stage.name == "compile" %} - {% if loop.length == 0 %} - null, {# compile_started_at #} - {% else %} - '{{ stage.started_at }}', {# compile_started_at #} - {% endif %} - {% endfor %} - - {% for stage in model.timing if stage.name == "execute" %} - {% if loop.length == 0 %} - null, {# query_completed_at #} - {% else %} - '{{ stage.completed_at }}', {# query_completed_at #} - {% endif %} - {% endfor %} - {% else %} - null, {# compile_started_at #} - null, {# query_completed_at #} - {% endif %} + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} {{ model.execution_time }}, {# total_node_runtime #} safe_cast('{{ model.adapter_response.rows_affected }}' as int64), @@ -172,26 +140,10 @@ '{{ model.thread_id }}', {# thread_id #} '{{ model.status }}', {# status #} - {% if model.timing != [] %} - {% for stage in model.timing if stage.name == "compile" %} - {% if loop.length == 0 %} - null, {# compile_started_at #} - {% else %} - '{{ stage.started_at }}', {# compile_started_at #} - {% endif %} - {% endfor %} - - {% for stage in model.timing if stage.name == "execute" %} - {% if loop.length == 0 %} - null, {# query_completed_at #} - {% else %} - '{{ stage.completed_at }}', {# query_completed_at #} - {% endif %} - {% endfor %} - {% else %} - null, {# compile_started_at #} - null, {# query_completed_at #} - {% endif %} + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} {{ model.execution_time }}, {# total_node_runtime #} try_cast('{{ model.adapter_response.rows_affected }}' as int), {# rows_affected #} From af2ed3136d79e1794c438abb6843672ded391a51 Mon Sep 17 00:00:00 2001 From: Gemma Down <52132406+glsdown@users.noreply.github.com> Date: Thu, 28 Sep 2023 14:42:20 +0100 Subject: [PATCH 20/31] Update version number (#387) --- README.md | 2 +- dbt_project.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a63bba3c..3f0818f7 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ See the generated [dbt docs site](https://brooklyn-data.github.io/dbt_artifacts/ ``` packages: - package: brooklyn-data/dbt_artifacts - version: 2.5.0 + version: 2.6.0 ``` :construction_worker: Make sure to fix at least the **minor** version, to avoid issues when a new release is open. See the notes on upgrading below for more detail. diff --git a/dbt_project.yml b/dbt_project.yml index 83a4ccad..4a031fa9 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: "dbt_artifacts" -version: "2.5.0" +version: "2.6.0" config-version: 2 require-dbt-version: [">=1.3.0", "<1.7.0"] profile: "dbt_artifacts" From 069d71ce1d6a041e201ce11b349ed38f34fe1cdd Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Fri, 29 Sep 2023 08:37:26 -0600 Subject: [PATCH 21/31] Add postgres support (#368) * Add macros for postgres * Add steps for postgres * Add postgres entry * Add a postgres service to action * Just test postgres for now * Fix duplicate section * Install correct package * Update host to localhost * Update profiles.yml to env vars * Add postgres env vars to step * Try again * Remove parse_json func * Another fix * Another fix * Alias the subquery * Add back other CDW [skip-ci] * Add postgres as supported DW [skip ci] * Test removing env vars * Another test * Add back DWs * Add missing CI tests * Formatting * Update to include new features --------- Co-authored-by: Gemma Down --- .github/workflows/ci_test_package.yml | 30 ++++++++- .github/workflows/main_test_package.yml | 15 ++++- README.md | 1 + integration_test_project/profiles.yml | 13 +++- .../tests/singular_test.sql | 2 +- .../upload_exposures.sql | 34 ++++++++++ .../upload_invocations.sql | 62 +++++++++++++++++++ .../upload_model_executions.sql | 41 ++++++++++++ .../upload_models.sql | 35 +++++++++++ .../upload_seed_executions.sql | 57 +++++++++++++++++ .../upload_seeds.sql | 31 ++++++++++ .../upload_snapshot_executions.sql | 57 +++++++++++++++++ .../upload_snapshots.sql | 33 ++++++++++ .../upload_sources.sql | 31 ++++++++++ .../upload_test_executions.sql | 55 ++++++++++++++++ .../upload_tests.sql | 28 +++++++++ .../insert_into_metadata_table.sql | 14 +++++ tox.ini | 40 ++++++++++++ 18 files changed, 573 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_test_package.yml b/.github/workflows/ci_test_package.yml index edd8cdc9..b02997ee 100644 --- a/.github/workflows/ci_test_package.yml +++ b/.github/workflows/ci_test_package.yml @@ -37,7 +37,7 @@ jobs: strategy: fail-fast: false # Don't fail one DWH if the others fail matrix: - warehouse: ["snowflake", "bigquery"] + warehouse: ["snowflake", "bigquery", "postgres"] runs-on: ubuntu-latest environment: name: Approve Integration Tests @@ -45,6 +45,19 @@ jobs: contents: "read" id-token: "write" + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: - name: Get latest release uses: rez0n/actions-github-release@main @@ -100,7 +113,7 @@ jobs: strategy: fail-fast: false # Don't fail one DWH if the others fail matrix: - warehouse: ["snowflake", "bigquery"] + warehouse: ["snowflake", "bigquery", "postgres"] # When supporting a new version, update the list here version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0"] runs-on: ubuntu-latest @@ -110,6 +123,19 @@ jobs: contents: "read" id-token: "write" + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: - uses: actions/setup-python@v4 with: diff --git a/.github/workflows/main_test_package.yml b/.github/workflows/main_test_package.yml index 9aa70ae3..4398c459 100644 --- a/.github/workflows/main_test_package.yml +++ b/.github/workflows/main_test_package.yml @@ -34,13 +34,26 @@ jobs: integration: strategy: matrix: - warehouse: ["snowflake", "bigquery"] + warehouse: ["snowflake", "bigquery", "postgres"] version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0"] runs-on: ubuntu-latest permissions: contents: "read" id-token: "write" + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: - name: Checkout uses: actions/checkout@v3 diff --git a/README.md b/README.md index 3f0818f7..17688961 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ The package currently supports - Spark :white_check_mark: - Snowflake :white_check_mark: - Google BigQuery :white_check_mark: +- Postgres :white_check_mark: Models included: diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml index e8548e3f..b24ad80d 100644 --- a/integration_test_project/profiles.yml +++ b/integration_test_project/profiles.yml @@ -2,8 +2,8 @@ # You should __NEVER__ check credentials into version control. Thanks for reading :) config: - send_anonymous_usage_stats: False - use_colors: True + send_anonymous_usage_stats: False + use_colors: True dbt_artifacts: target: snowflake @@ -43,3 +43,12 @@ dbt_artifacts: timeout_seconds: 300 priority: interactive retries: 1 + postgres: + type: postgres + host: localhost + user: postgres + password: postgres + port: 5432 + dbname: postgres + schema: public + threads: 8 diff --git a/integration_test_project/tests/singular_test.sql b/integration_test_project/tests/singular_test.sql index 67a2508d..a1e49e8c 100644 --- a/integration_test_project/tests/singular_test.sql +++ b/integration_test_project/tests/singular_test.sql @@ -1 +1 @@ -select 1 as failures from (select 2) where 1 = 2 +select 1 as failures from (select 2) as foo where 1 = 2 diff --git a/macros/upload_individual_datasets/upload_exposures.sql b/macros/upload_individual_datasets/upload_exposures.sql index 488f9ffd..9f0ec5d3 100644 --- a/macros/upload_individual_datasets/upload_exposures.sql +++ b/macros/upload_individual_datasets/upload_exposures.sql @@ -84,3 +84,37 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro postgres__get_exposures_dml_sql(exposures) -%} + {% if exposures != [] %} + + {% set exposure_values %} + {% for exposure in exposures -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + $${{ exposure.unique_id }}$$, {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + $${{ exposure.name }}$$, {# name #} + '{{ exposure.type }}', {# type #} + $${{ tojson(exposure.owner) }}$$, {# owner #} + '{{ exposure.maturity }}', {# maturity #} + $${{ exposure.original_file_path }}$$, {# path #} + $${{ exposure.description }}$$, {# description #} + '{{ exposure.url }}', {# url #} + '{{ exposure.package_name }}', {# package_name #} + $${{ tojson(exposure.depends_on.nodes) }}$$, {# depends_on_nodes #} + $${{ tojson(exposure.tags) }}$$, {# tags #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + $${{ tojson(exposure) }}$$ {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ exposure_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_individual_datasets/upload_invocations.sql b/macros/upload_individual_datasets/upload_invocations.sql index f3dcdd6b..21c5574c 100644 --- a/macros/upload_individual_datasets/upload_invocations.sql +++ b/macros/upload_individual_datasets/upload_invocations.sql @@ -159,3 +159,65 @@ {{ invocation_values }} {% endmacro -%} + +{% macro postgres__get_invocations_dml_sql() -%} + {% set invocation_values %} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ dbt_version }}', {# dbt_version #} + '{{ project_name }}', {# project_name #} + '{{ run_started_at }}', {# run_started_at #} + '{{ flags.WHICH }}', {# dbt_command #} + {{ flags.FULL_REFRESH }}, {# full_refresh_flag #} + '{{ target.profile_name }}', {# target_profile_name #} + '{{ target.name }}', {# target_name #} + '{{ target.schema }}', {# target_schema #} + {{ target.threads }}, {# target_threads #} + + '{{ env_var("DBT_CLOUD_PROJECT_ID", "") }}', {# dbt_cloud_project_id #} + '{{ env_var("DBT_CLOUD_JOB_ID", "") }}', {# dbt_cloud_job_id #} + '{{ env_var("DBT_CLOUD_RUN_ID", "") }}', {# dbt_cloud_run_id #} + '{{ env_var("DBT_CLOUD_RUN_REASON_CATEGORY", "") }}', {# dbt_cloud_run_reason_category #} + $${{ env_var('DBT_CLOUD_RUN_REASON', '') }}$$, {# dbt_cloud_run_reason #} + + {% if var('env_vars', none) %} + {% set env_vars_dict = {} %} + {% for env_variable in var('env_vars') %} + {% do env_vars_dict.update({env_variable: (env_var(env_variable, ''))}) %} + {% endfor %} + $${{ tojson(env_vars_dict) }}$$, {# env_vars #} + {% else %} + null, {# env_vars #} + {% endif %} + + {% if var('dbt_vars', none) %} + {% set dbt_vars_dict = {} %} + {% for dbt_var in var('dbt_vars') %} + {% do dbt_vars_dict.update({dbt_var: (var(dbt_var, ''))}) %} + {% endfor %} + $${{ tojson(dbt_vars_dict) }}$$, {# dbt_vars #} + {% else %} + null, {# dbt_vars #} + {% endif %} + + {% if invocation_args_dict.vars %} + {# vars - different format for pre v1.5 (yaml vs list) #} + {% if invocation_args_dict.vars is string %} + {# BigQuery does not handle the yaml-string from "--vars" well, when passed to "parse_json". Workaround is to parse the string, and then "tojson" will properly format the dict as a json-object. #} + {% set parsed_inv_args_vars = fromyaml(invocation_args_dict.vars) %} + {% do invocation_args_dict.update({'vars': parsed_inv_args_vars}) %} + {% endif %} + {% endif %} + + $${{ tojson(invocation_args_dict) }}$$, {# invocation_args #} + + {% set metadata_env = {} %} + {% for key, value in dbt_metadata_envs.items() %} + {% do metadata_env.update({key: value}) %} + {% endfor %} + $${{ tojson(metadata_env) }}$$ {# dbt_custom_envs #} + ) + {% endset %} + {{ invocation_values }} + +{% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_model_executions.sql b/macros/upload_individual_datasets/upload_model_executions.sql index fca386b2..bca26fea 100644 --- a/macros/upload_individual_datasets/upload_model_executions.sql +++ b/macros/upload_individual_datasets/upload_model_executions.sql @@ -162,3 +162,44 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro postgres__get_model_executions_dml_sql(models) -%} + {% if models != [] %} + {% set model_execution_values %} + {% for model in models -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + {{ config_full_refresh }}, {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} + + {{ model.execution_time }}, {# total_node_runtime #} + null, {# rows_affected #} + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + $${{ model.message }}$$, {# message #} + $${{ tojson(model.adapter_response) }}$$ {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ model_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_individual_datasets/upload_models.sql b/macros/upload_individual_datasets/upload_models.sql index f0748876..d8dc49fd 100644 --- a/macros/upload_individual_datasets/upload_models.sql +++ b/macros/upload_individual_datasets/upload_models.sql @@ -89,3 +89,38 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro postgres__get_models_dml_sql(models) -%} + {% if models != [] %} + {% set model_values %} + {% for model in models -%} + {% do model.pop('raw_code', None) %} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ model.database }}', {# database #} + '{{ model.schema }}', {# schema #} + '{{ model.name }}', {# name #} + '{{ tojson(model.depends_on.nodes) }}', {# depends_on_nodes #} + '{{ model.package_name }}', {# package_name #} + $${{ model.original_file_path | replace('\\', '\\\\') }}$$, {# path #} + '{{ model.checksum.checksum }}', {# checksum #} + '{{ model.config.materialized }}', {# materialization #} + '{{ tojson(model.tags) }}', {# tags #} + $${{ model.config.meta }}$$, {# meta #} + '{{ model.alias }}', {# alias #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + $${{ tojson(model) }}$$ {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ model_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_individual_datasets/upload_seed_executions.sql b/macros/upload_individual_datasets/upload_seed_executions.sql index 69c4540c..ca947ac2 100644 --- a/macros/upload_individual_datasets/upload_seed_executions.sql +++ b/macros/upload_individual_datasets/upload_seed_executions.sql @@ -208,3 +208,60 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro postgres__get_seed_executions_dml_sql(seeds) -%} + {% if seeds != [] %} + {% set seed_execution_values %} + {% for model in seeds -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + {{ config_full_refresh }}, {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% if model.timing != [] %} + {% for stage in model.timing if stage.name == "compile" %} + {% if loop.length == 0 %} + null, {# compile_started_at #} + {% else %} + '{{ stage.started_at }}', {# compile_started_at #} + {% endif %} + {% endfor %} + + {% for stage in model.timing if stage.name == "execute" %} + {% if loop.length == 0 %} + null, {# query_completed_at #} + {% else %} + '{{ stage.completed_at }}', {# query_completed_at #} + {% endif %} + {% endfor %} + {% else %} + null, {# compile_started_at #} + null, {# query_completed_at #} + {% endif %} + + {{ model.execution_time }}, {# total_node_runtime #} + null, -- rows_affected not available {# Databricks #} + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + $${{ model.message }}$$, {# message #} + $${{ tojson(model.adapter_response) }}$$ {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ seed_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_seeds.sql b/macros/upload_individual_datasets/upload_seeds.sql index 2f12507e..32e67383 100644 --- a/macros/upload_individual_datasets/upload_seeds.sql +++ b/macros/upload_individual_datasets/upload_seeds.sql @@ -78,3 +78,34 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro postgres__get_seeds_dml_sql(seeds) -%} + {% if seeds != [] %} + {% set seed_values %} + {% for seed in seeds -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ seed.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ seed.database }}', {# database #} + '{{ seed.schema }}', {# schema #} + '{{ seed.name }}', {# name #} + '{{ seed.package_name }}', {# package_name #} + '{{ seed.original_file_path | replace('\\', '\\\\') }}', {# path #} + '{{ seed.checksum.checksum }}', {# checksum #} + $${{ tojson(seed.config.meta) }}$$, {# meta #} + '{{ seed.alias }}', {# alias #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + $${{ tojson(seed) }}$$ {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ seed_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_individual_datasets/upload_snapshot_executions.sql b/macros/upload_individual_datasets/upload_snapshot_executions.sql index a9980b18..369348cd 100644 --- a/macros/upload_individual_datasets/upload_snapshot_executions.sql +++ b/macros/upload_individual_datasets/upload_snapshot_executions.sql @@ -208,3 +208,60 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro postgres__get_snapshot_executions_dml_sql(snapshots) -%} + {% if snapshots != [] %} + {% set snapshot_execution_values %} + {% for model in snapshots -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + {{ config_full_refresh }}, {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% if model.timing != [] %} + {% for stage in model.timing if stage.name == "compile" %} + {% if loop.length == 0 %} + null, {# compile_started_at #} + {% else %} + '{{ stage.started_at }}', {# compile_started_at #} + {% endif %} + {% endfor %} + + {% for stage in model.timing if stage.name == "execute" %} + {% if loop.length == 0 %} + null, {# query_completed_at #} + {% else %} + '{{ stage.completed_at }}', {# query_completed_at #} + {% endif %} + {% endfor %} + {% else %} + null, {# compile_started_at #} + null, {# query_completed_at #} + {% endif %} + + {{ model.execution_time }}, {# total_node_runtime #} + null, {# rows_affected #} + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + $${{ model.message }}$$, {# message #} + $${{ tojson(model.adapter_response) }}$$ {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ snapshot_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_snapshots.sql b/macros/upload_individual_datasets/upload_snapshots.sql index 82789e3a..0be6759b 100644 --- a/macros/upload_individual_datasets/upload_snapshots.sql +++ b/macros/upload_individual_datasets/upload_snapshots.sql @@ -86,3 +86,36 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro postgres__get_snapshots_dml_sql(snapshots) -%} + {% if snapshots != [] %} + {% set snapshot_values %} + {% for snapshot in snapshots -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ snapshot.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ snapshot.database }}', {# database #} + '{{ snapshot.schema }}', {# schema #} + '{{ snapshot.name }}', {# name #} + $${{ tojson(snapshot.depends_on.nodes) }}$$, {# depends_on_nodes #} + '{{ snapshot.package_name }}', {# package_name #} + '{{ snapshot.original_file_path | replace('\\', '\\\\') }}', {# path #} + '{{ snapshot.checksum.checksum }}', {# checksum #} + '{{ snapshot.config.strategy }}', {# strategy #} + $${{ tojson(snapshot.config.meta) }}$$, {# meta #} + '{{ snapshot.alias }}', {# alias #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + $${{ tojson(snapshot) }}$$ {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ snapshot_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_individual_datasets/upload_sources.sql b/macros/upload_individual_datasets/upload_sources.sql index 23208d6d..dbcb49e4 100644 --- a/macros/upload_individual_datasets/upload_sources.sql +++ b/macros/upload_individual_datasets/upload_sources.sql @@ -78,3 +78,34 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro postgres__get_sources_dml_sql(sources) -%} + {% if sources != [] %} + {% set source_values %} + {% for source in sources -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ source.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ source.database }}', {# database #} + '{{ source.schema }}', {# schema #} + '{{ source.source_name }}', {# source_name #} + '{{ source.loader }}', {# loader #} + '{{ source.name }}', {# name #} + '{{ source.identifier }}', {# identifier #} + $${{ source.loaded_at_field }}$$, {# loaded_at_field #} + $${{ tojson(source.freshness) }}$$, {# freshness #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + $${{ tojson(source) }}$$ {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ source_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_individual_datasets/upload_test_executions.sql b/macros/upload_individual_datasets/upload_test_executions.sql index 60986658..a42b03ba 100644 --- a/macros/upload_individual_datasets/upload_test_executions.sql +++ b/macros/upload_individual_datasets/upload_test_executions.sql @@ -125,3 +125,58 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro postgres__get_test_executions_dml_sql(tests) -%} + {% if tests != [] %} + {% set test_execution_values %} + {% for test in tests -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ test.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = test.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + {{ config_full_refresh }}, {# was_full_refresh #} + + '{{ test.thread_id }}', {# thread_id #} + '{{ test.status }}', {# status #} + + {% if test.timing != [] %} + {% for stage in test.timing if stage.name == "compile" %} + {% if loop.length == 0 %} + null, {# compile_started_at #} + {% else %} + '{{ stage.started_at }}', {# compile_started_at #} + {% endif %} + {% endfor %} + + {% for stage in test.timing if stage.name == "execute" %} + {% if loop.length == 0 %} + null, {# query_completed_at #} + {% else %} + '{{ stage.completed_at }}', {# query_completed_at #} + {% endif %} + {% endfor %} + {% else %} + null, {# compile_started_at #} + null, {# query_completed_at #} + {% endif %} + + {{ test.execution_time }}, {# total_node_runtime #} + null, {# rows_affected not available in Databricks #} + {{ 'null' if test.failures is none else test.failures }}, {# failures #} + $${{ test.message }}$$, {# message #} + $${{ tojson(test.adapter_response) }}$$ {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + + {%- endfor %} + {% endset %} + {{ test_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_tests.sql b/macros/upload_individual_datasets/upload_tests.sql index d29341ff..cf75e9ba 100644 --- a/macros/upload_individual_datasets/upload_tests.sql +++ b/macros/upload_individual_datasets/upload_tests.sql @@ -69,3 +69,31 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro postgres__get_tests_dml_sql(tests) -%} + {% if tests != [] %} + {% set test_values %} + {% for test in tests -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ test.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ test.name }}', {# name #} + $${{ tojson(test.depends_on.nodes) }}$$, {# depends_on_nodes #} + '{{ test.package_name }}', {# package_name #} + '{{ test.original_file_path | replace('\\', '\\\\') }}', {# test_path #} + $${{ tojson(test.tags) }}$$, {# tags #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + $${{ tojson(test) }}$$ {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ test_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_results/insert_into_metadata_table.sql b/macros/upload_results/insert_into_metadata_table.sql index 2ec4d5d5..24f1eb77 100644 --- a/macros/upload_results/insert_into_metadata_table.sql +++ b/macros/upload_results/insert_into_metadata_table.sql @@ -45,5 +45,19 @@ {%- endmacro %} +{%- endmacro %} + +{% macro postgres__insert_into_metadata_table(relation, fields, content) -%} + + {% set insert_into_table_query %} + insert into {{ relation }} {{ fields }} + values + {{ content }} + {% endset %} + + {% do run_query(insert_into_table_query) %} + +{%- endmacro %} + {% macro default__insert_into_metadata_table(relation, fields, content) -%} {%- endmacro %} diff --git a/tox.ini b/tox.ini index fbff0176..8add76ba 100644 --- a/tox.ini +++ b/tox.ini @@ -250,3 +250,43 @@ commands = dbt deps dbt build --exclude snapshot --target spark +[testenv:integration_postgres] +changedir = integration_test_project +deps = dbt-postgres~=1.6.0 +commands = + dbt clean + dbt deps + dbt build --target postgres + +[testenv:integration_postgres_1_3_0] +changedir = integration_test_project +deps = dbt-postgres~=1.3.0 +commands = + dbt clean + dbt deps + dbt build --target postgres + +[testenv:integration_postgres_1_4_0] +changedir = integration_test_project +deps = dbt-postgres~=1.4.0 +commands = + dbt clean + dbt deps + dbt build --target postgres + +[testenv:integration_postgres_1_5_0] +changedir = integration_test_project +deps = dbt-postgres~=1.5.0 +commands = + dbt clean + dbt deps + dbt build --target postgres + +[testenv:integration_postgres_1_6_0] +changedir = integration_test_project +deps = dbt-postgres~=1.6.0 +commands = + dbt clean + dbt deps + dbt build --target postgres + From 039004a6572d862e44ca51ece6f406dcd4a535e8 Mon Sep 17 00:00:00 2001 From: Dat Nguyen <103571964+il-dat@users.noreply.github.com> Date: Tue, 3 Oct 2023 15:18:22 +0700 Subject: [PATCH 22/31] Update get_column_name_lists.sql (#393) --- macros/upload_results/get_column_name_lists.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/macros/upload_results/get_column_name_lists.sql b/macros/upload_results/get_column_name_lists.sql index ea708467..7911e866 100644 --- a/macros/upload_results/get_column_name_lists.sql +++ b/macros/upload_results/get_column_name_lists.sql @@ -52,8 +52,8 @@ {% elif dataset == 'model_executions' %} ( - node_id, command_invocation_id, + node_id, run_started_at, was_full_refresh, thread_id, @@ -76,8 +76,8 @@ {% elif dataset == 'models' %} ( - node_id, command_invocation_id, + node_id, run_started_at, database, schema, From 5d2df3fe2a3877f693594d46bcb4f7bd126575b4 Mon Sep 17 00:00:00 2001 From: Gemma Down <52132406+glsdown@users.noreply.github.com> Date: Thu, 5 Oct 2023 12:41:09 +0100 Subject: [PATCH 23/31] Tidy up execution timings logic (#389) * Minor tidy up * Simplifying timing extract --- integration_test_project/example-env.sh | 1 + .../upload_seed_executions.sql | 72 ++++--------------- .../upload_snapshot_executions.sql | 72 ++++--------------- .../upload_test_executions.sql | 48 +++---------- 4 files changed, 33 insertions(+), 160 deletions(-) diff --git a/integration_test_project/example-env.sh b/integration_test_project/example-env.sh index 51450cf3..47cb0d67 100755 --- a/integration_test_project/example-env.sh +++ b/integration_test_project/example-env.sh @@ -18,6 +18,7 @@ export DBT_ENV_SPARK_DRIVER_PATH= # /Library/simba/spark/lib/libsparkodbc_sbu.dy export DBT_ENV_SPARK_ENDPOINT= # The endpoint ID from the Databricks HTTP path # dbt environment variables, change these +export DBT_VERSION="1_5_0" export DBT_CLOUD_PROJECT_ID= export DBT_CLOUD_JOB_ID= export DBT_CLOUD_RUN_ID= diff --git a/macros/upload_individual_datasets/upload_seed_executions.sql b/macros/upload_individual_datasets/upload_seed_executions.sql index ca947ac2..1ccbfe2a 100644 --- a/macros/upload_individual_datasets/upload_seed_executions.sql +++ b/macros/upload_individual_datasets/upload_seed_executions.sql @@ -38,26 +38,10 @@ '{{ model.thread_id }}', {# thread_id #} '{{ model.status }}', {# status #} - {% if model.timing != [] %} - {% for stage in model.timing if stage.name == "compile" %} - {% if loop.length == 0 %} - null, {# compile_started_at #} - {% else %} - '{{ stage.started_at }}', {# compile_started_at #} - {% endif %} - {% endfor %} - - {% for stage in model.timing if stage.name == "execute" %} - {% if loop.length == 0 %} - null, {# query_completed_at #} - {% else %} - '{{ stage.completed_at }}', {# query_completed_at #} - {% endif %} - {% endfor %} - {% else %} - null, {# compile_started_at #} - null, {# query_completed_at #} - {% endif %} + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} {{ model.execution_time }}, {# total_node_runtime #} null, -- rows_affected not available {# Only available in Snowflake #} @@ -95,26 +79,10 @@ '{{ model.thread_id }}', {# thread_id #} '{{ model.status }}', {# status #} - {% if model.timing != [] %} - {% for stage in model.timing if stage.name == "compile" %} - {% if loop.length == 0 %} - null, {# compile_started_at #} - {% else %} - '{{ stage.started_at }}', {# compile_started_at #} - {% endif %} - {% endfor %} - - {% for stage in model.timing if stage.name == "execute" %} - {% if loop.length == 0 %} - null, {# query_completed_at #} - {% else %} - '{{ stage.completed_at }}', {# query_completed_at #} - {% endif %} - {% endfor %} - {% else %} - null, {# compile_started_at #} - null, {# query_completed_at #} - {% endif %} + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} {{ model.execution_time }}, {# total_node_runtime #} null, -- rows_affected not available {# Databricks #} @@ -170,26 +138,10 @@ '{{ model.thread_id }}', {# thread_id #} '{{ model.status }}', {# status #} - {% if model.timing != [] %} - {% for stage in model.timing if stage.name == "compile" %} - {% if loop.length == 0 %} - null, {# compile_started_at #} - {% else %} - '{{ stage.started_at }}', {# compile_started_at #} - {% endif %} - {% endfor %} - - {% for stage in model.timing if stage.name == "execute" %} - {% if loop.length == 0 %} - null, {# query_completed_at #} - {% else %} - '{{ stage.completed_at }}', {# query_completed_at #} - {% endif %} - {% endfor %} - {% else %} - null, {# compile_started_at #} - null, {# query_completed_at #} - {% endif %} + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} {{ model.execution_time }}, {# total_node_runtime #} try_cast('{{ model.adapter_response.rows_affected }}' as int), {# rows_affected #} diff --git a/macros/upload_individual_datasets/upload_snapshot_executions.sql b/macros/upload_individual_datasets/upload_snapshot_executions.sql index 369348cd..2006b168 100644 --- a/macros/upload_individual_datasets/upload_snapshot_executions.sql +++ b/macros/upload_individual_datasets/upload_snapshot_executions.sql @@ -38,26 +38,10 @@ '{{ model.thread_id }}', {# thread_id #} '{{ model.status }}', {# status #} - {% if model.timing != [] %} - {% for stage in model.timing if stage.name == "compile" %} - {% if loop.length == 0 %} - null, {# compile_started_at #} - {% else %} - '{{ stage.started_at }}', {# compile_started_at #} - {% endif %} - {% endfor %} - - {% for stage in model.timing if stage.name == "execute" %} - {% if loop.length == 0 %} - null, {# query_completed_at #} - {% else %} - '{{ stage.completed_at }}', {# query_completed_at #} - {% endif %} - {% endfor %} - {% else %} - null, {# compile_started_at #} - null, {# query_completed_at #} - {% endif %} + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} {{ model.execution_time }}, {# total_node_runtime #} null, -- rows_affected not available {# Only available in Snowflake #} @@ -95,26 +79,10 @@ '{{ model.thread_id }}', {# thread_id #} '{{ model.status }}', {# status #} - {% if model.timing != [] %} - {% for stage in model.timing if stage.name == "compile" %} - {% if loop.length == 0 %} - null, {# compile_started_at #} - {% else %} - '{{ stage.started_at }}', {# compile_started_at #} - {% endif %} - {% endfor %} - - {% for stage in model.timing if stage.name == "execute" %} - {% if loop.length == 0 %} - null, {# query_completed_at #} - {% else %} - '{{ stage.completed_at }}', {# query_completed_at #} - {% endif %} - {% endfor %} - {% else %} - null, {# compile_started_at #} - null, {# query_completed_at #} - {% endif %} + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} {{ model.execution_time }}, {# total_node_runtime #} null, -- rows_affected not available {# Databricks #} @@ -170,26 +138,10 @@ '{{ model.thread_id }}', {# thread_id #} '{{ model.status }}', {# status #} - {% if model.timing != [] %} - {% for stage in model.timing if stage.name == "compile" %} - {% if loop.length == 0 %} - null, {# compile_started_at #} - {% else %} - '{{ stage.started_at }}', {# compile_started_at #} - {% endif %} - {% endfor %} - - {% for stage in model.timing if stage.name == "execute" %} - {% if loop.length == 0 %} - null, {# query_completed_at #} - {% else %} - '{{ stage.completed_at }}', {# query_completed_at #} - {% endif %} - {% endfor %} - {% else %} - null, {# compile_started_at #} - null, {# query_completed_at #} - {% endif %} + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} {{ model.execution_time }}, {# total_node_runtime #} try_cast('{{ model.adapter_response.rows_affected }}' as int), {# rows_affected #} diff --git a/macros/upload_individual_datasets/upload_test_executions.sql b/macros/upload_individual_datasets/upload_test_executions.sql index a42b03ba..ea3553ae 100644 --- a/macros/upload_individual_datasets/upload_test_executions.sql +++ b/macros/upload_individual_datasets/upload_test_executions.sql @@ -35,26 +35,10 @@ '{{ test.thread_id }}', {# thread_id #} '{{ test.status }}', {# status #} - {% if test.timing != [] %} - {% for stage in test.timing if stage.name == "compile" %} - {% if loop.length == 0 %} - null, {# compile_started_at #} - {% else %} - '{{ stage.started_at }}', {# compile_started_at #} - {% endif %} - {% endfor %} - - {% for stage in test.timing if stage.name == "execute" %} - {% if loop.length == 0 %} - null, {# query_completed_at #} - {% else %} - '{{ stage.completed_at }}', {# query_completed_at #} - {% endif %} - {% endfor %} - {% else %} - null, {# compile_started_at #} - null, {# query_completed_at #} - {% endif %} + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} {{ test.execution_time }}, {# total_node_runtime #} null, {# rows_affected not available in Databricks #} @@ -89,26 +73,10 @@ '{{ test.thread_id }}', {# thread_id #} '{{ test.status }}', {# status #} - {% if test.timing != [] %} - {% for stage in test.timing if stage.name == "compile" %} - {% if loop.length == 0 %} - null, {# compile_started_at #} - {% else %} - '{{ stage.started_at }}', {# compile_started_at #} - {% endif %} - {% endfor %} - - {% for stage in test.timing if stage.name == "execute" %} - {% if loop.length == 0 %} - null, {# query_completed_at #} - {% else %} - '{{ stage.completed_at }}', {# query_completed_at #} - {% endif %} - {% endfor %} - {% else %} - null, {# compile_started_at #} - null, {# query_completed_at #} - {% endif %} + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} {{ test.execution_time }}, {# total_node_runtime #} null, {# rows_affected not available in Databricks #} From 165945f9564230a1417d9498936c2a2a8fc70bf7 Mon Sep 17 00:00:00 2001 From: Gemma Down <52132406+glsdown@users.noreply.github.com> Date: Thu, 5 Oct 2023 13:01:15 +0100 Subject: [PATCH 24/31] Update version numbers (#394) --- README.md | 2 +- dbt_project.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 17688961..6c537660 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ See the generated [dbt docs site](https://brooklyn-data.github.io/dbt_artifacts/ ``` packages: - package: brooklyn-data/dbt_artifacts - version: 2.6.0 + version: 2.6.1 ``` :construction_worker: Make sure to fix at least the **minor** version, to avoid issues when a new release is open. See the notes on upgrading below for more detail. diff --git a/dbt_project.yml b/dbt_project.yml index 4a031fa9..ee9e85ca 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: "dbt_artifacts" -version: "2.6.0" +version: "2.6.1" config-version: 2 require-dbt-version: [">=1.3.0", "<1.7.0"] profile: "dbt_artifacts" From deeec8bcfd6462dcbd5bbc2ea878e811dd0f3484 Mon Sep 17 00:00:00 2001 From: Gemma Down <52132406+glsdown@users.noreply.github.com> Date: Thu, 16 Nov 2023 12:34:05 +0000 Subject: [PATCH 25/31] Add support for 1.7 (#403) --- .github/workflows/ci_lint_package.yml | 2 +- .github/workflows/ci_test_package.yml | 4 +- .github/workflows/main_lint_package.yml | 2 +- .github/workflows/main_test_package.yml | 2 +- .github/workflows/publish_docs_on_release.yml | 2 +- dbt_project.yml | 2 +- tox.ini | 45 ++++++++++++++++--- 7 files changed, 46 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci_lint_package.yml b/.github/workflows/ci_lint_package.yml index 9f1508f3..2563e0cf 100644 --- a/.github/workflows/ci_lint_package.yml +++ b/.github/workflows/ci_lint_package.yml @@ -50,7 +50,7 @@ jobs: architecture: "x64" - name: Install Python packages - run: python -m pip install dbt-snowflake~=1.6.0 sqlfluff-templater-dbt~=2.3.2 + run: python -m pip install dbt-snowflake~=1.7.0 sqlfluff-templater-dbt~=2.3.2 - name: Test database connection run: dbt debug diff --git a/.github/workflows/ci_test_package.yml b/.github/workflows/ci_test_package.yml index b02997ee..034a316f 100644 --- a/.github/workflows/ci_test_package.yml +++ b/.github/workflows/ci_test_package.yml @@ -21,7 +21,7 @@ env: DBT_ENV_SECRET_DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DBT_ENV_SECRET_GCP_PROJECT: ${{ secrets.GCP_PROJECT }} # Env var to test version - LAST_RELEASE_SUPPORTED_DBT_VERSION: 1_6_0 # A dbt version supported by both the last release and this one + LAST_RELEASE_SUPPORTED_DBT_VERSION: 1_7_0 # A dbt version supported by both the last release and this one # Env vars to test invocations model DBT_CLOUD_PROJECT_ID: 123 DBT_CLOUD_JOB_ID: ABC @@ -115,7 +115,7 @@ jobs: matrix: warehouse: ["snowflake", "bigquery", "postgres"] # When supporting a new version, update the list here - version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0"] + version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0", "1_7_0"] runs-on: ubuntu-latest environment: name: Approve Integration Tests diff --git a/.github/workflows/main_lint_package.yml b/.github/workflows/main_lint_package.yml index 7747f565..771d185f 100644 --- a/.github/workflows/main_lint_package.yml +++ b/.github/workflows/main_lint_package.yml @@ -46,7 +46,7 @@ jobs: architecture: "x64" - name: Install Python packages - run: python -m pip install dbt-snowflake~=1.6.0 sqlfluff-templater-dbt~=2.3.2 + run: python -m pip install dbt-snowflake~=1.7.0 sqlfluff-templater-dbt~=2.3.2 - name: Test database connection run: dbt debug diff --git a/.github/workflows/main_test_package.yml b/.github/workflows/main_test_package.yml index 4398c459..c8e503b5 100644 --- a/.github/workflows/main_test_package.yml +++ b/.github/workflows/main_test_package.yml @@ -35,7 +35,7 @@ jobs: strategy: matrix: warehouse: ["snowflake", "bigquery", "postgres"] - version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0"] + version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0", "1_7_0"] runs-on: ubuntu-latest permissions: contents: "read" diff --git a/.github/workflows/publish_docs_on_release.yml b/.github/workflows/publish_docs_on_release.yml index d558a464..b02ee057 100644 --- a/.github/workflows/publish_docs_on_release.yml +++ b/.github/workflows/publish_docs_on_release.yml @@ -39,7 +39,7 @@ jobs: uses: actions/checkout@v3 - name: Install Python packages - run: python -m pip install dbt-snowflake~=1.6.0 + run: python -m pip install dbt-snowflake~=1.7.0 - name: Test database connection run: dbt debug diff --git a/dbt_project.yml b/dbt_project.yml index ee9e85ca..7ddce641 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,7 +1,7 @@ name: "dbt_artifacts" version: "2.6.1" config-version: 2 -require-dbt-version: [">=1.3.0", "<1.7.0"] +require-dbt-version: [">=1.3.0", "<1.8.0"] profile: "dbt_artifacts" clean-targets: # folders to be removed by `dbt clean` diff --git a/tox.ini b/tox.ini index 8add76ba..542d6e21 100644 --- a/tox.ini +++ b/tox.ini @@ -36,7 +36,7 @@ rules = LT01,LT02,LT03,CP01,AL01,AL02,CP02,ST08,LT06,LT07,AM01,LT08,AL05,RF02,RF deps = sqlfluff-templater-dbt~=2.0.2 - dbt-snowflake~=1.6.0 + dbt-snowflake~=1.7.0 [sqlfluff:indentation] indent_unit = space @@ -114,13 +114,13 @@ commands = sqlfluff fix models --ignore parsing # Generate docs [testenv:generate_docs] -deps = dbt-snowflake~=1.6.0 +deps = dbt-snowflake~=1.7.0 commands = dbt docs generate --profiles-dir integration_test_project # Snowflake integration tests [testenv:integration_snowflake] changedir = integration_test_project -deps = dbt-snowflake~=1.6.0 +deps = dbt-snowflake~=1.7.0 commands = dbt clean dbt deps @@ -159,10 +159,18 @@ commands = dbt deps dbt build --target snowflake +[testenv:integration_snowflake_1_7_0] +changedir = integration_test_project +deps = dbt-snowflake~=1.7.0 +commands = + dbt clean + dbt deps + dbt build --target snowflake + # Databricks integration tests [testenv:integration_databricks] changedir = integration_test_project -deps = dbt-databricks~=1.6.0 +deps = dbt-databricks~=1.7.0 commands = dbt clean dbt deps @@ -200,10 +208,18 @@ commands = dbt deps dbt build --target databricks +[testenv:integration_databricks_1_7_0] +changedir = integration_test_project +deps = dbt-databricks~=1.7.0 +commands = + dbt clean + dbt deps + dbt build --target databricks + # Bigquery integration tests [testenv:integration_bigquery] changedir = integration_test_project -deps = dbt-bigquery~=1.6.0 +deps = dbt-bigquery~=1.7.0 commands = dbt clean dbt deps @@ -241,6 +257,14 @@ commands = dbt deps dbt build --target bigquery --vars '"my_var": "my value"' +[testenv:integration_bigquery_1_7_0] +changedir = integration_test_project +deps = dbt-bigquery~=1.7.0 +commands = + dbt clean + dbt deps + dbt build --target bigquery --vars '"my_var": "my value"' + # Spark integration test (disabled) [testenv:integration_spark] changedir = integration_test_project @@ -252,7 +276,7 @@ commands = [testenv:integration_postgres] changedir = integration_test_project -deps = dbt-postgres~=1.6.0 +deps = dbt-postgres~=1.7.0 commands = dbt clean dbt deps @@ -290,3 +314,12 @@ commands = dbt deps dbt build --target postgres +[testenv:integration_postgres_1_7_0] +changedir = integration_test_project +deps = dbt-postgres~=1.7.0 +commands = + dbt clean + dbt deps + dbt build --target postgres + + From 4220a9cea519c98a7d23ec8396d661c6fc8880da Mon Sep 17 00:00:00 2001 From: Gemma Down <52132406+glsdown@users.noreply.github.com> Date: Thu, 16 Nov 2023 12:40:12 +0000 Subject: [PATCH 26/31] Release v2.6.2 (#404) * Add support for 1.7 * Bump version number --- README.md | 2 +- dbt_project.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6c537660..6350d743 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ See the generated [dbt docs site](https://brooklyn-data.github.io/dbt_artifacts/ ``` packages: - package: brooklyn-data/dbt_artifacts - version: 2.6.1 + version: 2.6.2 ``` :construction_worker: Make sure to fix at least the **minor** version, to avoid issues when a new release is open. See the notes on upgrading below for more detail. diff --git a/dbt_project.yml b/dbt_project.yml index 7ddce641..60bd2117 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: "dbt_artifacts" -version: "2.6.1" +version: "2.6.2" config-version: 2 require-dbt-version: [">=1.3.0", "<1.8.0"] profile: "dbt_artifacts" From dba29174d7a61d3e71beb8b9a9b6e96621f8dab6 Mon Sep 17 00:00:00 2001 From: Matt Lam Date: Thu, 30 Nov 2023 18:02:54 -0800 Subject: [PATCH 27/31] Fix: typo in upload_results to determine batch size --- macros/upload_results/upload_results.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/upload_results/upload_results.sql b/macros/upload_results/upload_results.sql index fcadc199..4876b82c 100644 --- a/macros/upload_results/upload_results.sql +++ b/macros/upload_results/upload_results.sql @@ -19,7 +19,7 @@ {% set objects = dbt_artifacts.get_dataset_content(dataset) %} {# Upload in chunks to reduce query size #} - {% if dataset == 'model' %} + {% if dataset == 'models' %} {% set upload_limit = 50 if target.type == 'bigquery' else 100 %} {% else %} {% set upload_limit = 300 if target.type == 'bigquery' else 5000 %} From 2ee23c20941fbf59dac0708d459be24c08eb1c71 Mon Sep 17 00:00:00 2001 From: Mark Greenwood Date: Thu, 21 Dec 2023 16:23:26 +0000 Subject: [PATCH 28/31] Copy model object to leave original intact --- .../upload_models.sql | 81 ++++++++++--------- 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/macros/upload_individual_datasets/upload_models.sql b/macros/upload_individual_datasets/upload_models.sql index d8dc49fd..b5f9a8ec 100644 --- a/macros/upload_individual_datasets/upload_models.sql +++ b/macros/upload_individual_datasets/upload_models.sql @@ -24,22 +24,23 @@ {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(15)) }} from values {% for model in models -%} - {% do model.pop('raw_code', None) %} + {% set model_copy = model.copy() -%} + {% do model_copy.pop('raw_code', None) %} ( '{{ invocation_id }}', {# command_invocation_id #} - '{{ model.unique_id }}', {# node_id #} + '{{ model_copy.unique_id }}', {# node_id #} '{{ run_started_at }}', {# run_started_at #} - '{{ model.database }}', {# database #} - '{{ model.schema }}', {# schema #} - '{{ model.name }}', {# name #} - '{{ tojson(model.depends_on.nodes) | replace('\\', '\\\\') }}', {# depends_on_nodes #} - '{{ model.package_name }}', {# package_name #} - '{{ model.original_file_path | replace('\\', '\\\\') }}', {# path #} - '{{ model.checksum.checksum | replace('\\', '\\\\') }}', {# checksum #} - '{{ model.config.materialized }}', {# materialization #} - '{{ tojson(model.tags) }}', {# tags #} - '{{ tojson(model.config.meta) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}', {# meta #} - '{{ model.alias }}', {# alias #} + '{{ model_copy.database }}', {# database #} + '{{ model_copy.schema }}', {# schema #} + '{{ model_copy.name }}', {# name #} + '{{ tojson(model_copy.depends_on.nodes) | replace('\\', '\\\\') }}', {# depends_on_nodes #} + '{{ model_copy.package_name }}', {# package_name #} + '{{ model_copy.original_file_path | replace('\\', '\\\\') }}', {# path #} + '{{ model_copy.checksum.checksum | replace('\\', '\\\\') }}', {# checksum #} + '{{ model_copy.config.materialized }}', {# materialization #} + '{{ tojson(model_copy.tags) }}', {# tags #} + '{{ tojson(model_copy.config.meta) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}', {# meta #} + '{{ model_copy.alias }}', {# alias #} {% if var('dbt_artifacts_exclude_all_results', false) %} null {% else %} @@ -59,22 +60,23 @@ {% if models != [] %} {% set model_values %} {% for model in models -%} - {% do model.pop('raw_code', None) %} + {% set model_copy = model.copy() -%} + {% do model_copy.pop('raw_code', None) %} ( '{{ invocation_id }}', {# command_invocation_id #} - '{{ model.unique_id }}', {# node_id #} + '{{ model_copy.unique_id }}', {# node_id #} '{{ run_started_at }}', {# run_started_at #} - '{{ model.database }}', {# database #} - '{{ model.schema }}', {# schema #} - '{{ model.name }}', {# name #} - {{ tojson(model.depends_on.nodes) }}, {# depends_on_nodes #} - '{{ model.package_name }}', {# package_name #} - '{{ model.original_file_path | replace('\\', '\\\\') }}', {# path #} - '{{ model.checksum.checksum | replace('\\', '\\\\') }}', {# checksum #} - '{{ model.config.materialized }}', {# materialization #} - {{ tojson(model.tags) }}, {# tags #} - {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model.config.meta)) }}, {# meta #} - '{{ model.alias }}', {# alias #} + '{{ model_copy.database }}', {# database #} + '{{ model_copy.schema }}', {# schema #} + '{{ model_copy.name }}', {# name #} + {{ tojson(model_copy.depends_on.nodes) }}, {# depends_on_nodes #} + '{{ model_copy.package_name }}', {# package_name #} + '{{ model_copy.original_file_path | replace('\\', '\\\\') }}', {# path #} + '{{ model_copy.checksum.checksum | replace('\\', '\\\\') }}', {# checksum #} + '{{ model_copy.config.materialized }}', {# materialization #} + {{ tojson(model_copy.tags) }}, {# tags #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model_copy.config.meta)) }}, {# meta #} + '{{ model_copy.alias }}', {# alias #} {% if var('dbt_artifacts_exclude_all_results', false) %} null {% else %} @@ -94,22 +96,23 @@ {% if models != [] %} {% set model_values %} {% for model in models -%} - {% do model.pop('raw_code', None) %} + {% set model_copy = model.copy() -%} + {% do model_copy.pop('raw_code', None) %} ( '{{ invocation_id }}', {# command_invocation_id #} - '{{ model.unique_id }}', {# node_id #} + '{{ model_copy.unique_id }}', {# node_id #} '{{ run_started_at }}', {# run_started_at #} - '{{ model.database }}', {# database #} - '{{ model.schema }}', {# schema #} - '{{ model.name }}', {# name #} - '{{ tojson(model.depends_on.nodes) }}', {# depends_on_nodes #} - '{{ model.package_name }}', {# package_name #} - $${{ model.original_file_path | replace('\\', '\\\\') }}$$, {# path #} - '{{ model.checksum.checksum }}', {# checksum #} - '{{ model.config.materialized }}', {# materialization #} - '{{ tojson(model.tags) }}', {# tags #} - $${{ model.config.meta }}$$, {# meta #} - '{{ model.alias }}', {# alias #} + '{{ model_copy.database }}', {# database #} + '{{ model_copy.schema }}', {# schema #} + '{{ model_copy.name }}', {# name #} + '{{ tojson(model_copy.depends_on.nodes) }}', {# depends_on_nodes #} + '{{ model_copy.package_name }}', {# package_name #} + $${{ model_copy.original_file_path | replace('\\', '\\\\') }}$$, {# path #} + '{{ model_copy.checksum.checksum }}', {# checksum #} + '{{ model_copy.config.materialized }}', {# materialization #} + '{{ tojson(model_copy.tags) }}', {# tags #} + $${{ model_copy.config.meta }}$$, {# meta #} + '{{ model_copy.alias }}', {# alias #} {% if var('dbt_artifacts_exclude_all_results', false) %} null {% else %} From d71c39fa3e7a9082e19b0a920ecc01be9745fca2 Mon Sep 17 00:00:00 2001 From: Jared Rimmer <100997264+jared-rimmer@users.noreply.github.com> Date: Thu, 25 Jan 2024 11:28:38 +0000 Subject: [PATCH 29/31] Update macros/upload_results/upload_results.sql --- macros/upload_results/upload_results.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/upload_results/upload_results.sql b/macros/upload_results/upload_results.sql index 4876b82c..114a667d 100644 --- a/macros/upload_results/upload_results.sql +++ b/macros/upload_results/upload_results.sql @@ -18,7 +18,7 @@ {# Get the results that need to be uploaded #} {% set objects = dbt_artifacts.get_dataset_content(dataset) %} - {# Upload in chunks to reduce query size #} + {# Upload in chunks to reduce the query size #} {% if dataset == 'models' %} {% set upload_limit = 50 if target.type == 'bigquery' else 100 %} {% else %} From b51347b9b7fdade59740f93c5f55c51135d14820 Mon Sep 17 00:00:00 2001 From: Mark Greenwood Date: Thu, 25 Jan 2024 12:08:10 +0000 Subject: [PATCH 30/31] replace model for include all results --- macros/upload_individual_datasets/upload_models.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/upload_individual_datasets/upload_models.sql b/macros/upload_individual_datasets/upload_models.sql index b5f9a8ec..57b1caff 100644 --- a/macros/upload_individual_datasets/upload_models.sql +++ b/macros/upload_individual_datasets/upload_models.sql @@ -44,7 +44,7 @@ {% if var('dbt_artifacts_exclude_all_results', false) %} null {% else %} - '{{ tojson(model) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}' {# all_results #} + '{{ tojson(model_copy) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}' {# all_results #} {% endif %} ) {%- if not loop.last %},{%- endif %} From dff781bf18fdad10693e70cf50812ddfaba64c7a Mon Sep 17 00:00:00 2001 From: Mark Greenwood Date: Thu, 25 Jan 2024 12:10:26 +0000 Subject: [PATCH 31/31] replace model for include all results --- macros/upload_individual_datasets/upload_models.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/macros/upload_individual_datasets/upload_models.sql b/macros/upload_individual_datasets/upload_models.sql index 57b1caff..7570b06a 100644 --- a/macros/upload_individual_datasets/upload_models.sql +++ b/macros/upload_individual_datasets/upload_models.sql @@ -80,7 +80,7 @@ {% if var('dbt_artifacts_exclude_all_results', false) %} null {% else %} - {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_results #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model_copy) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_results #} {% endif %} ) {%- if not loop.last %},{%- endif %} @@ -116,7 +116,7 @@ {% if var('dbt_artifacts_exclude_all_results', false) %} null {% else %} - $${{ tojson(model) }}$$ {# all_results #} + $${{ tojson(model_copy) }}$$ {# all_results #} {% endif %} ) {%- if not loop.last %},{%- endif %}