try placeholder policy

Author: Phil-NHS

databricks.yml

@@ -29,25 +29,19 @@ bundle:
   name: DatabricksPOC
   uuid: ba682f8a-5d14-4ae7-a770-d6a359c0e835
 
+exclude:
+  - exclude-me-poc-test/*.yml
 include:
-  #   - 'variables/*.yml' if execute in order might be able to put vars in folders based off headings, may affect running in personal area before deploy?
-  # - resources/variables/serverless_usage_policy_constants.yml
-  # - resources/variables/env_serverless_usage_policy_constants.yml
   - resources/variables/*.yml
   - resources/environments/*.yml
   - resources/**/*.yml
   - resources/**/**/*.yml
+
 # ** double astrix doesnt seem to do sub directories
 # poc should show ods ingest, reporting, and tests jobs and pipeline
 
 # When this list get unmanageable we may want to consider a constants file but for similicity here is best for now
 variables:
-    # qqqq redeclaring so it loads them?
-  # serverless_usage_policies:
-  #   type: complex
-  # env_serverless_usage_policies:
-  #   type: complex
-
 
   # ============================================================
   # Environment Configuration
@@ -128,7 +122,6 @@ targets:
 ############## DBX UI and from local machine deploys
 
   personal: 
-   
     mode: development
     # Deploy under own user not service principle
     default: true
@@ -257,12 +250,14 @@ targets:
       env_serverless_usage_policies:
         env_analysis_batch_id: ${var.serverless_usage_policies.prod_analysis_batch_id}
         env_lh_live_id: ${var.serverless_usage_policies.prod_lh_live_id}
+        
         # Note: You can leave test IDs blank here or map them to a default 
         # since Prod doesn't have test policies. However frustratingly despite excluding resource/test the bundle validation check it anyway
-        env_test_all_id: "NA"
-        env_test_dq_id: "NA"
-        env_test_int_id: "NA"
-        env_test_unit_id: "NA"
+        
+        env_test_all_id: ${var.serverless_usage_policies.DUMMY_POLICY_FOR_BUNDLE_ID}
+        env_test_dq_id: ${var.serverless_usage_policies.DUMMY_POLICY_FOR_BUNDLE_ID}
+        env_test_int_id: ${var.serverless_usage_policies.DUMMY_POLICY_FOR_BUNDLE_ID}
+        env_test_unit_id: ${var.serverless_usage_policies.DUMMY_POLICY_FOR_BUNDLE_ID}
     permissions:
       - group_name: prod_env_users
         level: CAN_VIEW

docs/But why is it like that.md

@@ -15,3 +15,6 @@ from teamname_functions import core_func as core
 
 > this nice clean approach won't work for us because github wont have the same path and won't work for username paths. Which is why the approach in the project is to determine the existing path, and then path to the needed libraries based on it. Using the approach above will work until used in github or deployed to usernamed paths. Putting libraries outside of username workspaces would mean the libraries can't be easily changed with the work depending on them because they are outside of source control. Another option would have been through the ui but has similar issues [adding local library dependencys microsoft learn dbx](https://learn.microsoft.com/en-gb/azure/databricks/compute/serverless/dependencies#create-common-utilities-to-share-across-your-workspace) 
 
+## DUMMY_POLICY_FOR_BUNDLE_ID
+
+Without having databricks.yml per target the bundle validate validates all yml included files at the top level, even if they are sync excluded e.g. prod excluding test files. So this policy should not show up in usage. It also mean we need to declare IDs for policys we are not using 

exclude-me-poc-test/job-not-in-dabs.yml

@@ -0,0 +1,12 @@
+resources:
+  jobs:
+    fake_excluded_job:
+      name: "[${bundle.target}] fake exclude job"
+
+      usage_policy_id: ${var.env_serverless_usage_policies.fake_policy_id}       
+        
+      tasks:
+        - task_key: medallion_refresher
+          run_job_task:
+            job_id: ${resources.jobs.medallion_refresh.id}
+          timeout_seconds: 1800  # fail after 30 min instead of hanging

resources/variables/serverless_usage_policy_constants.yml

@@ -5,7 +5,14 @@ variables:
     description: "Map of Databricks Serverless Usage Policy IDs for cost attribution"
     type: complex
     default:
-
+      #----------------------------------------------------------
+      #-------#    Ugly Bundle Validate Handling    #--------
+      #---------------------------------------------------------
+      # Without having a databricks.yml per environment we need
+      # a dummy id because despite sync/exclude bundle validate will check
+      # for the value
+      #---------------------------------------------------------------
+      DUMMY_POLICY_FOR_BUNDLE_ID : "dd654d53-01e4-4fcf-b480-a17642b75fa4"
       # ---------------------------------------------------------------------------
       # Default / Cross-Environment Policies
       # ---------------------------------------------------------------------------