Enable re-discovery of resources during deployment

build.helpers.psm1

@@ -651,7 +651,14 @@ function Install-Protobuf {
         } elseif ($IsWindows) {
             if (Test-CommandAvailable -Name 'winget') {
                 Write-Verbose -Verbose "Using winget to install Protobuf"
-                winget install Google.Protobuf --accept-source-agreements --accept-package-agreements --source winget --silent
+                winget install Google.Protobuf --accept-source-agreements --accept-package-agreements --source winget --force
+                # need to add to PATH
+                $protocFolder = "$env:USERPROFILE\AppData\Local\Microsoft\WinGet\Packages\Google.Protobuf_Microsoft.Winget.Source_8wekyb3d8bbwe\bin"
+                if (Test-Path $protocFolder) {
+                    $env:PATH += ";$protocFolder"
+                } else {
+                    throw "protoc folder not found after installation: $protocFolder"
+                }
             } else {
                 Write-Warning "winget not found, please install Protobuf manually"
             }
@@ -665,7 +672,7 @@ function Install-Protobuf {
         }
 
         if ($LASTEXITCODE -ne 0) {
-            throw "Failed to install Protobuf"
+            throw "Failed to install Protobuf: $LASTEXITCODE"
         }
     }
 }

dsc/tests/dsc_discovery.tests.ps1

@@ -3,8 +3,6 @@
 
 Describe 'tests for resource discovery' {
     BeforeAll {
-        $env:DSC_RESOURCE_PATH = $testdrive
-
         $script:lookupTableFilePath = if ($IsWindows) {
             Join-Path $env:LocalAppData "dsc\AdaptedResourcesLookupTable.json"
         } else {
@@ -16,10 +14,6 @@ Describe 'tests for resource discovery' {
         Remove-Item -Path "$testdrive/test.dsc.resource.*" -ErrorAction SilentlyContinue
     }
 
-    AfterAll {
-        $env:DSC_RESOURCE_PATH = $null
-    }
-
     It 'Use DSC_RESOURCE_PATH instead of PATH when defined' {
         $resourceJson = @'
         {
@@ -31,75 +25,85 @@ Describe 'tests for resource discovery' {
             }
           }
 '@
-
-        Set-Content -Path "$testdrive/test.dsc.resource.json" -Value $resourceJson
-        $resources = dsc resource list | ConvertFrom-Json
-        $resources.Count | Should -Be 1
-        $resources.type | Should -BeExactly 'DSC/TestPathResource'
+        try {
+            $oldPath = $env:PATH
+            $env:DSC_RESOURCE_PATH = $testdrive
+            Set-Content -Path "$testdrive/test.dsc.resource.json" -Value $resourceJson
+            $resources = dsc resource list | ConvertFrom-Json
+            $resources.Count | Should -Be 1
+            $resources.type | Should -BeExactly 'DSC/TestPathResource'
+        }
+        finally {
+            $env:PATH = $oldPath
+            $env:DSC_RESOURCE_PATH = $null
+        }
     }
 
-    It 'support discovering <extension>' -TestCases @(
-        @{ extension = 'yaml' }
-        @{ extension = 'yml' }
-    ) {
-        param($extension)
-
-        $resourceYaml = @'
-        $schema: https://aka.ms/dsc/schemas/v3/bundled/resource/manifest.json
-        type: DSC/TestYamlResource
-        version: 0.1.0
-        get:
-          executable: dsc
+    Context 'Forced discovery using $testdrive' {
+        BeforeAll {
+            $env:DSC_RESOURCE_PATH = $testdrive
+        }
+
+        AfterAll {
+            $env:DSC_RESOURCE_PATH = $null
+        }
+
+        It 'support discovering <extension>' -TestCases @(
+            @{ extension = 'yaml' }
+            @{ extension = 'yml' }
+        ) {
+            param($extension)
+
+            $resourceYaml = @'
+            $schema: https://aka.ms/dsc/schemas/v3/bundled/resource/manifest.json
+            type: DSC/TestYamlResource
+            version: 0.1.0
+            get:
+              executable: dsc
 '@
 
-        Set-Content -Path "$testdrive/test.dsc.resource.$extension" -Value $resourceYaml
-        $resources = dsc resource list | ConvertFrom-Json
-        $resources.Count | Should -Be 1
-        $resources.type | Should -BeExactly 'DSC/TestYamlResource'
-    }
+            Set-Content -Path "$testdrive/test.dsc.resource.$extension" -Value $resourceYaml
+            $resources = dsc resource list | ConvertFrom-Json
+            $resources.Count | Should -Be 1
+            $resources.type | Should -BeExactly 'DSC/TestYamlResource'
+        }
 
-    It 'does not support discovering a file with an extension that is not json or yaml' {
-        param($extension)
+        It 'does not support discovering a file with an extension that is not json or yaml' {
+            param($extension)
 
-        $resourceInput = @'
-        $schema: https://aka.ms/dsc/schemas/v3/bundled/resource/manifest.json
-        type: DSC/TestYamlResource
-        version: 0.1.0
-        get:
-          executable: dsc
+            $resourceInput = @'
+            $schema: https://aka.ms/dsc/schemas/v3/bundled/resource/manifest.json
+            type: DSC/TestYamlResource
+            version: 0.1.0
+            get:
+              executable: dsc
 '@
 
-        Set-Content -Path "$testdrive/test.dsc.resource.txt" -Value $resourceInput
-        $resources = dsc resource list | ConvertFrom-Json
-        $resources.Count | Should -Be 0
-    }
+            Set-Content -Path "$testdrive/test.dsc.resource.txt" -Value $resourceInput
+            $resources = dsc resource list | ConvertFrom-Json
+            $resources.Count | Should -Be 0
+        }
 
-    It 'warns on invalid semver' {
-        $manifest = @'
-        {
-            "$schema": "https://aka.ms/dsc/schemas/v3/bundled/resource/manifest.json",
-            "type": "Test/InvalidSemver",
-            "version": "1.1.0..1",
-            "get": {
-                "executable": "dsctest"
-            },
-            "schema": {
-                "command": {
+        It 'warns on invalid semver' {
+            $manifest = @'
+            {
+                "$schema": "https://aka.ms/dsc/schemas/v3/bundled/resource/manifest.json",
+                "type": "Test/InvalidSemver",
+                "version": "1.1.0..1",
+                "get": {
                     "executable": "dsctest"
+                },
+                "schema": {
+                    "command": {
+                        "executable": "dsctest"
+                    }
                 }
             }
-        }
 '@
-        $oldPath = $env:DSC_RESOURCE_PATH
-        try {
-            $env:DSC_RESOURCE_PATH = $testdrive
             Set-Content -Path "$testdrive/test.dsc.resource.json" -Value $manifest
             $null = dsc resource list 2> "$testdrive/error.txt"
             "$testdrive/error.txt" | Should -FileContentMatchExactly 'WARN.*?does not use semver' -Because (Get-Content -Raw "$testdrive/error.txt")
         }
-        finally {
-            $env:DSC_RESOURCE_PATH = $oldPath
-        }
     }
 
     It 'Ensure List operation populates adapter lookup table' {
@@ -296,4 +300,47 @@ Describe 'tests for resource discovery' {
             $env:DSC_RESOURCE_PATH = $null
         }
     }
+
+    It 'Resource discovery can be set to <mode>' -TestCases @(
+        @{ namespace = 'Microsoft.DSC'; mode = 'preDeployment' }
+        @{ namespace = 'Microsoft.DSC'; mode = 'duringDeployment' }
+        @{ namespace = 'Ignore'; mode = 'ignore' }
+    ) {
+        param($namespace, $mode)
+
+        $guid = (New-Guid).Guid.Replace('-', '')
+        $manifestPath = Join-Path (Split-Path (Get-Command dscecho -ErrorAction Stop).Source -Parent) echo.dsc.resource.json
+
+        $config_yaml = @"
+        `$schema: https://aka.ms/dsc/schemas/v3/bundled/config/manifest.json
+        metadata:
+          ${namespace}:
+            resourceDiscovery: $mode
+        resources:
+        - type: Test/CopyResource
+          name: This should be found and executed
+          properties:
+            sourceFile: $manifestPath
+            typeName: "Test/$guid"
+        - type: Test/$guid
+          name: This is the new resource
+          properties:
+            output: Hello World
+"@
+        $out = dsc -l trace config get -i $config_yaml 2> "$testdrive/tracing.txt"
+        $traceLog = Get-Content -Raw -Path "$testdrive/tracing.txt"
+        if ($mode -ne 'duringDeployment') {
+            $LASTEXITCODE | Should -Be 2
+            $out | Should -BeNullOrEmpty
+            $traceLog | Should -Match "ERROR.*?Resource not found: Test/$guid"
+            $traceLog | Should -Not -Match "Invoking get for 'Test/CopyResource'"
+        } else {
+            $LASTEXITCODE | Should -Be 0 -Because (Get-Content -Raw -Path "$testdrive/tracing.txt")
+            $output = $out | ConvertFrom-Json
+            $output.results[0].result.actualState.typeName | Should -BeExactly "Test/$guid" -Because $out
+            $output.results[1].result.actualState.output | Should -BeExactly 'Hello World' -Because $out
+            $traceLog | Should -Match "Invoking get for 'Test/$guid'"
+            $traceLog | Should -Match "Skipping resource discovery due to 'resourceDiscovery' mode set to 'DuringDeployment'"
+        }
+    }
 }

lib/dsc-lib/locales/en-us.toml

@@ -84,6 +84,7 @@ skippingOutput = "Skipping output for '%{name}' due to condition evaluating to f
 secureOutputSkipped = "Secure output '%{name}' is skipped"
 outputTypeNotMatch = "Output '%{name}' type does not match expected type '%{expected_type}'"
 copyNotSupported = "Copy for output '%{name}' is currently not supported"
+skippingResourceDiscovery = "Skipping resource discovery due to 'resourceDiscovery' mode set to 'DuringDeployment'"
 
 [configure.parameters]
 importingParametersFromComplexInput = "Importing parameters from complex input"

lib/dsc-lib/src/configure/config_doc.rs

@@ -60,34 +60,46 @@ pub enum RestartRequired {
     Process(Process),
 }
 
+#[derive(Debug, Clone, PartialEq, Deserialize, Serialize, JsonSchema, DscRepoSchema)]
+#[serde(rename_all = "camelCase")]
+#[dsc_repo_schema(base_name = "resourceDiscovery", folder_path = "metadata/Microsoft.DSC")]
+pub enum ResourceDiscoveryMode {
+    PreDeployment,
+    DuringDeployment,
+}
+
 #[derive(Debug, Default, Clone, PartialEq, Deserialize, Serialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
 pub struct MicrosoftDscMetadata {
     /// The duration of the configuration operation
     #[serde(skip_serializing_if = "Option::is_none")]
     pub duration: Option<String>,
     /// The end time of the configuration operation
-    #[serde(rename = "endDatetime", skip_serializing_if = "Option::is_none")]
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub end_datetime: Option<String>,
     /// The type of execution
-    #[serde(rename = "executionType", skip_serializing_if = "Option::is_none")]
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub execution_type: Option<ExecutionKind>,
     /// The operation being performed
     #[serde(skip_serializing_if = "Option::is_none")]
     pub operation: Option<Operation>,
     /// Specify specific adapter type used for implicit operations
-    #[serde(rename = "requireAdapter", skip_serializing_if = "Option::is_none")]
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub require_adapter: Option<String>,
+    /// Indicates if resources are discovered pre-deployment or during deployment
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub resource_discovery: Option<ResourceDiscoveryMode>,
     /// Indicates what needs to be restarted after the configuration operation
-    #[serde(rename = "restartRequired", skip_serializing_if = "Option::is_none")]
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub restart_required: Option<Vec<RestartRequired>>,
     /// Copy loop context for resources expanded from copy loops
     #[serde(rename = "copyLoops", skip_serializing_if = "Option::is_none")]
     pub copy_loops: Option<Map<String, Value>>,
     /// The security context of the configuration operation, can be specified to be required
-    #[serde(rename = "securityContext", skip_serializing_if = "Option::is_none")]
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub security_context: Option<SecurityContextKind>,
     /// The start time of the configuration operation
-    #[serde(rename = "startDatetime", skip_serializing_if = "Option::is_none")]
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub start_datetime: Option<String>,
     /// Version of DSC
     #[serde(skip_serializing_if = "Option::is_none")]

lib/dsc-lib/src/configure/mod.rs

@@ -3,7 +3,7 @@
 
 use crate::configure::context::{Context, ProcessMode};
 use crate::configure::parameters::import_parameters;
-use crate::configure::{config_doc::{ExecutionKind, IntOrExpression, Metadata, Parameter, Resource, RestartRequired, ValueOrCopy}};
+use crate::configure::{config_doc::{ExecutionKind, IntOrExpression, Metadata, Parameter, Resource, ResourceDiscoveryMode, RestartRequired, ValueOrCopy}};
 use crate::discovery::discovery_trait::DiscoveryFilter;
 use crate::dscerror::DscError;
 use crate::dscresources::{
@@ -998,6 +998,7 @@ impl Configurator {
                     end_datetime: Some(end_datetime.to_rfc3339()),
                     execution_type: Some(self.context.execution_type.clone()),
                     operation: Some(operation),
+                    resource_discovery: None,
                     restart_required: self.context.restart_required.clone(),
                     security_context: Some(self.context.security_context.clone()),
                     start_datetime: Some(self.context.start_datetime.to_rfc3339()),
@@ -1013,29 +1014,52 @@ impl Configurator {
         let config: Configuration = serde_json::from_str(self.json.as_str())?;
         check_security_context(config.metadata.as_ref())?;
 
-        // Perform discovery of resources used in config
-        // create an array of DiscoveryFilter using the resource types and api_versions from the config
-        let mut discovery_filter: Vec<DiscoveryFilter> = Vec::new();
-        let config_copy = config.clone();
-        for resource in config_copy.resources {
-            let adapter = get_require_adapter_from_metadata(&resource.metadata);
-            let filter = DiscoveryFilter::new(&resource.resource_type, resource.api_version.as_deref(), adapter.as_deref());
-            if !discovery_filter.contains(&filter) {
-                discovery_filter.push(filter);
+        let mut skip_resource_validation = false;
+        if let Some(metadata) = &config.metadata {
+            if let Some(microsoft_metadata) = &metadata.microsoft {
+                if let Some(mode) = &microsoft_metadata.resource_discovery {
+                    if *mode == ResourceDiscoveryMode::DuringDeployment {
+                        debug!("{}", t!("configure.mod.skippingResourceDiscovery"));
+                        skip_resource_validation = true;
+                        self.discovery.refresh_cache = true;
+                    }
+                }
             }
-            // defer actual unrolling until parameters are available
-            if let Some(copy) = &resource.copy {
-                debug!("{}", t!("configure.mod.validateCopy", name = &copy.name, count = copy.count));
-                if copy.mode.is_some() {
-                    return Err(DscError::Validation(t!("configure.mod.copyModeNotSupported").to_string()));
+        }
+
+        if !skip_resource_validation {
+            // Perform discovery of resources used in config
+            // create an array of DiscoveryFilter using the resource types and api_versions from the config
+            let mut discovery_filter: Vec<DiscoveryFilter> = Vec::new();
+            let config_copy = config.clone();
+            for resource in config_copy.resources {
+                let adapter = get_require_adapter_from_metadata(&resource.metadata);
+                let filter = DiscoveryFilter::new(&resource.resource_type, resource.api_version.as_deref(), adapter.as_deref());
+                if !discovery_filter.contains(&filter) {
+                    discovery_filter.push(filter);
                 }
-                if copy.batch_size.is_some() {
-                    return Err(DscError::Validation(t!("configure.mod.copyBatchSizeNotSupported").to_string()));
+                // defer actual unrolling until parameters are available
+                if let Some(copy) = &resource.copy {
+                    debug!("{}", t!("configure.mod.validateCopy", name = &copy.name, count = copy.count));
+                    if copy.mode.is_some() {
+                        return Err(DscError::Validation(t!("configure.mod.copyModeNotSupported").to_string()));
+                    }
+                    if copy.batch_size.is_some() {
+                        return Err(DscError::Validation(t!("configure.mod.copyBatchSizeNotSupported").to_string()));
+                    }
                 }
             }
+            self.discovery.find_resources(&discovery_filter, self.progress_format)?;
+
+            // now check that each resource in the config was found
+            for resource in config.resources.iter() {
+                let adapter = get_require_adapter_from_metadata(&resource.metadata);
+                let Some(_dsc_resource) = self.discovery.find_resource(&DiscoveryFilter::new(&resource.resource_type, resource.api_version.as_deref(), adapter.as_deref()))? else {
+                    return Err(DscError::ResourceNotFound(resource.resource_type.to_string(), resource.api_version.as_deref().unwrap_or("").to_string()));
+                };
+            }
         }
 
-        self.discovery.find_resources(&discovery_filter, self.progress_format)?;
         self.config = config;
         Ok(())
     }