diff --git a/data/project_types/pupmod.yaml b/data/project_types/pupmod.yaml index 84d0b88..7225e5f 100644 --- a/data/project_types/pupmod.yaml +++ b/data/project_types/pupmod.yaml @@ -7,6 +7,14 @@ profile::obsoletes::files: - 'spec/fixtures/manifests/site.pp' # Not required since Puppet 3.x - '.pmtignore' # Now using .pdkignore +# Workflow files are bootstrap-strategy: laid down in full only when +# missing, then refreshed in place by the merge_github_workflows stage, +# which preserves Renovate-managed values (uses refs, ruby versions, +# runner/container images). See simp/puppetsync#50. +# (This key activates once profile::managed_file's strategy support is +# merged; it is inert before then.) +profile::github_actions::strategy: bootstrap + profile::github_actions::absent_action_files: - pr_glci.yml # PR-triggered GLCI actions - pr_glci_manual.yml # --> manual trigger for external contributors diff --git a/data/project_types/pupmod_skeleton.yaml b/data/project_types/pupmod_skeleton.yaml index 0a506cb..9f5d4f1 100644 --- a/data/project_types/pupmod_skeleton.yaml +++ b/data/project_types/pupmod_skeleton.yaml @@ -11,6 +11,10 @@ profile::obsoletes::files: - 'spec/fixtures/manifests/site.pp' # Not required since Puppet 3.x - '.pmtignore' # Now using .pdkignore +# Bootstrap + merge_github_workflows, same as project_type pupmod +# (see data/project_types/pupmod.yaml and simp/puppetsync#50) +profile::github_actions::strategy: bootstrap + profile::github_actions::absent_action_files: - pr_glci.yml # PR-triggered GLCI actions - pr_glci_manual.yml # --> manual trigger for external contributors diff --git a/dist/puppetsync/plans/init.pp b/dist/puppetsync/plans/init.pp index 28c1b3f..481e029 100644 --- a/dist/puppetsync/plans/init.pp +++ b/dist/puppetsync/plans/init.pp @@ -216,6 +216,63 @@ } } + $repos.puppetsync::pipeline_stage( + # -------------------------------------------------------------------------- + 'merge_github_workflows', + # -------------------------------------------------------------------------- + $opts + ) |$ok_repos, $stage_name| { + # Refresh existing workflow files from their baseline templates while + # preserving Renovate-managed values (action refs, image tags, ruby + # versions). Only files that exist in BOTH the repo and the template + # chain are merged; creating and removing workflow files remains + # profile::github_actions' job. See simp/puppetsync#50. + # + # NOTE: this stage only protects Renovate's values when + # profile::github_actions runs in bootstrap strategy (set per + # project_type in Hiera). With the profile still enforcing, the apply + # stage overwrites each workflow with pure template BEFORE this stage + # runs, so there is nothing left to preserve. The bootstrap flip for + # pupmod/pupmod_skeleton ships with this stage. + $gha_repos = $ok_repos.filter |$repo| { + $repo.facts['project_type'] in ['pupmod', 'pupmod_skeleton'] + } + run_task_with('puppetsync::merge_gha_workflows', + $gha_repos, + '_catch_errors' => true, + ) |$repo| { + $ptype = $repo.facts['project_type'] ? { + 'pupmod_skeleton' => 'pupmod', + default => $repo.facts['project_type'], + } + $target_module_name = $repo.facts.dig('module_metadata','name').lest || { + $repo.vars['mod_data']['repo_name'] + } + $wf_dir = "${repo.vars['repo_path']}/.github/workflows" + $existing_files = file::exists($wf_dir) ? { + true => dir::children($wf_dir).filter |$f| { $f =~ /\.yml$/ }, + default => [], + } + $workflows = $existing_files.map |$f| { + $action = $f.regsubst(/\.yml$/, '') + $template_path = find_file( + "profile/${ptype}/_github/workflows/${action}.${target_module_name}.yml", + "profile/${ptype}/_github/workflows/${action}.yml", + "profile/_github/workflows/${action}.${target_module_name}.yml", + "profile/_github/workflows/${action}.yml", + ) + $template_path ? { + undef => undef, + default => Hash({ 'path' => "${wf_dir}/${f}", 'template' => file::read($template_path) }), + } + }.filter |$wf| { $wf =~ NotUndef } + Hash({ + 'workflows' => $workflows, + 'preserve_keys' => $opts.dig('merge_github_workflows', 'preserve_keys'), + }) + } + } + $repos.puppetsync::pipeline_stage( # -------------------------------------------------------------------------- 'configure_renovate', diff --git a/dist/puppetsync/tasks/merge_gha_workflows.json b/dist/puppetsync/tasks/merge_gha_workflows.json new file mode 100644 index 0000000..fe7ed4b --- /dev/null +++ b/dist/puppetsync/tasks/merge_gha_workflows.json @@ -0,0 +1,14 @@ +{ + "description": "Refresh GitHub Actions workflow files from their baseline templates, preserving Renovate-managed values (action refs, container image tags, ruby versions, runner images, ...) already present in the repo. Template text is canonical; values are grafted via Psych node coordinates, so comments and formatting are never disturbed.", + "input_method": "stdin", + "parameters": { + "workflows": { + "description": "Workflow files to merge: [{path: , template: }, ...]", + "type": "Array[Hash]" + }, + "preserve_keys": { + "description": "YAML mapping keys whose scalar values are Renovate-managed and must be preserved from the existing file (default: uses, image, container, ruby-version, runs-on)", + "type": "Optional[Array[String[1]]]" + } + } +} diff --git a/dist/puppetsync/tasks/merge_gha_workflows.rb b/dist/puppetsync/tasks/merge_gha_workflows.rb new file mode 100644 index 0000000..9fdae18 --- /dev/null +++ b/dist/puppetsync/tasks/merge_gha_workflows.rb @@ -0,0 +1,134 @@ +#!/opt/puppetlabs/bolt/bin/ruby +# +# Refresh GitHub Actions workflow files from their baseline templates while +# preserving the Renovate-managed values already in the repo. +# +# The template text is canonical — structure, jobs, comments, and formatting +# all come from it byte-for-byte. The only things carried over from the +# existing file are the scalar values of `preserve_keys` (action refs, +# container image tags, ruby versions, ...), including any trailing +# comment — pinned-digest conventions keep the human-readable version there. +# +# Matching pairs occurrences of the same key + identity positionally, where +# the identity is the part of the value Renovate never changes: +# +# uses: actions/checkout@v5 -> identity 'actions/checkout' +# image: ghcr.io/foo/builder:8 -> identity 'ghcr.io/foo/builder' +# ruby-version: '3.2' -> no identity (paired per key) +# +# Values new in the template keep the template's value; entries that +# vanished from the template vanish from the file. Scalars are located via +# Psych's node line/column info, so this needs no YAML re-serialization: +# template comments and formatting can't be disturbed by construction. +# See simp/puppetsync#50. + +require 'json' +require 'psych' + +DEFAULT_PRESERVE_KEYS = %w[uses image container ruby-version runs-on].freeze + +# All scalar values of the given mapping keys, in document order, as +# [mapping_path, key, value_node] triples. The path contains mapping key +# names only (sequence positions are deliberately excluded, so steps can be +# reordered within a job without losing their values). +def preserved_scalars(node, keys, path = [], acc = []) + case node + when Psych::Nodes::Mapping + node.children.each_slice(2) do |key, value| + key_name = key.is_a?(Psych::Nodes::Scalar) ? key.value : '?' + acc << [path.join('.'), key_name, value] if keys.include?(key_name) && value.is_a?(Psych::Nodes::Scalar) + preserved_scalars(value, keys, path + [key_name], acc) + end + when Psych::Nodes::Stream, Psych::Nodes::Document, Psych::Nodes::Sequence + node.children.each { |child| preserved_scalars(child, keys, path, acc) } + end + acc +end + +# The part of a value Renovate never changes (nil when the whole value is +# the managed part, e.g. a bare version) +def identity(value) + if value.include?('@') + value.split('@', 2).first + elsif value.match?(%r{\A[\w./:-]+:[\w.-]+\z}) + # image:tag, including ported registries (registry:5000/foo:8) — the + # identity is everything before the final (tag) colon + value.rpartition(':').first + end +end + +# { [path, key, identity] => [raw rest-of-line starting at the scalar +# (value + trailing comment, exactly as written), ...] in document order } +def existing_values(text, keys) + lines = text.split("\n", -1) + preserved_scalars(Psych.parse(text), keys).each_with_object(Hash.new { |h, k| h[k] = [] }) do |(path, key, node), map| + next unless node.start_line == node.end_line + + map[[path, key, identity(node.value)]] << lines[node.start_line][node.start_column..] + end +end + +def merge_workflow(template, existing_text, keys) + existing = existing_values(existing_text, keys) + lines = template.split("\n", -1) + updated = [] + + # Pair occurrences positionally per [path, key, identity], so a value only + # carries over when the same job/step context still has it — per-occurrence + # differences (distinct versions per job, trailing comments) survive + # verbatim. Template entries with no counterpart in the existing file + # (a genuinely new or restored step) keep the template's value. + counters = Hash.new(0) + preserved_scalars(Psych.parse(template), keys).each do |path, key, node| + next unless node.start_line == node.end_line + + id = [path, key, identity(node.value)] + list = existing[id] + value = list[counters[id]] || list.last + counters[id] += 1 + next if value.nil? + + line = lines[node.start_line] + next if line[node.start_column..] == value + + lines[node.start_line] = line[0...node.start_column] + value + updated << "#{key}: #{value.split(/\s+#/).first}" + end + + [lines.join("\n"), updated] +end + +def merge_gha_workflows(workflows, keys) + results = {} + workflows.each do |wf| + path = wf.fetch('path') + template = wf.fetch('template') + + unless File.exist?(path) + File.write(path, template) + results[path] = { 'changed' => true, 'created' => true } + next + end + + existing_text = File.read(path) + merged, updated = merge_workflow(template, existing_text, keys) + + if merged == existing_text + results[path] = { 'changed' => false } + else + File.write(path, merged) + results[path] = { 'changed' => true, 'preserved_values' => updated } + end + end + + { 'changed' => results.values.any? { |r| r['changed'] }, 'files' => results } +end + +stdin = STDIN.read +params = JSON.parse(stdin) + +workflows = params['workflows'] +raise('No workflows given') unless workflows.is_a?(Array) +keys = params.fetch('preserve_keys', nil) || DEFAULT_PRESERVE_KEYS + +puts JSON.generate(merge_gha_workflows(workflows, keys)) diff --git a/spec/tasks/merge_gha_workflows_spec.rb b/spec/tasks/merge_gha_workflows_spec.rb new file mode 100644 index 0000000..cdc048d --- /dev/null +++ b/spec/tasks/merge_gha_workflows_spec.rb @@ -0,0 +1,244 @@ +require 'spec_helper' + +describe 'task: merge_gha_workflows' do + let(:real_template) do + File.read(File.join(REPO_ROOT, 'modules', 'profile', 'files', 'pupmod', '_github', 'workflows', 'pr_tests.yml')) + end + + around(:each) do |example| + Dir.mktmpdir do |dir| + @dir = dir + @wf = File.join(dir, 'pr_tests.yml') + example.run + end + end + + def run_merge(workflows) + run_task('merge_gha_workflows.rb', 'workflows' => workflows) + end + + it 'writes the template when the target does not exist' do + stdout, stderr, status = run_merge([{ 'path' => @wf, 'template' => real_template }]) + + expect(status).to be_success, stderr + expect(JSON.parse(stdout)['files'][@wf]).to include('changed' => true, 'created' => true) + expect(File.read(@wf)).to eq(real_template) + end + + it 'is a no-op when the target matches the template' do + File.write(@wf, real_template) + + stdout, stderr, status = run_merge([{ 'path' => @wf, 'template' => real_template }]) + + expect(status).to be_success, stderr + expect(JSON.parse(stdout)).to include('changed' => false) + expect(File.read(@wf)).to eq(real_template) + end + + it 'preserves Renovate-bumped action refs when refreshing the template' do + bumped = real_template.gsub('actions/checkout@v5', 'actions/checkout@v7') + raise 'munge failed' if bumped == real_template + File.write(@wf, bumped) + + stdout, stderr, status = run_merge([{ 'path' => @wf, 'template' => real_template }]) + + expect(status).to be_success, stderr + expect(JSON.parse(stdout)['files'][@wf]['changed']).to be false + expect(File.read(@wf)).to eq(bumped) # untouched: only refs differ, and they're preserved + end + + it 'restores template structure while grafting existing refs onto it' do + # Target: structurally stale (a job removed) AND Renovate-bumped + stale = real_template + .gsub('actions/checkout@v5', 'actions/checkout@v7') + .sub(/^ puppet-syntax:.*?(?=^ \w)/m, '') + raise 'munge failed' unless stale.length < real_template.length + File.write(@wf, stale) + + stdout, stderr, status = run_merge([{ 'path' => @wf, 'template' => real_template }]) + + expect(status).to be_success, stderr + expect(JSON.parse(stdout)['files'][@wf]['changed']).to be true + merged = File.read(@wf) + # Structure comes back from the template... + expect(merged).to include('puppet-syntax:') + # ...jobs that survived keep their bumped refs; the restored job takes + # the template's ref (it has no counterpart in the existing file) + expect(merged.scan('actions/checkout@v7').count).to eq(real_template.scan('actions/checkout@v5').count - 1) + expect(merged.scan('actions/checkout@v5').count).to eq(1) + # And apart from the grafted ref lines, output is byte-identical to the template + expect(merged.gsub('actions/checkout@v7', 'actions/checkout@v5')).to eq(real_template) + end + + it 'preserves pinned-digest refs with their trailing version comment' do + template = <<~YAML + jobs: + build: + steps: + - uses: actions/checkout@v5 + - name: setup + uses: ruby/setup-ruby@v1 + YAML + pinned = <<~YAML + jobs: + build: + steps: + - uses: actions/checkout@8edcb1bdb4e267140fa742c62e395cd74f332709 # v7.0.0 + - name: setup + uses: ruby/setup-ruby@ec106b438a1ff6ff109590de34ddc62c540232e0 # v1.244.0 + YAML + File.write(@wf, pinned) + + stdout, stderr, status = run_merge([{ 'path' => @wf, 'template' => template }]) + + expect(status).to be_success, stderr + expect(JSON.parse(stdout)['files'][@wf]['changed']).to be false + expect(File.read(@wf)).to eq(pinned) + end + + it 'uses the template ref for actions new to the template' do + existing = <<~YAML + jobs: + build: + steps: + - uses: actions/checkout@v7 + YAML + template = <<~YAML + jobs: + build: + steps: + - uses: actions/checkout@v5 + - uses: actions/upload-artifact@v4 + YAML + File.write(@wf, existing) + + stdout, stderr, status = run_merge([{ 'path' => @wf, 'template' => template }]) + + expect(status).to be_success, stderr + merged = File.read(@wf) + expect(merged).to include('actions/checkout@v7') # preserved + expect(merged).to include('actions/upload-artifact@v4') # from template + end + + it 'handles multiple workflow files in one invocation' do + other = File.join(@dir, 'tag_deploy.yml') + File.write(@wf, real_template.gsub('actions/checkout@v5', 'actions/checkout@v7')) + File.write(other, "jobs:\n x:\n steps:\n - uses: actions/checkout@v7\n") + + stdout, stderr, status = run_merge([ + { 'path' => @wf, 'template' => real_template }, + { 'path' => other, 'template' => "jobs:\n x:\n steps:\n - uses: actions/checkout@v5\n" }, + ]) + + expect(status).to be_success, stderr + result = JSON.parse(stdout) + expect(result['files'].keys).to contain_exactly(@wf, other) + expect(File.read(other)).to include('actions/checkout@v7') + end + + it 'preserves other Renovate-managed values: ruby versions and container image tags' do + template = <<~YAML + jobs: + spec: + container: + image: ghcr.io/simp/build:8.0.0 + steps: + - uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.2' + release: + container: ruby:3.2 + steps: + - uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.2' + YAML + # Renovate bumped each managed value — differently per job for ruby-version + bumped = <<~YAML + jobs: + spec: + container: + image: ghcr.io/simp/build:9.1.0 + steps: + - uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.4' + release: + container: ruby:3.3 + steps: + - uses: ruby/setup-ruby@v1 + with: + ruby-version: '4.0' + YAML + File.write(@wf, bumped) + + stdout, stderr, status = run_merge([{ 'path' => @wf, 'template' => template }]) + + expect(status).to be_success, stderr + expect(JSON.parse(stdout)['files'][@wf]['changed']).to be false + expect(File.read(@wf)).to eq(bumped) + end + + it 'grafts non-uses managed values onto refreshed template structure' do + template = <<~YAML + name: New Name + jobs: + spec: + container: + image: ghcr.io/simp/build:8.0.0 + steps: + - uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.2' + YAML + stale = template + .sub('New Name', 'Old Name') + .sub('ghcr.io/simp/build:8.0.0', 'ghcr.io/simp/build:9.1.0') + .sub("ruby-version: '3.2'", "ruby-version: '3.4'") + File.write(@wf, stale) + + stdout, stderr, status = run_merge([{ 'path' => @wf, 'template' => template }]) + + expect(status).to be_success, stderr + expect(JSON.parse(stdout)['files'][@wf]['changed']).to be true + merged = File.read(@wf) + expect(merged).to include('New Name') # structure from template + expect(merged).to include('ghcr.io/simp/build:9.1.0') # preserved + expect(merged).to include("ruby-version: '3.4'") # preserved + end + + it 'pairs ported-registry images by everything before the tag colon' do + template = <<~YAML + jobs: + build: + container: + image: registry.internal:5000/simp/builder:8.0.0 + YAML + bumped = template.sub(':8.0.0', ':9.2.0') + File.write(@wf, bumped) + + stdout, stderr, status = run_merge([{ 'path' => @wf, 'template' => template }]) + + expect(status).to be_success, stderr + expect(JSON.parse(stdout)['files'][@wf]['changed']).to be false + expect(File.read(@wf)).to eq(bumped) + end + + it 'is idempotent' do + stale = real_template.gsub('actions/checkout@v5', 'actions/checkout@v7').sub("name: PR Tests\n", "name: Old Name\n") + File.write(@wf, stale) + run_merge([{ 'path' => @wf, 'template' => real_template }]) + first_pass = File.read(@wf) + + stdout, stderr, status = run_merge([{ 'path' => @wf, 'template' => real_template }]) + + expect(status).to be_success, stderr + expect(JSON.parse(stdout)['changed']).to be false + expect(File.read(@wf)).to eq(first_pass) + end + + it 'fails when workflows param is missing' do + _stdout, _stderr, status = run_task('merge_gha_workflows.rb', {}) + expect(status).not_to be_success + end +end