diff --git a/README.md b/README.md index ff53566..6e0ad71 100644 --- a/README.md +++ b/README.md @@ -332,6 +332,33 @@ puppetsync::repos_config: # ... and so on ``` +#### Dynamic repolists (GitHub org inventory) + +Instead of hand-maintaining the repo list, a repolist file can define +`puppetsync::repos_source` to build it from the GitHub API at run time (see +`data/sync/repolists/github-org.yaml` for a ready-to-use example): + +```yaml +puppetsync::repos_source: + org: simp + include: ['pupmod-*', 'puppet-*', 'rubygem-*'] # name globs + include_forks: # forks the org maintains + - rubygem-simp-rspec-puppet-facts + - pupmod-voxpupuli-selinux +``` + +* Archived and empty repos are excluded; forks are excluded unless they + match `include_forks` +* Repos with the `puppetsync-ignore` GitHub topic are excluded — set the + topic to opt a repo out without touching puppetsync +* Each repo's branch comes from the API's default branch (so e.g. + `pupmod-voxpupuli-selinux` syncs `simp-master` automatically) +* The generated list is snapshotted to + `data/sync/repolists/generated-{CONFIG_NAME}.yaml`; run the + approve/merge plans with `repolist=generated-{CONFIG_NAME}` so they use + the exact inventory the sync ran against (commit the snapshot if the + session is handed off) + ### Plans Each plan: diff --git a/data/sync/repolists/github-org.yaml b/data/sync/repolists/github-org.yaml new file mode 100644 index 0000000..9e0832a --- /dev/null +++ b/data/sync/repolists/github-org.yaml @@ -0,0 +1,28 @@ +--- +# Dynamic inventory (simp/puppetsync#55): build the repo list from the +# GitHub API at run time instead of hand-maintaining it. When the sync plan +# runs with this repolist, it writes a snapshot of the generated list to +# data/sync/repolists/generated-.yaml — use `repolist=generated-` +# for the approve/merge plans so they operate on the exact same inventory. +# +# Filtering notes: +# - Archived repos and empty repos are always excluded (exclude_archived +# defaults to true) +# - Forks are excluded unless listed in include_forks — the escape hatch +# for forks the org actively maintains +# - Repos with the 'puppetsync-ignore' topic are excluded by default: set +# that topic on a repo in GitHub to opt it out without touching this file +# - Names are matched with globs; project_type filtering (the config's +# permitted_project_types) still applies after the clone, so a +# name-pattern false positive gets culled there +puppetsync::repos_source: + org: simp + include: + - 'pupmod-*' + - 'puppet-*' + - 'rubygem-*' + exclude: [] + # Forks that simp actively maintains: + include_forks: + - rubygem-simp-rspec-puppet-facts + - pupmod-voxpupuli-selinux diff --git a/dist/puppetsync/plans/init.pp b/dist/puppetsync/plans/init.pp index 28c1b3f..9ac6adc 100644 --- a/dist/puppetsync/plans/init.pp +++ b/dist/puppetsync/plans/init.pp @@ -56,6 +56,15 @@ # A Hash of repos and branches to to clone, modify, and PR. # By default, this is loaded from Hiera data based on the `config` parameter. # +# @param repos_source +# Optional dynamic-inventory configuration (`puppetsync::repos_source` in +# the repolist file): build `repos_config` from the GitHub API instead of a +# hand-maintained list. See `data/sync/repolists/github-org.yaml` and the +# `puppetsync::list_github_repos` task. The generated list is snapshotted +# to `data/sync/repolists/generated-.yaml` so the approve/merge +# plans can reuse the exact same inventory. Static `repos_config` entries +# are merged on top (they win for duplicate repo URLs). +# # @param extra_gem_path # Absolute path to a gem path with extra gems the bolt interpreter will to run # some of the Ruby tasks. This may be needed to provide rubygems for Jira, @@ -99,7 +108,8 @@ String[1] $config = 'latest', String[1] $repolist = 'latest', Hash $puppetsync_config = lookup('puppetsync::plan_config'), - Hash $repos_config = lookup('puppetsync::repos_config'), + Hash $repos_config = lookup('puppetsync::repos_config', Hash, undef, {}), + Optional[Hash] $repos_source = lookup('puppetsync::repos_source', Optional[Hash], undef, undef), Optional[String[1]] $puppet_role = $puppetsync_config.dig('puppetsync','puppet_role'), Stdlib::Absolutepath $extra_gem_path = "${project_dir}/.plan.gems", Sensitive[String[1]] $github_token = Sensitive(system::env('GITHUB_API_TOKEN')), @@ -110,7 +120,31 @@ 'clone_git_repos' => true, 'github_api_delay_seconds' => 5, } + getvar('puppetsync_config.puppetsync.plans.sync').lest || {{}} + $options - $repos = puppetsync::setup_project_repos( $puppetsync_config, $repos_config, $project_dir, $opts ) + + # Dynamic inventory (simp/puppetsync#55): when the repolist file defines + # `puppetsync::repos_source`, build the repo list from the GitHub API and + # snapshot it so approve/merge plans can reuse the exact same inventory. + if $repos_source =~ NotUndef and !$opts.dig('list_pipeline_stages') { + $listing = run_task('puppetsync::list_github_repos', 'localhost', + "List GitHub org '${repos_source['org']}' repos for dynamic inventory", + { + 'source' => $repos_source, + 'github_authtoken' => $github_token.unwrap, + } + ).first.value + + $effective_repos_config = $listing['repos_config'] + $repos_config + $snapshot_path = "${project_dir}/data/sync/repolists/generated-${config}.yaml" + file::write($snapshot_path, Hash({'puppetsync::repos_config' => $effective_repos_config}).to_yaml) + out::message( sprintf( + '== dynamic inventory: %d repos from GitHub org %s (snapshot: %s — use repolist=generated-%s for approve/merge)', + $effective_repos_config.size, $repos_source['org'], $snapshot_path, $config, + )) + } else { + $effective_repos_config = $repos_config + } + + $repos = puppetsync::setup_project_repos( $puppetsync_config, $effective_repos_config, $project_dir, $opts ) $feature_branch = getvar('puppetsync_config.git.feature_branch') # ---------------------------------------------------------------------------- diff --git a/dist/puppetsync/tasks/list_github_repos.json b/dist/puppetsync/tasks/list_github_repos.json new file mode 100644 index 0000000..a27d111 --- /dev/null +++ b/dist/puppetsync/tasks/list_github_repos.json @@ -0,0 +1,19 @@ +{ + "description": "Build a puppetsync repos_config hash from the GitHub API: list an org's repos and filter them (archived, forks with an include_forks escape hatch, name globs, topics). Branches come from each repo's default_branch.", + "input_method": "stdin", + "parameters": { + "source": { + "description": "Filter configuration: {org, include, exclude, include_forks, include_topics, exclude_topics, exclude_archived, exclude_forks}", + "type": "Hash" + }, + "github_authtoken": { + "description": "GitHub API token (optional but strongly recommended: unauthenticated requests are heavily rate-limited)", + "type": "Optional[String[1]]", + "sensitive": true + }, + "repos": { + "description": "Repo objects to filter instead of querying the API (for testing)", + "type": "Optional[Array[Hash]]" + } + } +} diff --git a/dist/puppetsync/tasks/list_github_repos.rb b/dist/puppetsync/tasks/list_github_repos.rb new file mode 100644 index 0000000..b5a0e53 --- /dev/null +++ b/dist/puppetsync/tasks/list_github_repos.rb @@ -0,0 +1,127 @@ +#!/opt/puppetlabs/bolt/bin/ruby +# +# Build a puppetsync repos_config hash dynamically from the GitHub API +# (simp/puppetsync#55), so repolists don't have to be hand-maintained. +# +# Filtering (all driven by the `source` parameter): +# +# - Archived repos are excluded (`exclude_archived`, default true) +# - Forks are excluded (`exclude_forks`, default true) UNLESS the repo +# matches an `include_forks` glob — the escape hatch for forks the org +# actively maintains (e.g. rubygem-simp-rspec-puppet-facts, +# pupmod-voxpupuli-selinux) +# - Repos with any topic in `exclude_topics` are excluded (default: +# ['puppetsync-ignore'] — set that topic on a repo in GitHub to opt it +# out without touching puppetsync) +# - The repo name must match an `include` glob, or the repo must have a +# topic in `include_topics` (defaults: ['*'] / none) +# - Repos matching an `exclude` glob are always excluded +# - Empty repos (size 0) are excluded — there is nothing to clone +# +# Each repo's branch comes from the API's default_branch (e.g. +# pupmod-voxpupuli-selinux uses 'simp-master'). +# +# NOTE: this intentionally does NOT decide what kind of project a repo is — +# `puppetsync::filter_permitted_repos` still applies project_type filtering +# after the clone, which is the safety net for name-pattern false positives. + +require 'json' +require 'net/http' +require 'uri' + +PER_PAGE = 100 + +# Transient failures (network errors, 5xx) are retried with backoff so a +# GitHub blip doesn't abort a whole (possibly scheduled) run; 4xx fails fast +def fetch_page(uri, token, attempts: 3) + attempt = 0 + begin + attempt += 1 + request = Net::HTTP::Get.new(uri) + request['Accept'] = 'application/vnd.github+json' + request['X-GitHub-Api-Version'] = '2022-11-28' + request['Authorization'] = "Bearer #{token}" if token && !token.empty? + response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) { |http| http.request(request) } + raise Net::HTTPFatalError.new("GitHub API returned #{response.code}", response) if response.code.start_with?('5') + raise("ERROR: GitHub API returned #{response.code} for #{uri.path}: #{response.body.to_s[0, 300]}") unless response.code == '200' + + response.body + rescue Net::HTTPFatalError, SocketError, SystemCallError, Net::OpenTimeout, Net::ReadTimeout, OpenSSL::SSL::SSLError => e + raise("ERROR: GitHub API request for #{uri.path} failed after #{attempts} attempts: #{e.message}") if attempt >= attempts + + warn "== retrying #{uri.path} after error (attempt #{attempt}/#{attempts}): #{e.message}" + sleep(2 * attempt) + retry + end +end + +def fetch_org_repos(org, token) + repos = [] + page = 1 + loop do + body = fetch_page(URI("https://api.github.com/orgs/#{org}/repos?type=all&per_page=#{PER_PAGE}&page=#{page}"), token) + batch = JSON.parse(body) + repos.concat(batch) + break if batch.size < PER_PAGE + + page += 1 + end + repos +end + +def glob_match?(name, globs) + globs.any? { |glob| File.fnmatch(glob, name) } +end + +def select_repos(repos, source) + include_globs = source.fetch('include', nil) || ['*'] + exclude_globs = source.fetch('exclude', nil) || [] + include_topics = source.fetch('include_topics', nil) || [] + exclude_topics = source.fetch('exclude_topics', nil) || ['puppetsync-ignore'] + exclude_archived = source.fetch('exclude_archived', true) + exclude_forks = source.fetch('exclude_forks', true) + include_forks = source.fetch('include_forks', nil) || [] + + repos.select do |repo| + name = repo['name'] + topics = repo['topics'] || [] + next false if exclude_archived && repo['archived'] + next false if repo['size'].to_i.zero? + next false if glob_match?(name, exclude_globs) + next false if topics.any? { |topic| exclude_topics.include?(topic) } + + # A fork listed in include_forks is an explicit allow-list entry: it + # both clears the fork gate AND bypasses the include filters below + # (listing it means "I want this fork", whatever its name) + explicitly_included_fork = repo['fork'] && glob_match?(name, include_forks) + next false if repo['fork'] && exclude_forks && !explicitly_included_fork + next true if explicitly_included_fork + + glob_match?(name, include_globs) || topics.any? { |topic| include_topics.include?(topic) } + end +end + +def repos_config(repos) + repos.sort_by { |repo| repo['name'] }.each_with_object({}) do |repo, config| + config["https://github.com/#{repo['full_name']}"] = { 'branch' => repo['default_branch'] } + end +end + +stdin = STDIN.read +params = JSON.parse(stdin) + +source = params['source'] +raise('No source given') unless source.is_a?(Hash) + +# `repos` bypasses the API for testing; normal runs fetch the org listing +repos = params['repos'] +if repos.nil? + org = source['org'] + raise("No org given in source") unless org + + repos = fetch_org_repos(org, params['github_authtoken']) +end + +selected = select_repos(repos, source) +warn "== selected #{selected.size} of #{repos.size} repos" +puts JSON.generate({ 'repos_config' => repos_config(selected), 'count' => selected.size }) diff --git a/spec/tasks/list_github_repos_spec.rb b/spec/tasks/list_github_repos_spec.rb new file mode 100644 index 0000000..9b65bb4 --- /dev/null +++ b/spec/tasks/list_github_repos_spec.rb @@ -0,0 +1,123 @@ +require 'spec_helper' + +describe 'task: list_github_repos' do + # Modeled on real simp-org repos (see simp/puppetsync#55) + let(:org_repos) do + [ + { 'name' => 'pupmod-simp-aide', 'full_name' => 'simp/pupmod-simp-aide', + 'fork' => false, 'archived' => false, 'default_branch' => 'master', 'topics' => [], 'size' => 500 }, + { 'name' => 'puppet-gpasswd', 'full_name' => 'simp/puppet-gpasswd', + 'fork' => false, 'archived' => false, 'default_branch' => 'master', 'topics' => [], 'size' => 200 }, + { 'name' => 'pupmod-simp-ntpd', 'full_name' => 'simp/pupmod-simp-ntpd', + 'fork' => false, 'archived' => true, 'default_branch' => 'master', 'topics' => [], 'size' => 300 }, + { 'name' => 'rubygem-simp-rspec-puppet-facts', 'full_name' => 'simp/rubygem-simp-rspec-puppet-facts', + 'fork' => true, 'archived' => false, 'default_branch' => 'master', 'topics' => [], 'size' => 100 }, + { 'name' => 'pupmod-voxpupuli-selinux', 'full_name' => 'simp/pupmod-voxpupuli-selinux', + 'fork' => true, 'archived' => false, 'default_branch' => 'simp-master', 'topics' => [], 'size' => 400 }, + { 'name' => 'puppetlabs-apache', 'full_name' => 'simp/puppetlabs-apache', + 'fork' => true, 'archived' => false, 'default_branch' => 'main', 'topics' => [], 'size' => 900 }, + { 'name' => 'pupmod-simp-optout', 'full_name' => 'simp/pupmod-simp-optout', + 'fork' => false, 'archived' => false, 'default_branch' => 'master', 'topics' => ['puppetsync-ignore'], 'size' => 100 }, + { 'name' => 'topic-tagged-tool', 'full_name' => 'simp/topic-tagged-tool', + 'fork' => false, 'archived' => false, 'default_branch' => 'main', 'topics' => ['simp-baseline'], 'size' => 50 }, + { 'name' => 'empty-repo', 'full_name' => 'simp/empty-repo', + 'fork' => false, 'archived' => false, 'default_branch' => 'main', 'topics' => [], 'size' => 0 }, + ] + end + + def run_list(source) + stdout, stderr, status = run_task('list_github_repos.rb', 'source' => source, 'repos' => org_repos) + expect(status).to be_success, stderr + JSON.parse(stdout) + end + + def urls(result) + result['repos_config'].keys + end + + it 'excludes archived repos, forks, empty and opted-out repos by default' do + result = run_list({ 'org' => 'simp' }) + + expect(urls(result)).to contain_exactly( + 'https://github.com/simp/pupmod-simp-aide', + 'https://github.com/simp/puppet-gpasswd', + 'https://github.com/simp/topic-tagged-tool', + ) + end + + it 'keeps forks matching include_forks (the maintained-fork exceptions)' do + result = run_list( + 'org' => 'simp', + 'include_forks' => ['rubygem-simp-rspec-puppet-facts', 'pupmod-voxpupuli-selinux'], + ) + + expect(urls(result)).to include( + 'https://github.com/simp/rubygem-simp-rspec-puppet-facts', + 'https://github.com/simp/pupmod-voxpupuli-selinux', + ) + expect(urls(result)).not_to include('https://github.com/simp/puppetlabs-apache') + end + + it 'include_forks alone suffices: listed forks bypass the include globs' do + # Mirrors the shipped github-org.yaml shape, but with include globs that + # do NOT match one of the fork exceptions + result = run_list( + 'org' => 'simp', + 'include' => ['pupmod-*'], + 'include_forks' => ['rubygem-simp-rspec-puppet-facts'], + ) + + expect(urls(result)).to include('https://github.com/simp/rubygem-simp-rspec-puppet-facts') + expect(urls(result)).not_to include('https://github.com/simp/puppet-gpasswd') # non-fork still needs include + end + + it 'takes each branch from the API default_branch' do + result = run_list( + 'org' => 'simp', + 'include_forks' => ['pupmod-voxpupuli-selinux'], + ) + + expect(result['repos_config']['https://github.com/simp/pupmod-voxpupuli-selinux']).to eq('branch' => 'simp-master') + expect(result['repos_config']['https://github.com/simp/pupmod-simp-aide']).to eq('branch' => 'master') + end + + it 'filters by include name globs without assuming pupmod-simp-*' do + result = run_list('org' => 'simp', 'include' => ['pupmod-*', 'puppet-*']) + + expect(urls(result)).to contain_exactly( + 'https://github.com/simp/pupmod-simp-aide', + 'https://github.com/simp/puppet-gpasswd', + ) + end + + it 'admits repos by topic even when name globs miss them' do + result = run_list('org' => 'simp', 'include' => ['pupmod-*'], 'include_topics' => ['simp-baseline']) + + expect(urls(result)).to include( + 'https://github.com/simp/pupmod-simp-aide', + 'https://github.com/simp/topic-tagged-tool', + ) + end + + it 'honors explicit exclude globs above all' do + result = run_list('org' => 'simp', 'exclude' => ['pupmod-simp-aide']) + + expect(urls(result)).not_to include('https://github.com/simp/pupmod-simp-aide') + end + + it 'can include archived repos when asked' do + result = run_list('org' => 'simp', 'exclude_archived' => false) + + expect(urls(result)).to include('https://github.com/simp/pupmod-simp-ntpd') + end + + it 'sorts output by repo name for stable snapshots' do + result = run_list('org' => 'simp') + expect(urls(result)).to eq(urls(result).sort_by { |u| u.split('/').last }) + end + + it 'fails without a source' do + _stdout, _stderr, status = run_task('list_github_repos.rb', 'repos' => []) + expect(status).not_to be_success + end +end