Build inventory dynamically from the GitHub API (#55)

README.md

@@ -332,6 +332,33 @@ puppetsync::repos_config:
   # ... and so on
 ```
 
+#### Dynamic repolists (GitHub org inventory)
+
+Instead of hand-maintaining the repo list, a repolist file can define
+`puppetsync::repos_source` to build it from the GitHub API at run time (see
+`data/sync/repolists/github-org.yaml` for a ready-to-use example):
+
+```yaml
+puppetsync::repos_source:
+  org: simp
+  include: ['pupmod-*', 'puppet-*', 'rubygem-*']  # name globs
+  include_forks:                                  # forks the org maintains
+    - rubygem-simp-rspec-puppet-facts
+    - pupmod-voxpupuli-selinux
+```
+
+* Archived and empty repos are excluded; forks are excluded unless they
+  match `include_forks`
+* Repos with the `puppetsync-ignore` GitHub topic are excluded — set the
+  topic to opt a repo out without touching puppetsync
+* Each repo's branch comes from the API's default branch (so e.g.
+  `pupmod-voxpupuli-selinux` syncs `simp-master` automatically)
+* The generated list is snapshotted to
+  `data/sync/repolists/generated-{CONFIG_NAME}.yaml`; run the
+  approve/merge plans with `repolist=generated-{CONFIG_NAME}` so they use
+  the exact inventory the sync ran against (commit the snapshot if the
+  session is handed off)
+
 ### Plans
 
 Each plan:

data/sync/repolists/github-org.yaml

@@ -0,0 +1,28 @@
+---
+# Dynamic inventory (simp/puppetsync#55): build the repo list from the
+# GitHub API at run time instead of hand-maintaining it. When the sync plan
+# runs with this repolist, it writes a snapshot of the generated list to
+# data/sync/repolists/generated-<config>.yaml — use `repolist=generated-<config>`
+# for the approve/merge plans so they operate on the exact same inventory.
+#
+# Filtering notes:
+#   - Archived repos and empty repos are always excluded (exclude_archived
+#     defaults to true)
+#   - Forks are excluded unless listed in include_forks — the escape hatch
+#     for forks the org actively maintains
+#   - Repos with the 'puppetsync-ignore' topic are excluded by default: set
+#     that topic on a repo in GitHub to opt it out without touching this file
+#   - Names are matched with globs; project_type filtering (the config's
+#     permitted_project_types) still applies after the clone, so a
+#     name-pattern false positive gets culled there
+puppetsync::repos_source:
+  org: simp
+  include:
+    - 'pupmod-*'
+    - 'puppet-*'
+    - 'rubygem-*'
+  exclude: []
+  # Forks that simp actively maintains:
+  include_forks:
+    - rubygem-simp-rspec-puppet-facts
+    - pupmod-voxpupuli-selinux

dist/puppetsync/plans/init.pp

@@ -56,6 +56,15 @@
 #   A Hash of repos and branches to to clone, modify, and PR.
 #   By default, this is loaded from Hiera data based on the `config` parameter.
 #
+# @param repos_source
+#   Optional dynamic-inventory configuration (`puppetsync::repos_source` in
+#   the repolist file): build `repos_config` from the GitHub API instead of a
+#   hand-maintained list. See `data/sync/repolists/github-org.yaml` and the
+#   `puppetsync::list_github_repos` task. The generated list is snapshotted
+#   to `data/sync/repolists/generated-<config>.yaml` so the approve/merge
+#   plans can reuse the exact same inventory. Static `repos_config` entries
+#   are merged on top (they win for duplicate repo URLs).
+#
 # @param extra_gem_path
 #   Absolute path to a gem path with extra gems the bolt interpreter will to run
 #   some of the Ruby tasks.  This may be needed to provide rubygems for Jira,
@@ -99,7 +108,8 @@
   String[1]            $config                 = 'latest',
   String[1]            $repolist               = 'latest',
   Hash                 $puppetsync_config      = lookup('puppetsync::plan_config'),
-  Hash                 $repos_config           = lookup('puppetsync::repos_config'),
+  Hash                 $repos_config           = lookup('puppetsync::repos_config', Hash, undef, {}),
+  Optional[Hash]       $repos_source           = lookup('puppetsync::repos_source', Optional[Hash], undef, undef),
   Optional[String[1]]  $puppet_role            = $puppetsync_config.dig('puppetsync','puppet_role'),
   Stdlib::Absolutepath $extra_gem_path         = "${project_dir}/.plan.gems",
   Sensitive[String[1]] $github_token           = Sensitive(system::env('GITHUB_API_TOKEN')),
@@ -110,7 +120,31 @@
     'clone_git_repos'          => true,
     'github_api_delay_seconds' => 5,
    } + getvar('puppetsync_config.puppetsync.plans.sync').lest || {{}} + $options
-  $repos = puppetsync::setup_project_repos( $puppetsync_config, $repos_config, $project_dir, $opts )
+
+  # Dynamic inventory (simp/puppetsync#55): when the repolist file defines
+  # `puppetsync::repos_source`, build the repo list from the GitHub API and
+  # snapshot it so approve/merge plans can reuse the exact same inventory.
+  if $repos_source =~ NotUndef and !$opts.dig('list_pipeline_stages') {
+    $listing = run_task('puppetsync::list_github_repos', 'localhost',
+      "List GitHub org '${repos_source['org']}' repos for dynamic inventory",
+      {
+        'source'           => $repos_source,
+        'github_authtoken' => $github_token.unwrap,
+      }
+    ).first.value
+
+    $effective_repos_config = $listing['repos_config'] + $repos_config
+    $snapshot_path = "${project_dir}/data/sync/repolists/generated-${config}.yaml"
+    file::write($snapshot_path, Hash({'puppetsync::repos_config' => $effective_repos_config}).to_yaml)
+    out::message( sprintf(
+      '== dynamic inventory: %d repos from GitHub org %s (snapshot: %s — use repolist=generated-%s for approve/merge)',
+      $effective_repos_config.size, $repos_source['org'], $snapshot_path, $config,
+    ))
+  } else {
+    $effective_repos_config = $repos_config
+  }
+
+  $repos = puppetsync::setup_project_repos( $puppetsync_config, $effective_repos_config, $project_dir, $opts )
   $feature_branch    = getvar('puppetsync_config.git.feature_branch')
 
   # ----------------------------------------------------------------------------

dist/puppetsync/tasks/list_github_repos.json

@@ -0,0 +1,19 @@
+{
+  "description": "Build a puppetsync repos_config hash from the GitHub API: list an org's repos and filter them (archived, forks with an include_forks escape hatch, name globs, topics). Branches come from each repo's default_branch.",
+  "input_method": "stdin",
+  "parameters": {
+    "source": {
+      "description": "Filter configuration: {org, include, exclude, include_forks, include_topics, exclude_topics, exclude_archived, exclude_forks}",
+      "type": "Hash"
+    },
+    "github_authtoken": {
+      "description": "GitHub API token (optional but strongly recommended: unauthenticated requests are heavily rate-limited)",
+      "type": "Optional[String[1]]",
+      "sensitive": true
+    },
+    "repos": {
+      "description": "Repo objects to filter instead of querying the API (for testing)",
+      "type": "Optional[Array[Hash]]"
+    }
+  }
+}

dist/puppetsync/tasks/list_github_repos.rb

@@ -0,0 +1,127 @@
+#!/opt/puppetlabs/bolt/bin/ruby
+#
+# Build a puppetsync repos_config hash dynamically from the GitHub API
+# (simp/puppetsync#55), so repolists don't have to be hand-maintained.
+#
+# Filtering (all driven by the `source` parameter):
+#
+#   - Archived repos are excluded (`exclude_archived`, default true)
+#   - Forks are excluded (`exclude_forks`, default true) UNLESS the repo
+#     matches an `include_forks` glob — the escape hatch for forks the org
+#     actively maintains (e.g. rubygem-simp-rspec-puppet-facts,
+#     pupmod-voxpupuli-selinux)
+#   - Repos with any topic in `exclude_topics` are excluded (default:
+#     ['puppetsync-ignore'] — set that topic on a repo in GitHub to opt it
+#     out without touching puppetsync)
+#   - The repo name must match an `include` glob, or the repo must have a
+#     topic in `include_topics` (defaults: ['*'] / none)
+#   - Repos matching an `exclude` glob are always excluded
+#   - Empty repos (size 0) are excluded — there is nothing to clone
+#
+# Each repo's branch comes from the API's default_branch (e.g.
+# pupmod-voxpupuli-selinux uses 'simp-master').
+#
+# NOTE: this intentionally does NOT decide what kind of project a repo is —
+# `puppetsync::filter_permitted_repos` still applies project_type filtering
+# after the clone, which is the safety net for name-pattern false positives.
+
+require 'json'
+require 'net/http'
+require 'uri'
+
+PER_PAGE = 100
+
+# Transient failures (network errors, 5xx) are retried with backoff so a
+# GitHub blip doesn't abort a whole (possibly scheduled) run; 4xx fails fast
+def fetch_page(uri, token, attempts: 3)
+  attempt = 0
+  begin
+    attempt += 1
+    request = Net::HTTP::Get.new(uri)
+    request['Accept'] = 'application/vnd.github+json'
+    request['X-GitHub-Api-Version'] = '2022-11-28'
+    request['Authorization'] = "Bearer #{token}" if token && !token.empty?
+    response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) { |http| http.request(request) }
+    raise Net::HTTPFatalError.new("GitHub API returned #{response.code}", response) if response.code.start_with?('5')
+    raise("ERROR: GitHub API returned #{response.code} for #{uri.path}: #{response.body.to_s[0, 300]}") unless response.code == '200'
+
+    response.body
+  rescue Net::HTTPFatalError, SocketError, SystemCallError, Net::OpenTimeout, Net::ReadTimeout, OpenSSL::SSL::SSLError => e
+    raise("ERROR: GitHub API request for #{uri.path} failed after #{attempts} attempts: #{e.message}") if attempt >= attempts
+
+    warn "== retrying #{uri.path} after error (attempt #{attempt}/#{attempts}): #{e.message}"
+    sleep(2 * attempt)
+    retry
+  end
+end
+
+def fetch_org_repos(org, token)
+  repos = []
+  page = 1
+  loop do
+    body = fetch_page(URI("https://api.github.com/orgs/#{org}/repos?type=all&per_page=#{PER_PAGE}&page=#{page}"), token)
+    batch = JSON.parse(body)
+    repos.concat(batch)
+    break if batch.size < PER_PAGE
+
+    page += 1
+  end
+  repos
+end
+
+def glob_match?(name, globs)
+  globs.any? { |glob| File.fnmatch(glob, name) }
+end
+
+def select_repos(repos, source)
+  include_globs    = source.fetch('include', nil) || ['*']
+  exclude_globs    = source.fetch('exclude', nil) || []
+  include_topics   = source.fetch('include_topics', nil) || []
+  exclude_topics   = source.fetch('exclude_topics', nil) || ['puppetsync-ignore']
+  exclude_archived = source.fetch('exclude_archived', true)
+  exclude_forks    = source.fetch('exclude_forks', true)
+  include_forks    = source.fetch('include_forks', nil) || []
+
+  repos.select do |repo|
+    name = repo['name']
+    topics = repo['topics'] || []
+    next false if exclude_archived && repo['archived']
+    next false if repo['size'].to_i.zero?
+    next false if glob_match?(name, exclude_globs)
+    next false if topics.any? { |topic| exclude_topics.include?(topic) }
+
+    # A fork listed in include_forks is an explicit allow-list entry: it
+    # both clears the fork gate AND bypasses the include filters below
+    # (listing it means "I want this fork", whatever its name)
+    explicitly_included_fork = repo['fork'] && glob_match?(name, include_forks)
+    next false if repo['fork'] && exclude_forks && !explicitly_included_fork
+    next true if explicitly_included_fork
+
+    glob_match?(name, include_globs) || topics.any? { |topic| include_topics.include?(topic) }
+  end
+end
+
+def repos_config(repos)
+  repos.sort_by { |repo| repo['name'] }.each_with_object({}) do |repo, config|
+    config["https://github.com/#{repo['full_name']}"] = { 'branch' => repo['default_branch'] }
+  end
+end
+
+stdin = STDIN.read
+params = JSON.parse(stdin)
+
+source = params['source']
+raise('No source given') unless source.is_a?(Hash)
+
+# `repos` bypasses the API for testing; normal runs fetch the org listing
+repos = params['repos']
+if repos.nil?
+  org = source['org']
+  raise("No org given in source") unless org
+
+  repos = fetch_org_repos(org, params['github_authtoken'])
+end
+
+selected = select_repos(repos, source)
+warn "== selected #{selected.size} of #{repos.size} repos"
+puts JSON.generate({ 'repos_config' => repos_config(selected), 'count' => selected.size })

spec/tasks/list_github_repos_spec.rb

@@ -0,0 +1,123 @@
+require 'spec_helper'
+
+describe 'task: list_github_repos' do
+  # Modeled on real simp-org repos (see simp/puppetsync#55)
+  let(:org_repos) do
+    [
+      { 'name' => 'pupmod-simp-aide', 'full_name' => 'simp/pupmod-simp-aide',
+        'fork' => false, 'archived' => false, 'default_branch' => 'master', 'topics' => [], 'size' => 500 },
+      { 'name' => 'puppet-gpasswd', 'full_name' => 'simp/puppet-gpasswd',
+        'fork' => false, 'archived' => false, 'default_branch' => 'master', 'topics' => [], 'size' => 200 },
+      { 'name' => 'pupmod-simp-ntpd', 'full_name' => 'simp/pupmod-simp-ntpd',
+        'fork' => false, 'archived' => true, 'default_branch' => 'master', 'topics' => [], 'size' => 300 },
+      { 'name' => 'rubygem-simp-rspec-puppet-facts', 'full_name' => 'simp/rubygem-simp-rspec-puppet-facts',
+        'fork' => true, 'archived' => false, 'default_branch' => 'master', 'topics' => [], 'size' => 100 },
+      { 'name' => 'pupmod-voxpupuli-selinux', 'full_name' => 'simp/pupmod-voxpupuli-selinux',
+        'fork' => true, 'archived' => false, 'default_branch' => 'simp-master', 'topics' => [], 'size' => 400 },
+      { 'name' => 'puppetlabs-apache', 'full_name' => 'simp/puppetlabs-apache',
+        'fork' => true, 'archived' => false, 'default_branch' => 'main', 'topics' => [], 'size' => 900 },
+      { 'name' => 'pupmod-simp-optout', 'full_name' => 'simp/pupmod-simp-optout',
+        'fork' => false, 'archived' => false, 'default_branch' => 'master', 'topics' => ['puppetsync-ignore'], 'size' => 100 },
+      { 'name' => 'topic-tagged-tool', 'full_name' => 'simp/topic-tagged-tool',
+        'fork' => false, 'archived' => false, 'default_branch' => 'main', 'topics' => ['simp-baseline'], 'size' => 50 },
+      { 'name' => 'empty-repo', 'full_name' => 'simp/empty-repo',
+        'fork' => false, 'archived' => false, 'default_branch' => 'main', 'topics' => [], 'size' => 0 },
+    ]
+  end
+
+  def run_list(source)
+    stdout, stderr, status = run_task('list_github_repos.rb', 'source' => source, 'repos' => org_repos)
+    expect(status).to be_success, stderr
+    JSON.parse(stdout)
+  end
+
+  def urls(result)
+    result['repos_config'].keys
+  end
+
+  it 'excludes archived repos, forks, empty and opted-out repos by default' do
+    result = run_list({ 'org' => 'simp' })
+
+    expect(urls(result)).to contain_exactly(
+      'https://github.com/simp/pupmod-simp-aide',
+      'https://github.com/simp/puppet-gpasswd',
+      'https://github.com/simp/topic-tagged-tool',
+    )
+  end
+
+  it 'keeps forks matching include_forks (the maintained-fork exceptions)' do
+    result = run_list(
+      'org' => 'simp',
+      'include_forks' => ['rubygem-simp-rspec-puppet-facts', 'pupmod-voxpupuli-selinux'],
+    )
+
+    expect(urls(result)).to include(
+      'https://github.com/simp/rubygem-simp-rspec-puppet-facts',
+      'https://github.com/simp/pupmod-voxpupuli-selinux',
+    )
+    expect(urls(result)).not_to include('https://github.com/simp/puppetlabs-apache')
+  end
+
+  it 'include_forks alone suffices: listed forks bypass the include globs' do
+    # Mirrors the shipped github-org.yaml shape, but with include globs that
+    # do NOT match one of the fork exceptions
+    result = run_list(
+      'org' => 'simp',
+      'include' => ['pupmod-*'],
+      'include_forks' => ['rubygem-simp-rspec-puppet-facts'],
+    )
+
+    expect(urls(result)).to include('https://github.com/simp/rubygem-simp-rspec-puppet-facts')
+    expect(urls(result)).not_to include('https://github.com/simp/puppet-gpasswd') # non-fork still needs include
+  end
+
+  it 'takes each branch from the API default_branch' do
+    result = run_list(
+      'org' => 'simp',
+      'include_forks' => ['pupmod-voxpupuli-selinux'],
+    )
+
+    expect(result['repos_config']['https://github.com/simp/pupmod-voxpupuli-selinux']).to eq('branch' => 'simp-master')
+    expect(result['repos_config']['https://github.com/simp/pupmod-simp-aide']).to eq('branch' => 'master')
+  end
+
+  it 'filters by include name globs without assuming pupmod-simp-*' do
+    result = run_list('org' => 'simp', 'include' => ['pupmod-*', 'puppet-*'])
+
+    expect(urls(result)).to contain_exactly(
+      'https://github.com/simp/pupmod-simp-aide',
+      'https://github.com/simp/puppet-gpasswd',
+    )
+  end
+
+  it 'admits repos by topic even when name globs miss them' do
+    result = run_list('org' => 'simp', 'include' => ['pupmod-*'], 'include_topics' => ['simp-baseline'])
+
+    expect(urls(result)).to include(
+      'https://github.com/simp/pupmod-simp-aide',
+      'https://github.com/simp/topic-tagged-tool',
+    )
+  end
+
+  it 'honors explicit exclude globs above all' do
+    result = run_list('org' => 'simp', 'exclude' => ['pupmod-simp-aide'])
+
+    expect(urls(result)).not_to include('https://github.com/simp/pupmod-simp-aide')
+  end
+
+  it 'can include archived repos when asked' do
+    result = run_list('org' => 'simp', 'exclude_archived' => false)
+
+    expect(urls(result)).to include('https://github.com/simp/pupmod-simp-ntpd')
+  end
+
+  it 'sorts output by repo name for stable snapshots' do
+    result = run_list('org' => 'simp')
+    expect(urls(result)).to eq(urls(result).sort_by { |u| u.split('/').last })
+  end
+
+  it 'fails without a source' do
+    _stdout, _stderr, status = run_task('list_github_repos.rb', 'repos' => [])
+    expect(status).not_to be_success
+  end
+end