From 687686cd2c0d9a3e81fbcba1c5803c239cf200bf Mon Sep 17 00:00:00 2001 From: mich-elle-luna Date: Wed, 10 Jun 2026 08:15:37 -0700 Subject: [PATCH 01/22] Add Redis repository memory workflow (demo) Adds a GitHub Actions workflow that stores commit and PR context in Redis as vector embeddings and surfaces related prior discussions back into GitHub via PR comments or workflow step summaries. Triggers on push to non-main branches, pull_request opened, and workflow_dispatch for manual testing. Requires MEMORY_REDIS_URL and OPENAI_API_KEY secrets. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/repository-memory.yml | 45 ++++++++ scripts/memory/collect_context.js | 96 ++++++++++++++++ scripts/memory/package.json | 13 +++ scripts/memory/post_comment.js | 100 ++++++++++++++++ scripts/memory/retrieve_memory.js | 144 ++++++++++++++++++++++++ 5 files changed, 398 insertions(+) create mode 100644 .github/workflows/repository-memory.yml create mode 100644 scripts/memory/collect_context.js create mode 100644 scripts/memory/package.json create mode 100644 scripts/memory/post_comment.js create mode 100644 scripts/memory/retrieve_memory.js diff --git a/.github/workflows/repository-memory.yml b/.github/workflows/repository-memory.yml new file mode 100644 index 0000000000..542ad6913d --- /dev/null +++ b/.github/workflows/repository-memory.yml @@ -0,0 +1,45 @@ +name: Redis Repository Memory + +on: + push: + branches-ignore: + - main + pull_request: + types: + - opened + workflow_dispatch: + +permissions: + pull-requests: write + contents: read + +jobs: + memory: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install dependencies + working-directory: scripts/memory + run: npm install + + - name: Collect context + run: node scripts/memory/collect_context.js + + - name: Retrieve and store memory + env: + REDIS_URL: ${{ secrets.MEMORY_REDIS_URL }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: node scripts/memory/retrieve_memory.js + + - name: Post result + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: node scripts/memory/post_comment.js diff --git a/scripts/memory/collect_context.js b/scripts/memory/collect_context.js new file mode 100644 index 0000000000..607c5921e9 --- /dev/null +++ b/scripts/memory/collect_context.js @@ -0,0 +1,96 @@ +'use strict'; + +const fs = require('fs'); +const { execSync } = require('child_process'); + +const eventName = process.env.GITHUB_EVENT_NAME; +const eventPath = process.env.GITHUB_EVENT_PATH; +const sha = process.env.GITHUB_SHA; +const ref = process.env.GITHUB_REF || ''; +const repo = process.env.GITHUB_REPOSITORY; + +if (!eventName || !eventPath) { + console.error('Missing required GitHub environment variables (GITHUB_EVENT_NAME, GITHUB_EVENT_PATH)'); + process.exit(1); +} + +const event = JSON.parse(fs.readFileSync(eventPath, 'utf8')); + +function getChangedFiles(commitSha) { + try { + return execSync(`git diff --name-only ${commitSha}^1 ${commitSha}`, { + encoding: 'utf8', + stdio: ['pipe', 'pipe', 'pipe'], + }) + .trim() + .split('\n') + .filter(Boolean); + } catch { + return []; + } +} + +let context; + +if (eventName === 'push') { + const headCommit = event.head_commit || {}; + const branch = ref.replace('refs/heads/', ''); + const title = (headCommit.message || '').split('\n')[0]; + + context = { + eventName, + type: 'push', + sha, + branch, + repo, + prNumber: null, + title, + body: headCommit.message || '', + labels: [], + changedFiles: getChangedFiles(sha), + author: headCommit.author?.name || headCommit.committer?.name || 'unknown', + timestamp: new Date().toISOString(), + }; +} else if (eventName === 'pull_request') { + const pr = event.pull_request; + context = { + eventName, + type: 'pull_request', + sha: pr.head.sha, + branch: pr.head.ref, + repo, + prNumber: pr.number, + title: pr.title, + body: pr.body || '', + labels: (pr.labels || []).map(l => l.name), + changedFiles: [], + author: pr.user?.login || 'unknown', + timestamp: new Date().toISOString(), + }; +} else if (eventName === 'workflow_dispatch') { + context = { + eventName, + type: 'workflow_dispatch', + sha, + branch: ref.replace('refs/heads/', ''), + repo, + prNumber: null, + title: 'Manual workflow dispatch', + body: '', + labels: [], + changedFiles: [], + author: event.sender?.login || 'unknown', + timestamp: new Date().toISOString(), + }; +} else { + console.error(`Unsupported event type: ${eventName}`); + process.exit(1); +} + +context.searchText = [context.title, context.body, context.changedFiles.join(' ')] + .join('\n') + .slice(0, 4000); + +fs.writeFileSync('/tmp/memory_context.json', JSON.stringify(context, null, 2)); + +console.log(`trigger=${eventName} repo=${repo} title="${context.title}" files=${context.changedFiles.length}`); diff --git a/scripts/memory/package.json b/scripts/memory/package.json new file mode 100644 index 0000000000..854d972d3b --- /dev/null +++ b/scripts/memory/package.json @@ -0,0 +1,13 @@ +{ + "name": "redis-repo-memory", + "version": "1.0.0", + "private": true, + "description": "GitHub Actions scripts for Redis-backed repository memory", + "engines": { + "node": ">=18" + }, + "dependencies": { + "openai": "^4.77.0", + "redis": "^4.7.0" + } +} diff --git a/scripts/memory/post_comment.js b/scripts/memory/post_comment.js new file mode 100644 index 0000000000..1b0959f41d --- /dev/null +++ b/scripts/memory/post_comment.js @@ -0,0 +1,100 @@ +'use strict'; + +const fs = require('fs'); + +const COMMENT_MARKER = ''; + +async function githubRequest(method, path, body, token) { + const res = await fetch(`https://api.github.com${path}`, { + method, + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + 'Content-Type': 'application/json', + }, + body: body ? JSON.stringify(body) : undefined, + }); + if (!res.ok) { + const text = await res.text(); + throw new Error(`GitHub API ${method} ${path} → ${res.status}: ${text}`); + } + return res.json(); +} + +function buildMarkdown(related, context, sourceUrl) { + const lines = [ + COMMENT_MARKER, + '## 🧠 Redis Repository Memory', + '', + ]; + + if (related.length === 0) { + lines.push("No related prior context found — this looks new to the repository's memory."); + } else { + lines.push(`Found **${related.length}** related item${related.length === 1 ? '' : 's'} from repository history:`); + lines.push(''); + for (const item of related) { + const label = item.type === 'pr_summary' ? 'PR' : item.type === 'issue_summary' ? 'Issue' : 'Commit'; + const link = item.sourceUrl ? `[${item.title}](${item.sourceUrl})` : `**${item.title}**`; + lines.push(`- **${label}** — ${link}`); + if (item.bodySummary) { + lines.push(` > ${item.bodySummary.split('\n')[0].slice(0, 140)}`); + } + } + } + + lines.push(''); + lines.push(`*Memory updated at [${context.sha?.slice(0, 7)}](${sourceUrl})*`); + return lines.join('\n'); +} + +async function postOrUpdatePrComment(repo, prNumber, body, token) { + const comments = await githubRequest('GET', `/repos/${repo}/issues/${prNumber}/comments`, null, token); + const existing = comments.find(c => typeof c.body === 'string' && c.body.includes(COMMENT_MARKER)); + + if (existing) { + await githubRequest('PATCH', `/repos/${repo}/issues/comments/${existing.id}`, { body }, token); + console.log(`updated_comment_id=${existing.id} pr=${prNumber}`); + } else { + const created = await githubRequest('POST', `/repos/${repo}/issues/${prNumber}/comments`, { body }, token); + console.log(`created_comment_id=${created.id} pr=${prNumber}`); + } +} + +async function main() { + const token = process.env.GITHUB_TOKEN; + const repo = process.env.GITHUB_REPOSITORY; + + if (!token) { + console.error('GITHUB_TOKEN is not set'); + process.exit(1); + } + if (!repo) { + console.error('GITHUB_REPOSITORY is not set'); + process.exit(1); + } + + const { context, related, sourceUrl } = JSON.parse( + fs.readFileSync('/tmp/memory_results.json', 'utf8') + ); + + const markdown = buildMarkdown(related, context, sourceUrl); + + if (context.prNumber) { + await postOrUpdatePrComment(repo, context.prNumber, markdown, token); + } else { + const summaryPath = process.env.GITHUB_STEP_SUMMARY; + if (summaryPath) { + fs.appendFileSync(summaryPath, markdown + '\n'); + } else { + process.stdout.write(markdown + '\n'); + } + console.log(`posted_to=step_summary related=${related.length}`); + } +} + +main().catch(err => { + console.error('post_comment failed:', err.message); + process.exit(1); +}); diff --git a/scripts/memory/retrieve_memory.js b/scripts/memory/retrieve_memory.js new file mode 100644 index 0000000000..4b1368fef7 --- /dev/null +++ b/scripts/memory/retrieve_memory.js @@ -0,0 +1,144 @@ +'use strict'; + +const fs = require('fs'); +const { createClient, SchemaFieldTypes, VectorAlgorithms } = require('redis'); +const OpenAI = require('openai'); + +const INDEX_NAME = 'repo_memory_idx'; +const KEY_PREFIX = 'memory:'; +const VECTOR_DIM = 1536; +const SIMILARITY_THRESHOLD = 0.7; // cosine distance [0,2]; lower = more similar +const TOP_K = 10; +const MAX_RESULTS = 5; + +async function ensureIndex(client) { + try { + await client.ft.info(INDEX_NAME); + } catch (err) { + if (!err.message?.includes('Unknown Index name')) throw err; + await client.ft.create( + INDEX_NAME, + { + type: { type: SchemaFieldTypes.TAG }, + repo: { type: SchemaFieldTypes.TAG }, + title: { type: SchemaFieldTypes.TEXT, WEIGHT: 2 }, + body_summary: { type: SchemaFieldTypes.TEXT }, + source_url: { type: SchemaFieldTypes.TEXT, NOSTEM: true }, + created_at: { type: SchemaFieldTypes.TEXT }, + tags: { type: SchemaFieldTypes.TAG, SEPARATOR: ',' }, + embedding: { + type: SchemaFieldTypes.VECTOR, + ALGORITHM: VectorAlgorithms.HNSW, + TYPE: 'FLOAT32', + DIM: VECTOR_DIM, + DISTANCE_METRIC: 'COSINE', + }, + }, + { ON: 'HASH', PREFIX: KEY_PREFIX } + ); + console.log(`created index: ${INDEX_NAME}`); + } +} + +async function main() { + const redisUrl = process.env.REDIS_URL; + const openaiKey = process.env.OPENAI_API_KEY; + + if (!redisUrl) { + console.error('REDIS_URL secret is not set'); + process.exit(1); + } + if (!openaiKey) { + console.error('OPENAI_API_KEY secret is not set'); + process.exit(1); + } + + const context = JSON.parse(fs.readFileSync('/tmp/memory_context.json', 'utf8')); + + const client = createClient({ url: redisUrl }); + client.on('error', err => console.error('Redis error:', err.message)); + await client.connect(); + + try { + const openai = new OpenAI({ apiKey: openaiKey }); + + await ensureIndex(client); + + const embeddingRes = await openai.embeddings.create({ + model: 'text-embedding-3-small', + input: context.searchText || context.title, + }); + const embeddingBuffer = Buffer.from( + new Float32Array(embeddingRes.data[0].embedding).buffer + ); + + const searchResult = await client.ft.search( + INDEX_NAME, + `*=>[KNN ${TOP_K} @embedding $BLOB AS score]`, + { + PARAMS: { BLOB: embeddingBuffer }, + RETURN: ['type', 'title', 'body_summary', 'source_url', 'created_at', 'score', 'repo'], + SORTBY: { BY: 'score', DIRECTION: 'ASC' }, + DIALECT: 2, + } + ); + + const related = searchResult.documents + .filter(doc => { + const score = parseFloat(doc.value.score ?? '2'); + return score < SIMILARITY_THRESHOLD && doc.value.repo === context.repo; + }) + .slice(0, MAX_RESULTS); + + // Store the current context as a memory artifact + const safeRepo = context.repo.replace(/\//g, '_'); + const itemId = context.prNumber + ? `${safeRepo}_pr_${context.prNumber}` + : `${safeRepo}_push_${context.sha?.slice(0, 8)}`; + + const sourceUrl = context.prNumber + ? `https://github.com/${context.repo}/pull/${context.prNumber}` + : `https://github.com/${context.repo}/commit/${context.sha}`; + + await client.hSet(`${KEY_PREFIX}${itemId}`, { + id: itemId, + type: context.type === 'pull_request' ? 'pr_summary' : 'push_summary', + repo: context.repo, + title: context.title, + body_summary: context.body.slice(0, 500), + source_url: sourceUrl, + created_at: context.timestamp, + tags: context.labels.join(','), + embedding: embeddingBuffer, + }); + + console.log(`stored=${KEY_PREFIX}${itemId} related_count=${related.length}`); + + fs.writeFileSync( + '/tmp/memory_results.json', + JSON.stringify( + { + context, + sourceUrl, + related: related.map(doc => ({ + type: doc.value.type, + title: doc.value.title, + bodySummary: doc.value.body_summary, + sourceUrl: doc.value.source_url, + createdAt: doc.value.created_at, + score: parseFloat(doc.value.score ?? '2'), + })), + }, + null, + 2 + ) + ); + } finally { + await client.quit(); + } +} + +main().catch(err => { + console.error('retrieve_memory failed:', err.message); + process.exit(1); +}); From 488a5b1a25baf111544a85171f6a97aecb0dcdce Mon Sep 17 00:00:00 2001 From: mich-elle-luna Date: Wed, 10 Jun 2026 08:21:13 -0700 Subject: [PATCH 02/22] Fix ensureIndex error string match for Redis 8 --- scripts/memory/retrieve_memory.js | 47 +++++++++++++++---------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/scripts/memory/retrieve_memory.js b/scripts/memory/retrieve_memory.js index 4b1368fef7..576e357360 100644 --- a/scripts/memory/retrieve_memory.js +++ b/scripts/memory/retrieve_memory.js @@ -12,32 +12,29 @@ const TOP_K = 10; const MAX_RESULTS = 5; async function ensureIndex(client) { - try { - await client.ft.info(INDEX_NAME); - } catch (err) { - if (!err.message?.includes('Unknown Index name')) throw err; - await client.ft.create( - INDEX_NAME, - { - type: { type: SchemaFieldTypes.TAG }, - repo: { type: SchemaFieldTypes.TAG }, - title: { type: SchemaFieldTypes.TEXT, WEIGHT: 2 }, - body_summary: { type: SchemaFieldTypes.TEXT }, - source_url: { type: SchemaFieldTypes.TEXT, NOSTEM: true }, - created_at: { type: SchemaFieldTypes.TEXT }, - tags: { type: SchemaFieldTypes.TAG, SEPARATOR: ',' }, - embedding: { - type: SchemaFieldTypes.VECTOR, - ALGORITHM: VectorAlgorithms.HNSW, - TYPE: 'FLOAT32', - DIM: VECTOR_DIM, - DISTANCE_METRIC: 'COSINE', - }, + const exists = await client.ft.info(INDEX_NAME).then(() => true).catch(() => false); + if (exists) return; + await client.ft.create( + INDEX_NAME, + { + type: { type: SchemaFieldTypes.TAG }, + repo: { type: SchemaFieldTypes.TAG }, + title: { type: SchemaFieldTypes.TEXT, WEIGHT: 2 }, + body_summary: { type: SchemaFieldTypes.TEXT }, + source_url: { type: SchemaFieldTypes.TEXT, NOSTEM: true }, + created_at: { type: SchemaFieldTypes.TEXT }, + tags: { type: SchemaFieldTypes.TAG, SEPARATOR: ',' }, + embedding: { + type: SchemaFieldTypes.VECTOR, + ALGORITHM: VectorAlgorithms.HNSW, + TYPE: 'FLOAT32', + DIM: VECTOR_DIM, + DISTANCE_METRIC: 'COSINE', }, - { ON: 'HASH', PREFIX: KEY_PREFIX } - ); - console.log(`created index: ${INDEX_NAME}`); - } + }, + { ON: 'HASH', PREFIX: KEY_PREFIX } + ); + console.log(`created index: ${INDEX_NAME}`); } async function main() { From 47262ffc1508f8398245c0dc6d0d136a5cd5a719 Mon Sep 17 00:00:00 2001 From: mich-elle-luna Date: Wed, 10 Jun 2026 08:34:25 -0700 Subject: [PATCH 03/22] Trigger workflow after OpenAI credits added From 7de52440f643b251af664c3158d9d1be0acbd01c Mon Sep 17 00:00:00 2001 From: mich-elle-luna Date: Wed, 10 Jun 2026 08:51:22 -0700 Subject: [PATCH 04/22] Remove non-memory workflows from test branch --- .github/workflows/autocomment_jira_link.yaml | 50 -- .../workflows/autocomment_staging_links.yaml | 171 ---- .github/workflows/cleanup_pr_folders.yaml | 77 -- .github/workflows/k8s_apis_sync.yaml | 282 ------- .github/workflows/main.yml | 793 ------------------ .github/workflows/rc_api_sync.yaml | 70 -- .../workflows/rc_supported_regions_sync.yaml | 74 -- .github/workflows/redis_docs_sync.yaml | 104 --- .../workflows/redis_modules_docs_sync.yaml | 84 -- .github/workflows/redisvl_docs_sync.yaml | 128 --- .github/workflows/update_command_pages.yaml | 74 -- 11 files changed, 1907 deletions(-) delete mode 100644 .github/workflows/autocomment_jira_link.yaml delete mode 100644 .github/workflows/autocomment_staging_links.yaml delete mode 100644 .github/workflows/cleanup_pr_folders.yaml delete mode 100644 .github/workflows/k8s_apis_sync.yaml delete mode 100644 .github/workflows/main.yml delete mode 100644 .github/workflows/rc_api_sync.yaml delete mode 100644 .github/workflows/rc_supported_regions_sync.yaml delete mode 100644 .github/workflows/redis_docs_sync.yaml delete mode 100644 .github/workflows/redis_modules_docs_sync.yaml delete mode 100644 .github/workflows/redisvl_docs_sync.yaml delete mode 100644 .github/workflows/update_command_pages.yaml diff --git a/.github/workflows/autocomment_jira_link.yaml b/.github/workflows/autocomment_jira_link.yaml deleted file mode 100644 index fc9d12dfc8..0000000000 --- a/.github/workflows/autocomment_jira_link.yaml +++ /dev/null @@ -1,50 +0,0 @@ -name: autocomment_jira_link - -on: - pull_request: - types: - - opened -permissions: - pull-requests: write - contents: read - -jobs: - auto-comment: - runs-on: ubuntu-latest - steps: - - name: 'Checkout' - uses: 'actions/checkout@v3' - - - name: Create Comment - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_NUMBER: ${{ github.event.pull_request.number }} - REPO: ${{ github.repository }} - BRANCH_NAME: ${{ github.event.pull_request.head.ref }} - run: | - generate_post_data() - { - cat </dev/null - - printf '%s\n' 'Comment written!' - fi \ No newline at end of file diff --git a/.github/workflows/autocomment_staging_links.yaml b/.github/workflows/autocomment_staging_links.yaml deleted file mode 100644 index 0d75c5cfed..0000000000 --- a/.github/workflows/autocomment_staging_links.yaml +++ /dev/null @@ -1,171 +0,0 @@ -name: autocomment_staging_links - -on: - pull_request: - paths: - - 'content/**' - types: - - opened - - synchronize -permissions: - pull-requests: write - contents: read - -jobs: - auto-comment: - runs-on: ubuntu-latest - steps: - - name: 'Checkout' - uses: 'actions/checkout@v3' - - - name: 'Install jq' - run: sudo apt-get install jq - - - name: Create Comment - env: - PR_NUMBER: ${{ github.event.pull_request.number }} - BRANCH_NAME: ${{ github.head_ref }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_REPOSITORY: ${{ github.repository }} - run: | - set -e - - CREATE_COMMENT_URL="https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" - UPDATE_COMMENT_URL="https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/comments" - FILES_URL="https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" - - # Check if comment already exists - EXISTING_COMMENT_JSON=$(curl -Ls \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" \ - $CREATE_COMMENT_URL \ - | jq -r '.[] | select(.user.login == "github-actions[bot]" and ((.body // "") | contains("Staging links:")))' - ) - EXISTING_COMMENT_ID=$(jq -r '.id' <<<"$EXISTING_COMMENT_JSON") - EXISTING_COMMENT=$(jq -r '.body' <<<"$EXISTING_COMMENT_JSON") - - # Get all changed md files - CHANGED_MD_FILES=$(curl -Ls \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" \ - $FILES_URL \ - | jq -r --arg prefix $BRANCH_NAME/ '.[] | select(((.filename | test("content/(?!.*embed).*\\.md")) and .status != "removed")) | ($prefix + .filename)' \ - | sed -E -e 's|(^[^/]+/)([^/]+/)|\1|' -e 's|^|https://redis.io/docs/staging/|' -e 's|(\/_?index)?\.md|\/|' \ - | sort \ - | uniq) - - # Get all changed image files - CHANGED_IMAGE_FILES=$(curl -Ls \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" \ - $FILES_URL \ - | jq -r '.[] | select(.filename | test("^static/images\/.+(.png|.svg|.gif|.)")) | .filename | sub("^static/";"")') - - # Get all changed embeds files - CHANGED_EMBED_FILES=$(curl -Ls \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" \ - $FILES_URL \ - | jq -r '.[] | select(.filename | test("^content/embeds\/.+.md")) | .filename | split("/")[-1]') - - if [[ -n "$CHANGED_IMAGE_FILES" ]] - then - # For each image, work out in which README it is present - MD_FILES_WITH_IMAGE=() - for CHANGED_IMAGE_FILE in "${CHANGED_IMAGE_FILES[@]}"; do - MD_FILE_WITH_IMAGE=$(grep -ro "$CHANGED_IMAGE_FILE" content \ - | sed -E -e 's|:.+||' -e "s|^content/|https://redis.io/docs/staging/$BRANCH_NAME/|" -e 's|(_?index)?\.md||' \ - | grep -v "https://redis.io/docs/staging/$BRANCH_NAME/embeds/" \ - | sort \ - | uniq) - MD_FILES_WITH_IMAGE+=($MD_FILE_WITH_IMAGE) - done - CHANGED_MD_FILES=$(printf "%s\n" "${CHANGED_MD_FILES}" "${MD_FILES_WITH_IMAGE[@]}" \ - | sort \ - | uniq) - fi - - if [[ -n "$CHANGED_EMBED_FILES" ]] - then - # For each embed, work out in which README it is present - MD_FILES_WITH_EMBED=() - for CHANGED_EMBED_FILE in "${CHANGED_EMBED_FILES[@]}"; do - MD_FILE_WITH_EMBED=$(grep -ro "< embed-md \"${CHANGED_EMBED_FILE}\" >" content \ - | sed -E -e 's|:.+||' -e "s|^content/|https://redis.io/docs/staging/$BRANCH_NAME/|" -e 's|(_?index)?\.md||' \ - | sort \ - | uniq) - MD_FILES_WITH_EMBED+=($MD_FILE_WITH_EMBED) - done - CHANGED_MD_FILES=$(printf "%s\n" "${CHANGED_MD_FILES}" "${MD_FILES_WITH_EMBED[@]}" \ - | sort \ - | uniq) - fi - - CHANGED_MD_FILES=$(printf "%s\n" "${CHANGED_MD_FILES}" \ - | xargs \ - | sed 's/ /
/g') - - if [[ -z "$CHANGED_MD_FILES" ]] - then - if [[ -z "$EXISTING_COMMENT_ID" ]] - then - printf '%s\n' 'Nothing to do!' - exit 0 - else - # If a comment already exists but there are no changed files in the PR anymore, delete the comment - curl -Ls \ - -X DELETE \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -H "Authorization: token ${GITHUB_TOKEN}" \ - "${UPDATE_COMMENT_URL}/${EXISTING_COMMENT_ID}" - printf '%s\n' 'Comment deleted!' - exit 0 - fi - fi - - COMMENT="Staging links:
$CHANGED_MD_FILES" - - printf '%s %s\n' 'Writing comment: ' "$COMMENT" - - generate_post_data() - { - cat </dev/null - - printf '%s\n' 'Comment updated!' - else - printf '%s\n' 'Nothing changed!' - fi - else - # Write comment on pull request - curl -Ls \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -H "Authorization: token ${GITHUB_TOKEN}" \ - $CREATE_COMMENT_URL \ - --data "$(generate_post_data)" 1>/dev/null - - printf '%s\n' 'Comment written!' - fi \ No newline at end of file diff --git a/.github/workflows/cleanup_pr_folders.yaml b/.github/workflows/cleanup_pr_folders.yaml deleted file mode 100644 index cbd89f55a5..0000000000 --- a/.github/workflows/cleanup_pr_folders.yaml +++ /dev/null @@ -1,77 +0,0 @@ -name: cleanup_pr_folders - -on: - pull_request: - types: - - closed - -jobs: - staging: - if: github.event.pull_request.merged == true - runs-on: ubuntu-latest - permissions: - contents: 'read' - id-token: 'write' - env: - STAGING_BUCKET: docs-staging-learn-redis-com - STAGING_PROJECT_ID: ${{ secrets.GCP_PROJECT_STAGING }} - STAGING_SERVICE_ACCOUNT: ${{ secrets.STAGING_SERVICE_ACCOUNT }} - STAGING_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.STAGING_WORKLOAD_IDENTITY_PROVIDER }} - - steps: - - name: 'Google auth' - uses: 'google-github-actions/auth@v2' - with: - project_id: '${{ env.STAGING_PROJECT_ID }}' - service_account: '${{ env.STAGING_SERVICE_ACCOUNT }}' - workload_identity_provider: '${{ env.STAGING_WORKLOAD_IDENTITY_PROVIDER }}' - - - name: 'Set up Cloud SDK' - uses: 'google-github-actions/setup-gcloud@v2' - with: - project_id: '${{ env.STAGING_PROJECT_ID }}' - version: '>= 363.0.0' - - - name: Remove pull request staging folder - env: - BRANCH_TO_CLEANUP: ${{ github.event.pull_request.head.ref }} - run: | - if gsutil -q stat "gs://${STAGING_BUCKET}/staging/${BRANCH_TO_CLEANUP}/*"; then - gsutil -q -m rm -r "gs://${STAGING_BUCKET}/staging/${BRANCH_TO_CLEANUP}/*" - echo "Removed gs://${STAGING_BUCKET}/staging/${BRANCH_TO_CLEANUP}" - fi - - prod: - if: github.event.pull_request.merged == true - runs-on: ubuntu-latest - permissions: - contents: 'read' - id-token: 'write' - env: - PROD_BUCKET: docs-prod-learn-redis-com - PROD_PROJECT_ID: ${{ secrets.GCP_PROJECT_PROD }} - PROD_SERVICE_ACCOUNT: ${{ secrets.PROD_SERVICE_ACCOUNT }} - PROD_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.PROD_WORKLOAD_IDENTITY_PROVIDER }} - - steps: - - name: 'Google auth' - uses: 'google-github-actions/auth@v2' - with: - project_id: '${{ env.PROD_PROJECT_ID }}' - service_account: '${{ env.PROD_SERVICE_ACCOUNT }}' - workload_identity_provider: '${{ env.PROD_WORKLOAD_IDENTITY_PROVIDER }}' - - - name: 'Set up Cloud SDK' - uses: 'google-github-actions/setup-gcloud@v2' - with: - project_id: '${{ env.PROD_PROJECT_ID }}' - version: '>= 363.0.0' - - - name: Remove pull request staging folder - env: - BRANCH_TO_CLEANUP: ${{ github.event.pull_request.head.ref }} - run: | - if gsutil -q stat "gs://${PROD_BUCKET}/staging/${BRANCH_TO_CLEANUP}/*"; then - gsutil -q -m rm -r "gs://${PROD_BUCKET}/staging/${BRANCH_TO_CLEANUP}/*" - echo "Removed gs://${PROD_BUCKET}/staging/${BRANCH_TO_CLEANUP}" - fi \ No newline at end of file diff --git a/.github/workflows/k8s_apis_sync.yaml b/.github/workflows/k8s_apis_sync.yaml deleted file mode 100644 index 55f516a625..0000000000 --- a/.github/workflows/k8s_apis_sync.yaml +++ /dev/null @@ -1,282 +0,0 @@ -name: k8s_apis_sync - -on: - workflow_dispatch: - inputs: - release: - type: string - required: true - description: '' - bucket_folder: - type: string - required: true - description: '' - -jobs: - k8s_apis_sync: - runs-on: ubuntu-latest - permissions: - id-token: write - contents: write - pull-requests: write - actions: write - steps: - - name: Generate a token - id: generate-token - uses: actions/create-github-app-token@v1 - with: - app-id: ${{ secrets.DOCS_APP_ID }} - private-key: ${{ secrets.DOCS_APP_PRIVATE_KEY }} - - - name: 'Checkout' - uses: actions/checkout@v3 - with: - token: ${{ steps.generate-token.outputs.token }} - - - name: 'Install crdoc' - run: |- - curl -LO https://github.com/fybrik/crdoc/releases/download/v0.6.3/crdoc_Linux_x86_64.tar.gz - tar -xvzf crdoc_Linux_x86_64.tar.gz - sudo mv crdoc /bin/crdoc - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: arn:aws:iam::022044324644:role/GitHubRedisDocsS3Access - aws-region: us-east-1 - - - name: 'Fetch release' - env: - RELEASE: ${{ github.event.inputs.release }} - BUCKET_FOLDER: ${{ github.event.inputs.bucket_folder }} - run: |- - aws s3 cp "s3://redislabs-k8s/${BUCKET_FOLDER}/redis-enterprise-operator-${RELEASE}.tar.gz" . - - tar xf "redis-enterprise-operator-${RELEASE}.tar.gz" - - - name: 'Generate READMEs' - run: |- - mkdir artifacts - mkdir templates - cp content/operate/kubernetes/reference/api/kubernetes-api-reference-template.tmpl templates/template.tmpl - - crdoc --resources crds/reaadb_crd.yaml --output artifacts/redis_enterprise_active_active_database_api.md --template templates/template.tmpl - sed -E -i 's/^### RedisEnterpriseActiveActiveDatabase\./### /g' artifacts/redis_enterprise_active_active_database_api.md - sed -E -i 's/^\[↩ Parent\]\(#redisenterpriseactiveactivedatabase/\[↩ Parent\]\(#/g' artifacts/redis_enterprise_active_active_database_api.md - sed -E -i 's/ _tmp.md && mv _tmp.md artifacts/redis_enterprise_active_active_database_api.md - - crdoc --resources crds/rec_crd.yaml --output artifacts/redis_enterprise_cluster_api.md --template templates/template.tmpl - sed -E -i 's/^### RedisEnterpriseCluster\./### /g' artifacts/redis_enterprise_cluster_api.md - sed -E -i 's/^\[↩ Parent\]\(#redisenterprisecluster/\[↩ Parent\]\(#/g' artifacts/redis_enterprise_cluster_api.md - sed -E -i 's/ _tmp.md && mv _tmp.md artifacts/redis_enterprise_cluster_api.md - - crdoc --resources crds/redb_crd.yaml --output artifacts/redis_enterprise_database_api.md --template templates/template.tmpl - sed -E -i 's/^### RedisEnterpriseDatabase\./### /g' artifacts/redis_enterprise_database_api.md - sed -E -i 's/^\[↩ Parent\]\(#redisenterprisedatabase/\[↩ Parent\]\(#/g' artifacts/redis_enterprise_database_api.md - sed -E -i 's/ _tmp.md && mv _tmp.md artifacts/redis_enterprise_database_api.md - - crdoc --resources crds/rerc_crd.yaml --output artifacts/redis_enterprise_remote_cluster_api.md --template templates/template.tmpl - sed -E -i 's/^### RedisEnterpriseRemoteCluster\./### /g' artifacts/redis_enterprise_remote_cluster_api.md - sed -E -i 's/^\[↩ Parent\]\(#redisenterpriseremotecluster/\[↩ Parent\]\(#/g' artifacts/redis_enterprise_remote_cluster_api.md - sed -E -i 's/ _tmp.md && mv _tmp.md artifacts/redis_enterprise_remote_cluster_api.md - - crdoc --resources crds/reacl_crd.yaml --output artifacts/redis_enterprise_acl_api.md --template templates/template.tmpl - sed -E -i 's/^### RedisEnterpriseACL\./### /g' artifacts/redis_enterprise_acl_api.md - sed -E -i 's/^\[↩ Parent\]\(#redisenterpriseacl/\[↩ Parent\]\(#/g' artifacts/redis_enterprise_acl_api.md - sed -E -i 's/ _tmp.md && mv _tmp.md artifacts/redis_enterprise_acl_api.md - - crdoc --resources crds/recrole_crd.yaml --output artifacts/redis_enterprise_cluster_role_api.md --template templates/template.tmpl - sed -E -i 's/^### RedisEnterpriseClusterRole\./### /g' artifacts/redis_enterprise_cluster_role_api.md - sed -E -i 's/^\[↩ Parent\]\(#redisenterpriseclusterrole/\[↩ Parent\]\(#/g' artifacts/redis_enterprise_cluster_role_api.md - sed -E -i 's/ _tmp.md && mv _tmp.md artifacts/redis_enterprise_cluster_role_api.md - - crdoc --resources crds/recrolebinding_crd.yaml --output artifacts/redis_enterprise_cluster_role_binding_api.md --template templates/template.tmpl - sed -E -i 's/^### RedisEnterpriseClusterRoleBinding\./### /g' artifacts/redis_enterprise_cluster_role_binding_api.md - sed -E -i 's/^\[↩ Parent\]\(#redisenterpriseclusterrolebinding/\[↩ Parent\]\(#/g' artifacts/redis_enterprise_cluster_role_binding_api.md - sed -E -i 's/ _tmp.md && mv _tmp.md artifacts/redis_enterprise_cluster_role_binding_api.md - - crdoc --resources crds/redbrole_crd.yaml --output artifacts/redis_enterprise_database_role_api.md --template templates/template.tmpl - sed -E -i 's/^### RedisEnterpriseDatabaseRole\./### /g' artifacts/redis_enterprise_database_role_api.md - sed -E -i 's/^\[↩ Parent\]\(#redisenterprisedatabaserole/\[↩ Parent\]\(#/g' artifacts/redis_enterprise_database_role_api.md - sed -E -i 's/ _tmp.md && mv _tmp.md artifacts/redis_enterprise_database_role_api.md - - crdoc --resources crds/redbrolebinding_crd.yaml --output artifacts/redis_enterprise_database_role_binding_api.md --template templates/template.tmpl - sed -E -i 's/^### RedisEnterpriseDatabaseRoleBinding\./### /g' artifacts/redis_enterprise_database_role_binding_api.md - sed -E -i 's/^\[↩ Parent\]\(#redisenterprisedatabaserolebinding/\[↩ Parent\]\(#/g' artifacts/redis_enterprise_database_role_binding_api.md - sed -E -i 's/ _tmp.md && mv _tmp.md artifacts/redis_enterprise_database_role_binding_api.md - - crdoc --resources crds/reuser_crd.yaml --output artifacts/redis_enterprise_user_api.md --template templates/template.tmpl - sed -E -i 's/^### RedisEnterpriseUser\./### /g' artifacts/redis_enterprise_user_api.md - sed -E -i 's/^\[↩ Parent\]\(#redisenterpriseuser/\[↩ Parent\]\(#/g' artifacts/redis_enterprise_user_api.md - sed -E -i 's/ _tmp.md && mv _tmp.md artifacts/redis_enterprise_user_api.md - - - name: 'Remove fields without description' - run: |- - file="artifacts/redis_enterprise_cluster_api.md" - empty_fields_file="content/operate/kubernetes/reference/api/redis_enterprise_cluster_empty_fields" - - # Read empty fields and process them - while IFS= read -r field; do - # Skip empty lines and comments - [[ -z "$field" || "$field" =~ ^# ]] && continue - - # Convert field path to section header pattern - section_pattern="${field}" - - # Remove subsections that go deeper than [index] level - # Keep: spec.extraEnvVars - # Keep: spec.extraEnvVars[index] - # Remove: spec.extraEnvVars[index].valueFrom - # Remove: spec.extraEnvVars[index].valueFrom.fieldRef - awk -v base_pattern="${section_pattern}" ' - BEGIN { skip = 0 } - /^### / { - # Check if this line matches a pattern we want to remove - section = $0 - gsub(/^### /, "", section) - - # If it starts with our base pattern - if (index(section, base_pattern) == 1) { - remainder = substr(section, length(base_pattern) + 1) - - # If remainder is empty, keep it (main section) - if (remainder == "") { - skip = 0 - } - # If remainder is just [index], keep it - else if (remainder == "[index]") { - skip = 0 - } - # If remainder starts with [index]. (has more after [index]), remove it - else if (index(remainder, "[index].") == 1) { - skip = 1 - next - } - # If remainder starts with . (direct subsection), remove it - else if (index(remainder, ".") == 1) { - skip = 1 - next - } - # If remainder starts with []. (direct subsection), remove it - else if (index(remainder, "[].") == 1) { - skip = 1 - next - } - # Otherwise keep it - else { - skip = 0 - } - } else { - # Different section, stop skipping - skip = 0 - } - } - - skip == 0 { print } - ' "$file" > "${file}.tmp" && mv "${file}.tmp" "$file" - - done < "$empty_fields_file" - - - name: 'Fix link titles' - run: |- - sed -E -i 's/linkTitle: RedisEnterpriseActiveActiveDatabase API Reference/linkTitle: REAADB API/g' artifacts/redis_enterprise_active_active_database_api.md - sed -E -i 's/linkTitle: RedisEnterpriseCluster API Reference/linkTitle: REC API/g' artifacts/redis_enterprise_cluster_api.md - sed -E -i 's/linkTitle: RedisEnterpriseDatabase API Reference/linkTitle: REDB API/g' artifacts/redis_enterprise_database_api.md - sed -E -i 's/linkTitle: RedisEnterpriseRemoteCluster API Reference/linkTitle: RERC API/g' artifacts/redis_enterprise_remote_cluster_api.md - sed -E -i 's/linkTitle: RedisEnterpriseACL API Reference/linkTitle: REACL API/g' artifacts/redis_enterprise_acl_api.md - sed -E -i 's/linkTitle: RedisEnterpriseClusterRole API Reference/linkTitle: RECROLE API/g' artifacts/redis_enterprise_cluster_role_api.md - sed -E -i 's/linkTitle: RedisEnterpriseClusterRoleBinding API Reference/linkTitle: RECROLEBINDING API/g' artifacts/redis_enterprise_cluster_role_binding_api.md - sed -E -i 's/linkTitle: RedisEnterpriseDatabaseRole API Reference/linkTitle: REDBROLE API/g' artifacts/redis_enterprise_database_role_api.md - sed -E -i 's/linkTitle: RedisEnterpriseDatabaseRoleBinding API Reference/linkTitle: REDBROLEBINDING API/g' artifacts/redis_enterprise_database_role_binding_api.md - sed -E -i 's/linkTitle: RedisEnterpriseUser API Reference/linkTitle: REUSER API/g' artifacts/redis_enterprise_user_api.md - - - name: 'Generate YAML snippets' - run: |- - function formatYamlSnippet() { - cat > "$2" << EOL - \`\`\`yaml - $(cat $1) - \`\`\` - EOL - } - - formatYamlSnippet admission-service.yaml content/embeds/k8s/admission-service.md - formatYamlSnippet admission/webhook.yaml content/embeds/k8s/admission_webhook.md - formatYamlSnippet examples/v1/rec.yaml content/embeds/k8s/rec.md - formatYamlSnippet examples/v1alpha1/reaadb.yaml content/embeds/k8s/reaadb.md - formatYamlSnippet examples/v1alpha1/redb.yaml content/embeds/k8s/redb.md - formatYamlSnippet examples/v1alpha1/rerc.yaml content/embeds/k8s/rerc.md - formatYamlSnippet log_collector/log_collector_role_all_mode.yaml content/embeds/k8s/log_collector_role_all_mode.md - formatYamlSnippet log_collector/log_collector_role_restricted_mode.yaml content/embeds/k8s/log_collector_role_restricted_mode.md - formatYamlSnippet multi-namespace-redb/operator_cluster_role_binding.yaml content/embeds/k8s/multi-ns_operator_cluster_role_binding.md - formatYamlSnippet multi-namespace-redb/operator_cluster_role.yaml content/embeds/k8s/multi-ns_operator_cluster_role.md - formatYamlSnippet multi-namespace-redb/role_binding.yaml content/embeds/k8s/multi-ns_role_binding.md - formatYamlSnippet multi-namespace-redb/role.yaml content/embeds/k8s/multi-ns_role.md - formatYamlSnippet openshift/admission-service.yaml content/embeds/k8s/openshift_admission-service.md - formatYamlSnippet openshift/rec_rhel.yaml content/embeds/k8s/openshift_rec.md - formatYamlSnippet openshift/role_binding.yaml content/embeds/k8s/openshift_role_binding.md - formatYamlSnippet openshift/role.yaml content/embeds/k8s/openshift_role.md - formatYamlSnippet openshift/scc.yaml content/embeds/k8s/openshift_scc.md - formatYamlSnippet openshift/service_account.yaml content/embeds/k8s/openshift_service_account.md - formatYamlSnippet rack_awareness/rack_aware_cluster_role_binding.yaml content/embeds/k8s/rack_aware_cluster_role_binding.md - formatYamlSnippet rack_awareness/rack_aware_cluster_role.yaml content/embeds/k8s/rack_aware_cluster_role.md - formatYamlSnippet rack_awareness/rack_aware_rec.yaml content/embeds/k8s/rack_aware_rec.md - formatYamlSnippet role_binding.yaml content/embeds/k8s/role_binding.md - formatYamlSnippet role.yaml content/embeds/k8s/role.md - formatYamlSnippet service_account.yaml content/embeds/k8s/service_account.md - - - name: 'Send pull request' - env: - GH_TOKEN: ${{ steps.generate-token.outputs.token }} - RELEASE: ${{ github.event.inputs.release }} - run: |- - BRANCH="k8s_apis_docs_${RELEASE}" - - # Setup git email and username - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git config user.name "github-actions[bot]" - - git checkout -b "${BRANCH}" - - cp artifacts/redis_enterprise_active_active_database_api.md content/operate/kubernetes/reference/api/ - cp artifacts/redis_enterprise_cluster_api.md content/operate/kubernetes/reference/api/ - cp artifacts/redis_enterprise_database_api.md content/operate/kubernetes/reference/api/ - cp artifacts/redis_enterprise_remote_cluster_api.md content/operate/kubernetes/reference/api/ - cp artifacts/redis_enterprise_acl_api.md content/operate/kubernetes/reference/api/ - cp artifacts/redis_enterprise_cluster_role_api.md content/operate/kubernetes/reference/api/ - cp artifacts/redis_enterprise_cluster_role_binding_api.md content/operate/kubernetes/reference/api/ - cp artifacts/redis_enterprise_database_role_api.md content/operate/kubernetes/reference/api/ - cp artifacts/redis_enterprise_database_role_binding_api.md content/operate/kubernetes/reference/api/ - cp artifacts/redis_enterprise_user_api.md content/operate/kubernetes/reference/api/ - - git add content/operate/kubernetes/reference/api/ - git add content/embeds/k8s/ - - git commit -m "k8s api docs ${RELEASE}" - git push origin "${BRANCH}" - - gh pr create \ - --body "k8s api docs ${RELEASE}" \ - --title "k8s api docs ${RELEASE}" \ - --head "$BRANCH" \ - --base "main" \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 5ea69b01af..0000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,793 +0,0 @@ -name: build_docs - -on: - push: - branches: - - '**' - -env: - HUGO_VERSION: 0.143.1 - BUCKET: docs-prod-learn-redis-com - -jobs: - # Discover versions and build the latest docs - setup_and_build_latest: - name: Setup and build latest docs - runs-on: ubuntu-latest - permissions: - contents: 'read' - outputs: - kubernetes_versions: ${{ steps.discover.outputs.kubernetes_versions }} - rs_versions: ${{ steps.discover.outputs.rs_versions }} - rdi_versions: ${{ steps.discover.outputs.rdi_versions }} - redisvl_versions: ${{ steps.discover.outputs.redisvl_versions }} - hugo_root_path: ${{ steps.set-paths.outputs.hugo_root_path }} - bucket_path: ${{ steps.set-paths.outputs.bucket_path }} - - steps: - - name: Free up disk space - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/ghc - sudo rm -rf /opt/hostedtoolcache/CodeQL - - - name: Validate the branch name - env: - BRANCH_NAME: ${{ github.ref_name }} - run: | - if [[ "$BRANCH_NAME" =~ ^[a-zA-Z0-9_.-]+$ ]]; then - echo "The branch name $BRANCH_NAME is valid." - else - echo "ERROR: Invalid branch name $BRANCH_NAME!" - exit 1 - fi - - - name: Install Hugo - run: | - wget -O ${{ runner.temp }}/hugo.deb https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-amd64.deb \ - && sudo dpkg -i ${{ runner.temp }}/hugo.deb - - - name: Check the branch out - uses: actions/checkout@v4 - - - name: Set paths - id: set-paths - run: | - if [[ "${{ github.ref_name }}" == "main" ]]; then - echo "hugo_root_path=docs/staging/dev" >> $GITHUB_OUTPUT - echo "bucket_path=staging/dev" >> $GITHUB_OUTPUT - elif [[ "${{ github.ref_name }}" == "latest" ]]; then - echo "hugo_root_path=docs/latest" >> $GITHUB_OUTPUT - echo "bucket_path=latest" >> $GITHUB_OUTPUT - elif [[ "${{ endsWith(github.ref_name, '-build') }}" == "true" ]]; then - path=$(echo "docs/version/${{ github.ref_name }}" | sed 's/-build$//') - bucket=$(echo "version/${{ github.ref_name }}" | sed 's/-build$//') - echo "hugo_root_path=${path}" >> $GITHUB_OUTPUT - echo "bucket_path=${bucket}" >> $GITHUB_OUTPUT - else - echo "hugo_root_path=docs/staging/${{ github.ref_name }}" >> $GITHUB_OUTPUT - echo "bucket_path=staging/${{ github.ref_name }}" >> $GITHUB_OUTPUT - fi - - - name: Discover versions - id: discover - run: | - kubernetes_versions=$(find content/operate/kubernetes/ -maxdepth 1 -type d -regex ".*[0-9-]" | awk -F/ '{print $NF}' | jq -R -s -c 'split("\n") | map(select(length > 0))') - rs_versions=$(find content/operate/rs/ -maxdepth 1 -type d -regex ".*[0-9-]" | awk -F/ '{print $NF}' | jq -R -s -c 'split("\n") | map(select(length > 0))') - rdi_versions=$(find content/integrate/redis-data-integration/ -maxdepth 1 -type d -regex ".*[0-9-]" | awk -F/ '{print $NF}' | jq -R -s -c 'split("\n") | map(select(length > 0))') - redisvl_versions=$(find content/develop/ai/redisvl/ -maxdepth 1 -type d -regex ".*[0-9-]" | awk -F/ '{print $NF}' | jq -R -s -c 'split("\n") | map(select(length > 0))') - - echo "kubernetes_versions=${kubernetes_versions}" >> $GITHUB_OUTPUT - echo "rs_versions=${rs_versions}" >> $GITHUB_OUTPUT - echo "rdi_versions=${rdi_versions}" >> $GITHUB_OUTPUT - echo "redisvl_versions=${redisvl_versions}" >> $GITHUB_OUTPUT - - echo "Discovered versions:" - echo " Kubernetes: ${kubernetes_versions}" - echo " RS: ${rs_versions}" - echo " RDI: ${rdi_versions}" - echo " RedisVL: ${redisvl_versions}" - - - name: Install dependencies - run: make deps - - - name: Build latest (without versioned content) - run: | - set -x - hugo_root_path="${{ steps.set-paths.outputs.hugo_root_path }}" - sed -i "s#baseURL = \"https://redis.io\"#baseURL = \"https://redis.io/${hugo_root_path}\"#g" config.toml - - # Remove all versioned directories for the latest build - kubernetes_versions='${{ steps.discover.outputs.kubernetes_versions }}' - rs_versions='${{ steps.discover.outputs.rs_versions }}' - rdi_versions='${{ steps.discover.outputs.rdi_versions }}' - redisvl_versions='${{ steps.discover.outputs.redisvl_versions }}' - - for version in $(echo "$kubernetes_versions" | jq -r '.[]'); do - rm -rf "content/operate/kubernetes/${version}" - done - for version in $(echo "$rs_versions" | jq -r '.[]'); do - rm -rf "content/operate/rs/${version}" - done - for version in $(echo "$rdi_versions" | jq -r '.[]'); do - rm -rf "content/integrate/redis-data-integration/${version}" - done - for version in $(echo "$redisvl_versions" | jq -r '.[]'); do - rm -rf "content/develop/ai/redisvl/${version}" - done - - # Create version files for docs-nav.html dropdowns - echo "$kubernetes_versions" | jq -r '.[]' > kubernetes-versions - echo "$rs_versions" | jq -r '.[]' > rs-versions - echo "$rdi_versions" | jq -r '.[]' > rdi-versions - echo "$redisvl_versions" | jq -r '.[]' > redisvl-versions - - make all - - - name: List client examples - run: ls "${{ github.workspace }}/examples" - - - name: List files to be published - run: ls "${{ github.workspace }}/public" - - - name: Upload latest artifact - uses: actions/upload-artifact@v4 - with: - name: build-latest - path: public/ - retention-days: 1 - - # Build Kubernetes versions in parallel - build_kubernetes: - name: Build Kubernetes ${{ matrix.version }} - needs: setup_and_build_latest - if: ${{ needs.setup_and_build_latest.outputs.kubernetes_versions != '[]' }} - runs-on: ubuntu-latest - permissions: - contents: 'read' - strategy: - fail-fast: false - matrix: - version: ${{ fromJson(needs.setup_and_build_latest.outputs.kubernetes_versions) }} - - steps: - - name: Free up disk space - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/ghc - sudo rm -rf /opt/hostedtoolcache/CodeQL - - - name: Install Hugo - run: | - wget -O ${{ runner.temp }}/hugo.deb https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-amd64.deb \ - && sudo dpkg -i ${{ runner.temp }}/hugo.deb - - - name: Check the branch out - uses: actions/checkout@v4 - - - name: Install dependencies - run: make deps - - - name: Generate components - run: make components - - - name: Build Kubernetes ${{ matrix.version }} - run: | - set -x - version="${{ matrix.version }}" - hugo_root_path="${{ needs.setup_and_build_latest.outputs.hugo_root_path }}" - - sed -i "s#baseURL = \"https://redis.io\"#baseURL = \"https://redis.io/${hugo_root_path}\"#g" config.toml - - # Remove all other kubernetes versions - kubernetes_versions='${{ needs.setup_and_build_latest.outputs.kubernetes_versions }}' - for v in $(echo "$kubernetes_versions" | jq -r '.[]'); do - if [[ "$v" != "$version" ]]; then - rm -rf "content/operate/kubernetes/${v}" - fi - done - - # relrefs should not include version - find "content/operate/kubernetes/${version}" -type f | while read file; do - awk '{gsub(/\(\{\{< ?relref "\/operate\/kubernetes\/'"${version}"'/, "({{< relref \"/operate/kubernetes") }1' "$file" > tmpfile && mv tmpfile "$file" - done - - rsync -a --delete-after "content/operate/kubernetes/${version}"/ content/operate/kubernetes/ - sed -i 's/id="versionSelectorKubernetesValue" class="version-selector-control">latest/id="versionSelectorKubernetesValue" class="version-selector-control">v'"${version}"'/' layouts/partials/docs-nav.html - sed -i 's/linkTitle: '"${version}"'/linkTitle: Redis for Kubernetes/' content/operate/kubernetes/_index.md - - # inject replace command in meta-links - sed -i "12i \{\{ \$gh_path = replaceRE \`\^operate\/kubernetes\/\` \"operate\/kubernetes\/${version}\/\" \$gh_path }}" layouts/partials/meta-links.html - - # Create version files for docs-nav.html dropdowns - echo '${{ needs.setup_and_build_latest.outputs.kubernetes_versions }}' | jq -r '.[]' > kubernetes-versions - echo '${{ needs.setup_and_build_latest.outputs.rs_versions }}' | jq -r '.[]' > rs-versions - echo '${{ needs.setup_and_build_latest.outputs.rdi_versions }}' | jq -r '.[]' > rdi-versions - echo '${{ needs.setup_and_build_latest.outputs.redisvl_versions }}' | jq -r '.[]' > redisvl-versions - - hugo -d "output" - - - name: Upload artifact - uses: actions/upload-artifact@v4 - with: - name: build-kubernetes-${{ matrix.version }} - path: output/ - retention-days: 1 - - # Build RS versions in parallel - build_rs: - name: Build RS ${{ matrix.version }} - needs: setup_and_build_latest - if: ${{ needs.setup_and_build_latest.outputs.rs_versions != '[]' }} - runs-on: ubuntu-latest - permissions: - contents: 'read' - strategy: - fail-fast: false - matrix: - version: ${{ fromJson(needs.setup_and_build_latest.outputs.rs_versions) }} - - steps: - - name: Free up disk space - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/ghc - sudo rm -rf /opt/hostedtoolcache/CodeQL - - - name: Install Hugo - run: | - wget -O ${{ runner.temp }}/hugo.deb https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-amd64.deb \ - && sudo dpkg -i ${{ runner.temp }}/hugo.deb - - - name: Check the branch out - uses: actions/checkout@v4 - - - name: Install dependencies - run: make deps - - - name: Generate components - run: make components - - - name: Build RS ${{ matrix.version }} - run: | - set -x - version="${{ matrix.version }}" - hugo_root_path="${{ needs.setup_and_build_latest.outputs.hugo_root_path }}" - - sed -i "s#baseURL = \"https://redis.io\"#baseURL = \"https://redis.io/${hugo_root_path}\"#g" config.toml - - # Remove all other rs versions - rs_versions='${{ needs.setup_and_build_latest.outputs.rs_versions }}' - for v in $(echo "$rs_versions" | jq -r '.[]'); do - if [[ "$v" != "$version" ]]; then - rm -rf "content/operate/rs/${v}" - fi - done - - # relrefs should not include version - find "content/operate/rs/${version}" -type f | while read file; do - awk '{gsub(/\(\{\{< ?relref "\/operate\/rs\/'"${version}"'/, "({{< relref \"/operate/rs") }1' "$file" > tmpfile && mv tmpfile "$file" - done - - rsync -a --delete-after "content/operate/rs/${version}"/ content/operate/rs/ - sed -i 's/id="versionSelectorRsValue" class="version-selector-control">latest/id="versionSelectorRsValue" class="version-selector-control">v'"${version}"'/' layouts/partials/docs-nav.html - sed -i 's/linkTitle: '"${version}"'/linkTitle: Redis Software/' content/operate/rs/_index.md - - # inject replace command in meta-links - sed -i "12i \{\{ \$gh_path = replaceRE \`\^operate\/rs\/\` \"operate\/rs\/${version}\/\" \$gh_path }}" layouts/partials/meta-links.html - - # Create version files for docs-nav.html dropdowns - echo '${{ needs.setup_and_build_latest.outputs.kubernetes_versions }}' | jq -r '.[]' > kubernetes-versions - echo '${{ needs.setup_and_build_latest.outputs.rs_versions }}' | jq -r '.[]' > rs-versions - echo '${{ needs.setup_and_build_latest.outputs.rdi_versions }}' | jq -r '.[]' > rdi-versions - echo '${{ needs.setup_and_build_latest.outputs.redisvl_versions }}' | jq -r '.[]' > redisvl-versions - - hugo -d "output" - - - name: Upload artifact - uses: actions/upload-artifact@v4 - with: - name: build-rs-${{ matrix.version }} - path: output/ - retention-days: 1 - - # Build RDI versions in parallel - build_rdi: - name: Build RDI ${{ matrix.version }} - needs: setup_and_build_latest - if: ${{ needs.setup_and_build_latest.outputs.rdi_versions != '[]' }} - runs-on: ubuntu-latest - permissions: - contents: 'read' - strategy: - fail-fast: false - matrix: - version: ${{ fromJson(needs.setup_and_build_latest.outputs.rdi_versions) }} - - steps: - - name: Free up disk space - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/ghc - sudo rm -rf /opt/hostedtoolcache/CodeQL - - - name: Install Hugo - run: | - wget -O ${{ runner.temp }}/hugo.deb https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-amd64.deb \ - && sudo dpkg -i ${{ runner.temp }}/hugo.deb - - - name: Check the branch out - uses: actions/checkout@v4 - - - name: Install dependencies - run: make deps - - - name: Generate components - run: make components - - - name: Build RDI ${{ matrix.version }} - run: | - set -x - version="${{ matrix.version }}" - hugo_root_path="${{ needs.setup_and_build_latest.outputs.hugo_root_path }}" - - sed -i "s#baseURL = \"https://redis.io\"#baseURL = \"https://redis.io/${hugo_root_path}\"#g" config.toml - - # Remove all other rdi versions - rdi_versions='${{ needs.setup_and_build_latest.outputs.rdi_versions }}' - for v in $(echo "$rdi_versions" | jq -r '.[]'); do - if [[ "$v" != "$version" ]]; then - rm -rf "content/integrate/redis-data-integration/${v}" - fi - done - - # relrefs should not include version - find "content/integrate/redis-data-integration/${version}" -type f | while read file; do - awk '{gsub(/\(\{\{< ?relref "\/integrate\/redis-data-integration\/'"${version}"'/, "({{< relref \"/integrate/redis-data-integration") }1' "$file" > tmpfile && mv tmpfile "$file" - done - - rsync -a --delete-after "content/integrate/redis-data-integration/${version}"/ content/integrate/redis-data-integration/ - sed -i 's/id="versionSelectorRedis-Data-IntegrationValue" class="version-selector-control">latest/id="versionSelectorRedis-Data-IntegrationValue" class="version-selector-control">v'"${version}"'/' layouts/partials/docs-nav.html - sed -i 's/linkTitle: '"${version}"'/linkTitle: Redis Data Integration/' content/integrate/redis-data-integration/_index.md - - # inject replace command in meta-links - sed -i "12i \{\{ \$gh_path = replaceRE \`\^integrate\/redis-data-integration\/\` \"integrate\/redis-data-integration\/${version}\/\" \$gh_path }}" layouts/partials/meta-links.html - - # Create version files for docs-nav.html dropdowns - echo '${{ needs.setup_and_build_latest.outputs.kubernetes_versions }}' | jq -r '.[]' > kubernetes-versions - echo '${{ needs.setup_and_build_latest.outputs.rs_versions }}' | jq -r '.[]' > rs-versions - echo '${{ needs.setup_and_build_latest.outputs.rdi_versions }}' | jq -r '.[]' > rdi-versions - echo '${{ needs.setup_and_build_latest.outputs.redisvl_versions }}' | jq -r '.[]' > redisvl-versions - - hugo -d "output" - - - name: Upload artifact - uses: actions/upload-artifact@v4 - with: - name: build-rdi-${{ matrix.version }} - path: output/ - retention-days: 1 - - # Build RedisVL versions in parallel - build_redisvl: - name: Build RedisVL ${{ matrix.version }} - needs: setup_and_build_latest - if: ${{ needs.setup_and_build_latest.outputs.redisvl_versions != '[]' }} - runs-on: ubuntu-latest - permissions: - contents: 'read' - strategy: - fail-fast: false - matrix: - version: ${{ fromJson(needs.setup_and_build_latest.outputs.redisvl_versions) }} - - steps: - - name: Free up disk space - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/ghc - sudo rm -rf /opt/hostedtoolcache/CodeQL - - - name: Install Hugo - run: | - wget -O ${{ runner.temp }}/hugo.deb https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-amd64.deb \ - && sudo dpkg -i ${{ runner.temp }}/hugo.deb - - - name: Check the branch out - uses: actions/checkout@v4 - - - name: Install dependencies - run: make deps - - - name: Generate components - run: make components - - - name: Build RedisVL ${{ matrix.version }} - run: | - set -x - version="${{ matrix.version }}" - hugo_root_path="${{ needs.setup_and_build_latest.outputs.hugo_root_path }}" - - sed -i "s#baseURL = \"https://redis.io\"#baseURL = \"https://redis.io/${hugo_root_path}\"#g" config.toml - - # Remove all other redisvl versions - redisvl_versions='${{ needs.setup_and_build_latest.outputs.redisvl_versions }}' - for v in $(echo "$redisvl_versions" | jq -r '.[]'); do - if [[ "$v" != "$version" ]]; then - rm -rf "content/develop/ai/redisvl/${v}" - fi - done - - # relrefs should not include version - find "content/develop/ai/redisvl/${version}" -type f | while read file; do - awk '{gsub(/\(\{\{< ?relref "\/develop\/ai\/redisvl\/'"${version}"'/, "({{< relref \"/develop/ai/redisvl") }1' "$file" > tmpfile && mv tmpfile "$file" - done - - rsync -a --delete-after "content/develop/ai/redisvl/${version}"/ content/develop/ai/redisvl/ - sed -i 's/id="versionSelectorRedisvlValue" class="version-selector-control">latest/id="versionSelectorRedisvlValue" class="version-selector-control">v'"${version}"'/' layouts/partials/docs-nav.html - sed -i 's/linkTitle: '"${version}"'/linkTitle: RedisVL/' content/develop/ai/redisvl/_index.md - - # inject replace command in meta-links - sed -i "12i \{\{ \$gh_path = replaceRE \`\^develop\/ai\/redisvl\/\` \"develop\/ai\/redisvl\/${version}\/\" \$gh_path }}" layouts/partials/meta-links.html - - # Create version files for docs-nav.html dropdowns - echo '${{ needs.setup_and_build_latest.outputs.kubernetes_versions }}' | jq -r '.[]' > kubernetes-versions - echo '${{ needs.setup_and_build_latest.outputs.rs_versions }}' | jq -r '.[]' > rs-versions - echo '${{ needs.setup_and_build_latest.outputs.rdi_versions }}' | jq -r '.[]' > rdi-versions - echo '${{ needs.setup_and_build_latest.outputs.redisvl_versions }}' | jq -r '.[]' > redisvl-versions - - hugo -d "output" - - - name: Upload artifact - uses: actions/upload-artifact@v4 - with: - name: build-redisvl-${{ matrix.version }} - path: output/ - retention-days: 1 - - # Deploy latest build to GCS - deploy_latest: - name: Deploy latest - needs: setup_and_build_latest - runs-on: ubuntu-latest - permissions: - contents: 'read' - id-token: 'write' - env: - PROD_PROJECT_ID: ${{ secrets.GCP_PROJECT_PROD }} - PROD_SERVICE_ACCOUNT: ${{ secrets.PROD_SERVICE_ACCOUNT }} - PROD_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.PROD_WORKLOAD_IDENTITY_PROVIDER }} - - steps: - - name: Download latest artifact - uses: actions/download-artifact@v4 - with: - name: build-latest - path: public/ - - - name: 'Google auth' - uses: 'google-github-actions/auth@v2' - with: - project_id: '${{ env.PROD_PROJECT_ID }}' - service_account: '${{ env.PROD_SERVICE_ACCOUNT }}' - workload_identity_provider: '${{ env.PROD_WORKLOAD_IDENTITY_PROVIDER }}' - - - name: 'Set up Cloud SDK' - uses: 'google-github-actions/setup-gcloud@v2' - with: - project_id: '${{ env.PROD_PROJECT_ID }}' - version: '>= 363.0.0' - - - name: Deploy latest to GCS - run: | - bucket_path="${{ needs.setup_and_build_latest.outputs.bucket_path }}" - gsutil -m rsync -r -c -j html -d public gs://$BUCKET/$bucket_path - gsutil -m rsync -r -c -j html -d public gs://$BUCKET/docs/$bucket_path - - # Deploy Kubernetes versions to GCS - deploy_kubernetes: - name: Deploy Kubernetes ${{ matrix.version }} - needs: - - setup_and_build_latest - - build_kubernetes - if: ${{ needs.setup_and_build_latest.outputs.kubernetes_versions != '[]' }} - runs-on: ubuntu-latest - permissions: - contents: 'read' - id-token: 'write' - strategy: - fail-fast: false - matrix: - version: ${{ fromJson(needs.setup_and_build_latest.outputs.kubernetes_versions) }} - env: - PROD_PROJECT_ID: ${{ secrets.GCP_PROJECT_PROD }} - PROD_SERVICE_ACCOUNT: ${{ secrets.PROD_SERVICE_ACCOUNT }} - PROD_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.PROD_WORKLOAD_IDENTITY_PROVIDER }} - - steps: - - name: Download artifact - uses: actions/download-artifact@v4 - with: - name: build-kubernetes-${{ matrix.version }} - path: output/ - - - name: 'Google auth' - uses: 'google-github-actions/auth@v2' - with: - project_id: '${{ env.PROD_PROJECT_ID }}' - service_account: '${{ env.PROD_SERVICE_ACCOUNT }}' - workload_identity_provider: '${{ env.PROD_WORKLOAD_IDENTITY_PROVIDER }}' - - - name: 'Set up Cloud SDK' - uses: 'google-github-actions/setup-gcloud@v2' - with: - project_id: '${{ env.PROD_PROJECT_ID }}' - version: '>= 363.0.0' - - - name: Deploy Kubernetes ${{ matrix.version }} to GCS - run: | - version="${{ matrix.version }}" - bucket_path="${{ needs.setup_and_build_latest.outputs.bucket_path }}" - - # Matrix Hugo builds can produce different CSS hashes than deploy_latest, - # so versioned pages 404 on /css/index.min.{hash}.css. Additively rsync - # this build's css/scss (no -d) so each hash coexists alongside the - # canonical hash uploaded by deploy_latest. Applied on both latest and - # staging. - upload_assets() { - local dest="$1" - for d in css scss; do - if [ -d "output/$d" ]; then - gsutil -m rsync -r -c -j css "output/$d" "${dest}/$d" - fi - done - } - - if [[ "${{ github.ref_name }}" == "latest" ]]; then - gsutil -m rsync -r -c -j html -d output/operate/kubernetes "gs://${BUCKET}/kubernetes-${version}/operate/kubernetes" - gsutil -m rsync -r -c -j html -d "output/operate/kubernetes/${version}" "gs://${BUCKET}/docs/${bucket_path}/operate/kubernetes/${version}" - upload_assets "gs://${BUCKET}/docs/${bucket_path}" - elif [[ "$bucket_path" == staging/* ]]; then - gsutil -m rsync -r -c -j html -d output/operate/kubernetes "gs://${BUCKET}/${bucket_path}/kubernetes-${version}/operate/kubernetes" - gsutil -m rsync -r -c -j html -d "output/operate/kubernetes/${version}" "gs://${BUCKET}/docs/${bucket_path}/operate/kubernetes/${version}" - upload_assets "gs://${BUCKET}/docs/${bucket_path}" - fi - - # Deploy RS versions to GCS - deploy_rs: - name: Deploy RS ${{ matrix.version }} - needs: - - setup_and_build_latest - - build_rs - if: ${{ needs.setup_and_build_latest.outputs.rs_versions != '[]' }} - runs-on: ubuntu-latest - permissions: - contents: 'read' - id-token: 'write' - strategy: - fail-fast: false - matrix: - version: ${{ fromJson(needs.setup_and_build_latest.outputs.rs_versions) }} - env: - PROD_PROJECT_ID: ${{ secrets.GCP_PROJECT_PROD }} - PROD_SERVICE_ACCOUNT: ${{ secrets.PROD_SERVICE_ACCOUNT }} - PROD_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.PROD_WORKLOAD_IDENTITY_PROVIDER }} - - steps: - - name: Download artifact - uses: actions/download-artifact@v4 - with: - name: build-rs-${{ matrix.version }} - path: output/ - - - name: 'Google auth' - uses: 'google-github-actions/auth@v2' - with: - project_id: '${{ env.PROD_PROJECT_ID }}' - service_account: '${{ env.PROD_SERVICE_ACCOUNT }}' - workload_identity_provider: '${{ env.PROD_WORKLOAD_IDENTITY_PROVIDER }}' - - - name: 'Set up Cloud SDK' - uses: 'google-github-actions/setup-gcloud@v2' - with: - project_id: '${{ env.PROD_PROJECT_ID }}' - version: '>= 363.0.0' - - - name: Deploy RS ${{ matrix.version }} to GCS - run: | - version="${{ matrix.version }}" - bucket_path="${{ needs.setup_and_build_latest.outputs.bucket_path }}" - - upload_assets() { - local dest="$1" - for d in css scss; do - if [ -d "output/$d" ]; then - gsutil -m rsync -r -c -j css "output/$d" "${dest}/$d" - fi - done - } - - if [[ "${{ github.ref_name }}" == "latest" ]]; then - gsutil -m rsync -r -c -j html -d output/operate/rs "gs://${BUCKET}/rs-${version}/operate/rs" - gsutil -m rsync -r -c -j html -d "output/operate/rs/${version}" "gs://${BUCKET}/docs/${bucket_path}/operate/rs/${version}" - upload_assets "gs://${BUCKET}/docs/${bucket_path}" - elif [[ "$bucket_path" == staging/* ]]; then - gsutil -m rsync -r -c -j html -d output/operate/rs "gs://${BUCKET}/${bucket_path}/rs-${version}/operate/rs" - gsutil -m rsync -r -c -j html -d "output/operate/rs/${version}" "gs://${BUCKET}/docs/${bucket_path}/operate/rs/${version}" - upload_assets "gs://${BUCKET}/docs/${bucket_path}" - fi - - # Deploy RDI versions to GCS - deploy_rdi: - name: Deploy RDI ${{ matrix.version }} - needs: - - setup_and_build_latest - - build_rdi - if: ${{ needs.setup_and_build_latest.outputs.rdi_versions != '[]' }} - runs-on: ubuntu-latest - permissions: - contents: 'read' - id-token: 'write' - strategy: - fail-fast: false - matrix: - version: ${{ fromJson(needs.setup_and_build_latest.outputs.rdi_versions) }} - env: - PROD_PROJECT_ID: ${{ secrets.GCP_PROJECT_PROD }} - PROD_SERVICE_ACCOUNT: ${{ secrets.PROD_SERVICE_ACCOUNT }} - PROD_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.PROD_WORKLOAD_IDENTITY_PROVIDER }} - - steps: - - name: Download artifact - uses: actions/download-artifact@v4 - with: - name: build-rdi-${{ matrix.version }} - path: output/ - - - name: 'Google auth' - uses: 'google-github-actions/auth@v2' - with: - project_id: '${{ env.PROD_PROJECT_ID }}' - service_account: '${{ env.PROD_SERVICE_ACCOUNT }}' - workload_identity_provider: '${{ env.PROD_WORKLOAD_IDENTITY_PROVIDER }}' - - - name: 'Set up Cloud SDK' - uses: 'google-github-actions/setup-gcloud@v2' - with: - project_id: '${{ env.PROD_PROJECT_ID }}' - version: '>= 363.0.0' - - - name: Deploy RDI ${{ matrix.version }} to GCS - run: | - version="${{ matrix.version }}" - bucket_path="${{ needs.setup_and_build_latest.outputs.bucket_path }}" - - upload_assets() { - local dest="$1" - for d in css scss; do - if [ -d "output/$d" ]; then - gsutil -m rsync -r -c -j css "output/$d" "${dest}/$d" - fi - done - } - - if [[ "${{ github.ref_name }}" == "latest" ]]; then - gsutil -m rsync -r -c -j html -d output/integrate/redis-data-integration "gs://${BUCKET}/redis-data-integration-${version}/integrate/redis-data-integration" - gsutil -m rsync -r -c -j html -d "output/integrate/redis-data-integration/${version}" "gs://${BUCKET}/docs/${bucket_path}/integrate/redis-data-integration/${version}" - upload_assets "gs://${BUCKET}/docs/${bucket_path}" - elif [[ "$bucket_path" == staging/* ]]; then - gsutil -m rsync -r -c -j html -d output/integrate/redis-data-integration "gs://${BUCKET}/${bucket_path}/redis-data-integration-${version}/integrate/redis-data-integration" - gsutil -m rsync -r -c -j html -d "output/integrate/redis-data-integration/${version}" "gs://${BUCKET}/docs/${bucket_path}/integrate/redis-data-integration/${version}" - upload_assets "gs://${BUCKET}/docs/${bucket_path}" - fi - - # Deploy RedisVL versions to GCS - deploy_redisvl: - name: Deploy RedisVL ${{ matrix.version }} - needs: - - setup_and_build_latest - - build_redisvl - if: ${{ needs.setup_and_build_latest.outputs.redisvl_versions != '[]' }} - runs-on: ubuntu-latest - permissions: - contents: 'read' - id-token: 'write' - strategy: - fail-fast: false - matrix: - version: ${{ fromJson(needs.setup_and_build_latest.outputs.redisvl_versions) }} - env: - PROD_PROJECT_ID: ${{ secrets.GCP_PROJECT_PROD }} - PROD_SERVICE_ACCOUNT: ${{ secrets.PROD_SERVICE_ACCOUNT }} - PROD_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.PROD_WORKLOAD_IDENTITY_PROVIDER }} - - steps: - - name: Download artifact - uses: actions/download-artifact@v4 - with: - name: build-redisvl-${{ matrix.version }} - path: output/ - - - name: 'Google auth' - uses: 'google-github-actions/auth@v2' - with: - project_id: '${{ env.PROD_PROJECT_ID }}' - service_account: '${{ env.PROD_SERVICE_ACCOUNT }}' - workload_identity_provider: '${{ env.PROD_WORKLOAD_IDENTITY_PROVIDER }}' - - - name: 'Set up Cloud SDK' - uses: 'google-github-actions/setup-gcloud@v2' - with: - project_id: '${{ env.PROD_PROJECT_ID }}' - version: '>= 363.0.0' - - - name: Deploy RedisVL ${{ matrix.version }} to GCS - run: | - version="${{ matrix.version }}" - bucket_path="${{ needs.setup_and_build_latest.outputs.bucket_path }}" - - upload_assets() { - local dest="$1" - for d in css scss; do - if [ -d "output/$d" ]; then - gsutil -m rsync -r -c -j css "output/$d" "${dest}/$d" - fi - done - } - - if [[ "${{ github.ref_name }}" == "latest" ]]; then - gsutil -m rsync -r -c -j html -d output/develop/ai/redisvl "gs://${BUCKET}/redisvl-${version}/develop/ai/redisvl" - gsutil -m rsync -r -c -j html -d "output/develop/ai/redisvl/${version}" "gs://${BUCKET}/docs/${bucket_path}/develop/ai/redisvl/${version}" - upload_assets "gs://${BUCKET}/docs/${bucket_path}" - elif [[ "$bucket_path" == staging/* ]]; then - gsutil -m rsync -r -c -j html -d output/develop/ai/redisvl "gs://${BUCKET}/${bucket_path}/redisvl-${version}/develop/ai/redisvl" - gsutil -m rsync -r -c -j html -d "output/develop/ai/redisvl/${version}" "gs://${BUCKET}/docs/${bucket_path}/develop/ai/redisvl/${version}" - upload_assets "gs://${BUCKET}/docs/${bucket_path}" - fi - - # Deploy custom 404 page (only on latest branch) - deploy_404: - name: Deploy 404 page - needs: setup_and_build_latest - if: ${{ github.ref_name == 'latest' }} - runs-on: ubuntu-latest - permissions: - contents: 'read' - id-token: 'write' - env: - PROD_PROJECT_ID: ${{ secrets.GCP_PROJECT_PROD }} - PROD_SERVICE_ACCOUNT: ${{ secrets.PROD_SERVICE_ACCOUNT }} - PROD_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.PROD_WORKLOAD_IDENTITY_PROVIDER }} - - steps: - - name: Download latest artifact - uses: actions/download-artifact@v4 - with: - name: build-latest - path: public/ - - - name: Prepare 404 page - run: | - sed -i "s#/docs/latest/scss/#https://storage.googleapis.com/$BUCKET/docs/latest/scss/#g" public/404.html - sed -i "s#/docs/latest/css/#https://storage.googleapis.com/$BUCKET/docs/latest/css/#g" public/404.html - - - name: 'Google auth' - uses: 'google-github-actions/auth@v2' - with: - project_id: '${{ env.PROD_PROJECT_ID }}' - service_account: '${{ env.PROD_SERVICE_ACCOUNT }}' - workload_identity_provider: '${{ env.PROD_WORKLOAD_IDENTITY_PROVIDER }}' - - - name: 'Set up Cloud SDK' - uses: 'google-github-actions/setup-gcloud@v2' - with: - project_id: '${{ env.PROD_PROJECT_ID }}' - version: '>= 363.0.0' - - - name: Deploy 404 page to GCS - run: gsutil cp public/404.html gs://$BUCKET/docs/ diff --git a/.github/workflows/rc_api_sync.yaml b/.github/workflows/rc_api_sync.yaml deleted file mode 100644 index a90b16545a..0000000000 --- a/.github/workflows/rc_api_sync.yaml +++ /dev/null @@ -1,70 +0,0 @@ -name: rc_api_sync - -on: - schedule: - - cron: '0 0 * * *' # run every day at midnight UTC time - workflow_dispatch: # or run on manual trigger - -jobs: - rc_api_sync: - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - actions: write - steps: - - name: 'Checkout' - uses: 'actions/checkout@v3' - - - name: 'Fetch openapi json file' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - branch="rc_api_sync" - spec_change=false - - # check if remote branch already exists - git fetch --all - set +e - git ls-remote --exit-code --heads origin "refs/heads/${branch}" - if [ "$?" -eq 0 ]; then - set -e - # if it does, create local branch off existing remote branch - git checkout -b "${branch}" "origin/${branch}" - git branch --set-upstream-to="origin/${branch}" "${branch}" - git pull - else - set -e - # otherwise, create local branch from main - git checkout -b "${branch}" - fi - - curl -Ls https://api.redislabs.com/v1/cloud-api-docs \ - | jq '(.. | .example? | try select(test("^{"))) |= fromjson' > content/operate/rc/api/api-reference/openapi.json - - spec_is_different=$(git diff content/operate/rc/api/api-reference/openapi.json) - - if [[ ! -z $spec_is_different ]]; then - spec_change=true - - git add "content/operate/rc/api/api-reference/openapi.json" - git config user.email "177626021+redisdocsapp[bot]@users.noreply.github.com" - git config user.name "redisdocsapp[bot]" - git commit -m "Update content/operate/rc/api/api-reference/openapi.json" - fi - - if [ "$spec_change" = true ] ; then - git push origin "${branch}" - - # If a pr is not already open, create one - set +e - gh search prs -R redis/docs --state open --match title "update rc openapi spec" | grep -q "update rc openapi spec" - if [ "$?" -eq 1 ]; then - set -e - gh pr create \ - --body "update rc openapi spec" \ - --title "update rc openapi spec" \ - --head "$branch" \ - --base "main" - fi - fi \ No newline at end of file diff --git a/.github/workflows/rc_supported_regions_sync.yaml b/.github/workflows/rc_supported_regions_sync.yaml deleted file mode 100644 index a930bb1ac3..0000000000 --- a/.github/workflows/rc_supported_regions_sync.yaml +++ /dev/null @@ -1,74 +0,0 @@ -name: rc_supported_regions_sync - -on: - schedule: - - cron: '0 0 * * 1' # run every Monday at midnight UTC time - workflow_dispatch: # or run on manual trigger - -jobs: - rc_supported_regions_sync: - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - actions: write - steps: - - name: 'Checkout' - uses: 'actions/checkout@v3' - - - name: 'Install dependencies' - run: pip3 install requests - - - name: 'Run rc_supported_regions_sync.py' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - RC_API_KEY: ${{ secrets.RC_API_KEY }} - RC_API_SECRET_KEY: ${{ secrets.RC_API_SECRET_KEY }} - run: | - branch="rc_supported_regions_sync" - regions_change=false - - # check if remote branch already exists - git fetch --all - set +e - git ls-remote --exit-code --heads origin "refs/heads/${branch}" - if [ "$?" -eq 0 ]; then - set -e - # if it does, create local branch off existing remote branch - git checkout -b "${branch}" "origin/${branch}" - git branch --set-upstream-to="origin/${branch}" "${branch}" - git pull - else - set -e - # otherwise, create local branch from main - git checkout -b "${branch}" - fi - - python3 build/rc_supported_regions_sync.py - - regions_are_different=$(git diff data/rc_supported_regions.json) - - if [[ ! -z $regions_are_different ]]; then - regions_change=true - - git add "data/rc_supported_regions.json" - git config user.email "177626021+redisdocsapp[bot]@users.noreply.github.com" - git config user.name "redisdocsapp[bot]" - git commit -m "Update data/rc_supported_regions.json" - fi - - if [ "$regions_change" = true ] ; then - git push origin "${branch}" - - # If a pr is not already open, create one - set +e - gh search prs -R redis/docs --state open --match title "update rc supported regions" | grep -q "update rc supported regions" - if [ "$?" -eq 1 ]; then - set -e - gh pr create \ - --body "update rc supported regions" \ - --title "update rc supported regions" \ - --head "$branch" \ - --base "main" - fi - fi diff --git a/.github/workflows/redis_docs_sync.yaml b/.github/workflows/redis_docs_sync.yaml deleted file mode 100644 index 7b1aedbcaf..0000000000 --- a/.github/workflows/redis_docs_sync.yaml +++ /dev/null @@ -1,104 +0,0 @@ -name: redis_docs_sync - -on: - workflow_dispatch: - inputs: - release: - type: string - required: true - description: '' - -jobs: - redis_docs_sync: - runs-on: ubuntu-latest - permissions: - contents: write - actions: write - steps: - - name: Generate a token - id: generate-token - uses: actions/create-github-app-token@v1 - with: - app-id: ${{ secrets.DOCS_APP_ID }} - private-key: ${{ secrets.DOCS_APP_PRIVATE_KEY }} - - - name: 'Checkout' - uses: actions/checkout@v3 - with: - token: ${{ steps.generate-token.outputs.token }} - - - name: 'Generate modules-api-ref.md and commands.json files and push if necessary' - env: - GH_TOKEN: ${{ steps.generate-token.outputs.token }} - RELEASE: ${{ github.event.inputs.release }} - run: |- - BRANCH="redis_docs_sync_${RELEASE}" - - # Generate modules-api-ref.md - gh repo clone redis/redis - pushd redis - git fetch origin --tags - git checkout "tags/${RELEASE}" - make 1>/dev/null - src/redis-server & - utils/generate-commands-json.py > generated-commands.json - utils/generate-module-api-doc.rb > generated-modules-api-ref.md - popd - - # check if remote branch already exists - git fetch --all - set +e - git ls-remote --exit-code --heads origin "refs/heads/${BRANCH}" - if [ "$?" -eq 0 ]; then - set -e - # if it does, create local branch off existing remote branch - git checkout -b "${BRANCH}" "origin/${BRANCH}" - git branch --set-upstream-to="origin/${BRANCH}" "${BRANCH}" - git pull - else - set -e - # otherwise, create local branch from main - git checkout -b "${BRANCH}" - fi - - mv redis/generated-modules-api-ref.md content/develop/reference/modules/modules-api-ref.md - mv redis/generated-commands.json data/commands_core.json - - # Apply frontmatter patch - git apply content/develop/reference/modules/modules-api-ref-frontmatter.patch - - # Check if there are any changes - commands_core_is_different=$(git diff data/commands_core.json) - modules_api_is_different=$(git diff content/develop/reference/modules/modules-api-ref.md) - - # If commands_core file has changed, commit it - if [[ ! -z $commands_core_is_different ]]; then - git add data/commands_core.json - git config user.email "177626021+redisdocsapp[bot]@users.noreply.github.com" - git config user.name "redisdocsapp[bot]" - git commit -m "Update commands_core.json for release ${RELEASE}" - fi - - # If modules-api-ref file has changed, commit it - if [[ ! -z $modules_api_is_different ]]; then - git add content/develop/reference/modules/modules-api-ref.md - git config user.email "177626021+redisdocsapp[bot]@users.noreply.github.com" - git config user.name "redisdocsapp[bot]" - git commit -m "Update modules-api-ref.md for release ${RELEASE}" - fi - - if [[ ! -z $commands_core_is_different || ! -z $modules_api_is_different ]]; then - git push origin "${BRANCH}" - fi - - # If a pr is not already open, create one - set +e - gh search prs -R redis/docs --state open --match title "redis docs sync ${RELEASE}" | grep -q "redis docs sync ${RELEASE}" - if [ "$?" -eq 1 ]; then - set -e - gh pr create \ - --body "redis docs sync ${RELEASE}" \ - --title "redis docs sync ${RELEASE}" \ - --head "$BRANCH" \ - --base "main" - fi \ No newline at end of file diff --git a/.github/workflows/redis_modules_docs_sync.yaml b/.github/workflows/redis_modules_docs_sync.yaml deleted file mode 100644 index b72d565311..0000000000 --- a/.github/workflows/redis_modules_docs_sync.yaml +++ /dev/null @@ -1,84 +0,0 @@ -name: redis_modules_docs_sync - -on: - schedule: - - cron: '0 0 * * *' # run every day at midnight UTC time - workflow_dispatch: # or run on manual trigger - -jobs: - redis_modules_docs_sync: - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - actions: write - steps: - - name: 'Checkout' - uses: 'actions/checkout@v3' - - - name: 'Fetch commands json files from modules' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - branch="redis_modules_docs_sync" - modules_list=("redisjson" "redisbloom" "redistimeseries" "redisearch") - module_change=false - - # check if remote branch already exists - git fetch --all - set +e - git ls-remote --exit-code --heads origin "refs/heads/${branch}" - if [ "$?" -eq 0 ]; then - set -e - # if it does, create local branch off existing remote branch - git checkout -b "${branch}" "origin/${branch}" - git branch --set-upstream-to="origin/${branch}" "${branch}" - git pull - else - set -e - # otherwise, create local branch from main - git checkout -b "${branch}" - fi - - for module in "${modules_list[@]}"; do - # Find latest version - module_version=$( - curl -Ls \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${GH_TOKEN}" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/${module}/${module}/releases/latest" \ - | jq -r '.tag_name' - ) - - # Fetch release - gh release download "$module_version" -R "${module}/${module}" -A zip -O "${module}.zip" - unzip "${module}.zip" -d "${module}" - cp ${module}/*/commands.json "data/commands_${module}.json" - - modules_commands_is_different=$(git diff "data/commands_${module}.json") - if [[ ! -z $modules_commands_is_different ]]; then - module_change=true - - git add "data/commands_${module}.json" - git config user.email "177626021+redisdocsapp[bot]@users.noreply.github.com" - git config user.name "redisdocsapp[bot]" - git commit -m "Update data/commands_${module}.json for release ${module_version}" - fi - done - - if [ "$module_change" = true ] ; then - git push origin "${branch}" - - # If a pr is not already open, create one - set +e - gh search prs -R redis/docs --state open --match title "redis modules docs sync" | grep -q "redis modules docs sync" - if [ "$?" -eq 1 ]; then - set -e - gh pr create \ - --body "redis modules docs sync" \ - --title "redis modules docs sync" \ - --head "$branch" \ - --base "main" - fi - fi \ No newline at end of file diff --git a/.github/workflows/redisvl_docs_sync.yaml b/.github/workflows/redisvl_docs_sync.yaml deleted file mode 100644 index ff5a08f916..0000000000 --- a/.github/workflows/redisvl_docs_sync.yaml +++ /dev/null @@ -1,128 +0,0 @@ -name: redisvl_docs_sync - -on: - schedule: - - cron: '0 0 * * *' # run every day at midnight UTC time - workflow_dispatch: # or run on manual trigger - -jobs: - define-version-matrix: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - id: set-matrix - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - OLDEST_SUPPORTED_VERSION="0.6.0" - versions=$(gh release -R redis/redis-vl-python list --json name,isLatest \ - | jq -r '.[] | [.name, .isLatest] | @tsv' \ - | sed "/${OLDEST_SUPPORTED_VERSION}/q" \ - | jq -Rnc '{include: [inputs | split("\t") | {version: .[0], isLatest: .[1]}]}') - - echo "${versions}" - echo "matrix=${versions}" >> "$GITHUB_OUTPUT" - - redisvl_docs_sync: - runs-on: ubuntu-latest - needs: define-version-matrix - strategy: - matrix: ${{ fromJSON(needs['define-version-matrix'].outputs.matrix) }} - fail-fast: false - permissions: - contents: write - pull-requests: write - actions: write - steps: - - name: 'Checkout' - uses: 'actions/checkout@v3' - - - name: 'Setup Python' - uses: 'actions/setup-python@v5' - with: - python-version: '3.11' - - - name: 'Install deps' - run: | - pip3 install \ - sphinx==8.1.3 \ - sphinx_design==0.7.0 \ - sphinx_copybutton==0.5.2 \ - nbsphinx==0.9.7 \ - sphinx_favicon==1.1.0 \ - myst_nb==1.4.0 \ - sphinx_markdown_builder==0.6.8 \ - jupyter==1.1.1 - - - name: 'Fetch redisvl repo' - run: | - git clone https://github.com/redis/redis-vl-python - - - name: 'Sync redisvl docs' - run: | - version="${{ matrix.version }}" - version="${version#v}" - latest_flag="" - if [ "${{ matrix.isLatest }}" = "true" ]; then - latest_flag="--latest" - fi - python3 build/redisvl_docs_sync.py \ - --version "${version}" \ - --repo redis-vl-python \ - ${latest_flag} - - - name: 'Create pull request if necessary' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - release="${{ matrix.version }}" - release="${release#v}" - branch="redisvl_docs_sync_${release}" - redisvl_change=false - - # check if remote branch already exists - git fetch --all - set +e - git ls-remote --exit-code --heads origin "refs/heads/${branch}" - if [ "$?" -eq 0 ]; then - set -e - # if it does, create local branch off existing remote branch - git checkout -f -b "${branch}" "origin/${branch}" - git branch --set-upstream-to="origin/${branch}" "${branch}" - git pull - else - set -e - # otherwise, create local branch from main - git checkout -b "${branch}" - fi - - set +e - latest_redisvl_release_is_different=$(git diff content/develop/ai/redisvl/ static/images/redisvl/) - new_release=$(git ls-files --others --directory | grep -E 'content/develop/ai/redisvl/'${release}'|static/images/redisvl/') - - if [[ -n "$latest_redisvl_release_is_different" || -n "$new_release" ]]; then - redisvl_change=true - - git add "content/develop/ai/redisvl/" - git add "static/images/redisvl/" - git config user.email "177626021+redisdocsapp[bot]@users.noreply.github.com" - git config user.name "redisdocsapp[bot]" - git commit -m "Update for redisvl ${release}" - fi - - if [ "$redisvl_change" = true ] ; then - git push origin "${branch}" - - # If a pr is not already open, create one - set +e - gh search prs -R redis/docs --state open --match title "update for redisvl ${release}" | grep -q "update for redisvl ${release}" - if [ "$?" -eq 1 ]; then - set -e - gh pr create \ - --body "update for redisvl ${release}" \ - --title "update for redisvl ${release}" \ - --head "$branch" \ - --base "main" - fi - fi diff --git a/.github/workflows/update_command_pages.yaml b/.github/workflows/update_command_pages.yaml deleted file mode 100644 index f503564040..0000000000 --- a/.github/workflows/update_command_pages.yaml +++ /dev/null @@ -1,74 +0,0 @@ -name: update_command_pages - -on: - push: - branches: ['main'] - paths: - - data/commands_core.json # run when data/commands_core.json is updated on main - workflow_dispatch: # or run on manual trigger - -jobs: - update_command_pages: - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - actions: write - steps: - - name: 'Checkout' - uses: 'actions/checkout@v3' - - - name: Install dependencies - run: make deps - - - name: 'Run build/cmd_tools.py update-all' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - branch="update_command_pages" - commands_change=false - - # check if remote branch already exists - git fetch --all - set +e - git ls-remote --exit-code --heads origin "refs/heads/${branch}" - if [ "$?" -eq 0 ]; then - set -e - # if it does, create local branch off existing remote branch - git checkout -b "${branch}" "origin/${branch}" - git branch --set-upstream-to="origin/${branch}" "${branch}" - git pull - else - set -e - # otherwise, create local branch from main - git checkout -b "${branch}" - fi - - python3 build/cmd_tools.py update-all - - commands_are_different=$(git diff "content/commands/") - - if [[ ! -z $commands_are_different ]]; then - commands_change=true - - git add "content/commands/" - git config user.email "177626021+redisdocsapp[bot]@users.noreply.github.com" - git config user.name "redisdocsapp[bot]" - git commit -m "Update content/commands/" - fi - - if [ "$commands_change" = true ] ; then - git push origin "${branch}" - - # If a pr is not already open, create one - set +e - gh search prs -R redis/docs --state open --match title "update command pages" | grep -q "update command pages" - if [ "$?" -eq 1 ]; then - set -e - gh pr create \ - --body "update command pages" \ - --title "update command pages" \ - --head "$branch" \ - --base "main" - fi - fi \ No newline at end of file From f3dc14377f3c2b05c7fd24f9ba293fcd409ad44d Mon Sep 17 00:00:00 2001 From: mich-elle-luna <153109578+mich-elle-luna@users.noreply.github.com> Date: Wed, 10 Jun 2026 08:59:40 -0700 Subject: [PATCH 05/22] Update description of hashes in documentation Clarified the usage of hashes in representing objects and their field limits. --- content/develop/data-types/hashes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/develop/data-types/hashes.md b/content/develop/data-types/hashes.md index 06153cf87f..7c13aac922 100644 --- a/content/develop/data-types/hashes.md +++ b/content/develop/data-types/hashes.md @@ -44,7 +44,7 @@ You can use hashes to represent basic objects and to store groupings of counters {{< /clients-example >}} -While hashes are handy to represent *objects*, actually the number of fields you can +While hashes are used to represent *objects*, actually the number of fields you can put inside a hash has no practical limits (other than available memory), so you can use hashes in many different ways inside your application. From 8e5238f2d219841905597ded8fd1c96e63d48d2d Mon Sep 17 00:00:00 2001 From: mich-elle-luna Date: Wed, 10 Jun 2026 09:04:20 -0700 Subject: [PATCH 06/22] Raise similarity threshold to 0.95 for demo testing --- scripts/memory/retrieve_memory.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/memory/retrieve_memory.js b/scripts/memory/retrieve_memory.js index 576e357360..21dbfc770e 100644 --- a/scripts/memory/retrieve_memory.js +++ b/scripts/memory/retrieve_memory.js @@ -7,7 +7,7 @@ const OpenAI = require('openai'); const INDEX_NAME = 'repo_memory_idx'; const KEY_PREFIX = 'memory:'; const VECTOR_DIM = 1536; -const SIMILARITY_THRESHOLD = 0.7; // cosine distance [0,2]; lower = more similar +const SIMILARITY_THRESHOLD = 0.95; // cosine distance [0,2]; lower = more similar const TOP_K = 10; const MAX_RESULTS = 5; From 4ccb0d8d2315dd4e9defb62631c1136454987efc Mon Sep 17 00:00:00 2001 From: mich-elle-luna <153109578+mich-elle-luna@users.noreply.github.com> Date: Wed, 10 Jun 2026 09:10:16 -0700 Subject: [PATCH 07/22] Clarify explanation of small hashes in memory --- content/develop/data-types/hashes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/develop/data-types/hashes.md b/content/develop/data-types/hashes.md index 7c13aac922..de60c6f735 100644 --- a/content/develop/data-types/hashes.md +++ b/content/develop/data-types/hashes.md @@ -70,7 +70,7 @@ as well, like [`HINCRBY`]({{< relref "/commands/hincrby" >}}): You can find the [full list of hash commands in the documentation]({{< relref "/commands#hash" >}}). -It is worth noting that small hashes (i.e., a few elements with small values) are +It is worth noting that small hashes (for example, a few elements with small values) are encoded in special way in memory that make them very memory efficient. ## Examples From 7f13bab8e075f9e84d39de4e68c41034187db3d2 Mon Sep 17 00:00:00 2001 From: mich-elle-luna Date: Wed, 10 Jun 2026 09:17:43 -0700 Subject: [PATCH 08/22] Tune threshold to 0.75 and add commit status for push events --- .github/workflows/repository-memory.yml | 1 + scripts/memory/post_comment.js | 17 +++++++++++++++++ scripts/memory/retrieve_memory.js | 2 +- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/repository-memory.yml b/.github/workflows/repository-memory.yml index 542ad6913d..9cfe55f3f0 100644 --- a/.github/workflows/repository-memory.yml +++ b/.github/workflows/repository-memory.yml @@ -12,6 +12,7 @@ on: permissions: pull-requests: write contents: read + statuses: write jobs: memory: diff --git a/scripts/memory/post_comment.js b/scripts/memory/post_comment.js index 1b0959f41d..87143027d4 100644 --- a/scripts/memory/post_comment.js +++ b/scripts/memory/post_comment.js @@ -49,6 +49,20 @@ function buildMarkdown(related, context, sourceUrl) { return lines.join('\n'); } +async function postCommitStatus(repo, sha, related, sourceUrl, token) { + const description = related.length === 0 + ? 'No related prior context found' + : `Found ${related.length} related item${related.length === 1 ? '' : 's'}: ${related[0].title}`.slice(0, 140); + + await githubRequest('POST', `/repos/${repo}/statuses/${sha}`, { + state: 'success', + context: '🧠 Redis Repository Memory', + description, + target_url: sourceUrl, + }, token); + console.log(`posted_commit_status sha=${sha?.slice(0, 7)} related=${related.length}`); +} + async function postOrUpdatePrComment(repo, prNumber, body, token) { const comments = await githubRequest('GET', `/repos/${repo}/issues/${prNumber}/comments`, null, token); const existing = comments.find(c => typeof c.body === 'string' && c.body.includes(COMMENT_MARKER)); @@ -84,6 +98,9 @@ async function main() { if (context.prNumber) { await postOrUpdatePrComment(repo, context.prNumber, markdown, token); } else { + // Post commit status so results are visible on the commit without opening Actions + await postCommitStatus(repo, context.sha, related, sourceUrl, token); + const summaryPath = process.env.GITHUB_STEP_SUMMARY; if (summaryPath) { fs.appendFileSync(summaryPath, markdown + '\n'); diff --git a/scripts/memory/retrieve_memory.js b/scripts/memory/retrieve_memory.js index 21dbfc770e..7c0298fdad 100644 --- a/scripts/memory/retrieve_memory.js +++ b/scripts/memory/retrieve_memory.js @@ -7,7 +7,7 @@ const OpenAI = require('openai'); const INDEX_NAME = 'repo_memory_idx'; const KEY_PREFIX = 'memory:'; const VECTOR_DIM = 1536; -const SIMILARITY_THRESHOLD = 0.95; // cosine distance [0,2]; lower = more similar +const SIMILARITY_THRESHOLD = 0.75; // cosine distance [0,2]; lower = more similar const TOP_K = 10; const MAX_RESULTS = 5; From 5b9d676f99cf60375847e9dadd9816f724ac99d7 Mon Sep 17 00:00:00 2001 From: mich-elle-luna Date: Wed, 10 Jun 2026 09:21:40 -0700 Subject: [PATCH 09/22] Fix commit status context field: remove emoji --- scripts/memory/post_comment.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/memory/post_comment.js b/scripts/memory/post_comment.js index 87143027d4..3639cc2a2f 100644 --- a/scripts/memory/post_comment.js +++ b/scripts/memory/post_comment.js @@ -56,7 +56,7 @@ async function postCommitStatus(repo, sha, related, sourceUrl, token) { await githubRequest('POST', `/repos/${repo}/statuses/${sha}`, { state: 'success', - context: '🧠 Redis Repository Memory', + context: 'Redis Repository Memory', description, target_url: sourceUrl, }, token); From 89cd584b74252e86b4b4ebeb0c76660353a0137d Mon Sep 17 00:00:00 2001 From: mich-elle-luna Date: Wed, 10 Jun 2026 09:36:07 -0700 Subject: [PATCH 10/22] Tune similarity threshold down to 0.60 --- scripts/memory/retrieve_memory.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/memory/retrieve_memory.js b/scripts/memory/retrieve_memory.js index 7c0298fdad..9e09e74267 100644 --- a/scripts/memory/retrieve_memory.js +++ b/scripts/memory/retrieve_memory.js @@ -7,7 +7,7 @@ const OpenAI = require('openai'); const INDEX_NAME = 'repo_memory_idx'; const KEY_PREFIX = 'memory:'; const VECTOR_DIM = 1536; -const SIMILARITY_THRESHOLD = 0.75; // cosine distance [0,2]; lower = more similar +const SIMILARITY_THRESHOLD = 0.60; // cosine distance [0,2]; lower = more similar const TOP_K = 10; const MAX_RESULTS = 5; From 63d2ff48f3d252f3b145c7948edbff4743c9e7e4 Mon Sep 17 00:00:00 2001 From: mich-elle-luna <153109578+mich-elle-luna@users.noreply.github.com> Date: Wed, 10 Jun 2026 09:40:46 -0700 Subject: [PATCH 11/22] Clarify description of Redis hashes video --- content/develop/data-types/hashes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/develop/data-types/hashes.md b/content/develop/data-types/hashes.md index de60c6f735..82e78d4fc8 100644 --- a/content/develop/data-types/hashes.md +++ b/content/develop/data-types/hashes.md @@ -201,5 +201,5 @@ In practice, your hashes are limited only by the overall memory on the VMs hosti ## Learn more -* [Redis Hashes Explained](https://www.youtube.com/watch?v=-KdITaRkQ-U) is a short, comprehensive video explainer covering Redis hashes. +* [Redis Hashes Explained](https://www.youtube.com/watch?v=-KdITaRkQ-U) is a short, comprehensive video that explains Redis hashes. * [Redis University's RU101](https://university.redis.com/courses/ru101/) covers Redis hashes in detail. From 582d7d401189298894d42c56f29a6618b5906b4c Mon Sep 17 00:00:00 2001 From: mich-elle-luna <153109578+mich-elle-luna@users.noreply.github.com> Date: Wed, 10 Jun 2026 09:52:39 -0700 Subject: [PATCH 12/22] Fix typo in hashes.md regarding hash limits --- content/develop/data-types/hashes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/develop/data-types/hashes.md b/content/develop/data-types/hashes.md index 82e78d4fc8..3ad6cd010b 100644 --- a/content/develop/data-types/hashes.md +++ b/content/develop/data-types/hashes.md @@ -45,7 +45,7 @@ You can use hashes to represent basic objects and to store groupings of counters {{< /clients-example >}} While hashes are used to represent *objects*, actually the number of fields you can -put inside a hash has no practical limits (other than available memory), so you can use +put inside a hash has no practical limits (other than available memory). You can use hashes in many different ways inside your application. The command [`HSET`]({{< relref "/commands/hset" >}}) sets multiple fields of the hash, while [`HGET`]({{< relref "/commands/hget" >}}) retrieves From b10009c531802b42059d5a51a1efb52f00435009 Mon Sep 17 00:00:00 2001 From: mich-elle-luna Date: Wed, 10 Jun 2026 12:25:58 -0700 Subject: [PATCH 13/22] Add seed script and upstream repo support for retrieve --- .github/workflows/repository-memory.yml | 1 + scripts/memory/retrieve_memory.js | 5 +- scripts/memory/seed.js | 179 ++++++++++++++++++++++++ 3 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 scripts/memory/seed.js diff --git a/.github/workflows/repository-memory.yml b/.github/workflows/repository-memory.yml index 9cfe55f3f0..f30f3b4b6f 100644 --- a/.github/workflows/repository-memory.yml +++ b/.github/workflows/repository-memory.yml @@ -38,6 +38,7 @@ jobs: env: REDIS_URL: ${{ secrets.MEMORY_REDIS_URL }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + UPSTREAM_REPO: redis/docs run: node scripts/memory/retrieve_memory.js - name: Post result diff --git a/scripts/memory/retrieve_memory.js b/scripts/memory/retrieve_memory.js index 9e09e74267..a12d02799f 100644 --- a/scripts/memory/retrieve_memory.js +++ b/scripts/memory/retrieve_memory.js @@ -80,10 +80,13 @@ async function main() { } ); + // Match memories from the current repo and optionally the upstream repo + // (set UPSTREAM_REPO=redis/docs in the workflow to include seeded upstream content). + const allowedRepos = new Set([context.repo, process.env.UPSTREAM_REPO].filter(Boolean)); const related = searchResult.documents .filter(doc => { const score = parseFloat(doc.value.score ?? '2'); - return score < SIMILARITY_THRESHOLD && doc.value.repo === context.repo; + return score < SIMILARITY_THRESHOLD && allowedRepos.has(doc.value.repo); }) .slice(0, MAX_RESULTS); diff --git a/scripts/memory/seed.js b/scripts/memory/seed.js new file mode 100644 index 0000000000..f992058abd --- /dev/null +++ b/scripts/memory/seed.js @@ -0,0 +1,179 @@ +'use strict'; + +/** + * One-time seeding script: fetches PRs from a GitHub repo for the past N days, + * batches embeddings, and stores them in the Redis memory index. + * + * Usage: + * GITHUB_TOKEN=... REDIS_URL=... OPENAI_API_KEY=... \ + * SEED_REPO=redis/docs DAYS_BACK=365 node scripts/memory/seed.js + * + * Optional env vars: + * SEED_REPO — repo to fetch from (default: redis/docs) + * DAYS_BACK — how many days of history to seed (default: 365) + * EMBEDDING_MODEL — embedding model (default: text-embedding-3-small) + * DRY_RUN=true — fetch and embed but do not write to Redis + */ + +const { createClient, SchemaFieldTypes, VectorAlgorithms } = require('redis'); +const OpenAI = require('openai'); + +const SEED_REPO = process.env.SEED_REPO || 'redis/docs'; +const DAYS_BACK = parseInt(process.env.DAYS_BACK) || 365; +const EMBED_MODEL = process.env.EMBEDDING_MODEL || 'text-embedding-3-small'; +const VECTOR_DIM = 1536; +const BATCH_SIZE = 100; // embeddings per OpenAI call +const INDEX_NAME = 'repo_memory_idx'; +const KEY_PREFIX = 'memory:'; +const DRY_RUN = process.env.DRY_RUN === 'true'; + +async function fetchPRs(repo, cutoff, token) { + const [owner, name] = repo.split('/'); + const headers = { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + }; + + const prs = []; + let page = 1; + + while (true) { + const url = `https://api.github.com/repos/${owner}/${name}/pulls?state=all&sort=created&direction=desc&per_page=100&page=${page}`; + const res = await fetch(url, { headers }); + if (!res.ok) throw new Error(`GitHub API ${res.status}: ${await res.text()}`); + + const batch = await res.json(); + if (!batch.length) break; + + let done = false; + for (const pr of batch) { + if (new Date(pr.created_at) < cutoff) { done = true; break; } + prs.push({ + id: `${repo.replace('/', '_')}_pr_${pr.number}`, + type: 'pr_summary', + repo, + title: pr.title, + bodySummary: (pr.body || '').slice(0, 500), + sourceUrl: pr.html_url, + createdAt: pr.created_at, + tags: (pr.labels || []).map(l => l.name).join(','), + searchText: `${pr.title}\n${(pr.body || '').slice(0, 1000)}`, + }); + } + + console.log(` page ${page}: fetched ${batch.length} PRs, kept ${prs.length} total`); + if (done || batch.length < 100) break; + page++; + } + + return prs; +} + +async function embedBatch(texts, openai) { + const res = await openai.embeddings.create({ model: EMBED_MODEL, input: texts }); + return res.data.map(d => d.embedding); +} + +async function ensureIndex(client) { + const exists = await client.ft.info(INDEX_NAME).then(() => true).catch(() => false); + if (exists) return; + await client.ft.create( + INDEX_NAME, + { + type: { type: SchemaFieldTypes.TAG }, + repo: { type: SchemaFieldTypes.TAG }, + title: { type: SchemaFieldTypes.TEXT, WEIGHT: 2 }, + body_summary: { type: SchemaFieldTypes.TEXT }, + source_url: { type: SchemaFieldTypes.TEXT, NOSTEM: true }, + created_at: { type: SchemaFieldTypes.TEXT }, + tags: { type: SchemaFieldTypes.TAG, SEPARATOR: ',' }, + embedding: { + type: SchemaFieldTypes.VECTOR, + ALGORITHM: VectorAlgorithms.HNSW, + TYPE: 'FLOAT32', + DIM: VECTOR_DIM, + DISTANCE_METRIC: 'COSINE', + }, + }, + { ON: 'HASH', PREFIX: KEY_PREFIX } + ); + console.log(`Created index: ${INDEX_NAME}`); +} + +async function main() { + const token = process.env.GITHUB_TOKEN; + const redisUrl = process.env.REDIS_URL; + const openaiKey = process.env.OPENAI_API_KEY; + + if (!token) { console.error('GITHUB_TOKEN is required'); process.exit(1); } + if (!redisUrl) { console.error('REDIS_URL is required'); process.exit(1); } + if (!openaiKey){ console.error('OPENAI_API_KEY is required'); process.exit(1); } + + const cutoff = new Date(Date.now() - DAYS_BACK * 24 * 60 * 60 * 1000); + console.log(`Seeding from ${SEED_REPO}, past ${DAYS_BACK} days (since ${cutoff.toISOString().slice(0, 10)})`); + if (DRY_RUN) console.log('DRY RUN — no writes to Redis'); + + console.log('\nFetching PRs from GitHub...'); + const prs = await fetchPRs(SEED_REPO, cutoff, token); + console.log(`Total PRs to seed: ${prs.length}`); + + if (!prs.length) { console.log('Nothing to seed.'); return; } + + const openai = new OpenAI({ apiKey: openaiKey }); + + // Embed in batches + console.log(`\nEmbedding in batches of ${BATCH_SIZE}...`); + const embeddings = []; + for (let i = 0; i < prs.length; i += BATCH_SIZE) { + const slice = prs.slice(i, i + BATCH_SIZE); + const vecs = await embedBatch(slice.map(p => p.searchText), openai); + embeddings.push(...vecs); + console.log(` embedded ${Math.min(i + BATCH_SIZE, prs.length)}/${prs.length}`); + } + + if (DRY_RUN) { + console.log('\nDry run complete — skipping Redis writes.'); + return; + } + + // Write to Redis + const client = createClient({ url: redisUrl }); + client.on('error', err => console.error('Redis error:', err.message)); + await client.connect(); + + try { + await ensureIndex(client); + + console.log('\nWriting to Redis...'); + let written = 0; + for (let i = 0; i < prs.length; i++) { + const pr = prs[i]; + const vec = embeddings[i]; + const buf = Buffer.from(new Float32Array(vec).buffer); + + await client.hSet(`${KEY_PREFIX}${pr.id}`, { + id: pr.id, + type: pr.type, + repo: pr.repo, + title: pr.title, + body_summary: pr.bodySummary, + source_url: pr.sourceUrl, + created_at: pr.createdAt, + tags: pr.tags, + embedding: buf, + }); + written++; + if (written % 100 === 0) console.log(` wrote ${written}/${prs.length}`); + } + + console.log(`\nDone. Seeded ${written} PRs from ${SEED_REPO} into ${INDEX_NAME}.`); + } finally { + await client.quit(); + } +} + +main().catch(err => { + console.error('Seed failed:', err.message); + process.exit(1); +}); From 6139df65eff232764339247dd844faff833ab002 Mon Sep 17 00:00:00 2001 From: mich-elle-luna <153109578+mich-elle-luna@users.noreply.github.com> Date: Thu, 11 Jun 2026 09:42:26 -0700 Subject: [PATCH 14/22] Fix grammar in replica-redb.md documentation --- content/operate/kubernetes/re-databases/replica-redb.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/operate/kubernetes/re-databases/replica-redb.md b/content/operate/kubernetes/re-databases/replica-redb.md index aea6a261d0..e8b663b06c 100644 --- a/content/operate/kubernetes/re-databases/replica-redb.md +++ b/content/operate/kubernetes/re-databases/replica-redb.md @@ -10,7 +10,7 @@ linkTitle: Create replica databases weight: 42 --- -You can configure a replica of a database by creating an item in +Configure a replica of a database by creating an item in the [`replicaSources`]({{< relref "/operate/kubernetes/reference/api/redis_enterprise_database_api#specreplicasources" >}}) section of the RedisEnterpriseDatabase (REDB) custom resource. {{< note >}} From 96ab17a32ae7f1180b4d4405c299c374fa4e7953 Mon Sep 17 00:00:00 2001 From: mich-elle-luna <153109578+mich-elle-luna@users.noreply.github.com> Date: Thu, 11 Jun 2026 09:47:10 -0700 Subject: [PATCH 15/22] Fix formatting of AS attribute explanation --- content/commands/ft.create.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/commands/ft.create.md b/content/commands/ft.create.md index 659ec25ba2..d477eecd2e 100644 --- a/content/commands/ft.create.md +++ b/content/commands/ft.create.md @@ -259,7 +259,7 @@ after the SCHEMA keyword, declares which fields to index: - `{identifier}` for hashes, is a field name within the hash. For JSON, the identifier is a JSON Path expression. - - `AS {attribute}` defines the attribute associated to the identifier. For example, you can use this feature to alias a complex JSONPath expression with more memorable (and easier to type) name. + - `AS {attribute}` defines the attribute associated to the identifier. For example, you can use this feature to alias a complex JSONPath expression with a more memorable (and easier to type) name. Field types are: From 492ae2fe9a9395ae7b32feac66a9d18fe2671b6a Mon Sep 17 00:00:00 2001 From: mich-elle-luna <153109578+mich-elle-luna@users.noreply.github.com> Date: Thu, 11 Jun 2026 09:49:20 -0700 Subject: [PATCH 16/22] Add example for retrying connections in Redis-py Added a new example demonstrating retry logic for connections. --- content/develop/clients/redis-py/connect.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/content/develop/clients/redis-py/connect.md b/content/develop/clients/redis-py/connect.md index 7fa8c78a8f..3aa7f3d92e 100644 --- a/content/develop/clients/redis-py/connect.md +++ b/content/develop/clients/redis-py/connect.md @@ -253,6 +253,10 @@ this behavior to suit your use case. See [Retries]({{< relref "/develop/clients/redis-py/produsage#retries" >}}) for more information about custom retry strategies, with example code. +## New example for retrying connections + +here is a concrete example of retry logic + ## Connect using Smart client handoffs (SCH) *Smart client handoffs (SCH)* is a feature of Redis Cloud and From cffa236adf88140e204474e4fc8439bb8557579a Mon Sep 17 00:00:00 2001 From: mich-elle-luna <153109578+mich-elle-luna@users.noreply.github.com> Date: Thu, 11 Jun 2026 09:55:36 -0700 Subject: [PATCH 17/22] Add section for new agent type in documentation Added a section for a new agent type in the documentation. --- content/develop/ai/agent-builder/_index.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/content/develop/ai/agent-builder/_index.md b/content/develop/ai/agent-builder/_index.md index 8b27ce90fa..f9af61676f 100644 --- a/content/develop/ai/agent-builder/_index.md +++ b/content/develop/ai/agent-builder/_index.md @@ -20,6 +20,9 @@ Use the interactive builder below to generate your custom AI agent code: {{< agent-builder >}} +## New agent type +new agent goes here + ## What are AI agents? AI agents are intelligent systems that can plan, remember, and take actions to help users accomplish goals. Unlike simple chatbots, agents can remember conversations, plan multi-step tasks, use external tools, and learn from interactions. From bdc25924e55103953a5a2e69d16822c54ab30c4c Mon Sep 17 00:00:00 2001 From: mich-elle-luna Date: Thu, 11 Jun 2026 10:11:56 -0700 Subject: [PATCH 18/22] Seed issues and comments; trigger on PR synchronize --- .github/workflows/repository-memory.yml | 1 + scripts/memory/seed.js | 256 ++++++++++++++++-------- 2 files changed, 176 insertions(+), 81 deletions(-) diff --git a/.github/workflows/repository-memory.yml b/.github/workflows/repository-memory.yml index f30f3b4b6f..2d39f95e25 100644 --- a/.github/workflows/repository-memory.yml +++ b/.github/workflows/repository-memory.yml @@ -7,6 +7,7 @@ on: pull_request: types: - opened + - synchronize workflow_dispatch: permissions: diff --git a/scripts/memory/seed.js b/scripts/memory/seed.js index f992058abd..4b70769689 100644 --- a/scripts/memory/seed.js +++ b/scripts/memory/seed.js @@ -1,8 +1,8 @@ 'use strict'; /** - * One-time seeding script: fetches PRs from a GitHub repo for the past N days, - * batches embeddings, and stores them in the Redis memory index. + * One-time seeding script: fetches PRs, issues, and issue comments from a GitHub + * repo for the past N days, batches embeddings, and stores them in the Redis index. * * Usage: * GITHUB_TOKEN=... REDIS_URL=... OPENAI_API_KEY=... \ @@ -18,56 +18,136 @@ const { createClient, SchemaFieldTypes, VectorAlgorithms } = require('redis'); const OpenAI = require('openai'); -const SEED_REPO = process.env.SEED_REPO || 'redis/docs'; -const DAYS_BACK = parseInt(process.env.DAYS_BACK) || 365; -const EMBED_MODEL = process.env.EMBEDDING_MODEL || 'text-embedding-3-small'; -const VECTOR_DIM = 1536; -const BATCH_SIZE = 100; // embeddings per OpenAI call -const INDEX_NAME = 'repo_memory_idx'; -const KEY_PREFIX = 'memory:'; -const DRY_RUN = process.env.DRY_RUN === 'true'; +const SEED_REPO = process.env.SEED_REPO || 'redis/docs'; +const DAYS_BACK = parseInt(process.env.DAYS_BACK) || 365; +const EMBED_MODEL = process.env.EMBEDDING_MODEL || 'text-embedding-3-small'; +const VECTOR_DIM = 1536; +const BATCH_SIZE = 100; +const INDEX_NAME = 'repo_memory_idx'; +const KEY_PREFIX = 'memory:'; +const DRY_RUN = process.env.DRY_RUN === 'true'; +const MIN_COMMENT_LENGTH = 80; // skip very short comments -async function fetchPRs(repo, cutoff, token) { - const [owner, name] = repo.split('/'); - const headers = { +function ghHeaders(token) { + return { Accept: 'application/vnd.github+json', Authorization: `Bearer ${token}`, 'X-GitHub-Api-Version': '2022-11-28', }; +} + +async function ghFetch(url, token) { + const res = await fetch(url, { headers: ghHeaders(token) }); + if (!res.ok) throw new Error(`GitHub API ${res.status}: ${await res.text()}`); + return res.json(); +} - const prs = []; +async function fetchPRs(repo, cutoff, token) { + const [owner, name] = repo.split('/'); + const safeRepo = repo.replace('/', '_'); + const items = []; let page = 1; while (true) { - const url = `https://api.github.com/repos/${owner}/${name}/pulls?state=all&sort=created&direction=desc&per_page=100&page=${page}`; - const res = await fetch(url, { headers }); - if (!res.ok) throw new Error(`GitHub API ${res.status}: ${await res.text()}`); - - const batch = await res.json(); + const batch = await ghFetch( + `https://api.github.com/repos/${owner}/${name}/pulls?state=all&sort=created&direction=desc&per_page=100&page=${page}`, + token + ); if (!batch.length) break; let done = false; for (const pr of batch) { if (new Date(pr.created_at) < cutoff) { done = true; break; } - prs.push({ - id: `${repo.replace('/', '_')}_pr_${pr.number}`, - type: 'pr_summary', + items.push({ + id: `${safeRepo}_pr_${pr.number}`, + type: 'pr_summary', repo, - title: pr.title, + title: pr.title, bodySummary: (pr.body || '').slice(0, 500), - sourceUrl: pr.html_url, - createdAt: pr.created_at, - tags: (pr.labels || []).map(l => l.name).join(','), - searchText: `${pr.title}\n${(pr.body || '').slice(0, 1000)}`, + sourceUrl: pr.html_url, + createdAt: pr.created_at, + tags: (pr.labels || []).map(l => l.name).join(','), + searchText: `${pr.title}\n${(pr.body || '').slice(0, 1000)}`, }); } + if (done || batch.length < 100) break; + page++; + } - console.log(` page ${page}: fetched ${batch.length} PRs, kept ${prs.length} total`); + console.log(` PRs: ${items.length}`); + return items; +} + +async function fetchIssues(repo, cutoff, token) { + const [owner, name] = repo.split('/'); + const safeRepo = repo.replace('/', '_'); + const items = []; + let page = 1; + + while (true) { + const batch = await ghFetch( + `https://api.github.com/repos/${owner}/${name}/issues?state=all&sort=created&direction=desc&per_page=100&page=${page}`, + token + ); + if (!batch.length) break; + + let done = false; + for (const issue of batch) { + if (new Date(issue.created_at) < cutoff) { done = true; break; } + if (issue.pull_request) continue; // skip PRs returned by the issues endpoint + items.push({ + id: `${safeRepo}_issue_${issue.number}`, + type: 'issue_summary', + repo, + title: issue.title, + bodySummary: (issue.body || '').slice(0, 500), + sourceUrl: issue.html_url, + createdAt: issue.created_at, + tags: (issue.labels || []).map(l => l.name).join(','), + searchText: `${issue.title}\n${(issue.body || '').slice(0, 1000)}`, + }); + } if (done || batch.length < 100) break; page++; } - return prs; + console.log(` Issues: ${items.length}`); + return items; +} + +async function fetchIssueComments(repo, cutoff, token) { + const [owner, name] = repo.split('/'); + const safeRepo = repo.replace('/', '_'); + const items = []; + let page = 1; + + while (true) { + const batch = await ghFetch( + `https://api.github.com/repos/${owner}/${name}/issues/comments?sort=created&direction=desc&per_page=100&page=${page}&since=${cutoff.toISOString()}`, + token + ); + if (!batch.length) break; + + for (const comment of batch) { + if ((comment.body || '').length < MIN_COMMENT_LENGTH) continue; + items.push({ + id: `${safeRepo}_comment_${comment.id}`, + type: 'issue_comment', + repo, + title: comment.body.split('\n')[0].slice(0, 120), + bodySummary: comment.body.slice(0, 500), + sourceUrl: comment.html_url, + createdAt: comment.created_at, + tags: '', + searchText: comment.body.slice(0, 1000), + }); + } + if (batch.length < 100) break; + page++; + } + + console.log(` Comments (>=${MIN_COMMENT_LENGTH} chars): ${items.length}`); + return items; } async function embedBatch(texts, openai) { @@ -101,75 +181,89 @@ async function ensureIndex(client) { console.log(`Created index: ${INDEX_NAME}`); } +async function seedItems(items, openai, client) { + if (!items.length) return 0; + + const embeddings = []; + for (let i = 0; i < items.length; i += BATCH_SIZE) { + const slice = items.slice(i, i + BATCH_SIZE); + const vecs = await embedBatch(slice.map(p => p.searchText), openai); + embeddings.push(...vecs); + process.stdout.write(`\r embedded ${Math.min(i + BATCH_SIZE, items.length)}/${items.length}`); + } + process.stdout.write('\n'); + + if (DRY_RUN) return items.length; + + let written = 0; + for (let i = 0; i < items.length; i++) { + const item = items[i]; + const buf = Buffer.from(new Float32Array(embeddings[i]).buffer); + await client.hSet(`${KEY_PREFIX}${item.id}`, { + id: item.id, + type: item.type, + repo: item.repo, + title: item.title, + body_summary: item.bodySummary, + source_url: item.sourceUrl, + created_at: item.createdAt, + tags: item.tags, + embedding: buf, + }); + written++; + if (written % 200 === 0) process.stdout.write(`\r wrote ${written}/${items.length}`); + } + if (items.length >= 200) process.stdout.write('\n'); + return written; +} + async function main() { - const token = process.env.GITHUB_TOKEN; - const redisUrl = process.env.REDIS_URL; + const token = process.env.GITHUB_TOKEN; + const redisUrl = process.env.REDIS_URL; const openaiKey = process.env.OPENAI_API_KEY; - if (!token) { console.error('GITHUB_TOKEN is required'); process.exit(1); } - if (!redisUrl) { console.error('REDIS_URL is required'); process.exit(1); } - if (!openaiKey){ console.error('OPENAI_API_KEY is required'); process.exit(1); } + if (!token) { console.error('GITHUB_TOKEN is required'); process.exit(1); } + if (!redisUrl) { console.error('REDIS_URL is required'); process.exit(1); } + if (!openaiKey) { console.error('OPENAI_API_KEY is required'); process.exit(1); } const cutoff = new Date(Date.now() - DAYS_BACK * 24 * 60 * 60 * 1000); console.log(`Seeding from ${SEED_REPO}, past ${DAYS_BACK} days (since ${cutoff.toISOString().slice(0, 10)})`); if (DRY_RUN) console.log('DRY RUN — no writes to Redis'); - console.log('\nFetching PRs from GitHub...'); - const prs = await fetchPRs(SEED_REPO, cutoff, token); - console.log(`Total PRs to seed: ${prs.length}`); + console.log('\nFetching from GitHub...'); + const [prs, issues, comments] = await Promise.all([ + fetchPRs(SEED_REPO, cutoff, token), + fetchIssues(SEED_REPO, cutoff, token), + fetchIssueComments(SEED_REPO, cutoff, token), + ]); - if (!prs.length) { console.log('Nothing to seed.'); return; } + const all = [...prs, ...issues, ...comments]; + console.log(`Total items to seed: ${all.length}`); + if (!all.length) { console.log('Nothing to seed.'); return; } const openai = new OpenAI({ apiKey: openaiKey }); - - // Embed in batches - console.log(`\nEmbedding in batches of ${BATCH_SIZE}...`); - const embeddings = []; - for (let i = 0; i < prs.length; i += BATCH_SIZE) { - const slice = prs.slice(i, i + BATCH_SIZE); - const vecs = await embedBatch(slice.map(p => p.searchText), openai); - embeddings.push(...vecs); - console.log(` embedded ${Math.min(i + BATCH_SIZE, prs.length)}/${prs.length}`); + const client = DRY_RUN ? null : createClient({ url: redisUrl }); + if (client) { + client.on('error', err => console.error('Redis error:', err.message)); + await client.connect(); } - if (DRY_RUN) { - console.log('\nDry run complete — skipping Redis writes.'); - return; - } + try { + if (client) await ensureIndex(client); - // Write to Redis - const client = createClient({ url: redisUrl }); - client.on('error', err => console.error('Redis error:', err.message)); - await client.connect(); + console.log(`\nEmbedding and writing PRs...`); + const nPRs = await seedItems(prs, openai, client); - try { - await ensureIndex(client); - - console.log('\nWriting to Redis...'); - let written = 0; - for (let i = 0; i < prs.length; i++) { - const pr = prs[i]; - const vec = embeddings[i]; - const buf = Buffer.from(new Float32Array(vec).buffer); - - await client.hSet(`${KEY_PREFIX}${pr.id}`, { - id: pr.id, - type: pr.type, - repo: pr.repo, - title: pr.title, - body_summary: pr.bodySummary, - source_url: pr.sourceUrl, - created_at: pr.createdAt, - tags: pr.tags, - embedding: buf, - }); - written++; - if (written % 100 === 0) console.log(` wrote ${written}/${prs.length}`); - } + console.log(`Embedding and writing issues...`); + const nIssues = await seedItems(issues, openai, client); + + console.log(`Embedding and writing comments...`); + const nComments = await seedItems(comments, openai, client); - console.log(`\nDone. Seeded ${written} PRs from ${SEED_REPO} into ${INDEX_NAME}.`); + const verb = DRY_RUN ? 'Would seed' : 'Seeded'; + console.log(`\nDone. ${verb} ${nPRs} PRs, ${nIssues} issues, ${nComments} comments from ${SEED_REPO}.`); } finally { - await client.quit(); + if (client) await client.quit(); } } From 79433ed81dbb7bc1bab8b3661b9d2a9fa799d536 Mon Sep 17 00:00:00 2001 From: mich-elle-luna <153109578+mich-elle-luna@users.noreply.github.com> Date: Thu, 11 Jun 2026 10:41:35 -0700 Subject: [PATCH 19/22] Add performance best practices section Added a section on performance best practices to enhance database efficiency. --- .../create-database/create-pro-database-existing.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/content/operate/rc/databases/create-database/create-pro-database-existing.md b/content/operate/rc/databases/create-database/create-pro-database-existing.md index 92b6a95b0c..b3880ca2c0 100644 --- a/content/operate/rc/databases/create-database/create-pro-database-existing.md +++ b/content/operate/rc/databases/create-database/create-pro-database-existing.md @@ -77,6 +77,10 @@ The **Performance** section lets you manage the maximum size, throughput, and ha | **Hashing policy** | Defines the [hashing policy]({{< relref "/operate/rc/databases/configuration/clustering#manage-the-hashing-policy" >}}). | | **OSS Cluster API** | Activates the [OSS Cluster API]({{< relref "/operate/rc/databases/configuration/clustering#oss-cluster-api" >}}) for a database.
After you select OSS Cluster API, you can select **Use external endpoint** if you want to use the external endpoint for the database. Selecting **Use external endpoint** will block the private endpoint for this database. | +### Best practice for performance + +add performance best practices + ## Durability section The **Durability** section helps you keep your database (and your data) available when problems occur. @@ -128,4 +132,4 @@ The available alerts vary according to the plan type. See [Configure alerts]({{< ## Billing unit types -{{< embed-md "rc-pro-billing-units.md" >}} \ No newline at end of file +{{< embed-md "rc-pro-billing-units.md" >}} From 554cfffa615c8cbe64481d7f3a8970128d9f8a63 Mon Sep 17 00:00:00 2001 From: mich-elle-luna Date: Thu, 11 Jun 2026 10:49:41 -0700 Subject: [PATCH 20/22] Post new PR comment on synchronize instead of updating existing --- scripts/memory/collect_context.js | 1 + scripts/memory/post_comment.js | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/memory/collect_context.js b/scripts/memory/collect_context.js index 607c5921e9..b7343b2624 100644 --- a/scripts/memory/collect_context.js +++ b/scripts/memory/collect_context.js @@ -55,6 +55,7 @@ if (eventName === 'push') { const pr = event.pull_request; context = { eventName, + action: event.action, type: 'pull_request', sha: pr.head.sha, branch: pr.head.ref, diff --git a/scripts/memory/post_comment.js b/scripts/memory/post_comment.js index 3639cc2a2f..453afd1893 100644 --- a/scripts/memory/post_comment.js +++ b/scripts/memory/post_comment.js @@ -96,7 +96,13 @@ async function main() { const markdown = buildMarkdown(related, context, sourceUrl); if (context.prNumber) { - await postOrUpdatePrComment(repo, context.prNumber, markdown, token); + if (context.action === 'synchronize') { + // New commits pushed — always post a fresh comment so each update is visible + const created = await githubRequest('POST', `/repos/${repo}/issues/${context.prNumber}/comments`, { body: markdown }, token); + console.log(`created_comment_id=${created.id} pr=${context.prNumber} action=synchronize`); + } else { + await postOrUpdatePrComment(repo, context.prNumber, markdown, token); + } } else { // Post commit status so results are visible on the commit without opening Actions await postCommitStatus(repo, context.sha, related, sourceUrl, token); From d4f29cb068d63110ffa3d7ca44dc6287fff19ed7 Mon Sep 17 00:00:00 2001 From: mich-elle-luna Date: Thu, 11 Jun 2026 11:22:59 -0700 Subject: [PATCH 21/22] Exclude current commit/PR from memory search results --- scripts/memory/retrieve_memory.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/memory/retrieve_memory.js b/scripts/memory/retrieve_memory.js index a12d02799f..6779469ab2 100644 --- a/scripts/memory/retrieve_memory.js +++ b/scripts/memory/retrieve_memory.js @@ -80,21 +80,24 @@ async function main() { } ); - // Match memories from the current repo and optionally the upstream repo - // (set UPSTREAM_REPO=redis/docs in the workflow to include seeded upstream content). + // Build current item ID so we can exclude it from results + const safeRepo = context.repo.replace(/\//g, '_'); + const itemId = context.prNumber + ? `${safeRepo}_pr_${context.prNumber}` + : `${safeRepo}_push_${context.sha?.slice(0, 8)}`; + const currentKey = `${KEY_PREFIX}${itemId}`; + + // Match memories from the current repo and optionally the upstream repo, + // excluding the current item so a PR/commit never surfaces itself. const allowedRepos = new Set([context.repo, process.env.UPSTREAM_REPO].filter(Boolean)); const related = searchResult.documents .filter(doc => { const score = parseFloat(doc.value.score ?? '2'); - return score < SIMILARITY_THRESHOLD && allowedRepos.has(doc.value.repo); + return score < SIMILARITY_THRESHOLD && allowedRepos.has(doc.value.repo) && doc.id !== currentKey; }) .slice(0, MAX_RESULTS); // Store the current context as a memory artifact - const safeRepo = context.repo.replace(/\//g, '_'); - const itemId = context.prNumber - ? `${safeRepo}_pr_${context.prNumber}` - : `${safeRepo}_push_${context.sha?.slice(0, 8)}`; const sourceUrl = context.prNumber ? `https://github.com/${context.repo}/pull/${context.prNumber}` From be9b249ee37342734dc5d8aaafd56bce5db7ea89 Mon Sep 17 00:00:00 2001 From: mich-elle-luna Date: Thu, 11 Jun 2026 11:31:07 -0700 Subject: [PATCH 22/22] Exclude push memory for current SHA from PR results --- scripts/memory/retrieve_memory.js | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/memory/retrieve_memory.js b/scripts/memory/retrieve_memory.js index 6779469ab2..e3c98663f9 100644 --- a/scripts/memory/retrieve_memory.js +++ b/scripts/memory/retrieve_memory.js @@ -85,15 +85,20 @@ async function main() { const itemId = context.prNumber ? `${safeRepo}_pr_${context.prNumber}` : `${safeRepo}_push_${context.sha?.slice(0, 8)}`; - const currentKey = `${KEY_PREFIX}${itemId}`; - - // Match memories from the current repo and optionally the upstream repo, - // excluding the current item so a PR/commit never surfaces itself. + // Exclude the current item AND the push memory for the same SHA — a commit + // pushed to a branch before a PR opens gets stored as _push_{sha} and would + // otherwise surface as a related item when the PR runs. + const excludeKeys = new Set([ + `${KEY_PREFIX}${itemId}`, + `${KEY_PREFIX}${safeRepo}_push_${context.sha?.slice(0, 8)}`, + ]); + + // Match memories from the current repo and optionally the upstream repo. const allowedRepos = new Set([context.repo, process.env.UPSTREAM_REPO].filter(Boolean)); const related = searchResult.documents .filter(doc => { const score = parseFloat(doc.value.score ?? '2'); - return score < SIMILARITY_THRESHOLD && allowedRepos.has(doc.value.repo) && doc.id !== currentKey; + return score < SIMILARITY_THRESHOLD && allowedRepos.has(doc.value.repo) && !excludeKeys.has(doc.id); }) .slice(0, MAX_RESULTS);