From bddf2570033442d41afd2d0cec404d70270f88bb Mon Sep 17 00:00:00 2001 From: ailegion Date: Tue, 15 Sep 2026 15:24:31 +0200 Subject: [PATCH 1/2] feat(pipeline-templates): support zip project templates added: projects.json manifest, merged into the template list (optional, failures ignored) added: optional "type" field on templates ("yaml" | "zip"), defaults to yaml added: pipeline-templates:apply-zip IPC channel (check / replace / skip) added: zip download and extraction to the project root with adm-zip added: strip the top-level folder from GitHub archive zips added: allow github.com and codeload.github.com for zip downloads added: block zip entries that would extract outside the project folder added: Replace / Skip / Cancel dialog listing conflicting files added: onApplied prop to refresh pipelines and file tree after extraction changed: zip templates skip the folder step, button reads "Apply Template" changed: zip template cards show "project root" as the target --- .../pipelineTemplates.ipcHandlers.ts | 97 +++++++++++- .../modals/createPipelineModal/index.tsx | 142 +++++++++++++++++- .../components/sidebar/project-sidebar.tsx | 4 + .../services/pipelineTemplates.service.ts | 15 ++ src/types/ipc.ts | 3 +- 5 files changed, 255 insertions(+), 6 deletions(-) diff --git a/src/main/ipcHandlers/pipelineTemplates.ipcHandlers.ts b/src/main/ipcHandlers/pipelineTemplates.ipcHandlers.ts index 2dc819d0..9125acea 100644 --- a/src/main/ipcHandlers/pipelineTemplates.ipcHandlers.ts +++ b/src/main/ipcHandlers/pipelineTemplates.ipcHandlers.ts @@ -1,23 +1,45 @@ import { ipcMain } from 'electron'; +import path from 'path'; +import fs from 'fs'; +import AdmZip from 'adm-zip'; const PIPELINE_TEMPLATES_MANIFEST_URL = 'https://raw.githubusercontent.com/rosettadb/dbt-studio-templates/main/pipelines.json'; +const PROJECT_TEMPLATES_MANIFEST_URL = + 'https://raw.githubusercontent.com/rosettadb/dbt-studio-templates/main/projects.json'; + +const RAW_GITHUB_URL_PATTERN = /^https:\/\/raw\.githubusercontent\.com\//; +const GITHUB_ARCHIVE_URL_PATTERN = + /^https:\/\/(github\.com|codeload\.github\.com)\//; export function registerPipelineTemplatesHandlers() { - // Fetch the public pipeline templates manifest + // Fetch the public pipeline and project templates manifests ipcMain.handle('pipeline-templates:list', async () => { const res = await fetch(PIPELINE_TEMPLATES_MANIFEST_URL); if (!res.ok) { throw new Error(`Failed to fetch pipeline templates: ${res.statusText}`); } - return res.json(); + const pipelineTemplates = await res.json(); + + // Project templates are optional — pipeline templates still show if this fails + let projectTemplates: unknown[] = []; + try { + const projectsRes = await fetch(PROJECT_TEMPLATES_MANIFEST_URL); + if (projectsRes.ok) { + projectTemplates = await projectsRes.json(); + } + } catch { + projectTemplates = []; + } + + return [...pipelineTemplates, ...projectTemplates]; }); // Fetch the raw YAML content for a specific template ipcMain.handle( 'pipeline-templates:fetch-content', async (event, url: string) => { - if (!/^https:\/\/raw\.githubusercontent\.com\//.test(url)) { + if (!RAW_GITHUB_URL_PATTERN.test(url)) { throw new Error( 'Refusing to fetch template content from an untrusted host', ); @@ -29,4 +51,73 @@ export function registerPipelineTemplatesHandlers() { return res.text(); }, ); + + // Download a zip template and extract it into the project root. + // 'check' writes nothing and returns the paths that already exist. + ipcMain.handle( + 'pipeline-templates:apply-zip', + async ( + event, + projectPath: string, + url: string, + mode: 'check' | 'replace' | 'skip', + ) => { + const isGithubArchive = GITHUB_ARCHIVE_URL_PATTERN.test(url); + if (!isGithubArchive && !RAW_GITHUB_URL_PATTERN.test(url)) { + throw new Error( + 'Refusing to fetch template content from an untrusted host', + ); + } + const res = await fetch(url); + if (!res.ok) { + throw new Error(`Failed to fetch template content: ${res.statusText}`); + } + const zip = new AdmZip(Buffer.from(await res.arrayBuffer())); + + // GitHub repo archives wrap everything in a single top-level folder + let stripPrefix = ''; + if (isGithubArchive) { + const topLevel = new Set( + zip.getEntries().map((entry) => entry.entryName.split('/')[0]), + ); + if (topLevel.size === 1) { + stripPrefix = `${[...topLevel][0]}/`; + } + } + + const root = path.resolve(projectPath); + const files = zip + .getEntries() + .filter((entry) => !entry.isDirectory) + .map((entry) => { + const relativePath = entry.entryName.slice(stripPrefix.length); + const target = path.resolve(root, relativePath); + if (!target.startsWith(root + path.sep)) { + throw new Error( + `Refusing to extract outside the project: ${entry.entryName}`, + ); + } + return { entry, relativePath, target }; + }); + + if (mode === 'check') { + return files + .filter((file) => fs.existsSync(file.target)) + .map((file) => file.relativePath); + } + + // eslint-disable-next-line no-restricted-syntax + for (const file of files) { + if (mode === 'skip' && fs.existsSync(file.target)) { + // eslint-disable-next-line no-continue + continue; + } + // eslint-disable-next-line no-await-in-loop + await fs.promises.mkdir(path.dirname(file.target), { recursive: true }); + // eslint-disable-next-line no-await-in-loop + await fs.promises.writeFile(file.target, file.entry.getData()); + } + return []; + }, + ); } diff --git a/src/renderer/components/modals/createPipelineModal/index.tsx b/src/renderer/components/modals/createPipelineModal/index.tsx index 3b056a9b..81f9e645 100644 --- a/src/renderer/components/modals/createPipelineModal/index.tsx +++ b/src/renderer/components/modals/createPipelineModal/index.tsx @@ -37,6 +37,7 @@ import { Project } from '../../../../types/backend'; import { listPipelineTemplates, fetchPipelineTemplateContent, + applyZipTemplate, RemotePipelineTemplate, } from '../../../services/pipelineTemplates.service'; import { PipelineFolderTree } from './PipelineFolderTree'; @@ -103,6 +104,7 @@ interface CreatePipelineModalProps { onClose: () => void; project: Project; onCreated: (filePath: string) => void; + onApplied?: () => void; } type ModalView = 'menu' | 'browse' | 'location'; @@ -112,6 +114,7 @@ export const CreatePipelineModal: React.FC = ({ onClose, project, onCreated, + onApplied, }) => { const theme = useTheme(); const isDark = theme.palette.mode === 'dark'; @@ -135,6 +138,10 @@ export const CreatePipelineModal: React.FC = ({ getContent: () => Promise; subdir: string; } | null>(null); + const [pendingZip, setPendingZip] = React.useState<{ + url: string; + conflicts: string[]; + } | null>(null); // Which view to return to when backing out of the location step. const [locationOrigin, setLocationOrigin] = React.useState<'menu' | 'browse'>( @@ -151,6 +158,7 @@ export const CreatePipelineModal: React.FC = ({ setSelectedTemplateId(null); setSelectedRemoteId(null); setPendingCreate(null); + setPendingZip(null); setRemoteTemplates([]); setRemoteError(null); setLocationOrigin('menu'); @@ -245,6 +253,53 @@ export const CreatePipelineModal: React.FC = ({ } }; + const finishZipApply = async (url: string, mode: 'replace' | 'skip') => { + await applyZipTemplate(project.path, url, mode); + toast.success('Template applied successfully.'); + onApplied?.(); + onClose(); + }; + + const applyZip = async (url: string) => { + setIsCreating(true); + try { + const conflicts = await applyZipTemplate(project.path, url, 'check'); + if (conflicts.length > 0) { + setPendingZip({ url, conflicts }); + return; + } + await finishZipApply(url, 'replace'); + } catch (error) { + const msg = + error instanceof Error ? error.message : 'Failed to apply template'; + toast.error(msg); + } finally { + setIsCreating(false); + } + }; + + const handleZipConflict = async (mode: 'replace' | 'skip') => { + if (!pendingZip) return; + setIsCreating(true); + try { + await finishZipApply(pendingZip.url, mode); + setPendingZip(null); + } catch (error) { + const msg = + error instanceof Error ? error.message : 'Failed to apply template'; + toast.error(msg); + } finally { + setIsCreating(false); + } + }; + + const selectedRemoteTemplate = remoteTemplates.find( + (t) => t.id === selectedRemoteId, + ); + // Zip templates are extracted into the project root, so no location step + const isZipSelected = + view === 'browse' && selectedRemoteTemplate?.type === 'zip'; + // Selected template info (from whichever list it came from), used by the // location step to know what to create once a directory is chosen. const selectedTemplateInfo = React.useMemo(() => { @@ -269,6 +324,10 @@ export const CreatePipelineModal: React.FC = ({ const handleContinueToLocation = () => { if (view === 'menu' && !selectedTemplateId) return; if (view === 'browse' && !selectedRemoteId) return; + if (isZipSelected && selectedRemoteTemplate) { + applyZip(selectedRemoteTemplate.url); + return; + } setLocationOrigin(view === 'browse' ? 'browse' : 'menu'); setSelectedSubdir(''); setFolderTreeReloadToken((prev) => prev + 1); @@ -329,7 +388,8 @@ export const CreatePipelineModal: React.FC = ({ ? isCreating : (isBrowseView ? !selectedRemoteId : !selectedTemplateId) || isCreating; const hasSelection = isBrowseView ? !!selectedRemoteId : !!selectedTemplateId; - const actionLabel = isLocationView ? 'Create Pipeline' : 'Next'; + let actionLabel = isLocationView ? 'Create Pipeline' : 'Next'; + if (isZipSelected) actionLabel = 'Apply Template'; const primaryAction = isLocationView ? handleCreate : handleContinueToLocation; @@ -1037,7 +1097,9 @@ export const CreatePipelineModal: React.FC = ({ opacity: 0.8, }} > - rosetta/pipelines/{template.fileName} + {template.type === 'zip' + ? 'project root' + : `rosetta/pipelines/${template.fileName}`} ); @@ -1217,6 +1279,82 @@ export const CreatePipelineModal: React.FC = ({ + + {/* Zip template conflict dialog */} + !isCreating && setPendingZip(null)} + fullWidth + maxWidth="sm" + PaperProps={{ + sx: { + borderRadius: 2.5, + backgroundImage: 'none', + boxShadow: isDark + ? '0 24px 48px rgba(0,0,0,0.6)' + : '0 24px 48px rgba(0,0,0,0.16)', + }, + }} + > + + + Files Already Exist + + + + {pendingZip?.conflicts.length} file(s) from this template already + exist in this project: + + + {pendingZip?.conflicts.map((file) =>
  • {file}
    • )} +     + + Replace overrides the existing files. Skip keeps them and only adds + new files. + +     + + + + + +     ); }; diff --git a/src/renderer/components/sidebar/project-sidebar.tsx b/src/renderer/components/sidebar/project-sidebar.tsx index 877463ad..9946ec65 100644 --- a/src/renderer/components/sidebar/project-sidebar.tsx +++ b/src/renderer/components/sidebar/project-sidebar.tsx @@ -365,6 +365,10 @@ const ExplorerTab: React.FC = ({ onClose={() => setCreatePipelineOpen(false)} project={project} onCreated={(filePath) => handlePipelineCreated(filePath)} + onApplied={async () => { + await refetchPipelines(); + await onRefreshFiles(); + }} /> )} diff --git a/src/renderer/services/pipelineTemplates.service.ts b/src/renderer/services/pipelineTemplates.service.ts index a43a245e..d6da1a81 100644 --- a/src/renderer/services/pipelineTemplates.service.ts +++ b/src/renderer/services/pipelineTemplates.service.ts @@ -7,6 +7,8 @@ export interface RemotePipelineTemplate { steps: string[]; fileName: string; url: string; + // 'zip' templates are extracted into the project root; defaults to 'yaml' + type?: 'yaml' | 'zip'; } export const listPipelineTemplates = async (): Promise< @@ -23,3 +25,16 @@ export const fetchPipelineTemplateContent = async ( url, ); }; + +export const applyZipTemplate = async ( + projectPath: string, + url: string, + mode: 'check' | 'replace' | 'skip', +): Promise => { + return window.electron.ipcRenderer.invoke( + 'pipeline-templates:apply-zip', + projectPath, + url, + mode, + ); +}; diff --git a/src/types/ipc.ts b/src/types/ipc.ts index 63e24cc3..fedabfbd 100644 --- a/src/types/ipc.ts +++ b/src/types/ipc.ts @@ -536,7 +536,8 @@ export type FlowfileChannels = export type PipelineTemplatesChannels = | 'pipeline-templates:list' - | 'pipeline-templates:fetch-content'; + | 'pipeline-templates:fetch-content' + | 'pipeline-templates:apply-zip'; export type SecondBrainChannels = | 'second-brain:status' From ac6fe934ffb9e9bb5071b9251d3fa07a1ddf0631 Mon Sep 17 00:00:00 2001 From: ailegion Date: Wed, 16 Sep 2026 13:04:00 +0200 Subject: [PATCH 2/2] fix(pipeline-templates): address review findings for zip templates - Ignore projects.json manifest when it is not an array so pipeline templates still load - Resolve the extraction root from the project registry by id instead of trusting a renderer-supplied path - Reject extraction through symlinks: lstat the deepest existing path component and verify its real path stays inside the project root - Await onApplied after applying a zip template so refresh failures are observed and the spinner stays until the file tree refetches --- .../pipelineTemplates.ipcHandlers.ts | 64 ++++++++++++++++++- .../modals/createPipelineModal/index.tsx | 16 +++-- .../services/pipelineTemplates.service.ts | 4 +- 3 files changed, 75 insertions(+), 9 deletions(-) diff --git a/src/main/ipcHandlers/pipelineTemplates.ipcHandlers.ts b/src/main/ipcHandlers/pipelineTemplates.ipcHandlers.ts index 9125acea..d758287e 100644 --- a/src/main/ipcHandlers/pipelineTemplates.ipcHandlers.ts +++ b/src/main/ipcHandlers/pipelineTemplates.ipcHandlers.ts @@ -2,6 +2,7 @@ import { ipcMain } from 'electron'; import path from 'path'; import fs from 'fs'; import AdmZip from 'adm-zip'; +import { ProjectsService } from '../services'; const PIPELINE_TEMPLATES_MANIFEST_URL = 'https://raw.githubusercontent.com/rosettadb/dbt-studio-templates/main/pipelines.json'; @@ -12,6 +13,44 @@ const RAW_GITHUB_URL_PATTERN = /^https:\/\/raw\.githubusercontent\.com\//; const GITHUB_ARCHIVE_URL_PATTERN = /^https:\/\/(github\.com|codeload\.github\.com)\//; +const lstatOrNull = async (p: string) => { + try { + return await fs.promises.lstat(p); + } catch { + return null; + } +}; + +// The lexical startsWith check on the target cannot see symlinks that already +// exist inside the project. Walk up to the deepest existing path component +// without following links, reject it if it is a symlink (including dangling +// ones, which writeFile would otherwise create through), and confirm its real +// location is still inside the real project root. +const assertNoSymlinkEscape = async ( + realRoot: string, + target: string, + entryName: string, +) => { + let existing = target; + let stat = await lstatOrNull(existing); + while (!stat) { + const parent = path.dirname(existing); + if (parent === existing) break; + existing = parent; + // eslint-disable-next-line no-await-in-loop + stat = await lstatOrNull(existing); + } + if (!stat || stat.isSymbolicLink()) { + throw new Error( + `Refusing to extract through a symlink in the project: ${entryName}`, + ); + } + const real = await fs.promises.realpath(existing); + if (real !== realRoot && !real.startsWith(realRoot + path.sep)) { + throw new Error(`Refusing to extract outside the project: ${entryName}`); + } +}; + export function registerPipelineTemplatesHandlers() { // Fetch the public pipeline and project templates manifests ipcMain.handle('pipeline-templates:list', async () => { @@ -26,7 +65,10 @@ export function registerPipelineTemplatesHandlers() { try { const projectsRes = await fetch(PROJECT_TEMPLATES_MANIFEST_URL); if (projectsRes.ok) { - projectTemplates = await projectsRes.json(); + const parsed = await projectsRes.json(); + if (Array.isArray(parsed)) { + projectTemplates = parsed; + } } } catch { projectTemplates = []; @@ -54,14 +96,22 @@ export function registerPipelineTemplatesHandlers() { // Download a zip template and extract it into the project root. // 'check' writes nothing and returns the paths that already exist. + // The project root is resolved from the main-process registry so the + // renderer cannot direct extraction to an arbitrary directory. ipcMain.handle( 'pipeline-templates:apply-zip', async ( event, - projectPath: string, + projectId: string, url: string, mode: 'check' | 'replace' | 'skip', ) => { + const projects = await ProjectsService.loadProjects(); + const project = projects.find((p) => p.id === projectId); + if (!project) { + throw new Error(`Project not found: ${projectId}`); + } + const isGithubArchive = GITHUB_ARCHIVE_URL_PATTERN.test(url); if (!isGithubArchive && !RAW_GITHUB_URL_PATTERN.test(url)) { throw new Error( @@ -85,7 +135,7 @@ export function registerPipelineTemplatesHandlers() { } } - const root = path.resolve(projectPath); + const root = path.resolve(project.path); const files = zip .getEntries() .filter((entry) => !entry.isDirectory) @@ -106,6 +156,8 @@ export function registerPipelineTemplatesHandlers() { .map((file) => file.relativePath); } + const realRoot = await fs.promises.realpath(root); + // eslint-disable-next-line no-restricted-syntax for (const file of files) { if (mode === 'skip' && fs.existsSync(file.target)) { @@ -113,6 +165,12 @@ export function registerPipelineTemplatesHandlers() { continue; } // eslint-disable-next-line no-await-in-loop + await assertNoSymlinkEscape( + realRoot, + file.target, + file.entry.entryName, + ); + // eslint-disable-next-line no-await-in-loop await fs.promises.mkdir(path.dirname(file.target), { recursive: true }); // eslint-disable-next-line no-await-in-loop await fs.promises.writeFile(file.target, file.entry.getData()); diff --git a/src/renderer/components/modals/createPipelineModal/index.tsx b/src/renderer/components/modals/createPipelineModal/index.tsx index 81f9e645..3087c8f2 100644 --- a/src/renderer/components/modals/createPipelineModal/index.tsx +++ b/src/renderer/components/modals/createPipelineModal/index.tsx @@ -104,7 +104,7 @@ interface CreatePipelineModalProps { onClose: () => void; project: Project; onCreated: (filePath: string) => void; - onApplied?: () => void; + onApplied?: () => void | Promise; } type ModalView = 'menu' | 'browse' | 'location'; @@ -254,16 +254,24 @@ export const CreatePipelineModal: React.FC = ({ }; const finishZipApply = async (url: string, mode: 'replace' | 'skip') => { - await applyZipTemplate(project.path, url, mode); + await applyZipTemplate(project.id, url, mode); toast.success('Template applied successfully.'); - onApplied?.(); + // The template is already on disk at this point, so a failed refresh is + // reported on its own and must not block closing the modal. + try { + await onApplied?.(); + } catch (error) { + const msg = + error instanceof Error ? error.message : 'Failed to refresh files'; + toast.error(`Template applied, but refresh failed: ${msg}`); + } onClose(); }; const applyZip = async (url: string) => { setIsCreating(true); try { - const conflicts = await applyZipTemplate(project.path, url, 'check'); + const conflicts = await applyZipTemplate(project.id, url, 'check'); if (conflicts.length > 0) { setPendingZip({ url, conflicts }); return; diff --git a/src/renderer/services/pipelineTemplates.service.ts b/src/renderer/services/pipelineTemplates.service.ts index d6da1a81..f47b46e8 100644 --- a/src/renderer/services/pipelineTemplates.service.ts +++ b/src/renderer/services/pipelineTemplates.service.ts @@ -27,13 +27,13 @@ export const fetchPipelineTemplateContent = async ( }; export const applyZipTemplate = async ( - projectPath: string, + projectId: string, url: string, mode: 'check' | 'replace' | 'skip', ): Promise => { return window.electron.ipcRenderer.invoke( 'pipeline-templates:apply-zip', - projectPath, + projectId, url, mode, );