diff --git a/app/api/github/import/route.ts b/app/api/github/import/route.ts new file mode 100644 index 00000000..c998c209 --- /dev/null +++ b/app/api/github/import/route.ts @@ -0,0 +1,267 @@ +import { NextRequest, NextResponse } from "next/server"; +import { auth } from "@/auth"; +import { db } from "@/lib/db"; +import { rateLimit } from "@/lib/api-utils"; +import { + buildTemplateFolderFromEntries, + getMonorepoCandidates, + hasPackageJson, + parseGitHubRepoUrl, + type GitHubContentEntry, + type GitHubFileEntry, + type GitHubRepoMetadata, +} from "./utils"; + +const GITHUB_API_BASE = "https://api.github.com"; + +function getGitHubHeaders() { + const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN; + + return { + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + ...(token ? { Authorization: `Bearer ${token}` } : {}), + }; +} + +async function readJsonResponse(response: Response, fallbackMessage: string): Promise { + const raw = await response.text(); + let parsed: unknown = null; + + try { + parsed = raw ? JSON.parse(raw) : null; + } catch { + parsed = null; + } + + if (!response.ok) { + const parsedObject = parsed && typeof parsed === "object" ? (parsed as Record) : null; + const messageFromJson = + parsedObject && typeof parsedObject.message === "string" + ? parsedObject.message + : parsedObject && typeof parsedObject.error === "string" + ? parsedObject.error + : undefined; + + const message = + messageFromJson || + raw?.slice(0, 180) || + fallbackMessage; + throw new Error(message); + } + + return parsed as T; +} + +async function fetchGitHubJson(url: string): Promise { + const response = await fetch(url, { + headers: getGitHubHeaders(), + cache: "no-store", + }); + + return readJsonResponse(response, "GitHub request failed"); +} + +async function fetchDirectoryEntries( + owner: string, + repo: string, + dirPath = "", + ref?: string +): Promise { + const pathSuffix = dirPath ? `/${encodeURIComponent(dirPath).replace(/%2F/g, "/")}` : ""; + const refQuery = ref ? `?ref=${encodeURIComponent(ref)}` : ""; + const url = `${GITHUB_API_BASE}/repos/${owner}/${repo}/contents${pathSuffix}${refQuery}`; + const response = await fetch(url, { + headers: getGitHubHeaders(), + cache: "no-store", + }); + + const data = await readJsonResponse( + response, + "Failed to fetch repository contents" + ); + + return Array.isArray(data) ? data : [data]; +} + +type GitHubFileContentResponse = { + path: string; + content?: string; + encoding?: string; + download_url?: string | null; + size?: number; +}; + +async function fetchFileContent( + owner: string, + repo: string, + entry: GitHubContentEntry, + ref?: string +): Promise { + if (entry.download_url) { + const downloadResponse = await fetch(entry.download_url, { + headers: getGitHubHeaders(), + cache: "no-store", + }); + + if (!downloadResponse.ok) { + throw new Error(`Failed to fetch file ${entry.path}`); + } + + return downloadResponse.text(); + } + + const fileUrl = `${GITHUB_API_BASE}/repos/${owner}/${repo}/contents/${encodeURIComponent(entry.path).replace(/%2F/g, "/")}${ref ? `?ref=${encodeURIComponent(ref)}` : ""}`; + const response = await fetch(fileUrl, { + headers: getGitHubHeaders(), + cache: "no-store", + }); + + const data = await readJsonResponse(response, `Failed to fetch file ${entry.path}`); + if (typeof data.content !== "string") { + throw new Error(`Failed to fetch file ${entry.path}`); + } + + if (data.encoding === "base64") { + return Buffer.from(data.content.replace(/\n/g, ""), "base64").toString("utf8"); + } + + return data.content; +} + +async function collectFiles( + owner: string, + repo: string, + dirPath = "", + ref?: string +): Promise { + const entries = await fetchDirectoryEntries(owner, repo, dirPath, ref); + const files: GitHubFileEntry[] = []; + + for (const entry of entries) { + if (entry.type === "dir") { + const nestedFiles = await collectFiles(owner, repo, entry.path, ref); + files.push(...nestedFiles); + continue; + } + + if (entry.type !== "file") { + continue; + } + + const content = await fetchFileContent(owner, repo, entry, ref); + + files.push({ + path: entry.path, + content, + size: entry.size, + }); + } + + return files; +} + +export async function POST(request: NextRequest) { + try { + const session = await auth(); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const { allowed, remaining } = await rateLimit(`github-import:${session.user.id}`, 8, 60_000); + if (!allowed) { + return NextResponse.json( + { error: "Rate limit exceeded. Please wait before importing again." }, + { + status: 429, + headers: { + "Retry-After": "60", + "X-RateLimit-Remaining": String(remaining), + }, + } + ); + } + + const body = await request.json(); + const repoUrl = typeof body?.repoUrl === "string" ? body.repoUrl.trim() : ""; + const selectedSubdir = typeof body?.subdir === "string" ? body.subdir.trim().replace(/^\/+|\/+$/g, "") : ""; + + if (!repoUrl) { + return NextResponse.json({ error: "Repository URL is required" }, { status: 400 }); + } + + const parsed = parseGitHubRepoUrl(repoUrl); + if (!parsed) { + return NextResponse.json({ error: "Please enter a valid public GitHub repository URL" }, { status: 400 }); + } + + const repoInfo = await fetchGitHubJson( + `${GITHUB_API_BASE}/repos/${parsed.owner}/${parsed.repo}` + ); + + const branch = parsed.branch || repoInfo.default_branch; + const importPath = selectedSubdir || parsed.path || ""; + const rootName = importPath ? importPath.split("/").filter(Boolean).pop()! : repoInfo.name; + + if (!importPath) { + const rootEntries = await fetchDirectoryEntries(parsed.owner, parsed.repo, "", branch); + const rootHasPackageJson = hasPackageJson(rootEntries); + + if (!rootHasPackageJson) { + const candidateDirs = getMonorepoCandidates(rootEntries); + const projectDirs: string[] = []; + + for (const candidate of candidateDirs) { + try { + const candidateEntries = await fetchDirectoryEntries(parsed.owner, parsed.repo, candidate, branch); + if (hasPackageJson(candidateEntries)) { + projectDirs.push(candidate); + } + } catch { + continue; + } + } + + if (projectDirs.length > 1) { + return NextResponse.json({ + needsSubdir: true, + subdirs: projectDirs, + }); + } + } + } + + const files = await collectFiles(parsed.owner, parsed.repo, importPath, branch); + const templateData = buildTemplateFolderFromEntries(rootName, files); + + const playground = await db.playground.create({ + data: { + title: repoInfo.name, + description: repoInfo.description || `Imported from ${repoInfo.full_name}`, + template: "STATIC", + userId: session.user.id, + }, + }); + + await db.templateFile.create({ + data: { + playgroundId: playground.id, + content: JSON.parse(JSON.stringify(templateData)), + }, + }); + + return NextResponse.json({ + playgroundId: playground.id, + title: playground.title, + repository: repoInfo.full_name, + }); + } catch (error) { + console.error("GitHub import error:", error); + return NextResponse.json( + { + error: error instanceof Error ? error.message : "Failed to import repository", + }, + { status: 500 } + ); + } +} diff --git a/app/api/github/import/utils.test.ts b/app/api/github/import/utils.test.ts new file mode 100644 index 00000000..ccf41a6e --- /dev/null +++ b/app/api/github/import/utils.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; +import { buildTemplateFolderFromEntries, parseGitHubRepoUrl } from "./utils"; + +describe("parseGitHubRepoUrl", () => { + it("parses a standard repository URL", () => { + expect(parseGitHubRepoUrl("https://github.com/vercel/next.js")).toEqual({ + owner: "vercel", + repo: "next.js", + path: undefined, + }); + }); + + it("parses a tree URL and preserves the nested path", () => { + expect(parseGitHubRepoUrl("https://github.com/vercel/next.js/tree/main/examples/blog")).toEqual({ + owner: "vercel", + repo: "next.js", + branch: "main", + path: "examples/blog", + }); + }); + + it("rejects non-GitHub URLs", () => { + expect(parseGitHubRepoUrl("https://gitlab.com/example/repo")).toBeNull(); + }); + + it("rejects issue URLs instead of treating them like subpaths", () => { + expect(parseGitHubRepoUrl("https://github.com/vercel/next.js/issues/123")).toBeNull(); + }); +}); + +describe("buildTemplateFolderFromEntries", () => { + it("creates a nested template folder from flat GitHub file entries", () => { + const result = buildTemplateFolderFromEntries("repo", [ + { path: "src/index.tsx", content: "console.log('hi');", size: 18 }, + { path: "src/components/Button.tsx", content: "export const Button = () => null;", size: 33 }, + { path: "package.json", content: "{}", size: 2 }, + ]); + + expect(result.folderName).toBe("repo"); + expect(result.items).toHaveLength(2); + + const srcFolder = result.items.find((item) => "folderName" in item && item.folderName === "src"); + expect(srcFolder && "folderName" in srcFolder).toBe(true); + }); + + it("skips binary files and hidden paths", () => { + const result = buildTemplateFolderFromEntries("repo", [ + { path: ".git/config", content: "ignore", size: 6 }, + { path: "assets/logo.png", content: "binary", size: 6 }, + { path: "src/app.ts", content: "export {}", size: 9 }, + ]); + + expect(result.items).toHaveLength(1); + }); +}); diff --git a/app/api/github/import/utils.ts b/app/api/github/import/utils.ts new file mode 100644 index 00000000..31d3d206 --- /dev/null +++ b/app/api/github/import/utils.ts @@ -0,0 +1,208 @@ +import type { TemplateFolder } from "@/modules/playground/lib/path-to-json"; + +const SKIP_EXTENSIONS = new Set([ + "png", + "jpg", + "jpeg", + "gif", + "bmp", + "ico", + "svg", + "woff", + "woff2", + "ttf", + "eot", + "otf", + "mp3", + "mp4", + "wav", + "avi", + "mov", + "zip", + "tar", + "gz", + "rar", + "pdf", + "doc", + "docx", + "xls", + "xlsx", + "exe", + "dll", + "so", + "dylib", + "pyc", + "class", + "o", +]); + +const SKIP_FOLDERS = new Set([ + "node_modules", + ".git", + ".next", + "dist", + "build", + "__pycache__", + ".cache", + ".DS_Store", +]); + +const MAX_SINGLE_FILE_SIZE = 500_000; + +export interface ParsedGitHubRepoUrl { + owner: string; + repo: string; + branch?: string; + path?: string; +} + +export interface GitHubRepoMetadata { + name: string; + full_name: string; + html_url: string; + description: string | null; + default_branch: string; +} + +export interface GitHubContentEntry { + path: string; + type: "file" | "dir" | string; + size?: number; + name?: string; + download_url?: string | null; +} + +export interface GitHubFileEntry { + path: string; + content: string; + size?: number; +} + +export function parseGitHubRepoUrl(repoUrl: string): ParsedGitHubRepoUrl | null { + try { + const parsed = new URL(repoUrl.trim()); + const host = parsed.hostname.replace(/^www\./, "").toLowerCase(); + + if (host !== "github.com") { + return null; + } + + const segments = parsed.pathname.split("/").filter(Boolean); + if (segments.length < 2) { + return null; + } + + const [owner, repo, mode, branchOrPath, ...rest] = segments; + + if (!owner || !repo) { + return null; + } + + if (mode === "tree" || mode === "blob" || mode === "raw") { + const path = rest.join("/"); + return { + owner, + repo, + branch: branchOrPath, + path: path || undefined, + }; + } + + if (segments.length > 2) { + return null; + } + + return { + owner, + repo, + }; + } catch { + return null; + } +} + +function isTemplateFolder(item: TemplateFolder | { filename: string }): item is TemplateFolder { + return "folderName" in item; +} + +function shouldSkipPath(pathValue: string): boolean { + const parts = pathValue.split("/").filter(Boolean); + + if (parts.length === 0) { + return true; + } + + if (parts.some((part) => SKIP_FOLDERS.has(part) || part.startsWith("."))) { + return true; + } + + const fileName = parts[parts.length - 1]; + if (fileName.startsWith(".")) { + return true; + } + + const ext = fileName.includes(".") ? fileName.split(".").pop()!.toLowerCase() : ""; + return SKIP_EXTENSIONS.has(ext); +} + +export function buildTemplateFolderFromEntries( + rootName: string, + entries: GitHubFileEntry[] +): TemplateFolder { + const root: TemplateFolder = { folderName: rootName, items: [] }; + + for (const entry of entries) { + const cleanPath = entry.path.replace(/^\/+/, ""); + + if (!cleanPath || shouldSkipPath(cleanPath)) { + continue; + } + + if (typeof entry.size === "number" && entry.size > MAX_SINGLE_FILE_SIZE) { + continue; + } + + const parts = cleanPath.split("/").filter(Boolean); + if (parts.length === 0) { + continue; + } + + const fileName = parts.pop()!; + const dotIndex = fileName.lastIndexOf("."); + const fileExtension = dotIndex > 0 ? fileName.slice(dotIndex + 1) : ""; + const filename = dotIndex > 0 ? fileName.slice(0, dotIndex) : fileName; + + let currentFolder = root; + for (const part of parts) { + let nextFolder = currentFolder.items.find( + (item) => isTemplateFolder(item) && item.folderName === part + ) as TemplateFolder | undefined; + + if (!nextFolder) { + nextFolder = { folderName: part, items: [] }; + currentFolder.items.push(nextFolder); + } + + currentFolder = nextFolder; + } + + currentFolder.items.push({ + filename, + fileExtension, + content: entry.content, + }); + } + + return root; +} + +export function hasPackageJson(entries: GitHubContentEntry[]): boolean { + return entries.some((entry) => entry.type === "file" && entry.name === "package.json"); +} + +export function getMonorepoCandidates(entries: GitHubContentEntry[]): string[] { + return entries + .filter((entry) => entry.type === "dir") + .map((entry) => entry.path) + .filter(Boolean); +} diff --git a/modules/dashboard/components/github-import-dialog.tsx b/modules/dashboard/components/github-import-dialog.tsx index 8f15a427..331ee668 100644 --- a/modules/dashboard/components/github-import-dialog.tsx +++ b/modules/dashboard/components/github-import-dialog.tsx @@ -68,7 +68,18 @@ const GithubImportDialog = ({ children }: { children: React.ReactNode }) => { body: JSON.stringify(body), }); - const data = await response.json(); + const responseText = await response.text(); + let data: { error?: string; needsSubdir?: boolean; subdirs?: string[]; playgroundId?: string } = {}; + + try { + data = responseText ? JSON.parse(responseText) : {}; + } catch { + throw new Error( + responseText.includes("