From 78590cfe52ab31697a16bec6df99349c3d0a3567 Mon Sep 17 00:00:00 2001 From: javi11 Date: Sun, 10 May 2026 16:57:29 +0200 Subject: [PATCH 1/3] feat(par2): generate single set per folder with relative paths in FileDesc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #219 — SABnzbd/NZBGet now reconstruct the original folder tree when downloading a postie-generated NZB, matching the behavior of nyuu. ## Root cause * SABnzbd's `quick_check_set()` reads PAR2 FileDesc filenames and calls `renamer(..., create_local_directories=True)` using those names. If the FileDesc contains a relative path (e.g. `folder2/file.txt`) the directory is recreated on disk. * par2go v0.0.7 hardcoded `filepath.Base(path)` in `quickScanFile`, so every FileDesc carried only the bare filename — no folder tree. * postie also generated a separate PAR2 set per file instead of one set for the whole folder, so there was no shared set to carry the tree. ## Changes ### `Par2Executor` interface — new `CreateSet` method `internal/par2/par2.go` and `internal/par2/binary.go` gain a `CreateSet(ctx, files, outputDir, setName)` entrypoint that generates **one PAR2 set covering all files** in a folder, with each FileDesc carrying the file's `RelativePath` (forward-slash separated). * `NativeExecutor.CreateSet` uses the new `par2go v0.0.8` `CreateWithNames` API to embed relative paths in FileDesc packets. * `BinaryExecutor.CreateSet` invokes parpar with `--filepath-base --filepath-format outrel` so parpar derives the relative path from each input's disk path. * `checkExistingPar2SetInPath` detects pre-existing sets by set name, avoiding redundant regeneration. ### Caller update — `pkg/postie/postie.go` `postFolder` now calls `CreateSet` with the folder name as the set name instead of two separate `CreateInDirectory` calls. ### Dependency bump `github.com/javi11/par2go` v0.0.7 → v0.0.8, which adds `InputFile` / `CreateWithNames` for per-file logical name overrides. ### Tests * `internal/par2/par2_integration_test.go` — integration test that creates a real multi-file folder tree, runs `NativeExecutor.CreateSet`, then parses the resulting PAR2 binary to verify each FileDesc contains the correct relative path (e.g. `folder2/file2.txt`). * `pkg/postie/postfolder_test.go` — `mockPar2Executor` gains a `CreateSet` stub so existing postFolder unit tests continue to compile and pass. --- go.mod | 2 +- go.sum | 6 +- internal/par2/binary.go | 187 ++++++++++++++++++++++ internal/par2/par2.go | 205 +++++++++++++++++++++++++ internal/par2/par2_integration_test.go | 139 +++++++++++++++++ pkg/postie/postfolder_test.go | 4 + pkg/postie/postie.go | 13 +- 7 files changed, 544 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index c6e8516..e2af672 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/javi11/nntppool/v4 v4.11.1 github.com/javi11/nxg v0.1.0 github.com/javi11/nzbparser v0.5.4 - github.com/javi11/par2go v0.0.7 + github.com/javi11/par2go v0.0.8 github.com/klauspost/compress v1.18.2 github.com/mattn/go-sqlite3 v1.14.32 github.com/mnightingale/rapidyenc v0.0.0-20251128204712-7aafef1eaf1c diff --git a/go.sum b/go.sum index e7fceed..e1b6229 100644 --- a/go.sum +++ b/go.sum @@ -345,16 +345,14 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jackmordaunt/icns v1.0.0 h1:RYSxplerf/l/DUd09AHtITwckkv/mqjVv4DjYdPmAMQ= github.com/jackmordaunt/icns v1.0.0/go.mod h1:7TTQVEuGzVVfOPPlLNHJIkzA6CoV7aH1Dv9dW351oOo= -github.com/javi11/nntppool/v4 v4.10.1 h1:NHoRniTnCRgpt/niljsWjA3gP5pYtNwxaqbLdr0Bcew= -github.com/javi11/nntppool/v4 v4.10.1/go.mod h1:+UtisJLDFLXBSkW9R6uCRgdp3lS/+6pLAk7L+Wt6LMw= github.com/javi11/nntppool/v4 v4.11.1 h1:581fZSPv+RyIKY2hI0GB0eZrm1rCbAp+HRP5nIp7kd0= github.com/javi11/nntppool/v4 v4.11.1/go.mod h1:+UtisJLDFLXBSkW9R6uCRgdp3lS/+6pLAk7L+Wt6LMw= github.com/javi11/nxg v0.1.0 h1:CTThldYlaVIPIhpkrMw0HcTD0NLrW1uYMoDILjjEOtM= github.com/javi11/nxg v0.1.0/go.mod h1:+GvYpp+y1oq+qBOWxFMvfTjtin/0zCeomWfjiPkiu8A= github.com/javi11/nzbparser v0.5.4 h1:0aYyORZipp7iX8eNpT/efnzCeVO+9C0sE2HWCGc/JaI= github.com/javi11/nzbparser v0.5.4/go.mod h1:ikF7WI3BUGs5IHQJmKzmtTkX29NZW5nvUdo6ZWFZgL4= -github.com/javi11/par2go v0.0.7 h1:30KMAvFHCBheMkg6b+D1Pi+xx5EvGeXZdfKeoLDv3vc= -github.com/javi11/par2go v0.0.7/go.mod h1:GXuKNiRmZGv4rGia0FR23016IbcrsTMvLHihytsSnMU= +github.com/javi11/par2go v0.0.8 h1:dnEaRIHIxJZ9dQg2AZ6h/mD9L11xFpdAfaA+vWg2pAI= +github.com/javi11/par2go v0.0.8/go.mod h1:GXuKNiRmZGv4rGia0FR23016IbcrsTMvLHihytsSnMU= github.com/jaypipes/ghw v0.13.0 h1:log8MXuB8hzTNnSktqpXMHc0c/2k/WgjOMSUtnI1RV4= github.com/jaypipes/ghw v0.13.0/go.mod h1:In8SsaDqlb1oTyrbmTC14uy+fbBMvp+xdqX51MidlD8= github.com/jaypipes/pcidb v1.0.1 h1:WB2zh27T3nwg8AE8ei81sNRb9yWBii3JGNJtT7K9Oic= diff --git a/internal/par2/binary.go b/internal/par2/binary.go index bb80ae0..6eb0079 100644 --- a/internal/par2/binary.go +++ b/internal/par2/binary.go @@ -41,6 +41,193 @@ func (b *BinaryExecutor) Create(ctx context.Context, files []fileinfo.FileInfo) return b.CreateInDirectory(ctx, files, "") } +// CreateSet bundles all input files into a single par2 set named setName, +// using parpar's --filepath-format=common so each FileDesc carries the +// file's relative path (within the common ancestor of the inputs). This +// is what lets SABnzbd / NZBGet recreate the folder tree on disk. +func (b *BinaryExecutor) CreateSet(ctx context.Context, files []fileinfo.FileInfo, outputDir, setName string) ([]string, error) { + if len(files) == 0 { + return nil, fmt.Errorf("par2: no input files for set %q", setName) + } + if setName == "" { + return nil, fmt.Errorf("par2: empty set name") + } + + inputs := make([]fileinfo.FileInfo, 0, len(files)) + for _, f := range files { + if filepath.Ext(f.Path) == ".par2" { + continue + } + inputs = append(inputs, f) + } + if len(inputs) == 0 { + return nil, fmt.Errorf("par2: no non-par2 input files for set %q", setName) + } + + dirPath := outputDir + if dirPath == "" { + if b.cfg.TempDir != "" { + dirPath = b.cfg.TempDir + } else { + dirPath = filepath.Dir(inputs[0].Path) + } + } + if err := os.MkdirAll(dirPath, 0755); err != nil { + return nil, fmt.Errorf("par2: create output dir %s: %w", dirPath, err) + } + + if existing, ok := checkExistingPar2SetInPath(ctx, setName, dirPath); ok { + return existing, nil + } + + totalSize := uint64(0) + maxFileSize := uint64(0) + for _, f := range inputs { + totalSize += f.Size + if f.Size > maxFileSize { + maxFileSize = f.Size + } + } + blockSize := calculateParBlockSize(totalSize, b.articleSize) + if maxFileSize > 0 && blockSize > maxFileSize { + blockSize = alignDown(maxFileSize, 4) + } + if blockSize < 4 { + slog.WarnContext(ctx, "Block size too small for PAR2 set creation, skipping", + "setName", setName, "totalSize", totalSize) + return nil, nil + } + totalSlices := 0 + for _, f := range inputs { + n := int(math.Ceil(float64(f.Size) / float64(blockSize))) + if n == 0 { + n = 1 + } + totalSlices += n + } + redundancyPct := parseRedundancyPercentage(b.cfg.Redundancy, totalSize, blockSize) + numRecovery := max(int(math.Ceil(float64(totalSlices)*redundancyPct/100.0)), 1) + + outputBase := filepath.Join(dirPath, setName) + + // Common ancestor of all input paths — used as parpar --filepath-base so + // the recorded display name is the relative path from that ancestor. + rootDir := commonParentDir(inputs) + + args := []string{ + "-s", fmt.Sprintf("%db", blockSize), + "-r", fmt.Sprintf("%d", numRecovery), + "-o", outputBase, + "--filepath-base", rootDir, + "--filepath-format", "outrel", + } + args = append(args, b.cfg.ParparExtraArgs...) + for _, f := range inputs { + args = append(args, f.Path) + } + + progressID := uuid.New() + progressName := fmt.Sprintf("PAR2: %s", setName) + var pg progress.Progress + if b.jobProgress != nil { + pg = b.jobProgress.AddProgress(progressID, progressName, progress.ProgressTypePar2Generation, 100) + } + + slog.InfoContext(ctx, "Invoking parpar binary for set", + "binary", b.cfg.ParparBinaryPath, "setName", setName, "files", len(inputs), + "outputBase", outputBase, "filepathBase", rootDir, + "blockSize", blockSize, "recoverySlices", numRecovery) + + cmd := exec.CommandContext(ctx, b.cfg.ParparBinaryPath, args...) + + stdoutPipe, err := cmd.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("parpar stdout pipe: %w", err) + } + var stderrBuf bytes.Buffer + cmd.Stderr = &stderrBuf + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("parpar start for set %s: %w", setName, err) + } + + var stdoutBuf bytes.Buffer + var lastPct int64 + scanner := bufio.NewScanner(io.TeeReader(stdoutPipe, &stdoutBuf)) + scanner.Split(splitOnCROrLF) + for scanner.Scan() { + if pg == nil { + continue + } + if m := parparProgressRe.FindStringSubmatch(scanner.Text()); len(m) > 1 { + if pct, parseErr := strconv.ParseFloat(m[1], 64); parseErr == nil { + newPct := int64(pct) + if newPct > lastPct { + pg.UpdateProgress(newPct - lastPct) + lastPct = newPct + } + } + } + } + + if err := cmd.Wait(); err != nil { + if ctx.Err() != nil { + slog.InfoContext(ctx, "Parpar set cancelled", "setName", setName) + return nil, ctx.Err() + } + combined := strings.TrimSpace(stdoutBuf.String()) + if s := strings.TrimSpace(stderrBuf.String()); s != "" { + if combined != "" { + combined += "\n" + } + combined += s + } + return nil, fmt.Errorf("parpar failed for set %s: %w\noutput: %s", setName, err, combined) + } + + if pg != nil && b.jobProgress != nil { + if lastPct < 100 { + pg.UpdateProgress(100 - lastPct) + } + b.jobProgress.FinishProgress(progressID) + } + + return collectPar2SetFiles(ctx, dirPath, setName, outputBase+".par2"), nil +} + +// commonParentDir returns the deepest directory that contains every input. +// Used as parpar's --filepath-base so the embedded display name is the +// relative path of each file beneath that directory. +func commonParentDir(files []fileinfo.FileInfo) string { + if len(files) == 0 { + return "" + } + dir := filepath.Dir(files[0].Path) + for _, f := range files[1:] { + dir = commonPrefixDir(dir, filepath.Dir(f.Path)) + } + return dir +} + +func commonPrefixDir(a, b string) string { + if a == b { + return a + } + aParts := strings.Split(filepath.Clean(a), string(filepath.Separator)) + bParts := strings.Split(filepath.Clean(b), string(filepath.Separator)) + n := len(aParts) + if len(bParts) < n { + n = len(bParts) + } + i := 0 + for i < n && aParts[i] == bParts[i] { + i++ + } + if i == 0 { + return string(filepath.Separator) + } + return strings.Join(aParts[:i], string(filepath.Separator)) +} + // CreateInDirectory creates PAR2 files in the specified output directory using the parpar binary. func (b *BinaryExecutor) CreateInDirectory(ctx context.Context, files []fileinfo.FileInfo, outputDir string) ([]string, error) { var all []string diff --git a/internal/par2/par2.go b/internal/par2/par2.go index 353fd08..37e5d77 100644 --- a/internal/par2/par2.go +++ b/internal/par2/par2.go @@ -30,6 +30,11 @@ const maxPar2Blocks = 32768 type Par2Executor interface { Create(ctx context.Context, files []fileinfo.FileInfo) ([]string, error) CreateInDirectory(ctx context.Context, files []fileinfo.FileInfo, outputDir string) ([]string, error) + // CreateSet bundles all input files into a single par2 set named setName, + // embedding each file's RelativePath (or basename when empty) in the + // FileDesc packet so downloaders such as SABnzbd can reconstruct the + // folder tree on disk after par2 verification. + CreateSet(ctx context.Context, files []fileinfo.FileInfo, outputDir, setName string) ([]string, error) } // NativeExecutor implements Par2Executor using the built-in Go PAR2 creator. @@ -203,6 +208,206 @@ func (p *NativeExecutor) CreateInDirectory(ctx context.Context, files []fileinfo return createdPar2Paths, nil } +// CreateSet bundles all input files into a single par2 set named setName. +// Each FileDesc packet records the file's RelativePath (or filepath.Base +// when empty) so SABnzbd / NZBGet can recreate the folder tree on disk. +func (p *NativeExecutor) CreateSet(ctx context.Context, files []fileinfo.FileInfo, outputDir, setName string) ([]string, error) { + if len(files) == 0 { + return nil, fmt.Errorf("par2: no input files for set %q", setName) + } + if setName == "" { + return nil, fmt.Errorf("par2: empty set name") + } + + // Filter out any par2 files defensively — callers should not include them. + inputs := make([]fileinfo.FileInfo, 0, len(files)) + for _, f := range files { + if filepath.Ext(f.Path) == ".par2" { + continue + } + inputs = append(inputs, f) + } + if len(inputs) == 0 { + return nil, fmt.Errorf("par2: no non-par2 input files for set %q", setName) + } + + dirPath := outputDir + if dirPath == "" { + if p.cfg.TempDir != "" { + dirPath = p.cfg.TempDir + } else { + dirPath = filepath.Dir(inputs[0].Path) + } + } + if err := os.MkdirAll(dirPath, 0755); err != nil { + return nil, fmt.Errorf("par2: create output dir %s: %w", dirPath, err) + } + + // Reuse existing set if already on disk. + if existing, ok := checkExistingPar2SetInPath(ctx, setName, dirPath); ok { + return existing, nil + } + + // Slice size: smallest size that yields ≤ maxPar2Blocks slices across all + // inputs while respecting SliceSize config and SIMD alignment. + totalSize := uint64(0) + maxFileSize := uint64(0) + for _, f := range inputs { + totalSize += f.Size + if f.Size > maxFileSize { + maxFileSize = f.Size + } + } + parBlockSize := p.computeSetBlockSize(totalSize, maxFileSize) + if parBlockSize < 4 { + slog.WarnContext(ctx, "Block size too small for PAR2 set creation, skipping", + "setName", setName, "totalSize", totalSize) + return nil, nil + } + + // Total input slices and recovery blocks + totalSlices := 0 + for _, f := range inputs { + n := int(math.Ceil(float64(f.Size) / float64(parBlockSize))) + if n == 0 { + n = 1 + } + totalSlices += n + } + redundancyPct := parseRedundancyPercentage(p.cfg.Redundancy, totalSize, parBlockSize) + numRecovery := max(int(math.Ceil(float64(totalSlices)*redundancyPct/100.0)), 1) + + par2Path := filepath.Join(dirPath, setName+".par2") + + progressID := uuid.New() + progressName := fmt.Sprintf("PAR2: %s", setName) + var pg progress.Progress + if p.jobProgress != nil { + pg = p.jobProgress.AddProgress(progressID, progressName, progress.ProgressTypePar2Generation, 100) + } + + par2Inputs := make([]par2go.InputFile, len(inputs)) + for i, f := range inputs { + name := f.RelativePath + if name == "" { + name = filepath.Base(f.Path) + } + // Forward slashes only — par2go validates this and downloaders rely on it. + name = filepath.ToSlash(name) + par2Inputs[i] = par2go.InputFile{Path: f.Path, Name: name} + } + + opts := par2go.Options{ + SliceSize: int(parBlockSize), + NumRecovery: numRecovery, + NumGoroutines: p.cfg.NumGoroutines, + MemoryLimit: p.cfg.MemoryLimit, + Creator: "Postie", + OnProgress: func(phase string, pct float64) { + if pg == nil { + return + } + var overallPct float64 + switch phase { + case "hashing": + overallPct = pct * 20 + case "encoding": + overallPct = 20 + pct*75 + case "writing": + overallPct = 95 + pct*5 + } + delta := int64(overallPct) - pg.GetCurrent() + if delta > 0 { + pg.UpdateProgress(delta) + } + }, + } + + slog.InfoContext(ctx, "Creating PAR2 set", + "setName", setName, + "files", len(par2Inputs), + "blockSize", parBlockSize, + "inputSlices", totalSlices, + "recoveryBlocks", numRecovery, + "redundancy", redundancyPct) + + if err := par2go.CreateWithNames(ctx, par2Path, par2Inputs, opts); err != nil { + if ctx.Err() == context.Canceled { + slog.InfoContext(ctx, "Par2 set creation cancelled", "setName", setName) + return nil, ctx.Err() + } + return nil, fmt.Errorf("failed to create par2 set %s: %w", setName, err) + } + + if p.jobProgress != nil { + p.jobProgress.FinishProgress(progressID) + } + + return collectPar2SetFiles(ctx, dirPath, setName, par2Path), nil +} + +// computeSetBlockSize picks a slice size for a multi-file par2 set such that +// the total slice count stays under maxPar2Blocks and SIMD alignment is +// preserved. Honors p.cfg.SliceSize when sane. +func (p *NativeExecutor) computeSetBlockSize(totalSize, maxFileSize uint64) uint64 { + var parBlockSize uint64 + if p.cfg.SliceSize > 0 && uint64(p.cfg.SliceSize) <= maxFileSize { + parBlockSize = uint64(p.cfg.SliceSize) + } else { + parBlockSize = calculateParBlockSize(totalSize, p.articleSize) + } + // Ensure block size yields ≤ maxPar2Blocks total slices. + if parBlockSize > 0 { + approxSlices := totalSize / parBlockSize + if approxSlices >= maxPar2Blocks { + parBlockSize = (totalSize / (maxPar2Blocks - 1)) + 1 + } + } + // SIMD-safe alignment when a single file is smaller than the block size. + if maxFileSize > 0 && parBlockSize > maxFileSize { + const simdSafeAlignment = uint64(128) + if maxFileSize < simdSafeAlignment { + return 0 + } + parBlockSize = (maxFileSize / simdSafeAlignment) * simdSafeAlignment + } + return alignDown(parBlockSize, 4) +} + +// checkExistingPar2SetInPath looks for an already-generated par2 set in +// dirPath named ".par2" plus any companion volume files. +func checkExistingPar2SetInPath(ctx context.Context, setName, dirPath string) ([]string, bool) { + main := filepath.Join(dirPath, setName+".par2") + if _, err := os.Stat(main); os.IsNotExist(err) { + return nil, false + } + paths := collectPar2SetFiles(ctx, dirPath, setName, main) + slog.InfoContext(ctx, "Found existing PAR2 set, skipping generation", + "setName", setName, "par2Files", len(paths)) + return paths, true +} + +// collectPar2SetFiles returns the main par2 path plus all companion volume +// files matching ".vol*.par2" in dirPath. +func collectPar2SetFiles(ctx context.Context, dirPath, setName, mainPath string) []string { + out := []string{mainPath} + entries, err := os.ReadDir(dirPath) + if err != nil { + slog.WarnContext(ctx, "Failed to read directory for par2 volumes", "error", err) + return out + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + name := entry.Name() + if strings.HasPrefix(name, setName) && strings.Contains(name, ".vol") && strings.HasSuffix(name, ".par2") { + out = append(out, filepath.Join(dirPath, name)) + } + } + return out +} + // createPar2ForFile creates PAR2 files for a single input file in the given directory. func (p *NativeExecutor) createPar2ForFile(ctx context.Context, file fileinfo.FileInfo, dirPath string) ([]string, error) { var parBlockSize uint64 diff --git a/internal/par2/par2_integration_test.go b/internal/par2/par2_integration_test.go index fbb2224..d029b67 100644 --- a/internal/par2/par2_integration_test.go +++ b/internal/par2/par2_integration_test.go @@ -6,9 +6,12 @@ package par2 // Run with: go test ./internal/par2/... -run "Integration" -v -timeout 120s import ( + "bytes" "context" + "encoding/binary" "os" "path/filepath" + "sort" "strings" "testing" @@ -385,3 +388,139 @@ func TestIntegration_NativeExecutor_SkipsInputPar2Files(t *testing.T) { t.Fatalf("Create: unexpected error: %v", err) } } + +// --------------------------------------------------------------------------- +// CreateSet — folder mode, FileDesc relative paths (issue #219) +// --------------------------------------------------------------------------- + +// extractFileDescNames parses a par2 file and returns the list of filenames +// embedded in FileDesc packets, in the order they appear. +func extractFileDescNames(t *testing.T, path string) []string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read par2 %s: %v", path, err) + } + magic := []byte{'P', 'A', 'R', '2', 0, 'P', 'K', 'T'} + fdType := []byte{'P', 'A', 'R', ' ', '2', '.', '0', 0, 'F', 'i', 'l', 'e', 'D', 'e', 's', 'c'} + var names []string + for off := 0; off+64 <= len(data); { + if !bytes.Equal(data[off:off+8], magic) { + off++ + continue + } + length := binary.LittleEndian.Uint64(data[off+8 : off+16]) + if length < 64 || off+int(length) > len(data) { + break + } + typ := data[off+48 : off+64] + if bytes.Equal(typ, fdType) { + // Body layout: fileID(16) hashFull(16) hash16k(16) fileSize(8) name(N) + body := data[off+64 : off+int(length)] + if len(body) > 56 { + name := strings.TrimRight(string(body[56:]), "\x00") + names = append(names, name) + } + } + off += int(length) + } + return names +} + +func TestIntegration_NativeExecutor_CreateSet_EmbedsRelativePaths(t *testing.T) { + root := t.TempDir() + // Build folder1-testpkg/{file1.txt, folder2/file2.txt, folder2/folder3/file3.txt} + pkgDir := filepath.Join(root, "folder1-testpkg") + mustMkdir(t, filepath.Join(pkgDir, "folder2", "folder3")) + file1 := createIntegrationTestFile(t, pkgDir, "file1.txt", 4096) + file2 := createIntegrationTestFile(t, filepath.Join(pkgDir, "folder2"), "file2.txt", 4096) + file3 := createIntegrationTestFile(t, filepath.Join(pkgDir, "folder2", "folder3"), "file3.txt", 4096) + + files := []fileinfo.FileInfo{ + {Path: file1, Size: 4096, RelativePath: "folder1-testpkg/file1.txt"}, + {Path: file2, Size: 4096, RelativePath: "folder1-testpkg/folder2/file2.txt"}, + {Path: file3, Size: 4096, RelativePath: "folder1-testpkg/folder2/folder3/file3.txt"}, + } + + cfg := &config.Par2Config{Redundancy: "10", SliceSize: 0, MemoryLimit: 4 * 1024 * 1024 * 1024} + executor := New(750_000, cfg, nil) + + outDir := filepath.Join(root, "out") + created, err := executor.CreateSet(context.Background(), files, outDir, "folder1-testpkg") + if err != nil { + t.Fatalf("CreateSet: %v", err) + } + if len(created) == 0 { + t.Fatal("CreateSet returned no files") + } + + mainPar2 := filepath.Join(outDir, "folder1-testpkg.par2") + if _, err := os.Stat(mainPar2); err != nil { + t.Fatalf("expected main par2 %s: %v", mainPar2, err) + } + + got := extractFileDescNames(t, mainPar2) + sort.Strings(got) + want := []string{ + "folder1-testpkg/file1.txt", + "folder1-testpkg/folder2/file2.txt", + "folder1-testpkg/folder2/folder3/file3.txt", + } + // FileDesc packets repeat across volume files; dedupe on read. + got = dedupe(got) + if !equalStrings(got, want) { + t.Errorf("FileDesc names: got %v, want %v", got, want) + } +} + +func TestIntegration_NativeExecutor_CreateSet_FallsBackToBasename(t *testing.T) { + root := t.TempDir() + file := createIntegrationTestFile(t, root, "loose.bin", 4096) + + files := []fileinfo.FileInfo{ + {Path: file, Size: 4096}, // no RelativePath + } + cfg := &config.Par2Config{Redundancy: "10", MemoryLimit: 4 * 1024 * 1024 * 1024} + executor := New(750_000, cfg, nil) + + outDir := filepath.Join(root, "out") + if _, err := executor.CreateSet(context.Background(), files, outDir, "loose"); err != nil { + t.Fatalf("CreateSet: %v", err) + } + got := dedupe(extractFileDescNames(t, filepath.Join(outDir, "loose.par2"))) + if !equalStrings(got, []string{"loose.bin"}) { + t.Errorf("FileDesc names: got %v, want [loose.bin]", got) + } +} + +func mustMkdir(t *testing.T, p string) { + t.Helper() + if err := os.MkdirAll(p, 0755); err != nil { + t.Fatalf("mkdir %s: %v", p, err) + } +} + +func dedupe(in []string) []string { + seen := map[string]bool{} + out := []string{} + for _, s := range in { + if !seen[s] { + seen[s] = true + out = append(out, s) + } + } + sort.Strings(out) + return out +} + +func equalStrings(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/pkg/postie/postfolder_test.go b/pkg/postie/postfolder_test.go index 72b2912..17d7976 100644 --- a/pkg/postie/postfolder_test.go +++ b/pkg/postie/postfolder_test.go @@ -81,6 +81,10 @@ func (m *mockPar2Executor) CreateInDirectory(_ context.Context, _ []fileinfo.Fil return created, nil } +func (m *mockPar2Executor) CreateSet(ctx context.Context, files []fileinfo.FileInfo, outputDir, _ string) ([]string, error) { + return m.CreateInDirectory(ctx, files, outputDir) +} + // ─── helpers ──────────────────────────────────────────────────────────────── func boolPtr(b bool) *bool { return &b } diff --git a/pkg/postie/postie.go b/pkg/postie/postie.go index 5f48069..886f4d2 100644 --- a/pkg/postie/postie.go +++ b/pkg/postie/postie.go @@ -498,9 +498,9 @@ func (p *Postie) postFolder(ctx context.Context, files []fileinfo.FileInfo, root slog.DebugContext(ctx, "Generating PAR2 files directly in output directory", "folder", folderName, "outputDir", par2OutputDir) } - // If par2OutputDir is empty, CreateInDirectory will use default behavior (temp/source dir) + // If par2OutputDir is empty, CreateSet will use default behavior (temp/source dir) - createdPar2Paths, err = p.par2runner.CreateInDirectory(ctx, files, par2OutputDir) + createdPar2Paths, err = p.par2runner.CreateSet(ctx, files, par2OutputDir, folderName) if err != nil { if !errors.Is(err, context.Canceled) { slog.ErrorContext(ctx, "Error during par2 creation. Upload will continue without par2.", "error", err) @@ -508,9 +508,7 @@ func (p *Postie) postFolder(ctx context.Context, files []fileinfo.FileInfo, root // Continue without PAR2 files } else { allFilePaths = append(allFilePaths, createdPar2Paths...) - for k, v := range buildPar2RelativePaths(files, createdPar2Paths) { - relativePaths[k] = v - } + // par2 set files live at the folder root — basenames in NZB subjects. } } } @@ -570,7 +568,7 @@ func (p *Postie) postFolder(ctx context.Context, files []fileinfo.FileInfo, root "folder", folderName, "outputDir", par2OutputDir) } - createdPar2Paths, err = p.par2runner.CreateInDirectory(ctx, files, par2OutputDir) + createdPar2Paths, err = p.par2runner.CreateSet(ctx, files, par2OutputDir, folderName) if err != nil { if !errors.Is(err, context.Canceled) { slog.ErrorContext(ctx, "Error during par2 creation. Upload will continue without par2.", "error", err) @@ -578,7 +576,8 @@ func (p *Postie) postFolder(ctx context.Context, files []fileinfo.FileInfo, root return nil } - par2RelPaths := buildPar2RelativePaths(files, createdPar2Paths) + // par2 set files live at the folder root — basenames in NZB subjects. + par2RelPaths := map[string]string{} if err := p.poster.PostWithRelativePaths(ctx, createdPar2Paths, rootDir, nzbGen, par2RelPaths); err != nil { if !errors.Is(err, context.Canceled) { slog.ErrorContext(ctx, "Error during upload of par2 files. Upload will continue without par2.", "error", err) From 3b7c01319488fe35ddb1f07cb34d1c47555f9732 Mon Sep 17 00:00:00 2001 From: javi11 Date: Sun, 10 May 2026 16:58:03 +0200 Subject: [PATCH 2/3] chore: ignore .cocoindex_code directory --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index a35dc62..c877a51 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,5 @@ postie-cli # ClaudPoint checkpoint system .checkpoints/ example/ +# CocoIndex Code (ccc) +/.cocoindex_code/ From 41be88ba5f1298c472e26d13a5fe4011d06213e8 Mon Sep 17 00:00:00 2001 From: javi11 Date: Mon, 11 May 2026 08:53:52 +0200 Subject: [PATCH 3/3] fix(par2): use explicit folderDir as filepath-base in CreateSet - Add `folderDir string` parameter to `Par2Executor.CreateSet` interface so both executors receive the on-disk root of the folder being posted. - NativeExecutor: compute each file's FileDesc name via `filepath.Rel(folderDir, file.Path)` instead of using RelativePath verbatim. RelativePath includes the top-level folder name as a prefix (e.g. "ShowS01/extras/bonus.mkv") which would cause SABnzbd to double-nest the reconstructed directory tree. The Rel-based path produces "extras/bonus.mkv" as SABnzbd expects. - BinaryExecutor: pass folderDir directly as --filepath-base instead of the fragile commonParentDir() derivation, which silently dropped the subdirectory prefix when all input files were in the same subdir. Remove the now-unused commonParentDir / commonPrefixDir helpers. - postie.go: both CreateSet call sites now derive `folderDir = filepath.Join(rootDir, folderName)` and forward it. - Integration test: update expected FileDesc names to reflect the fix and add TestIntegration_NativeExecutor_CreateSet_AllFilesInSubdir to guard the single-subdirectory edge case. --- internal/par2/binary.go | 53 +++++--------------------- internal/par2/par2.go | 25 +++++++----- internal/par2/par2_integration_test.go | 52 +++++++++++++++++++++---- pkg/postie/postfolder_test.go | 2 +- pkg/postie/postie.go | 9 +++-- 5 files changed, 76 insertions(+), 65 deletions(-) diff --git a/internal/par2/binary.go b/internal/par2/binary.go index 6eb0079..0ac227e 100644 --- a/internal/par2/binary.go +++ b/internal/par2/binary.go @@ -43,9 +43,10 @@ func (b *BinaryExecutor) Create(ctx context.Context, files []fileinfo.FileInfo) // CreateSet bundles all input files into a single par2 set named setName, // using parpar's --filepath-format=common so each FileDesc carries the -// file's relative path (within the common ancestor of the inputs). This -// is what lets SABnzbd / NZBGet recreate the folder tree on disk. -func (b *BinaryExecutor) CreateSet(ctx context.Context, files []fileinfo.FileInfo, outputDir, setName string) ([]string, error) { +// folderDir as --filepath-base so parpar records each file's path relative +// to folderDir in the FileDesc packet, letting SABnzbd / NZBGet recreate +// the folder tree inside the job directory on disk. +func (b *BinaryExecutor) CreateSet(ctx context.Context, files []fileinfo.FileInfo, outputDir, setName, folderDir string) ([]string, error) { if len(files) == 0 { return nil, fmt.Errorf("par2: no input files for set %q", setName) } @@ -110,15 +111,15 @@ func (b *BinaryExecutor) CreateSet(ctx context.Context, files []fileinfo.FileInf outputBase := filepath.Join(dirPath, setName) - // Common ancestor of all input paths — used as parpar --filepath-base so - // the recorded display name is the relative path from that ancestor. - rootDir := commonParentDir(inputs) - + // Use folderDir as --filepath-base so parpar records each file's path + // relative to the folder root (e.g. "extras/bonus.mkv"). This is the + // explicit folder root passed by the caller, which is always correct + // regardless of whether all input files happen to share a subdirectory. args := []string{ "-s", fmt.Sprintf("%db", blockSize), "-r", fmt.Sprintf("%d", numRecovery), "-o", outputBase, - "--filepath-base", rootDir, + "--filepath-base", folderDir, "--filepath-format", "outrel", } args = append(args, b.cfg.ParparExtraArgs...) @@ -135,7 +136,7 @@ func (b *BinaryExecutor) CreateSet(ctx context.Context, files []fileinfo.FileInf slog.InfoContext(ctx, "Invoking parpar binary for set", "binary", b.cfg.ParparBinaryPath, "setName", setName, "files", len(inputs), - "outputBase", outputBase, "filepathBase", rootDir, + "outputBase", outputBase, "filepathBase", folderDir, "blockSize", blockSize, "recoverySlices", numRecovery) cmd := exec.CommandContext(ctx, b.cfg.ParparBinaryPath, args...) @@ -194,40 +195,6 @@ func (b *BinaryExecutor) CreateSet(ctx context.Context, files []fileinfo.FileInf return collectPar2SetFiles(ctx, dirPath, setName, outputBase+".par2"), nil } -// commonParentDir returns the deepest directory that contains every input. -// Used as parpar's --filepath-base so the embedded display name is the -// relative path of each file beneath that directory. -func commonParentDir(files []fileinfo.FileInfo) string { - if len(files) == 0 { - return "" - } - dir := filepath.Dir(files[0].Path) - for _, f := range files[1:] { - dir = commonPrefixDir(dir, filepath.Dir(f.Path)) - } - return dir -} - -func commonPrefixDir(a, b string) string { - if a == b { - return a - } - aParts := strings.Split(filepath.Clean(a), string(filepath.Separator)) - bParts := strings.Split(filepath.Clean(b), string(filepath.Separator)) - n := len(aParts) - if len(bParts) < n { - n = len(bParts) - } - i := 0 - for i < n && aParts[i] == bParts[i] { - i++ - } - if i == 0 { - return string(filepath.Separator) - } - return strings.Join(aParts[:i], string(filepath.Separator)) -} - // CreateInDirectory creates PAR2 files in the specified output directory using the parpar binary. func (b *BinaryExecutor) CreateInDirectory(ctx context.Context, files []fileinfo.FileInfo, outputDir string) ([]string, error) { var all []string diff --git a/internal/par2/par2.go b/internal/par2/par2.go index 37e5d77..7abaa65 100644 --- a/internal/par2/par2.go +++ b/internal/par2/par2.go @@ -30,11 +30,12 @@ const maxPar2Blocks = 32768 type Par2Executor interface { Create(ctx context.Context, files []fileinfo.FileInfo) ([]string, error) CreateInDirectory(ctx context.Context, files []fileinfo.FileInfo, outputDir string) ([]string, error) - // CreateSet bundles all input files into a single par2 set named setName, - // embedding each file's RelativePath (or basename when empty) in the - // FileDesc packet so downloaders such as SABnzbd can reconstruct the - // folder tree on disk after par2 verification. - CreateSet(ctx context.Context, files []fileinfo.FileInfo, outputDir, setName string) ([]string, error) + // CreateSet bundles all input files into a single par2 set named setName. + // folderDir is the on-disk root of the folder being posted (e.g. + // "/ShowS01"). Each FileDesc packet records the path of the + // file relative to folderDir (e.g. "extras/bonus.mkv") so downloaders + // such as SABnzbd can recreate the folder tree inside the job directory. + CreateSet(ctx context.Context, files []fileinfo.FileInfo, outputDir, setName, folderDir string) ([]string, error) } // NativeExecutor implements Par2Executor using the built-in Go PAR2 creator. @@ -209,9 +210,10 @@ func (p *NativeExecutor) CreateInDirectory(ctx context.Context, files []fileinfo } // CreateSet bundles all input files into a single par2 set named setName. -// Each FileDesc packet records the file's RelativePath (or filepath.Base -// when empty) so SABnzbd / NZBGet can recreate the folder tree on disk. -func (p *NativeExecutor) CreateSet(ctx context.Context, files []fileinfo.FileInfo, outputDir, setName string) ([]string, error) { +// folderDir is the on-disk root of the folder being posted. Each FileDesc +// packet records filepath.Rel(folderDir, file.Path) so SABnzbd / NZBGet +// can recreate the exact folder tree inside the job directory on disk. +func (p *NativeExecutor) CreateSet(ctx context.Context, files []fileinfo.FileInfo, outputDir, setName, folderDir string) ([]string, error) { if len(files) == 0 { return nil, fmt.Errorf("par2: no input files for set %q", setName) } @@ -288,8 +290,11 @@ func (p *NativeExecutor) CreateSet(ctx context.Context, files []fileinfo.FileInf par2Inputs := make([]par2go.InputFile, len(inputs)) for i, f := range inputs { - name := f.RelativePath - if name == "" { + // Compute the path relative to folderDir (e.g. "extras/bonus.mkv"). + // SABnzbd already creates a job folder named after the NZB title, so + // FileDesc must NOT include the top-level folder name as a prefix. + name, relErr := filepath.Rel(folderDir, f.Path) + if relErr != nil || name == "" || name == "." { name = filepath.Base(f.Path) } // Forward slashes only — par2go validates this and downloaders rely on it. diff --git a/internal/par2/par2_integration_test.go b/internal/par2/par2_integration_test.go index d029b67..11c84bc 100644 --- a/internal/par2/par2_integration_test.go +++ b/internal/par2/par2_integration_test.go @@ -446,7 +446,10 @@ func TestIntegration_NativeExecutor_CreateSet_EmbedsRelativePaths(t *testing.T) executor := New(750_000, cfg, nil) outDir := filepath.Join(root, "out") - created, err := executor.CreateSet(context.Background(), files, outDir, "folder1-testpkg") + // folderDir is the on-disk root of the folder being posted. + // FileDesc names must be relative to this dir (no top-level prefix). + folderDir := pkgDir + created, err := executor.CreateSet(context.Background(), files, outDir, "folder1-testpkg", folderDir) if err != nil { t.Fatalf("CreateSet: %v", err) } @@ -459,15 +462,47 @@ func TestIntegration_NativeExecutor_CreateSet_EmbedsRelativePaths(t *testing.T) t.Fatalf("expected main par2 %s: %v", mainPar2, err) } - got := extractFileDescNames(t, mainPar2) + got := dedupe(extractFileDescNames(t, mainPar2)) sort.Strings(got) + // SABnzbd already creates a job folder named "folder1-testpkg" from the + // NZB title; FileDesc paths must be relative to that folder root so files + // land at the correct depth (no double-nesting). want := []string{ - "folder1-testpkg/file1.txt", - "folder1-testpkg/folder2/file2.txt", - "folder1-testpkg/folder2/folder3/file3.txt", + "file1.txt", + "folder2/file2.txt", + "folder2/folder3/file3.txt", } - // FileDesc packets repeat across volume files; dedupe on read. - got = dedupe(got) + if !equalStrings(got, want) { + t.Errorf("FileDesc names: got %v, want %v", got, want) + } +} + +// TestIntegration_NativeExecutor_CreateSet_AllFilesInSubdir verifies that when +// all input files happen to live in one subdirectory (not spread across the +// folder root), the FileDesc still carries the subdir prefix — not bare basenames. +func TestIntegration_NativeExecutor_CreateSet_AllFilesInSubdir(t *testing.T) { + root := t.TempDir() + pkgDir := filepath.Join(root, "ShowS01") + mustMkdir(t, filepath.Join(pkgDir, "extras")) + fileA := createIntegrationTestFile(t, filepath.Join(pkgDir, "extras"), "bonus.mkv", 4096) + fileB := createIntegrationTestFile(t, filepath.Join(pkgDir, "extras"), "deleted.mkv", 4096) + + files := []fileinfo.FileInfo{ + {Path: fileA, Size: 4096, RelativePath: "ShowS01/extras/bonus.mkv"}, + {Path: fileB, Size: 4096, RelativePath: "ShowS01/extras/deleted.mkv"}, + } + cfg := &config.Par2Config{Redundancy: "10", MemoryLimit: 4 * 1024 * 1024 * 1024} + executor := New(750_000, cfg, nil) + + outDir := filepath.Join(root, "out") + folderDir := pkgDir // /ShowS01 + if _, err := executor.CreateSet(context.Background(), files, outDir, "ShowS01", folderDir); err != nil { + t.Fatalf("CreateSet: %v", err) + } + got := dedupe(extractFileDescNames(t, filepath.Join(outDir, "ShowS01.par2"))) + sort.Strings(got) + // Both files are in extras/ — FileDesc must preserve the subdir prefix. + want := []string{"extras/bonus.mkv", "extras/deleted.mkv"} if !equalStrings(got, want) { t.Errorf("FileDesc names: got %v, want %v", got, want) } @@ -484,7 +519,8 @@ func TestIntegration_NativeExecutor_CreateSet_FallsBackToBasename(t *testing.T) executor := New(750_000, cfg, nil) outDir := filepath.Join(root, "out") - if _, err := executor.CreateSet(context.Background(), files, outDir, "loose"); err != nil { + // folderDir == root: filepath.Rel(root, file) == "loose.bin" + if _, err := executor.CreateSet(context.Background(), files, outDir, "loose", root); err != nil { t.Fatalf("CreateSet: %v", err) } got := dedupe(extractFileDescNames(t, filepath.Join(outDir, "loose.par2"))) diff --git a/pkg/postie/postfolder_test.go b/pkg/postie/postfolder_test.go index 17d7976..87cdc64 100644 --- a/pkg/postie/postfolder_test.go +++ b/pkg/postie/postfolder_test.go @@ -81,7 +81,7 @@ func (m *mockPar2Executor) CreateInDirectory(_ context.Context, _ []fileinfo.Fil return created, nil } -func (m *mockPar2Executor) CreateSet(ctx context.Context, files []fileinfo.FileInfo, outputDir, _ string) ([]string, error) { +func (m *mockPar2Executor) CreateSet(ctx context.Context, files []fileinfo.FileInfo, outputDir, _, _ string) ([]string, error) { return m.CreateInDirectory(ctx, files, outputDir) } diff --git a/pkg/postie/postie.go b/pkg/postie/postie.go index 886f4d2..853f41d 100644 --- a/pkg/postie/postie.go +++ b/pkg/postie/postie.go @@ -499,8 +499,10 @@ func (p *Postie) postFolder(ctx context.Context, files []fileinfo.FileInfo, root "folder", folderName, "outputDir", par2OutputDir) } // If par2OutputDir is empty, CreateSet will use default behavior (temp/source dir) - - createdPar2Paths, err = p.par2runner.CreateSet(ctx, files, par2OutputDir, folderName) + // folderDir is the on-disk root of the folder; FileDesc paths are computed + // relative to it so SABnzbd recreates the tree inside the job folder. + folderDir := filepath.Join(rootDir, folderName) + createdPar2Paths, err = p.par2runner.CreateSet(ctx, files, par2OutputDir, folderName, folderDir) if err != nil { if !errors.Is(err, context.Canceled) { slog.ErrorContext(ctx, "Error during par2 creation. Upload will continue without par2.", "error", err) @@ -568,7 +570,8 @@ func (p *Postie) postFolder(ctx context.Context, files []fileinfo.FileInfo, root "folder", folderName, "outputDir", par2OutputDir) } - createdPar2Paths, err = p.par2runner.CreateSet(ctx, files, par2OutputDir, folderName) + folderDir := filepath.Join(rootDir, folderName) + createdPar2Paths, err = p.par2runner.CreateSet(ctx, files, par2OutputDir, folderName, folderDir) if err != nil { if !errors.Is(err, context.Canceled) { slog.ErrorContext(ctx, "Error during par2 creation. Upload will continue without par2.", "error", err)