From 9fd5004f35cd6d01a9cb2ebb95668044f0f9d34c Mon Sep 17 00:00:00 2001 From: Yan Song Date: Sat, 6 May 2023 11:16:25 +0000 Subject: [PATCH 1/4] solver: fix unnecessary force compression for provenance Signed-off-by: Yan Song --- solver/llbsolver/proc/provenance.go | 5 +++-- solver/llbsolver/proc/sbom.go | 3 ++- solver/llbsolver/provenance.go | 4 +++- solver/llbsolver/solver.go | 6 +++--- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/solver/llbsolver/proc/provenance.go b/solver/llbsolver/proc/provenance.go index 1af3af196..2edbdc39e 100644 --- a/solver/llbsolver/proc/provenance.go +++ b/solver/llbsolver/proc/provenance.go @@ -11,11 +11,12 @@ import ( "github.com/moby/buildkit/solver" "github.com/moby/buildkit/solver/llbsolver" "github.com/moby/buildkit/solver/result" + "github.com/moby/buildkit/util/compression" "github.com/pkg/errors" ) func ProvenanceProcessor(attrs map[string]string) llbsolver.Processor { - return func(ctx context.Context, res *llbsolver.Result, s *llbsolver.Solver, j *solver.Job) (*llbsolver.Result, error) { + return func(ctx context.Context, res *llbsolver.Result, s *llbsolver.Solver, j *solver.Job, comp compression.Config) (*llbsolver.Result, error) { ps, err := exptypes.ParsePlatforms(res.Metadata) if err != nil { return nil, err @@ -41,7 +42,7 @@ func ProvenanceProcessor(attrs map[string]string) llbsolver.Processor { return nil, errors.Errorf("could not find ref %s", p.ID) } - pc, err := llbsolver.NewProvenanceCreator(ctx, cp, ref, attrs, j) + pc, err := llbsolver.NewProvenanceCreator(ctx, cp, ref, attrs, j, comp) if err != nil { return nil, err } diff --git a/solver/llbsolver/proc/sbom.go b/solver/llbsolver/proc/sbom.go index 2d7e969ba..c0837218a 100644 --- a/solver/llbsolver/proc/sbom.go +++ b/solver/llbsolver/proc/sbom.go @@ -10,11 +10,12 @@ import ( "github.com/moby/buildkit/solver" "github.com/moby/buildkit/solver/llbsolver" "github.com/moby/buildkit/solver/result" + "github.com/moby/buildkit/util/compression" "github.com/pkg/errors" ) func SBOMProcessor(scannerRef string, useCache bool) llbsolver.Processor { - return func(ctx context.Context, res *llbsolver.Result, s *llbsolver.Solver, j *solver.Job) (*llbsolver.Result, error) { + return func(ctx context.Context, res *llbsolver.Result, s *llbsolver.Solver, j *solver.Job, comp compression.Config) (*llbsolver.Result, error) { // skip sbom generation if we already have an sbom if sbom.HasSBOM(res.Result) { return res, nil diff --git a/solver/llbsolver/provenance.go b/solver/llbsolver/provenance.go index b30581c85..f02e958e7 100644 --- a/solver/llbsolver/provenance.go +++ b/solver/llbsolver/provenance.go @@ -20,6 +20,7 @@ import ( "github.com/moby/buildkit/solver/llbsolver/provenance" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/source" + "github.com/moby/buildkit/util/compression" "github.com/moby/buildkit/worker" digest "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" @@ -372,7 +373,7 @@ type ProvenanceCreator struct { addLayers func() error } -func NewProvenanceCreator(ctx context.Context, cp *provenance.Capture, res solver.ResultProxy, attrs map[string]string, j *solver.Job) (*ProvenanceCreator, error) { +func NewProvenanceCreator(ctx context.Context, cp *provenance.Capture, res solver.ResultProxy, attrs map[string]string, j *solver.Job, comp compression.Config) (*ProvenanceCreator, error) { var reproducible bool if v, ok := attrs["reproducible"]; ok { b, err := strconv.ParseBool(v) @@ -449,6 +450,7 @@ func NewProvenanceCreator(ctx context.Context, cp *provenance.Capture, res solve ResolveRemotes: resolveRemotes, Mode: solver.CacheExportModeRemoteOnly, ExportRoots: true, + CompressionOpt: &comp, }); err != nil { return err } diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index 2f7ba61e5..ffe7ed570 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -93,7 +93,7 @@ type Solver struct { // Processor defines a processing function to be applied after solving, but // before exporting -type Processor func(ctx context.Context, result *Result, s *Solver, j *solver.Job) (*Result, error) +type Processor func(ctx context.Context, result *Result, s *Solver, j *solver.Job, comp compression.Config) (*Result, error) func New(opt Opt) (*Solver, error) { s := &Solver{ @@ -200,7 +200,7 @@ func (s *Solver) recordBuildHistory(ctx context.Context, id string, req frontend } makeProvenance := func(res solver.ResultProxy, cap *provenance.Capture) (*controlapi.Descriptor, func(), error) { - prc, err := NewProvenanceCreator(ctx2, cap, res, attrs, j) + prc, err := NewProvenanceCreator(ctx2, cap, res, attrs, j, exp.Exporter.Config().Compression()) if err != nil { return nil, nil, err } @@ -501,7 +501,7 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro } for _, post := range post { - res2, err := post(ctx, resProv, s, j) + res2, err := post(ctx, resProv, s, j, exp.Exporter.Config().Compression()) if err != nil { return nil, err } From 76aeaa55ff4e1c575644bffbf0a43bea92215fbe Mon Sep 17 00:00:00 2001 From: Yan Song Date: Sat, 6 May 2023 11:22:27 +0000 Subject: [PATCH 2/4] nydus: set default compression option to avoid recomputing Signed-off-by: Yan Song --- nydus/util/util.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/nydus/util/util.go b/nydus/util/util.go index f65cfe520..6524ebf78 100644 --- a/nydus/util/util.go +++ b/nydus/util/util.go @@ -15,12 +15,14 @@ type compressorKey struct{} type chunkDictDigestKey struct{} func WithContext(ctx context.Context, fsVersion string, compressor string, chunkDictDigest digest.Digest) context.Context { - if fsVersion == "" { - fsVersion = "5" + if fsVersion != "" { + ctx = context.WithValue(ctx, fsVersionKey{}, fsVersion) + } + + if compressor != "" { + ctx = context.WithValue(ctx, compressorKey{}, compressor) } - ctx = context.WithValue(ctx, fsVersionKey{}, fsVersion) - ctx = context.WithValue(ctx, compressorKey{}, compressor) if chunkDictDigest != "" { ctx = context.WithValue(ctx, chunkDictDigestKey{}, chunkDictDigest.String()) } @@ -29,8 +31,8 @@ func WithContext(ctx context.Context, fsVersion string, compressor string, chunk } func GetContext(ctx context.Context) (string, string, string) { - fsVersion := "" - compressor := "" + fsVersion := "5" + compressor := "lz4_block" chunkDictDigest := "" ctxValue := ctx.Value(fsVersionKey{}) From b2ef881dd4b2201bc5b630a0af91ea33c13d45bc Mon Sep 17 00:00:00 2001 From: Yan Song Date: Thu, 25 May 2023 09:18:46 +0000 Subject: [PATCH 3/4] nydus: use rafs v6 as default fs version - Use `--nydus-fs-version=6` as default; - Use `--nydus-compressor=zstd` as default; - Upgrade suggested nydus version to v2.1.6; Signed-off-by: Yan Song --- Dockerfile | 2 +- docs/nydus.md | 4 ++-- nydus/nydus.go | 4 ++-- nydus/util/util.go | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8869138b5..9c8a54f19 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ ARG CNI_VERSION=v1.1.1 ARG STARGZ_SNAPSHOTTER_VERSION=v0.13.0 ARG NERDCTL_VERSION=v1.0.0 ARG DNSNAME_VERSION=v1.3.1 -ARG NYDUS_VERSION=v2.1.0 +ARG NYDUS_VERSION=v2.1.6 ARG ALPINE_VERSION=3.17 diff --git a/docs/nydus.md b/docs/nydus.md index 0334c76c5..80c39a8ef 100644 --- a/docs/nydus.md +++ b/docs/nydus.md @@ -16,7 +16,7 @@ go build -tags=nydus -o ./bin/buildkitd ./cmd/buildkitd ### Building Nydus with BuildKit -Download `nydus-image` binary from [nydus release page](https://github.com/dragonflyoss/image-service/releases) (require v2.1.1 or higher), then put the `nydus-image` binary path into $PATH or specifying it on `NYDUS_BUILDER` environment variable for buildkitd: +Download `nydus-image` binary from [nydus release page](https://github.com/dragonflyoss/image-service/releases) (require v2.1.6 or higher), then put the `nydus-image` binary path into $PATH or specifying it on `NYDUS_BUILDER` environment variable for buildkitd: ``` env NYDUS_BUILDER=/path/to/nydus-image buildkitd ... @@ -33,7 +33,7 @@ buildctl build ... \ Available options: -- `nydus-fs-version`: Specify nydus image filesystem version, possible values: `5`, `6`, default `5`; +- `nydus-fs-version`: Specify nydus image filesystem version, possible values: `5`, `6`, default `6`; - `nydus-compressor`: Specify nydus image compressor, possible values: `none`, `lz4_block`, `zstd`, default `lz4_block`; - `nydus-chunk-dict-image`: Specify nydus chunk dict image reference for data de-duplication; diff --git a/nydus/nydus.go b/nydus/nydus.go index 2d808d2f0..5e80b6671 100644 --- a/nydus/nydus.go +++ b/nydus/nydus.go @@ -170,10 +170,10 @@ func loadChunkDict(ctx context.Context, registryHosts docker.RegistryHosts, sm * } if bootstrapDesc.Annotations[nydusify.LayerAnnotationFSVersion] == "" { - bootstrapDesc.Annotations[nydusify.LayerAnnotationFSVersion] = "5" + bootstrapDesc.Annotations[nydusify.LayerAnnotationFSVersion] = "6" } if nydusFSVersion == "" { - nydusFSVersion = "5" + nydusFSVersion = "6" } if bootstrapDesc.Annotations[nydusify.LayerAnnotationFSVersion] != nydusFSVersion { diff --git a/nydus/util/util.go b/nydus/util/util.go index 6524ebf78..22f58f641 100644 --- a/nydus/util/util.go +++ b/nydus/util/util.go @@ -31,8 +31,8 @@ func WithContext(ctx context.Context, fsVersion string, compressor string, chunk } func GetContext(ctx context.Context) (string, string, string) { - fsVersion := "5" - compressor := "lz4_block" + fsVersion := "6" + compressor := "zstd" chunkDictDigest := "" ctxValue := ctx.Value(fsVersionKey{}) From 62f6a4c63a3d274c8481c084091ce8f3d9769cd4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Sep 2023 18:22:44 +0000 Subject: [PATCH 4/4] build(deps): bump docker/setup-buildx-action from 2 to 3 Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 2 to 3. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 18 +++++++++--------- .github/workflows/buildx-image.yml | 2 +- .github/workflows/dockerd.yml | 4 ++-- .github/workflows/validate.yml | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d8d97ec4e..ecf961dc8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -50,7 +50,7 @@ jobs: uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: version: ${{ env.BUILDX_VERSION }} driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} @@ -109,7 +109,7 @@ jobs: uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: version: ${{ env.BUILDX_VERSION }} driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} @@ -166,7 +166,7 @@ jobs: uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: version: ${{ env.BUILDX_VERSION }} driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} @@ -197,7 +197,7 @@ jobs: uses: crazy-max/ghaction-github-runtime@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: version: ${{ env.BUILDX_VERSION }} driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} @@ -220,7 +220,7 @@ jobs: uses: crazy-max/ghaction-github-runtime@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: version: ${{ env.BUILDX_VERSION }} driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} @@ -313,7 +313,7 @@ jobs: uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: version: ${{ env.BUILDX_VERSION }} driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} @@ -373,7 +373,7 @@ jobs: uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: version: ${{ env.BUILDX_VERSION }} driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} @@ -409,7 +409,7 @@ jobs: uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: version: ${{ env.BUILDX_VERSION }} driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} @@ -481,7 +481,7 @@ jobs: uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: version: ${{ env.BUILDX_VERSION }} driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} diff --git a/.github/workflows/buildx-image.yml b/.github/workflows/buildx-image.yml index d9c655480..509069ba0 100644 --- a/.github/workflows/buildx-image.yml +++ b/.github/workflows/buildx-image.yml @@ -45,7 +45,7 @@ jobs: steps: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: version: ${{ env.BUILDX_VERSION }} buildkitd-flags: --debug diff --git a/.github/workflows/dockerd.yml b/.github/workflows/dockerd.yml index f760f55e6..564725707 100644 --- a/.github/workflows/dockerd.yml +++ b/.github/workflows/dockerd.yml @@ -43,7 +43,7 @@ jobs: return true; - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: version: ${{ env.BUILDX_VERSION }} driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} @@ -111,7 +111,7 @@ jobs: uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: version: ${{ env.BUILDX_VERSION }} driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 0042f1c9a..f6d18b97c 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -38,7 +38,7 @@ jobs: uses: actions/checkout@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: version: ${{ env.BUILDX_VERSION }} driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}