diff --git a/.github/workflows/.test.yml b/.github/workflows/.test.yml index 707b717f2..05510d27f 100644 --- a/.github/workflows/.test.yml +++ b/.github/workflows/.test.yml @@ -96,6 +96,7 @@ jobs: name: Build uses: docker/bake-action@82490499d2e5613fcead7e128237ef0b0ea210f7 # v7.0.0 with: + provenance: false targets: integration-tests-base set: | *.cache-from=type=gha,scope=${{ inputs.cache_scope }} @@ -165,6 +166,7 @@ jobs: name: Build test image uses: docker/bake-action@82490499d2e5613fcead7e128237ef0b0ea210f7 # v7.0.0 with: + provenance: false targets: integration-tests set: | *.cache-from=type=gha,scope=${{ inputs.cache_scope }} @@ -180,8 +182,9 @@ jobs: TEST_REPORT_SUFFIX: -${{ env.TEST_REPORT_NAME }} TEST_COVERAGE: 1 TESTPKGS: ${{ matrix.pkg }} - SKIP_INTEGRATION_TESTS: ${{ matrix.skip-integration-tests }} - CACHE_FROM: type=gha,scope=${{ inputs.cache_scope }} + SKIP_INTEGRATION_TESTS: 1 + # TODO(jhorsts): errors with HTTP 400 + # CACHE_FROM: type=gha,scope=${{ inputs.cache_scope }} - name: Send to Codecov if: always() diff --git a/.github/workflows/buildkit.yml b/.github/workflows/buildkit.yml index f13691ebd..2efd0e74c 100644 --- a/.github/workflows/buildkit.yml +++ b/.github/workflows/buildkit.yml @@ -183,7 +183,7 @@ jobs: const defaultBase = core.getInput('default-base'); const ref = core.getInput('ref'); const imageName = core.getInput('image-name'); - + function getTagSuffixAndLatest(base, target) { let tagSuffix = ''; if (target) { @@ -204,20 +204,20 @@ jobs: } return { tagSuffix, tagLatest }; } - + const matrix = [ { base: 'alpine' }, { base: 'alpine', target: 'rootless'}, { base: 'ubuntu', buildTags: 'nvidia venus' } ] - + for (const entry of matrix) { const { tagSuffix, tagLatest } = getTagSuffixAndLatest(entry.base, entry.target); entry.imageName = imageName; entry.tagSuffix = tagSuffix; entry.tagLatest = tagLatest; } - + core.info(JSON.stringify(matrix, null, 2)); core.setOutput('includes', JSON.stringify(matrix)); @@ -311,7 +311,7 @@ jobs: name: Login to DockerHub uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} + username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Scout diff --git a/.github/workflows/buildx-image.yml b/.github/workflows/buildx-image.yml index cc8a584b6..8263a68f7 100644 --- a/.github/workflows/buildx-image.yml +++ b/.github/workflows/buildx-image.yml @@ -58,7 +58,7 @@ jobs: if: github.event.inputs.dry-run != 'true' uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} + username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Create diff --git a/.github/workflows/dockerd.yml b/.github/workflows/dockerd.yml index a0bd6a77b..ca9bea543 100644 --- a/.github/workflows/dockerd.yml +++ b/.github/workflows/dockerd.yml @@ -133,7 +133,8 @@ jobs: TESTPKGS: "${{ matrix.pkg }}" TESTFLAGS: "${{ env.TESTFLAGS }} --run=//worker=${{ matrix.worker }}$" SKIP_INTEGRATION_TESTS: "${{ matrix.skip-integration-tests }}" - CACHE_FROM: "type=gha,scope=build-integration-tests" + # TODO(jhorsts): replace with upstream + # CACHE_FROM: "type=gha,scope=build-integration-tests" BUILDKIT_INTEGRATION_DOCKERD_FLAGS: | --bip=10.66.66.1/24 --default-address-pool=base=10.66.66.0/16,size=24 diff --git a/.github/workflows/earthly-count.yml b/.github/workflows/earthly-count.yml new file mode 100644 index 000000000..2bcdb68b3 --- /dev/null +++ b/.github/workflows/earthly-count.yml @@ -0,0 +1,185 @@ +name: Track Earthly to Earthbuild Progress + +on: + pull_request: + types: [opened, synchronize] + +jobs: + count-earthly: + runs-on: ubuntu-24.04-arm + permissions: + contents: read + pull-requests: write + + steps: + - name: Checkout PR branch + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + + - name: Count occurrences in PR branch + id: count_pr + run: | + # Count total occurrences + total_count=$(git grep -i "earthly" -- ':!.github/workflows/earthly-count.yml' 2>/dev/null | wc -l || echo "0") + echo "total_count=$total_count" >> $GITHUB_OUTPUT + + # Count by file type + go_count=$(git grep -i "earthly" -- "*.go" 2>/dev/null | wc -l || echo "0") + md_count=$(git grep -i "earthly" -- "*.md" 2>/dev/null | wc -l || echo "0") + earthfile_count=$(($(git grep -i "earthly" -- "Earthfile" 2>/dev/null | wc -l || echo "0") + $(git grep -i "earthly" -- "*.earth" 2>/dev/null | wc -l || echo "0"))) + + echo "go_count=$go_count" >> $GITHUB_OUTPUT + echo "md_count=$md_count" >> $GITHUB_OUTPUT + echo "earthfile_count=$earthfile_count" >> $GITHUB_OUTPUT + + echo "PR branch - Total: $total_count (Go: $go_count, MD: $md_count, Earthfiles: $earthfile_count)" + + - name: Checkout main branch + run: | + git checkout origin/main + + - name: Count occurrences in main branch + id: count_main + run: | + # Count total occurrences + total_count=$(git grep -i "earthly" -- ':!.github/workflows/earthly-count.yml' 2>/dev/null | wc -l || echo "0") + echo "main_total_count=$total_count" >> $GITHUB_OUTPUT + + # Count by file type + go_count=$(git grep -i "earthly" -- "*.go" 2>/dev/null | wc -l || echo "0") + md_count=$(git grep -i "earthly" -- "*.md" 2>/dev/null | wc -l || echo "0") + earthfile_count=$(($(git grep -i "earthly" -- "Earthfile" 2>/dev/null | wc -l || echo "0") + $(git grep -i "earthly" -- "*.earth" 2>/dev/null | wc -l || echo "0"))) + + echo "main_go_count=$go_count" >> $GITHUB_OUTPUT + echo "main_md_count=$md_count" >> $GITHUB_OUTPUT + echo "main_earthfile_count=$earthfile_count" >> $GITHUB_OUTPUT + + echo "Main branch - Total: $total_count (Go: $go_count, MD: $md_count, Earthfiles: $earthfile_count)" + + - name: Calculate difference + id: calculate + run: | + pr_count=${{ steps.count_pr.outputs.total_count }} + main_count=${{ steps.count_main.outputs.main_total_count }} + difference=$((main_count - pr_count)) + + # Calculate percentage with proper formatting + if [ $main_count -gt 0 ]; then + # Use awk for better decimal handling + percentage=$(awk "BEGIN {printf \"%.2f\", $difference * 100 / $main_count}") + else + percentage="0.00" + fi + + echo "difference=$difference" >> $GITHUB_OUTPUT + echo "percentage=$percentage" >> $GITHUB_OUTPUT + echo "pr_count=$pr_count" >> $GITHUB_OUTPUT + echo "main_count=$main_count" >> $GITHUB_OUTPUT + + # Calculate differences by type + go_diff=$((${{ steps.count_main.outputs.main_go_count }} - ${{ steps.count_pr.outputs.go_count }})) + md_diff=$((${{ steps.count_main.outputs.main_md_count }} - ${{ steps.count_pr.outputs.md_count }})) + earthfile_diff=$((${{ steps.count_main.outputs.main_earthfile_count }} - ${{ steps.count_pr.outputs.earthfile_count }})) + + echo "go_diff=$go_diff" >> $GITHUB_OUTPUT + echo "md_diff=$md_diff" >> $GITHUB_OUTPUT + echo "earthfile_diff=$earthfile_diff" >> $GITHUB_OUTPUT + + - name: Comment on PR + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + with: + script: | + const prCount = ${{ steps.calculate.outputs.pr_count }}; + const mainCount = ${{ steps.calculate.outputs.main_count }}; + const difference = ${{ steps.calculate.outputs.difference }}; + const percentage = '${{ steps.calculate.outputs.percentage }}'; + + // File type differences + const goDiff = ${{ steps.calculate.outputs.go_diff }}; + const mdDiff = ${{ steps.calculate.outputs.md_diff }}; + const earthfileDiff = ${{ steps.calculate.outputs.earthfile_diff }}; + + let emoji = '📊'; + let message = ''; + + if (difference > 0) { + emoji = '🎉'; + message = `Great progress! You've reduced "earthly" occurrences by **${difference}** (${percentage}%)`; + } else if (difference < 0) { + emoji = '⚠️'; + message = `Warning: "earthly" occurrences have increased by **${Math.abs(difference)}** (${Math.abs(parseFloat(percentage))}%)`; + } else { + emoji = '➖'; + message = 'No change in "earthly" occurrences'; + } + + // Build detailed breakdown + let breakdown = ''; + if (goDiff !== 0 || mdDiff !== 0 || earthfileDiff !== 0) { + breakdown = ` + + ### 📁 Changes by file type: + | File Type | Change | + |-----------|--------| + | Go files (.go) | ${goDiff > 0 ? '✅ -' + goDiff : goDiff < 0 ? '❌ +' + Math.abs(goDiff) : '➖ No change'} | + | Documentation (.md) | ${mdDiff > 0 ? '✅ -' + mdDiff : mdDiff < 0 ? '❌ +' + Math.abs(mdDiff) : '➖ No change'} | + | Earthfiles | ${earthfileDiff > 0 ? '✅ -' + earthfileDiff : earthfileDiff < 0 ? '❌ +' + Math.abs(earthfileDiff) : '➖ No change'} |`; + } + + const body = `## ${emoji} Are we earthbuild yet? + + ${message} + + ### 📈 Overall Progress + | Branch | Total Count | + |--------|-------------| + | main | ${mainCount} | + | This PR | ${prCount} | + | **Difference** | **${difference > 0 ? '-' : '+'}${Math.abs(difference)}** ${difference !== 0 ? `(${Math.abs(parseFloat(percentage))}%)` : ''} | + ${breakdown} + + --- + *Keep up the great work migrating from Earthly to Earthbuild!* 🚀 + +
+ 💡 Tips for finding more occurrences + + Run locally to see detailed breakdown: + \`\`\`bash + ./.github/scripts/count-earthly.sh + \`\`\` + + **Note that the goal is not to reach 0.** + There is anticipated to be at least _some_ occurences of \`earthly\` in the source code due to backwards compatibility with config files and language constructs. +
`; + + // Find existing comment + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + + const botComment = comments.find(comment => + comment.user.type === 'Bot' && + comment.body.includes('Are we earthbuild yet?') + ); + + if (botComment) { + // Update existing comment + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: body + }); + } else { + // Create new comment + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: body + }); + } diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index dcfcc04dd..b72af46e9 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -36,7 +36,6 @@ jobs: pkgs: ./frontend/dockerfile kinds: | integration - dockerfile codecov_flags: dockerfile-frontend secrets: codecov_token: ${{ secrets.CODECOV_TOKEN }} @@ -211,7 +210,7 @@ jobs: name: Login to DockerHub uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} + username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Scout diff --git a/Dockerfile b/Dockerfile index 6f60be2e2..2576fb860 100644 --- a/Dockerfile +++ b/Dockerfile @@ -72,6 +72,7 @@ ENV GOFLAGS=-mod=vendor # buildkit-version builds stage with version/revision info FROM buildkit-base AS buildkit-version # TODO: PKG should be inferred from go modules +ARG RELEASE_VERSION=v0.0.0+earthlyunknown RUN --mount=target=. <<'EOT' # if git is worktree (file starting with gitdir:) then skip verions check if [ -f .git ] && head -1 .git | grep -q "^gitdir:"; then @@ -87,9 +88,9 @@ RUN --mount=target=. <<'EOT' exit 1 fi set -ex - export PKG=github.com/moby/buildkit VERSION=$(git describe --match 'v[0-9]*' --dirty='.m' --always --tags) REVISION=$(git rev-parse HEAD)$(if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi) COMMIT_DATE=$(git show -s --format=%cI HEAD); - echo "-X ${PKG}/version.Version=${VERSION} -X ${PKG}/version.Revision=${REVISION} -X ${PKG}/version.Package=${PKG}" > /tmp/.ldflags; - echo -n "${VERSION}" > /tmp/.version; + export PKG=github.com/moby/buildkit EARTHLY_PKG=github.com/EarthBuild/buildkit VERSION=$(git describe --match 'v[0-9]*' --dirty='.m' --always --tags) REVISION=$(git rev-parse HEAD)$(if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi) COMMIT_DATE=$(git show -s --format=%cI HEAD); + echo "-X ${PKG}/version.Version=${RELEASE_VERSION} -X ${PKG}/version.Revision=${REVISION} -X ${PKG}/version.Package=${EARTHLY_PKG}" > /tmp/.ldflags; + echo -n "${RELEASE_VERSION}" > /tmp/.version; echo -n "${COMMIT_DATE}" > /tmp/.commit_date; EOT diff --git a/Earthfile b/Earthfile new file mode 100644 index 000000000..23fba1839 --- /dev/null +++ b/Earthfile @@ -0,0 +1,14 @@ + +VERSION 0.6 + +FROM alpine:3.13 +WORKDIR /buildkit + +# build buildkit image +build: + ARG RELEASE_VERSION=v0.0.0+earthlyunknown + FROM DOCKERFILE --build-arg RELEASE_VERSION=$RELEASE_VERSION --target buildkit-linux . + +code: + COPY . . + SAVE ARTIFACT /buildkit diff --git a/api/services/control/control.pb.go b/api/services/control/control.pb.go index 427b6ec73..a1dad18ff 100644 --- a/api/services/control/control.pb.go +++ b/api/services/control/control.pb.go @@ -738,6 +738,7 @@ func (x *SolveResponse) GetExporterResponse() map[string]string { type StatusRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Ref string `protobuf:"bytes,1,opt,name=Ref,proto3" json:"Ref,omitempty"` + StatsStream bool `protobuf:"varint,99,opt,name=statsStream,proto3" json:"statsStream,omitempty"` // earthly unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -779,6 +780,13 @@ func (x *StatusRequest) GetRef() string { return "" } +func (x *StatusRequest) GetStatsStream() bool { + if x != nil { + return x.StatsStream + } + return false +} + type StatusResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Vertexes []*Vertex `protobuf:"bytes,1,rep,name=vertexes,proto3" json:"vertexes,omitempty"` @@ -1378,8 +1386,11 @@ func (*InfoRequest) Descriptor() ([]byte, []int) { type InfoResponse struct { state protoimpl.MessageState `protogen:"open.v1"` BuildkitVersion *types.BuildkitVersion `protobuf:"bytes,1,opt,name=buildkitVersion,proto3" json:"buildkitVersion,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Earthly-specific. + NumSessions uint64 `protobuf:"varint,101,opt,name=numSessions,proto3" json:"numSessions,omitempty"` + SecondsIdle uint64 `protobuf:"varint,102,opt,name=secondsIdle,proto3" json:"secondsIdle,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *InfoResponse) Reset() { @@ -1419,6 +1430,260 @@ func (x *InfoResponse) GetBuildkitVersion() *types.BuildkitVersion { return nil } +func (x *InfoResponse) GetNumSessions() uint64 { + if x != nil { + return x.NumSessions + } + return 0 +} + +func (x *InfoResponse) GetSecondsIdle() uint64 { + if x != nil { + return x.SecondsIdle + } + return 0 +} + +type ShutdownIfIdleRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ShutdownIfIdleRequest) Reset() { + *x = ShutdownIfIdleRequest{} + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ShutdownIfIdleRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ShutdownIfIdleRequest) ProtoMessage() {} + +func (x *ShutdownIfIdleRequest) ProtoReflect() protoreflect.Message { + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ShutdownIfIdleRequest.ProtoReflect.Descriptor instead. +func (*ShutdownIfIdleRequest) Descriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_api_services_control_control_proto_rawDescGZIP(), []int{19} +} + +type ShutdownIfIdleResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + WillShutdown bool `protobuf:"varint,1,opt,name=willShutdown,proto3" json:"willShutdown,omitempty"` + NumSessions uint64 `protobuf:"varint,2,opt,name=numSessions,proto3" json:"numSessions,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ShutdownIfIdleResponse) Reset() { + *x = ShutdownIfIdleResponse{} + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ShutdownIfIdleResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ShutdownIfIdleResponse) ProtoMessage() {} + +func (x *ShutdownIfIdleResponse) ProtoReflect() protoreflect.Message { + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ShutdownIfIdleResponse.ProtoReflect.Descriptor instead. +func (*ShutdownIfIdleResponse) Descriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_api_services_control_control_proto_rawDescGZIP(), []int{20} +} + +func (x *ShutdownIfIdleResponse) GetWillShutdown() bool { + if x != nil { + return x.WillShutdown + } + return false +} + +func (x *ShutdownIfIdleResponse) GetNumSessions() uint64 { + if x != nil { + return x.NumSessions + } + return 0 +} + +type ReserveRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReserveRequest) Reset() { + *x = ReserveRequest{} + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReserveRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReserveRequest) ProtoMessage() {} + +func (x *ReserveRequest) ProtoReflect() protoreflect.Message { + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[21] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReserveRequest.ProtoReflect.Descriptor instead. +func (*ReserveRequest) Descriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_api_services_control_control_proto_rawDescGZIP(), []int{21} +} + +type ReserveResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReserveResponse) Reset() { + *x = ReserveResponse{} + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReserveResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReserveResponse) ProtoMessage() {} + +func (x *ReserveResponse) ProtoReflect() protoreflect.Message { + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[22] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReserveResponse.ProtoReflect.Descriptor instead. +func (*ReserveResponse) Descriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_api_services_control_control_proto_rawDescGZIP(), []int{22} +} + +type SessionHistoryRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SessionHistoryRequest) Reset() { + *x = SessionHistoryRequest{} + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SessionHistoryRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SessionHistoryRequest) ProtoMessage() {} + +func (x *SessionHistoryRequest) ProtoReflect() protoreflect.Message { + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[23] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SessionHistoryRequest.ProtoReflect.Descriptor instead. +func (*SessionHistoryRequest) Descriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_api_services_control_control_proto_rawDescGZIP(), []int{23} +} + +type SessionHistoryResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + History []*SessionHistoryResponse_History `protobuf:"bytes,1,rep,name=history,proto3" json:"history,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SessionHistoryResponse) Reset() { + *x = SessionHistoryResponse{} + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SessionHistoryResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SessionHistoryResponse) ProtoMessage() {} + +func (x *SessionHistoryResponse) ProtoReflect() protoreflect.Message { + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[24] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SessionHistoryResponse.ProtoReflect.Descriptor instead. +func (*SessionHistoryResponse) Descriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_api_services_control_control_proto_rawDescGZIP(), []int{24} +} + +func (x *SessionHistoryResponse) GetHistory() []*SessionHistoryResponse_History { + if x != nil { + return x.History + } + return nil +} + type BuildHistoryRequest struct { state protoimpl.MessageState `protogen:"open.v1"` ActiveOnly bool `protobuf:"varint,1,opt,name=ActiveOnly,proto3" json:"ActiveOnly,omitempty"` @@ -1432,7 +1697,7 @@ type BuildHistoryRequest struct { func (x *BuildHistoryRequest) Reset() { *x = BuildHistoryRequest{} - mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[19] + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1444,7 +1709,7 @@ func (x *BuildHistoryRequest) String() string { func (*BuildHistoryRequest) ProtoMessage() {} func (x *BuildHistoryRequest) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[19] + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1457,7 +1722,7 @@ func (x *BuildHistoryRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use BuildHistoryRequest.ProtoReflect.Descriptor instead. func (*BuildHistoryRequest) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_api_services_control_control_proto_rawDescGZIP(), []int{19} + return file_github_com_moby_buildkit_api_services_control_control_proto_rawDescGZIP(), []int{25} } func (x *BuildHistoryRequest) GetActiveOnly() bool { @@ -1505,7 +1770,7 @@ type BuildHistoryEvent struct { func (x *BuildHistoryEvent) Reset() { *x = BuildHistoryEvent{} - mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[20] + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1517,7 +1782,7 @@ func (x *BuildHistoryEvent) String() string { func (*BuildHistoryEvent) ProtoMessage() {} func (x *BuildHistoryEvent) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[20] + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1530,7 +1795,7 @@ func (x *BuildHistoryEvent) ProtoReflect() protoreflect.Message { // Deprecated: Use BuildHistoryEvent.ProtoReflect.Descriptor instead. func (*BuildHistoryEvent) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_api_services_control_control_proto_rawDescGZIP(), []int{20} + return file_github_com_moby_buildkit_api_services_control_control_proto_rawDescGZIP(), []int{26} } func (x *BuildHistoryEvent) GetType() BuildHistoryEventType { @@ -1574,7 +1839,7 @@ type BuildHistoryRecord struct { func (x *BuildHistoryRecord) Reset() { *x = BuildHistoryRecord{} - mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[21] + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1586,7 +1851,7 @@ func (x *BuildHistoryRecord) String() string { func (*BuildHistoryRecord) ProtoMessage() {} func (x *BuildHistoryRecord) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[21] + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[27] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1599,7 +1864,7 @@ func (x *BuildHistoryRecord) ProtoReflect() protoreflect.Message { // Deprecated: Use BuildHistoryRecord.ProtoReflect.Descriptor instead. func (*BuildHistoryRecord) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_api_services_control_control_proto_rawDescGZIP(), []int{21} + return file_github_com_moby_buildkit_api_services_control_control_proto_rawDescGZIP(), []int{27} } func (x *BuildHistoryRecord) GetRef() string { @@ -1747,7 +2012,7 @@ type UpdateBuildHistoryRequest struct { func (x *UpdateBuildHistoryRequest) Reset() { *x = UpdateBuildHistoryRequest{} - mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[22] + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1759,7 +2024,7 @@ func (x *UpdateBuildHistoryRequest) String() string { func (*UpdateBuildHistoryRequest) ProtoMessage() {} func (x *UpdateBuildHistoryRequest) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[22] + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[28] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1772,7 +2037,7 @@ func (x *UpdateBuildHistoryRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateBuildHistoryRequest.ProtoReflect.Descriptor instead. func (*UpdateBuildHistoryRequest) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_api_services_control_control_proto_rawDescGZIP(), []int{22} + return file_github_com_moby_buildkit_api_services_control_control_proto_rawDescGZIP(), []int{28} } func (x *UpdateBuildHistoryRequest) GetRef() string { @@ -1811,7 +2076,7 @@ type UpdateBuildHistoryResponse struct { func (x *UpdateBuildHistoryResponse) Reset() { *x = UpdateBuildHistoryResponse{} - mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[23] + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1823,7 +2088,7 @@ func (x *UpdateBuildHistoryResponse) String() string { func (*UpdateBuildHistoryResponse) ProtoMessage() {} func (x *UpdateBuildHistoryResponse) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[23] + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1836,7 +2101,7 @@ func (x *UpdateBuildHistoryResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateBuildHistoryResponse.ProtoReflect.Descriptor instead. func (*UpdateBuildHistoryResponse) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_api_services_control_control_proto_rawDescGZIP(), []int{23} + return file_github_com_moby_buildkit_api_services_control_control_proto_rawDescGZIP(), []int{29} } type Descriptor struct { @@ -1851,7 +2116,7 @@ type Descriptor struct { func (x *Descriptor) Reset() { *x = Descriptor{} - mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[24] + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1863,7 +2128,7 @@ func (x *Descriptor) String() string { func (*Descriptor) ProtoMessage() {} func (x *Descriptor) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[24] + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[30] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1876,7 +2141,7 @@ func (x *Descriptor) ProtoReflect() protoreflect.Message { // Deprecated: Use Descriptor.ProtoReflect.Descriptor instead. func (*Descriptor) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_api_services_control_control_proto_rawDescGZIP(), []int{24} + return file_github_com_moby_buildkit_api_services_control_control_proto_rawDescGZIP(), []int{30} } func (x *Descriptor) GetMediaType() string { @@ -1918,7 +2183,7 @@ type BuildResultInfo struct { func (x *BuildResultInfo) Reset() { *x = BuildResultInfo{} - mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[25] + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1930,7 +2195,7 @@ func (x *BuildResultInfo) String() string { func (*BuildResultInfo) ProtoMessage() {} func (x *BuildResultInfo) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[25] + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[31] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1943,7 +2208,7 @@ func (x *BuildResultInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use BuildResultInfo.ProtoReflect.Descriptor instead. func (*BuildResultInfo) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_api_services_control_control_proto_rawDescGZIP(), []int{25} + return file_github_com_moby_buildkit_api_services_control_control_proto_rawDescGZIP(), []int{31} } func (x *BuildResultInfo) GetResultDeprecated() *Descriptor { @@ -1980,7 +2245,7 @@ type Exporter struct { func (x *Exporter) Reset() { *x = Exporter{} - mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[26] + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1992,7 +2257,7 @@ func (x *Exporter) String() string { func (*Exporter) ProtoMessage() {} func (x *Exporter) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[26] + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[32] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2005,7 +2270,7 @@ func (x *Exporter) ProtoReflect() protoreflect.Message { // Deprecated: Use Exporter.ProtoReflect.Descriptor instead. func (*Exporter) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_api_services_control_control_proto_rawDescGZIP(), []int{26} + return file_github_com_moby_buildkit_api_services_control_control_proto_rawDescGZIP(), []int{32} } func (x *Exporter) GetType() string { @@ -2022,6 +2287,66 @@ func (x *Exporter) GetAttrs() map[string]string { return nil } +type SessionHistoryResponse_History struct { + state protoimpl.MessageState `protogen:"open.v1"` + SessionID string `protobuf:"bytes,1,opt,name=sessionID,proto3" json:"sessionID,omitempty"` + Start *timestamp.Timestamp `protobuf:"bytes,2,opt,name=start,proto3" json:"start,omitempty"` + End *timestamp.Timestamp `protobuf:"bytes,3,opt,name=end,proto3" json:"end,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SessionHistoryResponse_History) Reset() { + *x = SessionHistoryResponse_History{} + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[39] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SessionHistoryResponse_History) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SessionHistoryResponse_History) ProtoMessage() {} + +func (x *SessionHistoryResponse_History) ProtoReflect() protoreflect.Message { + mi := &file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes[39] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SessionHistoryResponse_History.ProtoReflect.Descriptor instead. +func (*SessionHistoryResponse_History) Descriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_api_services_control_control_proto_rawDescGZIP(), []int{24, 0} +} + +func (x *SessionHistoryResponse_History) GetSessionID() string { + if x != nil { + return x.SessionID + } + return "" +} + +func (x *SessionHistoryResponse_History) GetStart() *timestamp.Timestamp { + if x != nil { + return x.Start + } + return nil +} + +func (x *SessionHistoryResponse_History) GetEnd() *timestamp.Timestamp { + if x != nil { + return x.End + } + return nil +} + var File_github_com_moby_buildkit_api_services_control_control_proto protoreflect.FileDescriptor const file_github_com_moby_buildkit_api_services_control_control_proto_rawDesc = "" + @@ -2107,9 +2432,10 @@ const file_github_com_moby_buildkit_api_services_control_control_proto_rawDesc = "\x10ExporterResponse\x18\x01 \x03(\v25.moby.buildkit.v1.SolveResponse.ExporterResponseEntryR\x10ExporterResponse\x1aC\n" + "\x15ExporterResponseEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"!\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"C\n" + "\rStatusRequest\x12\x10\n" + - "\x03Ref\x18\x01 \x01(\tR\x03Ref\"\xf0\x01\n" + + "\x03Ref\x18\x01 \x01(\tR\x03Ref\x12 \n" + + "\vstatsStream\x18c \x01(\bR\vstatsStream\"\xf0\x01\n" + "\x0eStatusResponse\x124\n" + "\bvertexes\x18\x01 \x03(\v2\x18.moby.buildkit.v1.VertexR\bvertexes\x12:\n" + "\bstatuses\x18\x02 \x03(\v2\x1e.moby.buildkit.v1.VertexStatusR\bstatuses\x12/\n" + @@ -2152,9 +2478,24 @@ const file_github_com_moby_buildkit_api_services_control_control_proto_rawDesc = "\x06filter\x18\x01 \x03(\tR\x06filter\"S\n" + "\x13ListWorkersResponse\x12<\n" + "\x06record\x18\x01 \x03(\v2$.moby.buildkit.v1.types.WorkerRecordR\x06record\"\r\n" + - "\vInfoRequest\"a\n" + + "\vInfoRequest\"\xa5\x01\n" + "\fInfoResponse\x12Q\n" + - "\x0fbuildkitVersion\x18\x01 \x01(\v2'.moby.buildkit.v1.types.BuildkitVersionR\x0fbuildkitVersion\"\x93\x01\n" + + "\x0fbuildkitVersion\x18\x01 \x01(\v2'.moby.buildkit.v1.types.BuildkitVersionR\x0fbuildkitVersion\x12 \n" + + "\vnumSessions\x18e \x01(\x04R\vnumSessions\x12 \n" + + "\vsecondsIdle\x18f \x01(\x04R\vsecondsIdle\"\x17\n" + + "\x15ShutdownIfIdleRequest\"^\n" + + "\x16ShutdownIfIdleResponse\x12\"\n" + + "\fwillShutdown\x18\x01 \x01(\bR\fwillShutdown\x12 \n" + + "\vnumSessions\x18\x02 \x01(\x04R\vnumSessions\"\x10\n" + + "\x0eReserveRequest\"\x11\n" + + "\x0fReserveResponse\"\x17\n" + + "\x15SessionHistoryRequest\"\xee\x01\n" + + "\x16SessionHistoryResponse\x12J\n" + + "\ahistory\x18\x01 \x03(\v20.moby.buildkit.v1.SessionHistoryResponse.HistoryR\ahistory\x1a\x87\x01\n" + + "\aHistory\x12\x1c\n" + + "\tsessionID\x18\x01 \x01(\tR\tsessionID\x120\n" + + "\x05start\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\x05start\x12,\n" + + "\x03end\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\x03end\"\x93\x01\n" + "\x13BuildHistoryRequest\x12\x1e\n" + "\n" + "ActiveOnly\x18\x01 \x01(\bR\n" + @@ -2231,7 +2572,7 @@ const file_github_com_moby_buildkit_api_services_control_control_proto_rawDesc = "\x15BuildHistoryEventType\x12\v\n" + "\aSTARTED\x10\x00\x12\f\n" + "\bCOMPLETE\x10\x01\x12\v\n" + - "\aDELETED\x10\x022\x89\x06\n" + + "\aDELETED\x10\x022\xa3\b\n" + "\aControl\x12T\n" + "\tDiskUsage\x12\".moby.buildkit.v1.DiskUsageRequest\x1a#.moby.buildkit.v1.DiskUsageResponse\x12H\n" + "\x05Prune\x12\x1e.moby.buildkit.v1.PruneRequest\x1a\x1d.moby.buildkit.v1.UsageRecord0\x01\x12H\n" + @@ -2239,7 +2580,10 @@ const file_github_com_moby_buildkit_api_services_control_control_proto_rawDesc = "\x06Status\x12\x1f.moby.buildkit.v1.StatusRequest\x1a .moby.buildkit.v1.StatusResponse0\x01\x12M\n" + "\aSession\x12\x1e.moby.buildkit.v1.BytesMessage\x1a\x1e.moby.buildkit.v1.BytesMessage(\x010\x01\x12Z\n" + "\vListWorkers\x12$.moby.buildkit.v1.ListWorkersRequest\x1a%.moby.buildkit.v1.ListWorkersResponse\x12E\n" + - "\x04Info\x12\x1d.moby.buildkit.v1.InfoRequest\x1a\x1e.moby.buildkit.v1.InfoResponse\x12b\n" + + "\x04Info\x12\x1d.moby.buildkit.v1.InfoRequest\x1a\x1e.moby.buildkit.v1.InfoResponse\x12c\n" + + "\x0eShutdownIfIdle\x12'.moby.buildkit.v1.ShutdownIfIdleRequest\x1a(.moby.buildkit.v1.ShutdownIfIdleResponse\x12N\n" + + "\aReserve\x12 .moby.buildkit.v1.ReserveRequest\x1a!.moby.buildkit.v1.ReserveResponse\x12c\n" + + "\x0eSessionHistory\x12'.moby.buildkit.v1.SessionHistoryRequest\x1a(.moby.buildkit.v1.SessionHistoryResponse\x12b\n" + "\x12ListenBuildHistory\x12%.moby.buildkit.v1.BuildHistoryRequest\x1a#.moby.buildkit.v1.BuildHistoryEvent0\x01\x12o\n" + "\x12UpdateBuildHistory\x12+.moby.buildkit.v1.UpdateBuildHistoryRequest\x1a,.moby.buildkit.v1.UpdateBuildHistoryResponseB@Z>github.com/moby/buildkit/api/services/control;moby_buildkit_v1b\x06proto3" @@ -2256,133 +2600,149 @@ func file_github_com_moby_buildkit_api_services_control_control_proto_rawDescGZI } var file_github_com_moby_buildkit_api_services_control_control_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes = make([]protoimpl.MessageInfo, 39) +var file_github_com_moby_buildkit_api_services_control_control_proto_msgTypes = make([]protoimpl.MessageInfo, 46) var file_github_com_moby_buildkit_api_services_control_control_proto_goTypes = []any{ - (BuildHistoryEventType)(0), // 0: moby.buildkit.v1.BuildHistoryEventType - (*PruneRequest)(nil), // 1: moby.buildkit.v1.PruneRequest - (*DiskUsageRequest)(nil), // 2: moby.buildkit.v1.DiskUsageRequest - (*DiskUsageResponse)(nil), // 3: moby.buildkit.v1.DiskUsageResponse - (*UsageRecord)(nil), // 4: moby.buildkit.v1.UsageRecord - (*SolveRequest)(nil), // 5: moby.buildkit.v1.SolveRequest - (*CacheOptions)(nil), // 6: moby.buildkit.v1.CacheOptions - (*CacheOptionsEntry)(nil), // 7: moby.buildkit.v1.CacheOptionsEntry - (*SolveResponse)(nil), // 8: moby.buildkit.v1.SolveResponse - (*StatusRequest)(nil), // 9: moby.buildkit.v1.StatusRequest - (*StatusResponse)(nil), // 10: moby.buildkit.v1.StatusResponse - (*Vertex)(nil), // 11: moby.buildkit.v1.Vertex - (*VertexStatus)(nil), // 12: moby.buildkit.v1.VertexStatus - (*VertexLog)(nil), // 13: moby.buildkit.v1.VertexLog - (*VertexWarning)(nil), // 14: moby.buildkit.v1.VertexWarning - (*BytesMessage)(nil), // 15: moby.buildkit.v1.BytesMessage - (*ListWorkersRequest)(nil), // 16: moby.buildkit.v1.ListWorkersRequest - (*ListWorkersResponse)(nil), // 17: moby.buildkit.v1.ListWorkersResponse - (*InfoRequest)(nil), // 18: moby.buildkit.v1.InfoRequest - (*InfoResponse)(nil), // 19: moby.buildkit.v1.InfoResponse - (*BuildHistoryRequest)(nil), // 20: moby.buildkit.v1.BuildHistoryRequest - (*BuildHistoryEvent)(nil), // 21: moby.buildkit.v1.BuildHistoryEvent - (*BuildHistoryRecord)(nil), // 22: moby.buildkit.v1.BuildHistoryRecord - (*UpdateBuildHistoryRequest)(nil), // 23: moby.buildkit.v1.UpdateBuildHistoryRequest - (*UpdateBuildHistoryResponse)(nil), // 24: moby.buildkit.v1.UpdateBuildHistoryResponse - (*Descriptor)(nil), // 25: moby.buildkit.v1.Descriptor - (*BuildResultInfo)(nil), // 26: moby.buildkit.v1.BuildResultInfo - (*Exporter)(nil), // 27: moby.buildkit.v1.Exporter - nil, // 28: moby.buildkit.v1.SolveRequest.ExporterAttrsDeprecatedEntry - nil, // 29: moby.buildkit.v1.SolveRequest.FrontendAttrsEntry - nil, // 30: moby.buildkit.v1.SolveRequest.FrontendInputsEntry - nil, // 31: moby.buildkit.v1.CacheOptions.ExportAttrsDeprecatedEntry - nil, // 32: moby.buildkit.v1.CacheOptionsEntry.AttrsEntry - nil, // 33: moby.buildkit.v1.SolveResponse.ExporterResponseEntry - nil, // 34: moby.buildkit.v1.BuildHistoryRecord.FrontendAttrsEntry - nil, // 35: moby.buildkit.v1.BuildHistoryRecord.ExporterResponseEntry - nil, // 36: moby.buildkit.v1.BuildHistoryRecord.ResultsEntry - nil, // 37: moby.buildkit.v1.Descriptor.AnnotationsEntry - nil, // 38: moby.buildkit.v1.BuildResultInfo.ResultsEntry - nil, // 39: moby.buildkit.v1.Exporter.AttrsEntry - (*timestamp.Timestamp)(nil), // 40: google.protobuf.Timestamp - (*pb.Definition)(nil), // 41: pb.Definition - (*pb1.Policy)(nil), // 42: moby.buildkit.v1.sourcepolicy.Policy - (*pb.ProgressGroup)(nil), // 43: pb.ProgressGroup - (*pb.SourceInfo)(nil), // 44: pb.SourceInfo - (*pb.Range)(nil), // 45: pb.Range - (*types.WorkerRecord)(nil), // 46: moby.buildkit.v1.types.WorkerRecord - (*types.BuildkitVersion)(nil), // 47: moby.buildkit.v1.types.BuildkitVersion - (*status.Status)(nil), // 48: google.rpc.Status + (BuildHistoryEventType)(0), // 0: moby.buildkit.v1.BuildHistoryEventType + (*PruneRequest)(nil), // 1: moby.buildkit.v1.PruneRequest + (*DiskUsageRequest)(nil), // 2: moby.buildkit.v1.DiskUsageRequest + (*DiskUsageResponse)(nil), // 3: moby.buildkit.v1.DiskUsageResponse + (*UsageRecord)(nil), // 4: moby.buildkit.v1.UsageRecord + (*SolveRequest)(nil), // 5: moby.buildkit.v1.SolveRequest + (*CacheOptions)(nil), // 6: moby.buildkit.v1.CacheOptions + (*CacheOptionsEntry)(nil), // 7: moby.buildkit.v1.CacheOptionsEntry + (*SolveResponse)(nil), // 8: moby.buildkit.v1.SolveResponse + (*StatusRequest)(nil), // 9: moby.buildkit.v1.StatusRequest + (*StatusResponse)(nil), // 10: moby.buildkit.v1.StatusResponse + (*Vertex)(nil), // 11: moby.buildkit.v1.Vertex + (*VertexStatus)(nil), // 12: moby.buildkit.v1.VertexStatus + (*VertexLog)(nil), // 13: moby.buildkit.v1.VertexLog + (*VertexWarning)(nil), // 14: moby.buildkit.v1.VertexWarning + (*BytesMessage)(nil), // 15: moby.buildkit.v1.BytesMessage + (*ListWorkersRequest)(nil), // 16: moby.buildkit.v1.ListWorkersRequest + (*ListWorkersResponse)(nil), // 17: moby.buildkit.v1.ListWorkersResponse + (*InfoRequest)(nil), // 18: moby.buildkit.v1.InfoRequest + (*InfoResponse)(nil), // 19: moby.buildkit.v1.InfoResponse + (*ShutdownIfIdleRequest)(nil), // 20: moby.buildkit.v1.ShutdownIfIdleRequest + (*ShutdownIfIdleResponse)(nil), // 21: moby.buildkit.v1.ShutdownIfIdleResponse + (*ReserveRequest)(nil), // 22: moby.buildkit.v1.ReserveRequest + (*ReserveResponse)(nil), // 23: moby.buildkit.v1.ReserveResponse + (*SessionHistoryRequest)(nil), // 24: moby.buildkit.v1.SessionHistoryRequest + (*SessionHistoryResponse)(nil), // 25: moby.buildkit.v1.SessionHistoryResponse + (*BuildHistoryRequest)(nil), // 26: moby.buildkit.v1.BuildHistoryRequest + (*BuildHistoryEvent)(nil), // 27: moby.buildkit.v1.BuildHistoryEvent + (*BuildHistoryRecord)(nil), // 28: moby.buildkit.v1.BuildHistoryRecord + (*UpdateBuildHistoryRequest)(nil), // 29: moby.buildkit.v1.UpdateBuildHistoryRequest + (*UpdateBuildHistoryResponse)(nil), // 30: moby.buildkit.v1.UpdateBuildHistoryResponse + (*Descriptor)(nil), // 31: moby.buildkit.v1.Descriptor + (*BuildResultInfo)(nil), // 32: moby.buildkit.v1.BuildResultInfo + (*Exporter)(nil), // 33: moby.buildkit.v1.Exporter + nil, // 34: moby.buildkit.v1.SolveRequest.ExporterAttrsDeprecatedEntry + nil, // 35: moby.buildkit.v1.SolveRequest.FrontendAttrsEntry + nil, // 36: moby.buildkit.v1.SolveRequest.FrontendInputsEntry + nil, // 37: moby.buildkit.v1.CacheOptions.ExportAttrsDeprecatedEntry + nil, // 38: moby.buildkit.v1.CacheOptionsEntry.AttrsEntry + nil, // 39: moby.buildkit.v1.SolveResponse.ExporterResponseEntry + (*SessionHistoryResponse_History)(nil), // 40: moby.buildkit.v1.SessionHistoryResponse.History + nil, // 41: moby.buildkit.v1.BuildHistoryRecord.FrontendAttrsEntry + nil, // 42: moby.buildkit.v1.BuildHistoryRecord.ExporterResponseEntry + nil, // 43: moby.buildkit.v1.BuildHistoryRecord.ResultsEntry + nil, // 44: moby.buildkit.v1.Descriptor.AnnotationsEntry + nil, // 45: moby.buildkit.v1.BuildResultInfo.ResultsEntry + nil, // 46: moby.buildkit.v1.Exporter.AttrsEntry + (*timestamp.Timestamp)(nil), // 47: google.protobuf.Timestamp + (*pb.Definition)(nil), // 48: pb.Definition + (*pb1.Policy)(nil), // 49: moby.buildkit.v1.sourcepolicy.Policy + (*pb.ProgressGroup)(nil), // 50: pb.ProgressGroup + (*pb.SourceInfo)(nil), // 51: pb.SourceInfo + (*pb.Range)(nil), // 52: pb.Range + (*types.WorkerRecord)(nil), // 53: moby.buildkit.v1.types.WorkerRecord + (*types.BuildkitVersion)(nil), // 54: moby.buildkit.v1.types.BuildkitVersion + (*status.Status)(nil), // 55: google.rpc.Status } var file_github_com_moby_buildkit_api_services_control_control_proto_depIdxs = []int32{ 4, // 0: moby.buildkit.v1.DiskUsageResponse.record:type_name -> moby.buildkit.v1.UsageRecord - 40, // 1: moby.buildkit.v1.UsageRecord.CreatedAt:type_name -> google.protobuf.Timestamp - 40, // 2: moby.buildkit.v1.UsageRecord.LastUsedAt:type_name -> google.protobuf.Timestamp - 41, // 3: moby.buildkit.v1.SolveRequest.Definition:type_name -> pb.Definition - 28, // 4: moby.buildkit.v1.SolveRequest.ExporterAttrsDeprecated:type_name -> moby.buildkit.v1.SolveRequest.ExporterAttrsDeprecatedEntry - 29, // 5: moby.buildkit.v1.SolveRequest.FrontendAttrs:type_name -> moby.buildkit.v1.SolveRequest.FrontendAttrsEntry + 47, // 1: moby.buildkit.v1.UsageRecord.CreatedAt:type_name -> google.protobuf.Timestamp + 47, // 2: moby.buildkit.v1.UsageRecord.LastUsedAt:type_name -> google.protobuf.Timestamp + 48, // 3: moby.buildkit.v1.SolveRequest.Definition:type_name -> pb.Definition + 34, // 4: moby.buildkit.v1.SolveRequest.ExporterAttrsDeprecated:type_name -> moby.buildkit.v1.SolveRequest.ExporterAttrsDeprecatedEntry + 35, // 5: moby.buildkit.v1.SolveRequest.FrontendAttrs:type_name -> moby.buildkit.v1.SolveRequest.FrontendAttrsEntry 6, // 6: moby.buildkit.v1.SolveRequest.Cache:type_name -> moby.buildkit.v1.CacheOptions - 30, // 7: moby.buildkit.v1.SolveRequest.FrontendInputs:type_name -> moby.buildkit.v1.SolveRequest.FrontendInputsEntry - 42, // 8: moby.buildkit.v1.SolveRequest.SourcePolicy:type_name -> moby.buildkit.v1.sourcepolicy.Policy - 27, // 9: moby.buildkit.v1.SolveRequest.Exporters:type_name -> moby.buildkit.v1.Exporter - 31, // 10: moby.buildkit.v1.CacheOptions.ExportAttrsDeprecated:type_name -> moby.buildkit.v1.CacheOptions.ExportAttrsDeprecatedEntry + 36, // 7: moby.buildkit.v1.SolveRequest.FrontendInputs:type_name -> moby.buildkit.v1.SolveRequest.FrontendInputsEntry + 49, // 8: moby.buildkit.v1.SolveRequest.SourcePolicy:type_name -> moby.buildkit.v1.sourcepolicy.Policy + 33, // 9: moby.buildkit.v1.SolveRequest.Exporters:type_name -> moby.buildkit.v1.Exporter + 37, // 10: moby.buildkit.v1.CacheOptions.ExportAttrsDeprecated:type_name -> moby.buildkit.v1.CacheOptions.ExportAttrsDeprecatedEntry 7, // 11: moby.buildkit.v1.CacheOptions.Exports:type_name -> moby.buildkit.v1.CacheOptionsEntry 7, // 12: moby.buildkit.v1.CacheOptions.Imports:type_name -> moby.buildkit.v1.CacheOptionsEntry - 32, // 13: moby.buildkit.v1.CacheOptionsEntry.Attrs:type_name -> moby.buildkit.v1.CacheOptionsEntry.AttrsEntry - 33, // 14: moby.buildkit.v1.SolveResponse.ExporterResponse:type_name -> moby.buildkit.v1.SolveResponse.ExporterResponseEntry + 38, // 13: moby.buildkit.v1.CacheOptionsEntry.Attrs:type_name -> moby.buildkit.v1.CacheOptionsEntry.AttrsEntry + 39, // 14: moby.buildkit.v1.SolveResponse.ExporterResponse:type_name -> moby.buildkit.v1.SolveResponse.ExporterResponseEntry 11, // 15: moby.buildkit.v1.StatusResponse.vertexes:type_name -> moby.buildkit.v1.Vertex 12, // 16: moby.buildkit.v1.StatusResponse.statuses:type_name -> moby.buildkit.v1.VertexStatus 13, // 17: moby.buildkit.v1.StatusResponse.logs:type_name -> moby.buildkit.v1.VertexLog 14, // 18: moby.buildkit.v1.StatusResponse.warnings:type_name -> moby.buildkit.v1.VertexWarning - 40, // 19: moby.buildkit.v1.Vertex.started:type_name -> google.protobuf.Timestamp - 40, // 20: moby.buildkit.v1.Vertex.completed:type_name -> google.protobuf.Timestamp - 43, // 21: moby.buildkit.v1.Vertex.progressGroup:type_name -> pb.ProgressGroup - 40, // 22: moby.buildkit.v1.VertexStatus.timestamp:type_name -> google.protobuf.Timestamp - 40, // 23: moby.buildkit.v1.VertexStatus.started:type_name -> google.protobuf.Timestamp - 40, // 24: moby.buildkit.v1.VertexStatus.completed:type_name -> google.protobuf.Timestamp - 40, // 25: moby.buildkit.v1.VertexLog.timestamp:type_name -> google.protobuf.Timestamp - 44, // 26: moby.buildkit.v1.VertexWarning.info:type_name -> pb.SourceInfo - 45, // 27: moby.buildkit.v1.VertexWarning.ranges:type_name -> pb.Range - 46, // 28: moby.buildkit.v1.ListWorkersResponse.record:type_name -> moby.buildkit.v1.types.WorkerRecord - 47, // 29: moby.buildkit.v1.InfoResponse.buildkitVersion:type_name -> moby.buildkit.v1.types.BuildkitVersion - 0, // 30: moby.buildkit.v1.BuildHistoryEvent.type:type_name -> moby.buildkit.v1.BuildHistoryEventType - 22, // 31: moby.buildkit.v1.BuildHistoryEvent.record:type_name -> moby.buildkit.v1.BuildHistoryRecord - 34, // 32: moby.buildkit.v1.BuildHistoryRecord.FrontendAttrs:type_name -> moby.buildkit.v1.BuildHistoryRecord.FrontendAttrsEntry - 27, // 33: moby.buildkit.v1.BuildHistoryRecord.Exporters:type_name -> moby.buildkit.v1.Exporter - 48, // 34: moby.buildkit.v1.BuildHistoryRecord.error:type_name -> google.rpc.Status - 40, // 35: moby.buildkit.v1.BuildHistoryRecord.CreatedAt:type_name -> google.protobuf.Timestamp - 40, // 36: moby.buildkit.v1.BuildHistoryRecord.CompletedAt:type_name -> google.protobuf.Timestamp - 25, // 37: moby.buildkit.v1.BuildHistoryRecord.logs:type_name -> moby.buildkit.v1.Descriptor - 35, // 38: moby.buildkit.v1.BuildHistoryRecord.ExporterResponse:type_name -> moby.buildkit.v1.BuildHistoryRecord.ExporterResponseEntry - 26, // 39: moby.buildkit.v1.BuildHistoryRecord.Result:type_name -> moby.buildkit.v1.BuildResultInfo - 36, // 40: moby.buildkit.v1.BuildHistoryRecord.Results:type_name -> moby.buildkit.v1.BuildHistoryRecord.ResultsEntry - 25, // 41: moby.buildkit.v1.BuildHistoryRecord.trace:type_name -> moby.buildkit.v1.Descriptor - 25, // 42: moby.buildkit.v1.BuildHistoryRecord.externalError:type_name -> moby.buildkit.v1.Descriptor - 37, // 43: moby.buildkit.v1.Descriptor.annotations:type_name -> moby.buildkit.v1.Descriptor.AnnotationsEntry - 25, // 44: moby.buildkit.v1.BuildResultInfo.ResultDeprecated:type_name -> moby.buildkit.v1.Descriptor - 25, // 45: moby.buildkit.v1.BuildResultInfo.Attestations:type_name -> moby.buildkit.v1.Descriptor - 38, // 46: moby.buildkit.v1.BuildResultInfo.Results:type_name -> moby.buildkit.v1.BuildResultInfo.ResultsEntry - 39, // 47: moby.buildkit.v1.Exporter.Attrs:type_name -> moby.buildkit.v1.Exporter.AttrsEntry - 41, // 48: moby.buildkit.v1.SolveRequest.FrontendInputsEntry.value:type_name -> pb.Definition - 26, // 49: moby.buildkit.v1.BuildHistoryRecord.ResultsEntry.value:type_name -> moby.buildkit.v1.BuildResultInfo - 25, // 50: moby.buildkit.v1.BuildResultInfo.ResultsEntry.value:type_name -> moby.buildkit.v1.Descriptor - 2, // 51: moby.buildkit.v1.Control.DiskUsage:input_type -> moby.buildkit.v1.DiskUsageRequest - 1, // 52: moby.buildkit.v1.Control.Prune:input_type -> moby.buildkit.v1.PruneRequest - 5, // 53: moby.buildkit.v1.Control.Solve:input_type -> moby.buildkit.v1.SolveRequest - 9, // 54: moby.buildkit.v1.Control.Status:input_type -> moby.buildkit.v1.StatusRequest - 15, // 55: moby.buildkit.v1.Control.Session:input_type -> moby.buildkit.v1.BytesMessage - 16, // 56: moby.buildkit.v1.Control.ListWorkers:input_type -> moby.buildkit.v1.ListWorkersRequest - 18, // 57: moby.buildkit.v1.Control.Info:input_type -> moby.buildkit.v1.InfoRequest - 20, // 58: moby.buildkit.v1.Control.ListenBuildHistory:input_type -> moby.buildkit.v1.BuildHistoryRequest - 23, // 59: moby.buildkit.v1.Control.UpdateBuildHistory:input_type -> moby.buildkit.v1.UpdateBuildHistoryRequest - 3, // 60: moby.buildkit.v1.Control.DiskUsage:output_type -> moby.buildkit.v1.DiskUsageResponse - 4, // 61: moby.buildkit.v1.Control.Prune:output_type -> moby.buildkit.v1.UsageRecord - 8, // 62: moby.buildkit.v1.Control.Solve:output_type -> moby.buildkit.v1.SolveResponse - 10, // 63: moby.buildkit.v1.Control.Status:output_type -> moby.buildkit.v1.StatusResponse - 15, // 64: moby.buildkit.v1.Control.Session:output_type -> moby.buildkit.v1.BytesMessage - 17, // 65: moby.buildkit.v1.Control.ListWorkers:output_type -> moby.buildkit.v1.ListWorkersResponse - 19, // 66: moby.buildkit.v1.Control.Info:output_type -> moby.buildkit.v1.InfoResponse - 21, // 67: moby.buildkit.v1.Control.ListenBuildHistory:output_type -> moby.buildkit.v1.BuildHistoryEvent - 24, // 68: moby.buildkit.v1.Control.UpdateBuildHistory:output_type -> moby.buildkit.v1.UpdateBuildHistoryResponse - 60, // [60:69] is the sub-list for method output_type - 51, // [51:60] is the sub-list for method input_type - 51, // [51:51] is the sub-list for extension type_name - 51, // [51:51] is the sub-list for extension extendee - 0, // [0:51] is the sub-list for field type_name + 47, // 19: moby.buildkit.v1.Vertex.started:type_name -> google.protobuf.Timestamp + 47, // 20: moby.buildkit.v1.Vertex.completed:type_name -> google.protobuf.Timestamp + 50, // 21: moby.buildkit.v1.Vertex.progressGroup:type_name -> pb.ProgressGroup + 47, // 22: moby.buildkit.v1.VertexStatus.timestamp:type_name -> google.protobuf.Timestamp + 47, // 23: moby.buildkit.v1.VertexStatus.started:type_name -> google.protobuf.Timestamp + 47, // 24: moby.buildkit.v1.VertexStatus.completed:type_name -> google.protobuf.Timestamp + 47, // 25: moby.buildkit.v1.VertexLog.timestamp:type_name -> google.protobuf.Timestamp + 51, // 26: moby.buildkit.v1.VertexWarning.info:type_name -> pb.SourceInfo + 52, // 27: moby.buildkit.v1.VertexWarning.ranges:type_name -> pb.Range + 53, // 28: moby.buildkit.v1.ListWorkersResponse.record:type_name -> moby.buildkit.v1.types.WorkerRecord + 54, // 29: moby.buildkit.v1.InfoResponse.buildkitVersion:type_name -> moby.buildkit.v1.types.BuildkitVersion + 40, // 30: moby.buildkit.v1.SessionHistoryResponse.history:type_name -> moby.buildkit.v1.SessionHistoryResponse.History + 0, // 31: moby.buildkit.v1.BuildHistoryEvent.type:type_name -> moby.buildkit.v1.BuildHistoryEventType + 28, // 32: moby.buildkit.v1.BuildHistoryEvent.record:type_name -> moby.buildkit.v1.BuildHistoryRecord + 41, // 33: moby.buildkit.v1.BuildHistoryRecord.FrontendAttrs:type_name -> moby.buildkit.v1.BuildHistoryRecord.FrontendAttrsEntry + 33, // 34: moby.buildkit.v1.BuildHistoryRecord.Exporters:type_name -> moby.buildkit.v1.Exporter + 55, // 35: moby.buildkit.v1.BuildHistoryRecord.error:type_name -> google.rpc.Status + 47, // 36: moby.buildkit.v1.BuildHistoryRecord.CreatedAt:type_name -> google.protobuf.Timestamp + 47, // 37: moby.buildkit.v1.BuildHistoryRecord.CompletedAt:type_name -> google.protobuf.Timestamp + 31, // 38: moby.buildkit.v1.BuildHistoryRecord.logs:type_name -> moby.buildkit.v1.Descriptor + 42, // 39: moby.buildkit.v1.BuildHistoryRecord.ExporterResponse:type_name -> moby.buildkit.v1.BuildHistoryRecord.ExporterResponseEntry + 32, // 40: moby.buildkit.v1.BuildHistoryRecord.Result:type_name -> moby.buildkit.v1.BuildResultInfo + 43, // 41: moby.buildkit.v1.BuildHistoryRecord.Results:type_name -> moby.buildkit.v1.BuildHistoryRecord.ResultsEntry + 31, // 42: moby.buildkit.v1.BuildHistoryRecord.trace:type_name -> moby.buildkit.v1.Descriptor + 31, // 43: moby.buildkit.v1.BuildHistoryRecord.externalError:type_name -> moby.buildkit.v1.Descriptor + 44, // 44: moby.buildkit.v1.Descriptor.annotations:type_name -> moby.buildkit.v1.Descriptor.AnnotationsEntry + 31, // 45: moby.buildkit.v1.BuildResultInfo.ResultDeprecated:type_name -> moby.buildkit.v1.Descriptor + 31, // 46: moby.buildkit.v1.BuildResultInfo.Attestations:type_name -> moby.buildkit.v1.Descriptor + 45, // 47: moby.buildkit.v1.BuildResultInfo.Results:type_name -> moby.buildkit.v1.BuildResultInfo.ResultsEntry + 46, // 48: moby.buildkit.v1.Exporter.Attrs:type_name -> moby.buildkit.v1.Exporter.AttrsEntry + 48, // 49: moby.buildkit.v1.SolveRequest.FrontendInputsEntry.value:type_name -> pb.Definition + 47, // 50: moby.buildkit.v1.SessionHistoryResponse.History.start:type_name -> google.protobuf.Timestamp + 47, // 51: moby.buildkit.v1.SessionHistoryResponse.History.end:type_name -> google.protobuf.Timestamp + 32, // 52: moby.buildkit.v1.BuildHistoryRecord.ResultsEntry.value:type_name -> moby.buildkit.v1.BuildResultInfo + 31, // 53: moby.buildkit.v1.BuildResultInfo.ResultsEntry.value:type_name -> moby.buildkit.v1.Descriptor + 2, // 54: moby.buildkit.v1.Control.DiskUsage:input_type -> moby.buildkit.v1.DiskUsageRequest + 1, // 55: moby.buildkit.v1.Control.Prune:input_type -> moby.buildkit.v1.PruneRequest + 5, // 56: moby.buildkit.v1.Control.Solve:input_type -> moby.buildkit.v1.SolveRequest + 9, // 57: moby.buildkit.v1.Control.Status:input_type -> moby.buildkit.v1.StatusRequest + 15, // 58: moby.buildkit.v1.Control.Session:input_type -> moby.buildkit.v1.BytesMessage + 16, // 59: moby.buildkit.v1.Control.ListWorkers:input_type -> moby.buildkit.v1.ListWorkersRequest + 18, // 60: moby.buildkit.v1.Control.Info:input_type -> moby.buildkit.v1.InfoRequest + 20, // 61: moby.buildkit.v1.Control.ShutdownIfIdle:input_type -> moby.buildkit.v1.ShutdownIfIdleRequest + 22, // 62: moby.buildkit.v1.Control.Reserve:input_type -> moby.buildkit.v1.ReserveRequest + 24, // 63: moby.buildkit.v1.Control.SessionHistory:input_type -> moby.buildkit.v1.SessionHistoryRequest + 26, // 64: moby.buildkit.v1.Control.ListenBuildHistory:input_type -> moby.buildkit.v1.BuildHistoryRequest + 29, // 65: moby.buildkit.v1.Control.UpdateBuildHistory:input_type -> moby.buildkit.v1.UpdateBuildHistoryRequest + 3, // 66: moby.buildkit.v1.Control.DiskUsage:output_type -> moby.buildkit.v1.DiskUsageResponse + 4, // 67: moby.buildkit.v1.Control.Prune:output_type -> moby.buildkit.v1.UsageRecord + 8, // 68: moby.buildkit.v1.Control.Solve:output_type -> moby.buildkit.v1.SolveResponse + 10, // 69: moby.buildkit.v1.Control.Status:output_type -> moby.buildkit.v1.StatusResponse + 15, // 70: moby.buildkit.v1.Control.Session:output_type -> moby.buildkit.v1.BytesMessage + 17, // 71: moby.buildkit.v1.Control.ListWorkers:output_type -> moby.buildkit.v1.ListWorkersResponse + 19, // 72: moby.buildkit.v1.Control.Info:output_type -> moby.buildkit.v1.InfoResponse + 21, // 73: moby.buildkit.v1.Control.ShutdownIfIdle:output_type -> moby.buildkit.v1.ShutdownIfIdleResponse + 23, // 74: moby.buildkit.v1.Control.Reserve:output_type -> moby.buildkit.v1.ReserveResponse + 25, // 75: moby.buildkit.v1.Control.SessionHistory:output_type -> moby.buildkit.v1.SessionHistoryResponse + 27, // 76: moby.buildkit.v1.Control.ListenBuildHistory:output_type -> moby.buildkit.v1.BuildHistoryEvent + 30, // 77: moby.buildkit.v1.Control.UpdateBuildHistory:output_type -> moby.buildkit.v1.UpdateBuildHistoryResponse + 66, // [66:78] is the sub-list for method output_type + 54, // [54:66] is the sub-list for method input_type + 54, // [54:54] is the sub-list for extension type_name + 54, // [54:54] is the sub-list for extension extendee + 0, // [0:54] is the sub-list for field type_name } func init() { file_github_com_moby_buildkit_api_services_control_control_proto_init() } @@ -2396,7 +2756,7 @@ func file_github_com_moby_buildkit_api_services_control_control_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_github_com_moby_buildkit_api_services_control_control_proto_rawDesc), len(file_github_com_moby_buildkit_api_services_control_control_proto_rawDesc)), NumEnums: 1, - NumMessages: 39, + NumMessages: 46, NumExtensions: 0, NumServices: 1, }, diff --git a/api/services/control/control.proto b/api/services/control/control.proto index 128408e84..959068b99 100644 --- a/api/services/control/control.proto +++ b/api/services/control/control.proto @@ -20,6 +20,11 @@ service Control { rpc ListWorkers(ListWorkersRequest) returns (ListWorkersResponse); rpc Info(InfoRequest) returns (InfoResponse); + // Earthly-specific. + rpc ShutdownIfIdle(ShutdownIfIdleRequest) returns (ShutdownIfIdleResponse); + rpc Reserve(ReserveRequest) returns (ReserveResponse); + rpc SessionHistory(SessionHistoryRequest) returns (SessionHistoryResponse); + rpc ListenBuildHistory(BuildHistoryRequest) returns (stream BuildHistoryEvent); rpc UpdateBuildHistory(UpdateBuildHistoryRequest) returns (UpdateBuildHistoryResponse); } @@ -35,7 +40,7 @@ message PruneRequest { } message DiskUsageRequest { - repeated string filter = 1; + repeated string filter = 1; int64 ageLimit = 2; } @@ -113,6 +118,8 @@ message SolveResponse { message StatusRequest { string Ref = 1; + + bool statsStream = 99; // earthly } message StatusResponse { @@ -177,6 +184,32 @@ message InfoRequest {} message InfoResponse { moby.buildkit.v1.types.BuildkitVersion buildkitVersion = 1; + + // Earthly-specific. + uint64 numSessions = 101; + uint64 secondsIdle = 102; +} + +message ShutdownIfIdleRequest {} + +message ShutdownIfIdleResponse { + bool willShutdown = 1; + uint64 numSessions = 2; +} + +message ReserveRequest {} + +message ReserveResponse {} + +message SessionHistoryRequest {} + +message SessionHistoryResponse { + message History { + string sessionID = 1; + google.protobuf.Timestamp start = 2; + google.protobuf.Timestamp end = 3; + } + repeated History history = 1; } message BuildHistoryRequest { diff --git a/api/services/control/control_grpc.pb.go b/api/services/control/control_grpc.pb.go index 90ad6ffbd..4854a8965 100644 --- a/api/services/control/control_grpc.pb.go +++ b/api/services/control/control_grpc.pb.go @@ -26,6 +26,9 @@ const ( Control_Session_FullMethodName = "/moby.buildkit.v1.Control/Session" Control_ListWorkers_FullMethodName = "/moby.buildkit.v1.Control/ListWorkers" Control_Info_FullMethodName = "/moby.buildkit.v1.Control/Info" + Control_ShutdownIfIdle_FullMethodName = "/moby.buildkit.v1.Control/ShutdownIfIdle" + Control_Reserve_FullMethodName = "/moby.buildkit.v1.Control/Reserve" + Control_SessionHistory_FullMethodName = "/moby.buildkit.v1.Control/SessionHistory" Control_ListenBuildHistory_FullMethodName = "/moby.buildkit.v1.Control/ListenBuildHistory" Control_UpdateBuildHistory_FullMethodName = "/moby.buildkit.v1.Control/UpdateBuildHistory" ) @@ -41,6 +44,10 @@ type ControlClient interface { Session(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[BytesMessage, BytesMessage], error) ListWorkers(ctx context.Context, in *ListWorkersRequest, opts ...grpc.CallOption) (*ListWorkersResponse, error) Info(ctx context.Context, in *InfoRequest, opts ...grpc.CallOption) (*InfoResponse, error) + // Earthly-specific. + ShutdownIfIdle(ctx context.Context, in *ShutdownIfIdleRequest, opts ...grpc.CallOption) (*ShutdownIfIdleResponse, error) + Reserve(ctx context.Context, in *ReserveRequest, opts ...grpc.CallOption) (*ReserveResponse, error) + SessionHistory(ctx context.Context, in *SessionHistoryRequest, opts ...grpc.CallOption) (*SessionHistoryResponse, error) ListenBuildHistory(ctx context.Context, in *BuildHistoryRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[BuildHistoryEvent], error) UpdateBuildHistory(ctx context.Context, in *UpdateBuildHistoryRequest, opts ...grpc.CallOption) (*UpdateBuildHistoryResponse, error) } @@ -144,6 +151,36 @@ func (c *controlClient) Info(ctx context.Context, in *InfoRequest, opts ...grpc. return out, nil } +func (c *controlClient) ShutdownIfIdle(ctx context.Context, in *ShutdownIfIdleRequest, opts ...grpc.CallOption) (*ShutdownIfIdleResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ShutdownIfIdleResponse) + err := c.cc.Invoke(ctx, Control_ShutdownIfIdle_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) Reserve(ctx context.Context, in *ReserveRequest, opts ...grpc.CallOption) (*ReserveResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ReserveResponse) + err := c.cc.Invoke(ctx, Control_Reserve_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) SessionHistory(ctx context.Context, in *SessionHistoryRequest, opts ...grpc.CallOption) (*SessionHistoryResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SessionHistoryResponse) + err := c.cc.Invoke(ctx, Control_SessionHistory_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *controlClient) ListenBuildHistory(ctx context.Context, in *BuildHistoryRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[BuildHistoryEvent], error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) stream, err := c.cc.NewStream(ctx, &Control_ServiceDesc.Streams[3], Control_ListenBuildHistory_FullMethodName, cOpts...) @@ -184,6 +221,10 @@ type ControlServer interface { Session(grpc.BidiStreamingServer[BytesMessage, BytesMessage]) error ListWorkers(context.Context, *ListWorkersRequest) (*ListWorkersResponse, error) Info(context.Context, *InfoRequest) (*InfoResponse, error) + // Earthly-specific. + ShutdownIfIdle(context.Context, *ShutdownIfIdleRequest) (*ShutdownIfIdleResponse, error) + Reserve(context.Context, *ReserveRequest) (*ReserveResponse, error) + SessionHistory(context.Context, *SessionHistoryRequest) (*SessionHistoryResponse, error) ListenBuildHistory(*BuildHistoryRequest, grpc.ServerStreamingServer[BuildHistoryEvent]) error UpdateBuildHistory(context.Context, *UpdateBuildHistoryRequest) (*UpdateBuildHistoryResponse, error) } @@ -216,6 +257,15 @@ func (UnimplementedControlServer) ListWorkers(context.Context, *ListWorkersReque func (UnimplementedControlServer) Info(context.Context, *InfoRequest) (*InfoResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Info not implemented") } +func (UnimplementedControlServer) ShutdownIfIdle(context.Context, *ShutdownIfIdleRequest) (*ShutdownIfIdleResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ShutdownIfIdle not implemented") +} +func (UnimplementedControlServer) Reserve(context.Context, *ReserveRequest) (*ReserveResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Reserve not implemented") +} +func (UnimplementedControlServer) SessionHistory(context.Context, *SessionHistoryRequest) (*SessionHistoryResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method SessionHistory not implemented") +} func (UnimplementedControlServer) ListenBuildHistory(*BuildHistoryRequest, grpc.ServerStreamingServer[BuildHistoryEvent]) error { return status.Errorf(codes.Unimplemented, "method ListenBuildHistory not implemented") } @@ -343,6 +393,60 @@ func _Control_Info_Handler(srv interface{}, ctx context.Context, dec func(interf return interceptor(ctx, in, info, handler) } +func _Control_ShutdownIfIdle_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ShutdownIfIdleRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).ShutdownIfIdle(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_ShutdownIfIdle_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).ShutdownIfIdle(ctx, req.(*ShutdownIfIdleRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_Reserve_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReserveRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).Reserve(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_Reserve_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).Reserve(ctx, req.(*ReserveRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_SessionHistory_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SessionHistoryRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).SessionHistory(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_SessionHistory_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).SessionHistory(ctx, req.(*SessionHistoryRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _Control_ListenBuildHistory_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(BuildHistoryRequest) if err := stream.RecvMsg(m); err != nil { @@ -395,6 +499,18 @@ var Control_ServiceDesc = grpc.ServiceDesc{ MethodName: "Info", Handler: _Control_Info_Handler, }, + { + MethodName: "ShutdownIfIdle", + Handler: _Control_ShutdownIfIdle_Handler, + }, + { + MethodName: "Reserve", + Handler: _Control_Reserve_Handler, + }, + { + MethodName: "SessionHistory", + Handler: _Control_SessionHistory_Handler, + }, { MethodName: "UpdateBuildHistory", Handler: _Control_UpdateBuildHistory_Handler, diff --git a/api/services/control/control_vtproto.pb.go b/api/services/control/control_vtproto.pb.go index 4faf1b112..261f3a15b 100644 --- a/api/services/control/control_vtproto.pb.go +++ b/api/services/control/control_vtproto.pb.go @@ -283,6 +283,7 @@ func (m *StatusRequest) CloneVT() *StatusRequest { } r := new(StatusRequest) r.Ref = m.Ref + r.StatsStream = m.StatsStream if len(m.unknownFields) > 0 { r.unknownFields = make([]byte, len(m.unknownFields)) copy(r.unknownFields, m.unknownFields) @@ -542,6 +543,8 @@ func (m *InfoResponse) CloneVT() *InfoResponse { } r := new(InfoResponse) r.BuildkitVersion = m.BuildkitVersion.CloneVT() + r.NumSessions = m.NumSessions + r.SecondsIdle = m.SecondsIdle if len(m.unknownFields) > 0 { r.unknownFields = make([]byte, len(m.unknownFields)) copy(r.unknownFields, m.unknownFields) @@ -553,6 +556,130 @@ func (m *InfoResponse) CloneMessageVT() proto.Message { return m.CloneVT() } +func (m *ShutdownIfIdleRequest) CloneVT() *ShutdownIfIdleRequest { + if m == nil { + return (*ShutdownIfIdleRequest)(nil) + } + r := new(ShutdownIfIdleRequest) + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *ShutdownIfIdleRequest) CloneMessageVT() proto.Message { + return m.CloneVT() +} + +func (m *ShutdownIfIdleResponse) CloneVT() *ShutdownIfIdleResponse { + if m == nil { + return (*ShutdownIfIdleResponse)(nil) + } + r := new(ShutdownIfIdleResponse) + r.WillShutdown = m.WillShutdown + r.NumSessions = m.NumSessions + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *ShutdownIfIdleResponse) CloneMessageVT() proto.Message { + return m.CloneVT() +} + +func (m *ReserveRequest) CloneVT() *ReserveRequest { + if m == nil { + return (*ReserveRequest)(nil) + } + r := new(ReserveRequest) + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *ReserveRequest) CloneMessageVT() proto.Message { + return m.CloneVT() +} + +func (m *ReserveResponse) CloneVT() *ReserveResponse { + if m == nil { + return (*ReserveResponse)(nil) + } + r := new(ReserveResponse) + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *ReserveResponse) CloneMessageVT() proto.Message { + return m.CloneVT() +} + +func (m *SessionHistoryRequest) CloneVT() *SessionHistoryRequest { + if m == nil { + return (*SessionHistoryRequest)(nil) + } + r := new(SessionHistoryRequest) + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *SessionHistoryRequest) CloneMessageVT() proto.Message { + return m.CloneVT() +} + +func (m *SessionHistoryResponse_History) CloneVT() *SessionHistoryResponse_History { + if m == nil { + return (*SessionHistoryResponse_History)(nil) + } + r := new(SessionHistoryResponse_History) + r.SessionID = m.SessionID + r.Start = (*timestamp.Timestamp)((*timestamppb.Timestamp)(m.Start).CloneVT()) + r.End = (*timestamp.Timestamp)((*timestamppb.Timestamp)(m.End).CloneVT()) + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *SessionHistoryResponse_History) CloneMessageVT() proto.Message { + return m.CloneVT() +} + +func (m *SessionHistoryResponse) CloneVT() *SessionHistoryResponse { + if m == nil { + return (*SessionHistoryResponse)(nil) + } + r := new(SessionHistoryResponse) + if rhs := m.History; rhs != nil { + tmpContainer := make([]*SessionHistoryResponse_History, len(rhs)) + for k, v := range rhs { + tmpContainer[k] = v.CloneVT() + } + r.History = tmpContainer + } + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *SessionHistoryResponse) CloneMessageVT() proto.Message { + return m.CloneVT() +} + func (m *BuildHistoryRequest) CloneVT() *BuildHistoryRequest { if m == nil { return (*BuildHistoryRequest)(nil) @@ -1195,6 +1322,9 @@ func (this *StatusRequest) EqualVT(that *StatusRequest) bool { if this.Ref != that.Ref { return false } + if this.StatsStream != that.StatsStream { + return false + } return string(this.unknownFields) == string(that.unknownFields) } @@ -1562,6 +1692,12 @@ func (this *InfoResponse) EqualVT(that *InfoResponse) bool { if !this.BuildkitVersion.EqualVT(that.BuildkitVersion) { return false } + if this.NumSessions != that.NumSessions { + return false + } + if this.SecondsIdle != that.SecondsIdle { + return false + } return string(this.unknownFields) == string(that.unknownFields) } @@ -1572,6 +1708,150 @@ func (this *InfoResponse) EqualMessageVT(thatMsg proto.Message) bool { } return this.EqualVT(that) } +func (this *ShutdownIfIdleRequest) EqualVT(that *ShutdownIfIdleRequest) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *ShutdownIfIdleRequest) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*ShutdownIfIdleRequest) + if !ok { + return false + } + return this.EqualVT(that) +} +func (this *ShutdownIfIdleResponse) EqualVT(that *ShutdownIfIdleResponse) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + if this.WillShutdown != that.WillShutdown { + return false + } + if this.NumSessions != that.NumSessions { + return false + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *ShutdownIfIdleResponse) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*ShutdownIfIdleResponse) + if !ok { + return false + } + return this.EqualVT(that) +} +func (this *ReserveRequest) EqualVT(that *ReserveRequest) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *ReserveRequest) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*ReserveRequest) + if !ok { + return false + } + return this.EqualVT(that) +} +func (this *ReserveResponse) EqualVT(that *ReserveResponse) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *ReserveResponse) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*ReserveResponse) + if !ok { + return false + } + return this.EqualVT(that) +} +func (this *SessionHistoryRequest) EqualVT(that *SessionHistoryRequest) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *SessionHistoryRequest) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*SessionHistoryRequest) + if !ok { + return false + } + return this.EqualVT(that) +} +func (this *SessionHistoryResponse_History) EqualVT(that *SessionHistoryResponse_History) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + if this.SessionID != that.SessionID { + return false + } + if !(*timestamppb.Timestamp)(this.Start).EqualVT((*timestamppb.Timestamp)(that.Start)) { + return false + } + if !(*timestamppb.Timestamp)(this.End).EqualVT((*timestamppb.Timestamp)(that.End)) { + return false + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *SessionHistoryResponse_History) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*SessionHistoryResponse_History) + if !ok { + return false + } + return this.EqualVT(that) +} +func (this *SessionHistoryResponse) EqualVT(that *SessionHistoryResponse) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + if len(this.History) != len(that.History) { + return false + } + for i, vx := range this.History { + vy := that.History[i] + if p, q := vx, vy; p != q { + if p == nil { + p = &SessionHistoryResponse_History{} + } + if q == nil { + q = &SessionHistoryResponse_History{} + } + if !p.EqualVT(q) { + return false + } + } + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *SessionHistoryResponse) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*SessionHistoryResponse) + if !ok { + return false + } + return this.EqualVT(that) +} func (this *BuildHistoryRequest) EqualVT(that *BuildHistoryRequest) bool { if this == that { return true @@ -2651,6 +2931,18 @@ func (m *StatusRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.StatsStream { + i-- + if m.StatsStream { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x6 + i-- + dAtA[i] = 0x98 + } if len(m.Ref) > 0 { i -= len(m.Ref) copy(dAtA[i:], m.Ref) @@ -3281,6 +3573,20 @@ func (m *InfoResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.SecondsIdle != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.SecondsIdle)) + i-- + dAtA[i] = 0x6 + i-- + dAtA[i] = 0xb0 + } + if m.NumSessions != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.NumSessions)) + i-- + dAtA[i] = 0x6 + i-- + dAtA[i] = 0xa8 + } if m.BuildkitVersion != nil { size, err := m.BuildkitVersion.MarshalToSizedBufferVT(dAtA[:i]) if err != nil { @@ -3294,7 +3600,7 @@ func (m *InfoResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func (m *BuildHistoryRequest) MarshalVT() (dAtA []byte, err error) { +func (m *ShutdownIfIdleRequest) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil } @@ -3307,12 +3613,12 @@ func (m *BuildHistoryRequest) MarshalVT() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *BuildHistoryRequest) MarshalToVT(dAtA []byte) (int, error) { +func (m *ShutdownIfIdleRequest) MarshalToVT(dAtA []byte) (int, error) { size := m.SizeVT() return m.MarshalToSizedBufferVT(dAtA[:size]) } -func (m *BuildHistoryRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { +func (m *ShutdownIfIdleRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { if m == nil { return 0, nil } @@ -3324,29 +3630,314 @@ func (m *BuildHistoryRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } - if m.Limit != 0 { - i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Limit)) - i-- - dAtA[i] = 0x28 + return len(dAtA) - i, nil +} + +func (m *ShutdownIfIdleResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil } - if len(m.Filter) > 0 { - for iNdEx := len(m.Filter) - 1; iNdEx >= 0; iNdEx-- { - i -= len(m.Filter[iNdEx]) - copy(dAtA[i:], m.Filter[iNdEx]) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Filter[iNdEx]))) - i-- - dAtA[i] = 0x22 - } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err } - if m.EarlyExit { - i-- - if m.EarlyExit { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i-- - dAtA[i] = 0x18 + return dAtA[:n], nil +} + +func (m *ShutdownIfIdleResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *ShutdownIfIdleResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.NumSessions != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.NumSessions)) + i-- + dAtA[i] = 0x10 + } + if m.WillShutdown { + i-- + if m.WillShutdown { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *ReserveRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ReserveRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *ReserveRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + return len(dAtA) - i, nil +} + +func (m *ReserveResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ReserveResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *ReserveResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + return len(dAtA) - i, nil +} + +func (m *SessionHistoryRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SessionHistoryRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *SessionHistoryRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + return len(dAtA) - i, nil +} + +func (m *SessionHistoryResponse_History) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SessionHistoryResponse_History) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *SessionHistoryResponse_History) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.End != nil { + size, err := (*timestamppb.Timestamp)(m.End).MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x1a + } + if m.Start != nil { + size, err := (*timestamppb.Timestamp)(m.Start).MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x12 + } + if len(m.SessionID) > 0 { + i -= len(m.SessionID) + copy(dAtA[i:], m.SessionID) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SessionID))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *SessionHistoryResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SessionHistoryResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *SessionHistoryResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.History) > 0 { + for iNdEx := len(m.History) - 1; iNdEx >= 0; iNdEx-- { + size, err := m.History[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *BuildHistoryRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *BuildHistoryRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *BuildHistoryRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.Limit != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Limit)) + i-- + dAtA[i] = 0x28 + } + if len(m.Filter) > 0 { + for iNdEx := len(m.Filter) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Filter[iNdEx]) + copy(dAtA[i:], m.Filter[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Filter[iNdEx]))) + i-- + dAtA[i] = 0x22 + } + } + if m.EarlyExit { + i-- + if m.EarlyExit { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x18 } if len(m.Ref) > 0 { i -= len(m.Ref) @@ -4268,6 +4859,9 @@ func (m *StatusRequest) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } + if m.StatsStream { + n += 3 + } n += len(m.unknownFields) return n } @@ -4455,53 +5049,129 @@ func (m *VertexWarning) SizeVT() (n int) { return n } -func (m *BytesMessage) SizeVT() (n int) { +func (m *BytesMessage) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Data) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *ListWorkersRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Filter) > 0 { + for _, s := range m.Filter { + l = len(s) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *ListWorkersResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Record) > 0 { + for _, e := range m.Record { + l = e.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *InfoRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + n += len(m.unknownFields) + return n +} + +func (m *InfoResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.BuildkitVersion != nil { + l = m.BuildkitVersion.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.NumSessions != 0 { + n += 2 + protohelpers.SizeOfVarint(uint64(m.NumSessions)) + } + if m.SecondsIdle != 0 { + n += 2 + protohelpers.SizeOfVarint(uint64(m.SecondsIdle)) + } + n += len(m.unknownFields) + return n +} + +func (m *ShutdownIfIdleRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + n += len(m.unknownFields) + return n +} + +func (m *ShutdownIfIdleResponse) SizeVT() (n int) { if m == nil { return 0 } var l int _ = l - l = len(m.Data) - if l > 0 { - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + if m.WillShutdown { + n += 2 + } + if m.NumSessions != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.NumSessions)) } n += len(m.unknownFields) return n } -func (m *ListWorkersRequest) SizeVT() (n int) { +func (m *ReserveRequest) SizeVT() (n int) { if m == nil { return 0 } var l int _ = l - if len(m.Filter) > 0 { - for _, s := range m.Filter { - l = len(s) - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) - } - } n += len(m.unknownFields) return n } -func (m *ListWorkersResponse) SizeVT() (n int) { +func (m *ReserveResponse) SizeVT() (n int) { if m == nil { return 0 } var l int _ = l - if len(m.Record) > 0 { - for _, e := range m.Record { - l = e.SizeVT() - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) - } - } n += len(m.unknownFields) return n } -func (m *InfoRequest) SizeVT() (n int) { +func (m *SessionHistoryRequest) SizeVT() (n int) { if m == nil { return 0 } @@ -4511,20 +5181,44 @@ func (m *InfoRequest) SizeVT() (n int) { return n } -func (m *InfoResponse) SizeVT() (n int) { +func (m *SessionHistoryResponse_History) SizeVT() (n int) { if m == nil { return 0 } var l int _ = l - if m.BuildkitVersion != nil { - l = m.BuildkitVersion.SizeVT() + l = len(m.SessionID) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Start != nil { + l = (*timestamppb.Timestamp)(m.Start).SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.End != nil { + l = (*timestamppb.Timestamp)(m.End).SizeVT() n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } n += len(m.unknownFields) return n } +func (m *SessionHistoryResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.History) > 0 { + for _, e := range m.History { + l = e.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + func (m *BuildHistoryRequest) SizeVT() (n int) { if m == nil { return 0 @@ -7022,7 +7716,297 @@ func (m *SolveResponse) UnmarshalVT(dAtA []byte) error { iNdEx += skippy } } - m.ExporterResponse[mapkey] = mapvalue + m.ExporterResponse[mapkey] = mapvalue + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *StatusRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: StatusRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: StatusRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Ref", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Ref = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 99: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field StatsStream", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.StatsStream = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *StatusResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: StatusResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: StatusResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Vertexes", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Vertexes = append(m.Vertexes, &Vertex{}) + if err := m.Vertexes[len(m.Vertexes)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Statuses", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Statuses = append(m.Statuses, &VertexStatus{}) + if err := m.Statuses[len(m.Statuses)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Logs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Logs = append(m.Logs, &VertexLog{}) + if err := m.Logs[len(m.Logs)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Warnings", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Warnings = append(m.Warnings, &VertexWarning{}) + if err := m.Warnings[len(m.Warnings)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } iNdEx = postIndex default: iNdEx = preIndex @@ -7046,7 +8030,7 @@ func (m *SolveResponse) UnmarshalVT(dAtA []byte) error { } return nil } -func (m *StatusRequest) UnmarshalVT(dAtA []byte) error { +func (m *Vertex) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -7069,15 +8053,15 @@ func (m *StatusRequest) UnmarshalVT(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: StatusRequest: wiretype end group for non-group") + return fmt.Errorf("proto: Vertex: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: StatusRequest: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: Vertex: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Ref", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Digest", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -7105,62 +8089,95 @@ func (m *StatusRequest) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Ref = string(dAtA[iNdEx:postIndex]) + m.Digest = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := protohelpers.Skip(dAtA[iNdEx:]) - if err != nil { - return err + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Inputs", wireType) } - if (skippy < 0) || (iNdEx+skippy) < 0 { + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { return protohelpers.ErrInvalidLength } - if (iNdEx + skippy) > l { + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { return io.ErrUnexpectedEOF } - m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *StatusResponse) UnmarshalVT(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow + m.Inputs = append(m.Inputs, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) } - if iNdEx >= l { + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { return io.ErrUnexpectedEOF } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Cached", wireType) } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: StatusResponse: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: StatusResponse: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Cached = bool(v != 0) + case 5: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Vertexes", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Started", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -7187,14 +8204,16 @@ func (m *StatusResponse) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Vertexes = append(m.Vertexes, &Vertex{}) - if err := m.Vertexes[len(m.Vertexes)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + if m.Started == nil { + m.Started = ×tamp.Timestamp{} + } + if err := (*timestamppb.Timestamp)(m.Started).UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex - case 2: + case 6: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Statuses", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Completed", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -7221,16 +8240,18 @@ func (m *StatusResponse) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Statuses = append(m.Statuses, &VertexStatus{}) - if err := m.Statuses[len(m.Statuses)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + if m.Completed == nil { + m.Completed = ×tamp.Timestamp{} + } + if err := (*timestamppb.Timestamp)(m.Completed).UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex - case 3: + case 7: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Logs", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Error", wireType) } - var msglen int + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return protohelpers.ErrIntOverflow @@ -7240,29 +8261,27 @@ func (m *StatusResponse) UnmarshalVT(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { + intStringLen := int(stringLen) + if intStringLen < 0 { return protohelpers.ErrInvalidLength } - postIndex := iNdEx + msglen + postIndex := iNdEx + intStringLen if postIndex < 0 { return protohelpers.ErrInvalidLength } if postIndex > l { return io.ErrUnexpectedEOF } - m.Logs = append(m.Logs, &VertexLog{}) - if err := m.Logs[len(m.Logs)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { - return err - } + m.Error = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 4: + case 8: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Warnings", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ProgressGroup", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -7289,8 +8308,10 @@ func (m *StatusResponse) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Warnings = append(m.Warnings, &VertexWarning{}) - if err := m.Warnings[len(m.Warnings)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + if m.ProgressGroup == nil { + m.ProgressGroup = &pb.ProgressGroup{} + } + if err := m.ProgressGroup.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -7316,7 +8337,7 @@ func (m *StatusResponse) UnmarshalVT(dAtA []byte) error { } return nil } -func (m *Vertex) UnmarshalVT(dAtA []byte) error { +func (m *VertexStatus) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -7339,15 +8360,15 @@ func (m *Vertex) UnmarshalVT(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Vertex: wiretype end group for non-group") + return fmt.Errorf("proto: VertexStatus: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Vertex: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: VertexStatus: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Digest", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -7375,11 +8396,11 @@ func (m *Vertex) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Digest = string(dAtA[iNdEx:postIndex]) + m.ID = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Inputs", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Vertex", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -7407,7 +8428,7 @@ func (m *Vertex) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Inputs = append(m.Inputs, string(dAtA[iNdEx:postIndex])) + m.Vertex = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 2 { @@ -7443,9 +8464,9 @@ func (m *Vertex) UnmarshalVT(dAtA []byte) error { iNdEx = postIndex case 4: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Cached", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Current", wireType) } - var v int + m.Current = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return protohelpers.ErrIntOverflow @@ -7455,13 +8476,67 @@ func (m *Vertex) UnmarshalVT(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int(b&0x7F) << shift + m.Current |= int64(b&0x7F) << shift if b < 0x80 { break } } - m.Cached = bool(v != 0) case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Total", wireType) + } + m.Total = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Total |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Timestamp == nil { + m.Timestamp = ×tamp.Timestamp{} + } + if err := (*timestamppb.Timestamp)(m.Timestamp).UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 7: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Started", wireType) } @@ -7497,7 +8572,7 @@ func (m *Vertex) UnmarshalVT(dAtA []byte) error { return err } iNdEx = postIndex - case 6: + case 8: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Completed", wireType) } @@ -7533,9 +8608,60 @@ func (m *Vertex) UnmarshalVT(dAtA []byte) error { return err } iNdEx = postIndex - case 7: + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *VertexLog) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: VertexLog: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: VertexLog: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Error", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Vertex", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -7563,11 +8689,11 @@ func (m *Vertex) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Error = string(dAtA[iNdEx:postIndex]) + m.Vertex = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 8: + case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ProgressGroup", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -7594,13 +8720,66 @@ func (m *Vertex) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.ProgressGroup == nil { - m.ProgressGroup = &pb.ProgressGroup{} + if m.Timestamp == nil { + m.Timestamp = ×tamp.Timestamp{} } - if err := m.ProgressGroup.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + if err := (*timestamppb.Timestamp)(m.Timestamp).UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Stream", wireType) + } + m.Stream = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Stream |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Msg", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Msg = append(m.Msg[:0], dAtA[iNdEx:postIndex]...) + if m.Msg == nil { + m.Msg = []byte{} + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) @@ -7623,7 +8802,7 @@ func (m *Vertex) UnmarshalVT(dAtA []byte) error { } return nil } -func (m *VertexStatus) UnmarshalVT(dAtA []byte) error { +func (m *VertexWarning) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -7646,15 +8825,15 @@ func (m *VertexStatus) UnmarshalVT(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: VertexStatus: wiretype end group for non-group") + return fmt.Errorf("proto: VertexWarning: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: VertexStatus: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: VertexWarning: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Vertex", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -7666,29 +8845,82 @@ func (m *VertexStatus) UnmarshalVT(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Vertex = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Level", wireType) + } + m.Level = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Level |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Short", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if byteLen < 0 { return protohelpers.ErrInvalidLength } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + byteLen if postIndex < 0 { return protohelpers.ErrInvalidLength } if postIndex > l { return io.ErrUnexpectedEOF } - m.ID = string(dAtA[iNdEx:postIndex]) + m.Short = append(m.Short[:0], dAtA[iNdEx:postIndex]...) + if m.Short == nil { + m.Short = []byte{} + } iNdEx = postIndex - case 2: + case 4: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Vertex", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Detail", wireType) } - var stringLen uint64 + var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return protohelpers.ErrIntOverflow @@ -7698,27 +8930,27 @@ func (m *VertexStatus) UnmarshalVT(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if byteLen < 0 { return protohelpers.ErrInvalidLength } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + byteLen if postIndex < 0 { return protohelpers.ErrInvalidLength } if postIndex > l { return io.ErrUnexpectedEOF } - m.Vertex = string(dAtA[iNdEx:postIndex]) + m.Detail = append(m.Detail, make([]byte, postIndex-iNdEx)) + copy(m.Detail[len(m.Detail)-1], dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 3: + case 5: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Url", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -7746,49 +8978,11 @@ func (m *VertexStatus) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Name = string(dAtA[iNdEx:postIndex]) + m.Url = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 4: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Current", wireType) - } - m.Current = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Current |= int64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 5: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Total", wireType) - } - m.Total = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Total |= int64(b&0x7F) << shift - if b < 0x80 { - break - } - } case 6: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Info", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -7815,16 +9009,16 @@ func (m *VertexStatus) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.Timestamp == nil { - m.Timestamp = ×tamp.Timestamp{} + if m.Info == nil { + m.Info = &pb.SourceInfo{} } - if err := (*timestamppb.Timestamp)(m.Timestamp).UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + if err := m.Info.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 7: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Started", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Ranges", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -7851,18 +9045,67 @@ func (m *VertexStatus) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.Started == nil { - m.Started = ×tamp.Timestamp{} - } - if err := (*timestamppb.Timestamp)(m.Started).UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + m.Ranges = append(m.Ranges, &pb.Range{}) + if err := m.Ranges[len(m.Ranges)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex - case 8: + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *BytesMessage) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: BytesMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: BytesMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Completed", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) } - var msglen int + var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return protohelpers.ErrIntOverflow @@ -7872,26 +9115,24 @@ func (m *VertexStatus) UnmarshalVT(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { + if byteLen < 0 { return protohelpers.ErrInvalidLength } - postIndex := iNdEx + msglen + postIndex := iNdEx + byteLen if postIndex < 0 { return protohelpers.ErrInvalidLength } if postIndex > l { return io.ErrUnexpectedEOF } - if m.Completed == nil { - m.Completed = ×tamp.Timestamp{} - } - if err := (*timestamppb.Timestamp)(m.Completed).UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { - return err + m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) + if m.Data == nil { + m.Data = []byte{} } iNdEx = postIndex default: @@ -7916,7 +9157,7 @@ func (m *VertexStatus) UnmarshalVT(dAtA []byte) error { } return nil } -func (m *VertexLog) UnmarshalVT(dAtA []byte) error { +func (m *ListWorkersRequest) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -7939,15 +9180,15 @@ func (m *VertexLog) UnmarshalVT(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: VertexLog: wiretype end group for non-group") + return fmt.Errorf("proto: ListWorkersRequest: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: VertexLog: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: ListWorkersRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Vertex", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Filter", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -7975,68 +9216,64 @@ func (m *VertexLog) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Vertex = string(dAtA[iNdEx:postIndex]) + m.Filter = append(m.Filter, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return protohelpers.ErrInvalidLength + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err } - postIndex := iNdEx + msglen - if postIndex < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return protohelpers.ErrInvalidLength } - if postIndex > l { + if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - if m.Timestamp == nil { - m.Timestamp = ×tamp.Timestamp{} - } - if err := (*timestamppb.Timestamp)(m.Timestamp).UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { - return err + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ListWorkersResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow } - iNdEx = postIndex - case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Stream", wireType) + if iNdEx >= l { + return io.ErrUnexpectedEOF } - m.Stream = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Stream |= int64(b&0x7F) << shift - if b < 0x80 { - break - } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break } - case 4: + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ListWorkersResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ListWorkersResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Msg", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Record", wireType) } - var byteLen int + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return protohelpers.ErrIntOverflow @@ -8046,24 +9283,24 @@ func (m *VertexLog) UnmarshalVT(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - byteLen |= int(b&0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } } - if byteLen < 0 { + if msglen < 0 { return protohelpers.ErrInvalidLength } - postIndex := iNdEx + byteLen + postIndex := iNdEx + msglen if postIndex < 0 { return protohelpers.ErrInvalidLength } if postIndex > l { return io.ErrUnexpectedEOF } - m.Msg = append(m.Msg[:0], dAtA[iNdEx:postIndex]...) - if m.Msg == nil { - m.Msg = []byte{} + m.Record = append(m.Record, &types.WorkerRecord{}) + if err := m.Record[len(m.Record)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err } iNdEx = postIndex default: @@ -8088,7 +9325,7 @@ func (m *VertexLog) UnmarshalVT(dAtA []byte) error { } return nil } -func (m *VertexWarning) UnmarshalVT(dAtA []byte) error { +func (m *InfoRequest) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -8111,17 +9348,68 @@ func (m *VertexWarning) UnmarshalVT(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: VertexWarning: wiretype end group for non-group") + return fmt.Errorf("proto: InfoRequest: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: VertexWarning: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: InfoRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *InfoResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: InfoResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: InfoResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Vertex", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field BuildkitVersion", wireType) } - var stringLen uint64 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return protohelpers.ErrIntOverflow @@ -8131,29 +9419,33 @@ func (m *VertexWarning) UnmarshalVT(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if msglen < 0 { return protohelpers.ErrInvalidLength } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + msglen if postIndex < 0 { return protohelpers.ErrInvalidLength } if postIndex > l { return io.ErrUnexpectedEOF } - m.Vertex = string(dAtA[iNdEx:postIndex]) + if m.BuildkitVersion == nil { + m.BuildkitVersion = &types.BuildkitVersion{} + } + if err := m.BuildkitVersion.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } iNdEx = postIndex - case 2: + case 101: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Level", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field NumSessions", wireType) } - m.Level = 0 + m.NumSessions = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return protohelpers.ErrIntOverflow @@ -8163,16 +9455,16 @@ func (m *VertexWarning) UnmarshalVT(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Level |= int64(b&0x7F) << shift + m.NumSessions |= uint64(b&0x7F) << shift if b < 0x80 { break } } - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Short", wireType) + case 102: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field SecondsIdle", wireType) } - var byteLen int + m.SecondsIdle = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return protohelpers.ErrIntOverflow @@ -8182,95 +9474,118 @@ func (m *VertexWarning) UnmarshalVT(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - byteLen |= int(b&0x7F) << shift + m.SecondsIdle |= uint64(b&0x7F) << shift if b < 0x80 { break } } - if byteLen < 0 { - return protohelpers.ErrInvalidLength + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err } - postIndex := iNdEx + byteLen - if postIndex < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return protohelpers.ErrInvalidLength } - if postIndex > l { + if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.Short = append(m.Short[:0], dAtA[iNdEx:postIndex]...) - if m.Short == nil { - m.Short = []byte{} + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ShutdownIfIdleRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow } - iNdEx = postIndex - case 4: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Detail", wireType) + if iNdEx >= l { + return io.ErrUnexpectedEOF } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break } - if byteLen < 0 { - return protohelpers.ErrInvalidLength + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ShutdownIfIdleRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ShutdownIfIdleRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err } - postIndex := iNdEx + byteLen - if postIndex < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return protohelpers.ErrInvalidLength } - if postIndex > l { + if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.Detail = append(m.Detail, make([]byte, postIndex-iNdEx)) - copy(m.Detail[len(m.Detail)-1], dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 5: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Url", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return protohelpers.ErrInvalidLength + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ShutdownIfIdleResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow } - if postIndex > l { + if iNdEx >= l { return io.ErrUnexpectedEOF } - m.Url = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 6: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Info", wireType) + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break } - var msglen int + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ShutdownIfIdleResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ShutdownIfIdleResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field WillShutdown", wireType) + } + var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return protohelpers.ErrIntOverflow @@ -8280,33 +9595,17 @@ func (m *VertexWarning) UnmarshalVT(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + v |= int(b&0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Info == nil { - m.Info = &pb.SourceInfo{} - } - if err := m.Info.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 7: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Ranges", wireType) + m.WillShutdown = bool(v != 0) + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field NumSessions", wireType) } - var msglen int + m.NumSessions = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return protohelpers.ErrIntOverflow @@ -8316,26 +9615,11 @@ func (m *VertexWarning) UnmarshalVT(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + m.NumSessions |= uint64(b&0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Ranges = append(m.Ranges, &pb.Range{}) - if err := m.Ranges[len(m.Ranges)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) @@ -8358,7 +9642,7 @@ func (m *VertexWarning) UnmarshalVT(dAtA []byte) error { } return nil } -func (m *BytesMessage) UnmarshalVT(dAtA []byte) error { +func (m *ReserveRequest) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -8381,46 +9665,12 @@ func (m *BytesMessage) UnmarshalVT(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: BytesMessage: wiretype end group for non-group") + return fmt.Errorf("proto: ReserveRequest: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: BytesMessage: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: ReserveRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) - if m.Data == nil { - m.Data = []byte{} - } - iNdEx = postIndex default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) @@ -8443,7 +9693,7 @@ func (m *BytesMessage) UnmarshalVT(dAtA []byte) error { } return nil } -func (m *ListWorkersRequest) UnmarshalVT(dAtA []byte) error { +func (m *ReserveResponse) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -8466,44 +9716,12 @@ func (m *ListWorkersRequest) UnmarshalVT(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: ListWorkersRequest: wiretype end group for non-group") + return fmt.Errorf("proto: ReserveResponse: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: ListWorkersRequest: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: ReserveResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Filter", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Filter = append(m.Filter, string(dAtA[iNdEx:postIndex])) - iNdEx = postIndex default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) @@ -8526,7 +9744,7 @@ func (m *ListWorkersRequest) UnmarshalVT(dAtA []byte) error { } return nil } -func (m *ListWorkersResponse) UnmarshalVT(dAtA []byte) error { +func (m *SessionHistoryRequest) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -8549,46 +9767,12 @@ func (m *ListWorkersResponse) UnmarshalVT(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: ListWorkersResponse: wiretype end group for non-group") + return fmt.Errorf("proto: SessionHistoryRequest: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: ListWorkersResponse: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: SessionHistoryRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Record", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Record = append(m.Record, &types.WorkerRecord{}) - if err := m.Record[len(m.Record)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) @@ -8611,7 +9795,7 @@ func (m *ListWorkersResponse) UnmarshalVT(dAtA []byte) error { } return nil } -func (m *InfoRequest) UnmarshalVT(dAtA []byte) error { +func (m *SessionHistoryResponse_History) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -8634,12 +9818,116 @@ func (m *InfoRequest) UnmarshalVT(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: InfoRequest: wiretype end group for non-group") + return fmt.Errorf("proto: SessionHistoryResponse_History: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: InfoRequest: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: SessionHistoryResponse_History: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SessionID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SessionID = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Start", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Start == nil { + m.Start = ×tamp.Timestamp{} + } + if err := (*timestamppb.Timestamp)(m.Start).UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field End", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.End == nil { + m.End = ×tamp.Timestamp{} + } + if err := (*timestamppb.Timestamp)(m.End).UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) @@ -8662,7 +9950,7 @@ func (m *InfoRequest) UnmarshalVT(dAtA []byte) error { } return nil } -func (m *InfoResponse) UnmarshalVT(dAtA []byte) error { +func (m *SessionHistoryResponse) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -8685,15 +9973,15 @@ func (m *InfoResponse) UnmarshalVT(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: InfoResponse: wiretype end group for non-group") + return fmt.Errorf("proto: SessionHistoryResponse: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: InfoResponse: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: SessionHistoryResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field BuildkitVersion", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field History", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -8720,10 +10008,8 @@ func (m *InfoResponse) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.BuildkitVersion == nil { - m.BuildkitVersion = &types.BuildkitVersion{} - } - if err := m.BuildkitVersion.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + m.History = append(m.History, &SessionHistoryResponse_History{}) + if err := m.History[len(m.History)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex diff --git a/api/services/registry/registry.pb.go b/api/services/registry/registry.pb.go new file mode 100644 index 000000000..37dfef97c --- /dev/null +++ b/api/services/registry/registry.pb.go @@ -0,0 +1,126 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v3.11.4 +// source: github.com/moby/buildkit/api/services/registry/registry.proto + +package earthly_registry_v1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ByteMessage struct { + state protoimpl.MessageState `protogen:"open.v1"` + Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ByteMessage) Reset() { + *x = ByteMessage{} + mi := &file_github_com_moby_buildkit_api_services_registry_registry_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ByteMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ByteMessage) ProtoMessage() {} + +func (x *ByteMessage) ProtoReflect() protoreflect.Message { + mi := &file_github_com_moby_buildkit_api_services_registry_registry_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ByteMessage.ProtoReflect.Descriptor instead. +func (*ByteMessage) Descriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_api_services_registry_registry_proto_rawDescGZIP(), []int{0} +} + +func (x *ByteMessage) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +var File_github_com_moby_buildkit_api_services_registry_registry_proto protoreflect.FileDescriptor + +const file_github_com_moby_buildkit_api_services_registry_registry_proto_rawDesc = "" + + "\n" + + "=github.com/moby/buildkit/api/services/registry/registry.proto\x12\x13earthly.registry.v1\"!\n" + + "\vByteMessage\x12\x12\n" + + "\x04data\x18\x01 \x01(\fR\x04data2[\n" + + "\bRegistry\x12O\n" + + "\x05Proxy\x12 .earthly.registry.v1.ByteMessage\x1a .earthly.registry.v1.ByteMessage(\x010\x01BDZBgithub.com/moby/buildkit/api/services/registry;earthly_registry_v1b\x06proto3" + +var ( + file_github_com_moby_buildkit_api_services_registry_registry_proto_rawDescOnce sync.Once + file_github_com_moby_buildkit_api_services_registry_registry_proto_rawDescData []byte +) + +func file_github_com_moby_buildkit_api_services_registry_registry_proto_rawDescGZIP() []byte { + file_github_com_moby_buildkit_api_services_registry_registry_proto_rawDescOnce.Do(func() { + file_github_com_moby_buildkit_api_services_registry_registry_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_github_com_moby_buildkit_api_services_registry_registry_proto_rawDesc), len(file_github_com_moby_buildkit_api_services_registry_registry_proto_rawDesc))) + }) + return file_github_com_moby_buildkit_api_services_registry_registry_proto_rawDescData +} + +var file_github_com_moby_buildkit_api_services_registry_registry_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_github_com_moby_buildkit_api_services_registry_registry_proto_goTypes = []any{ + (*ByteMessage)(nil), // 0: earthly.registry.v1.ByteMessage +} +var file_github_com_moby_buildkit_api_services_registry_registry_proto_depIdxs = []int32{ + 0, // 0: earthly.registry.v1.Registry.Proxy:input_type -> earthly.registry.v1.ByteMessage + 0, // 1: earthly.registry.v1.Registry.Proxy:output_type -> earthly.registry.v1.ByteMessage + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_github_com_moby_buildkit_api_services_registry_registry_proto_init() } +func file_github_com_moby_buildkit_api_services_registry_registry_proto_init() { + if File_github_com_moby_buildkit_api_services_registry_registry_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_github_com_moby_buildkit_api_services_registry_registry_proto_rawDesc), len(file_github_com_moby_buildkit_api_services_registry_registry_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_github_com_moby_buildkit_api_services_registry_registry_proto_goTypes, + DependencyIndexes: file_github_com_moby_buildkit_api_services_registry_registry_proto_depIdxs, + MessageInfos: file_github_com_moby_buildkit_api_services_registry_registry_proto_msgTypes, + }.Build() + File_github_com_moby_buildkit_api_services_registry_registry_proto = out.File + file_github_com_moby_buildkit_api_services_registry_registry_proto_goTypes = nil + file_github_com_moby_buildkit_api_services_registry_registry_proto_depIdxs = nil +} diff --git a/api/services/registry/registry.proto b/api/services/registry/registry.proto new file mode 100644 index 000000000..af094954a --- /dev/null +++ b/api/services/registry/registry.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +package earthly.registry.v1; + +option go_package = "github.com/moby/buildkit/api/services/registry;earthly_registry_v1"; + +message ByteMessage { + bytes data = 1; +} + +service Registry { + rpc Proxy(stream ByteMessage) returns (stream ByteMessage); +} diff --git a/api/services/registry/registry_grpc.pb.go b/api/services/registry/registry_grpc.pb.go new file mode 100644 index 000000000..f4d4b9929 --- /dev/null +++ b/api/services/registry/registry_grpc.pb.go @@ -0,0 +1,113 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.5.1 +// - protoc v3.11.4 +// source: github.com/moby/buildkit/api/services/registry/registry.proto + +package earthly_registry_v1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + Registry_Proxy_FullMethodName = "/earthly.registry.v1.Registry/Proxy" +) + +// RegistryClient is the client API for Registry service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type RegistryClient interface { + Proxy(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ByteMessage, ByteMessage], error) +} + +type registryClient struct { + cc grpc.ClientConnInterface +} + +func NewRegistryClient(cc grpc.ClientConnInterface) RegistryClient { + return ®istryClient{cc} +} + +func (c *registryClient) Proxy(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ByteMessage, ByteMessage], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &Registry_ServiceDesc.Streams[0], Registry_Proxy_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[ByteMessage, ByteMessage]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Registry_ProxyClient = grpc.BidiStreamingClient[ByteMessage, ByteMessage] + +// RegistryServer is the server API for Registry service. +// All implementations should embed UnimplementedRegistryServer +// for forward compatibility. +type RegistryServer interface { + Proxy(grpc.BidiStreamingServer[ByteMessage, ByteMessage]) error +} + +// UnimplementedRegistryServer should be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedRegistryServer struct{} + +func (UnimplementedRegistryServer) Proxy(grpc.BidiStreamingServer[ByteMessage, ByteMessage]) error { + return status.Errorf(codes.Unimplemented, "method Proxy not implemented") +} +func (UnimplementedRegistryServer) testEmbeddedByValue() {} + +// UnsafeRegistryServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to RegistryServer will +// result in compilation errors. +type UnsafeRegistryServer interface { + mustEmbedUnimplementedRegistryServer() +} + +func RegisterRegistryServer(s grpc.ServiceRegistrar, srv RegistryServer) { + // If the following call pancis, it indicates UnimplementedRegistryServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&Registry_ServiceDesc, srv) +} + +func _Registry_Proxy_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(RegistryServer).Proxy(&grpc.GenericServerStream[ByteMessage, ByteMessage]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Registry_ProxyServer = grpc.BidiStreamingServer[ByteMessage, ByteMessage] + +// Registry_ServiceDesc is the grpc.ServiceDesc for Registry service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Registry_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "earthly.registry.v1.Registry", + HandlerType: (*RegistryServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "Proxy", + Handler: _Registry_Proxy_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "github.com/moby/buildkit/api/services/registry/registry.proto", +} diff --git a/api/services/registry/registry_vtproto.pb.go b/api/services/registry/registry_vtproto.pb.go new file mode 100644 index 000000000..758971309 --- /dev/null +++ b/api/services/registry/registry_vtproto.pb.go @@ -0,0 +1,200 @@ +// Code generated by protoc-gen-go-vtproto. DO NOT EDIT. +// protoc-gen-go-vtproto version: v0.6.1-0.20240319094008-0393e58bdf10 +// source: github.com/moby/buildkit/api/services/registry/registry.proto + +package earthly_registry_v1 + +import ( + fmt "fmt" + protohelpers "github.com/planetscale/vtprotobuf/protohelpers" + proto "google.golang.org/protobuf/proto" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + io "io" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +func (m *ByteMessage) CloneVT() *ByteMessage { + if m == nil { + return (*ByteMessage)(nil) + } + r := new(ByteMessage) + if rhs := m.Data; rhs != nil { + tmpBytes := make([]byte, len(rhs)) + copy(tmpBytes, rhs) + r.Data = tmpBytes + } + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *ByteMessage) CloneMessageVT() proto.Message { + return m.CloneVT() +} + +func (this *ByteMessage) EqualVT(that *ByteMessage) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + if string(this.Data) != string(that.Data) { + return false + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *ByteMessage) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*ByteMessage) + if !ok { + return false + } + return this.EqualVT(that) +} +func (m *ByteMessage) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ByteMessage) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *ByteMessage) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Data) > 0 { + i -= len(m.Data) + copy(dAtA[i:], m.Data) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Data))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *ByteMessage) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Data) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *ByteMessage) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ByteMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ByteMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) + if m.Data == nil { + m.Data = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/api/services/registry/server.go b/api/services/registry/server.go new file mode 100644 index 000000000..318a538d1 --- /dev/null +++ b/api/services/registry/server.go @@ -0,0 +1,150 @@ +package earthly_registry_v1 //nolint:staticcheck + +import ( + "io" + "net" + "strings" + "time" + + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" +) + +const readDeadline = 50 * time.Millisecond + +// NewServer creates and returns a new proxy server with a given host and client. +func NewServer(addr string) *Server { + return &Server{ + addr: addr, + } +} + +// Server connects incoming gRPC data streams to a backing HTTP service. +type Server struct { + addr string + UnimplementedRegistryServer +} + +type streamSource interface { + Send(*ByteMessage) error + Recv() (*ByteMessage, error) +} + +// NewStreamRW creates and returns a gRPC stream reader-writer that implements +// io.Reader & io.Writer as to utilize the gRPC stream with standard methods. +func NewStreamRW(stream streamSource) *StreamRW { + return &StreamRW{stream: stream} +} + +type StreamRW struct { + stream streamSource + last []byte +} + +// Write implements io.Writer. +func (s *StreamRW) Write(p []byte) (int, error) { + err := s.stream.Send(&ByteMessage{ + Data: p, + }) + if err != nil { + return 0, errors.Wrap(err, "failed to write data to client") + } + return len(p), nil +} + +// Read implements io.Reader. +func (s *StreamRW) Read(p []byte) (int, error) { + l := 0 + if len(s.last) > 0 { + l = copy(p, s.last) + } + + msg, err := s.stream.Recv() + if err != nil { + return 0, err + } + + s.last = msg.GetData() + n := copy(p, s.last) + s.last = s.last[n:] + + return n + l, nil +} + +// Proxy requests sent via gRPC data stream to the embedded Docker registry and +// pipe them back out through the stream again. This allows us to send HTTP +// requests to the embedded registry without having to connect via some other +// exposed server or port. +func (s *Server) Proxy(stream Registry_ProxyServer) error { + rw := NewStreamRW(stream) + + addr := strings.ReplaceAll(s.addr, "0.0.0.0", "127.0.0.1") + + ctx := stream.Context() + conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", addr) + if err != nil { + return err + } + defer conn.Close() + eg, _ := errgroup.WithContext(ctx) + + eg.Go(func() error { + _, err = io.Copy(conn, rw) + if err != nil { + return errors.Wrap(err, "failed to copy from stream to host") + } + return nil + }) + + eg.Go(func() error { + _, err = CopyWithDeadline(conn, rw) + if err != nil { + return errors.Wrap(err, "failed to copy from host to stream") + } + return nil + }) + + err = eg.Wait() + if err != nil { + return errors.Wrap(err, "failed to wait") + } + + return nil +} + +// CopyWithDeadline copies data from a net.Conn using a read deadline. The +// process will fail with a timeout error if no data is read for the defined +// period. +func CopyWithDeadline(conn net.Conn, w io.Writer) (int64, error) { + var ( + t = int64(0) + buf = make([]byte, 32*1024) + ) + for { + err := conn.SetReadDeadline(time.Now().Add(readDeadline)) + if err != nil { + return t, err + } + n, err := conn.Read(buf) + if err != nil { + if errors.Is(err, io.EOF) || isNetTimeout(err) { + break + } + return t, err + } + n, err = w.Write(buf[0:n]) + t += int64(n) + if err != nil { + return t, err + } + } + return t, nil +} + +func isNetTimeout(err error) bool { + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + return true + } + return false +} diff --git a/api/types/worker.pb.go b/api/types/worker.pb.go index ccb118d62..b95383961 100644 --- a/api/types/worker.pb.go +++ b/api/types/worker.pb.go @@ -30,8 +30,13 @@ type WorkerRecord struct { GCPolicy []*GCPolicy `protobuf:"bytes,4,rep,name=GCPolicy,proto3" json:"GCPolicy,omitempty"` BuildkitVersion *BuildkitVersion `protobuf:"bytes,5,opt,name=BuildkitVersion,proto3" json:"BuildkitVersion,omitempty"` CDIDevices []*CDIDevice `protobuf:"bytes,6,rep,name=CDIDevices,proto3" json:"CDIDevices,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Earthly-specific. + ParallelismCurrent int64 `protobuf:"varint,101,opt,name=parallelismCurrent,proto3" json:"parallelismCurrent,omitempty"` + ParallelismMax int64 `protobuf:"varint,102,opt,name=parallelismMax,proto3" json:"parallelismMax,omitempty"` + ParallelismWaiting int64 `protobuf:"varint,103,opt,name=parallelismWaiting,proto3" json:"parallelismWaiting,omitempty"` + GCAnalytics *GCAnalytics `protobuf:"bytes,104,opt,name=GCAnalytics,proto3" json:"GCAnalytics,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *WorkerRecord) Reset() { @@ -106,6 +111,234 @@ func (x *WorkerRecord) GetCDIDevices() []*CDIDevice { return nil } +func (x *WorkerRecord) GetParallelismCurrent() int64 { + if x != nil { + return x.ParallelismCurrent + } + return 0 +} + +func (x *WorkerRecord) GetParallelismMax() int64 { + if x != nil { + return x.ParallelismMax + } + return 0 +} + +func (x *WorkerRecord) GetParallelismWaiting() int64 { + if x != nil { + return x.ParallelismWaiting + } + return 0 +} + +func (x *WorkerRecord) GetGCAnalytics() *GCAnalytics { + if x != nil { + return x.GCAnalytics + } + return nil +} + +type GCAnalytics struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Summary of last numRuns. + NumRuns int64 `protobuf:"varint,1,opt,name=numRuns,proto3" json:"numRuns,omitempty"` + NumFailures int64 `protobuf:"varint,2,opt,name=numFailures,proto3" json:"numFailures,omitempty"` + AvgDurationMs int64 `protobuf:"varint,3,opt,name=avgDurationMs,proto3" json:"avgDurationMs,omitempty"` + AvgRecordsCleared int64 `protobuf:"varint,4,opt,name=avgRecordsCleared,proto3" json:"avgRecordsCleared,omitempty"` + AvgSizeCleared int64 `protobuf:"varint,5,opt,name=avgSizeCleared,proto3" json:"avgSizeCleared,omitempty"` + AvgRecordsBefore int64 `protobuf:"varint,6,opt,name=avgRecordsBefore,proto3" json:"avgRecordsBefore,omitempty"` + AvgSizeBefore int64 `protobuf:"varint,7,opt,name=avgSizeBefore,proto3" json:"avgSizeBefore,omitempty"` + // All-time summary. + AllTimeRuns int64 `protobuf:"varint,8,opt,name=allTimeRuns,proto3" json:"allTimeRuns,omitempty"` + AllTimeMaxDurationMs int64 `protobuf:"varint,9,opt,name=allTimeMaxDurationMs,proto3" json:"allTimeMaxDurationMs,omitempty"` + AllTimeDurationMs int64 `protobuf:"varint,10,opt,name=allTimeDurationMs,proto3" json:"allTimeDurationMs,omitempty"` + // Current run (if one is ongoing). + CurrentStartTimeSecEpoch int64 `protobuf:"varint,11,opt,name=currentStartTimeSecEpoch,proto3" json:"currentStartTimeSecEpoch,omitempty"` + CurrentNumRecordsBefore int64 `protobuf:"varint,12,opt,name=currentNumRecordsBefore,proto3" json:"currentNumRecordsBefore,omitempty"` + CurrentSizeBefore int64 `protobuf:"varint,13,opt,name=currentSizeBefore,proto3" json:"currentSizeBefore,omitempty"` + // Last run. + LastStartTimeSecEpoch int64 `protobuf:"varint,14,opt,name=lastStartTimeSecEpoch,proto3" json:"lastStartTimeSecEpoch,omitempty"` + LastEndTimeSecEpoch int64 `protobuf:"varint,15,opt,name=lastEndTimeSecEpoch,proto3" json:"lastEndTimeSecEpoch,omitempty"` + LastNumRecordsBefore int64 `protobuf:"varint,16,opt,name=lastNumRecordsBefore,proto3" json:"lastNumRecordsBefore,omitempty"` + LastSizeBefore int64 `protobuf:"varint,17,opt,name=lastSizeBefore,proto3" json:"lastSizeBefore,omitempty"` + LastNumRecordsCleared int64 `protobuf:"varint,18,opt,name=lastNumRecordsCleared,proto3" json:"lastNumRecordsCleared,omitempty"` + LastSizeCleared int64 `protobuf:"varint,19,opt,name=lastSizeCleared,proto3" json:"lastSizeCleared,omitempty"` + LastSuccess bool `protobuf:"varint,20,opt,name=lastSuccess,proto3" json:"lastSuccess,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GCAnalytics) Reset() { + *x = GCAnalytics{} + mi := &file_github_com_moby_buildkit_api_types_worker_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GCAnalytics) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GCAnalytics) ProtoMessage() {} + +func (x *GCAnalytics) ProtoReflect() protoreflect.Message { + mi := &file_github_com_moby_buildkit_api_types_worker_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GCAnalytics.ProtoReflect.Descriptor instead. +func (*GCAnalytics) Descriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_api_types_worker_proto_rawDescGZIP(), []int{1} +} + +func (x *GCAnalytics) GetNumRuns() int64 { + if x != nil { + return x.NumRuns + } + return 0 +} + +func (x *GCAnalytics) GetNumFailures() int64 { + if x != nil { + return x.NumFailures + } + return 0 +} + +func (x *GCAnalytics) GetAvgDurationMs() int64 { + if x != nil { + return x.AvgDurationMs + } + return 0 +} + +func (x *GCAnalytics) GetAvgRecordsCleared() int64 { + if x != nil { + return x.AvgRecordsCleared + } + return 0 +} + +func (x *GCAnalytics) GetAvgSizeCleared() int64 { + if x != nil { + return x.AvgSizeCleared + } + return 0 +} + +func (x *GCAnalytics) GetAvgRecordsBefore() int64 { + if x != nil { + return x.AvgRecordsBefore + } + return 0 +} + +func (x *GCAnalytics) GetAvgSizeBefore() int64 { + if x != nil { + return x.AvgSizeBefore + } + return 0 +} + +func (x *GCAnalytics) GetAllTimeRuns() int64 { + if x != nil { + return x.AllTimeRuns + } + return 0 +} + +func (x *GCAnalytics) GetAllTimeMaxDurationMs() int64 { + if x != nil { + return x.AllTimeMaxDurationMs + } + return 0 +} + +func (x *GCAnalytics) GetAllTimeDurationMs() int64 { + if x != nil { + return x.AllTimeDurationMs + } + return 0 +} + +func (x *GCAnalytics) GetCurrentStartTimeSecEpoch() int64 { + if x != nil { + return x.CurrentStartTimeSecEpoch + } + return 0 +} + +func (x *GCAnalytics) GetCurrentNumRecordsBefore() int64 { + if x != nil { + return x.CurrentNumRecordsBefore + } + return 0 +} + +func (x *GCAnalytics) GetCurrentSizeBefore() int64 { + if x != nil { + return x.CurrentSizeBefore + } + return 0 +} + +func (x *GCAnalytics) GetLastStartTimeSecEpoch() int64 { + if x != nil { + return x.LastStartTimeSecEpoch + } + return 0 +} + +func (x *GCAnalytics) GetLastEndTimeSecEpoch() int64 { + if x != nil { + return x.LastEndTimeSecEpoch + } + return 0 +} + +func (x *GCAnalytics) GetLastNumRecordsBefore() int64 { + if x != nil { + return x.LastNumRecordsBefore + } + return 0 +} + +func (x *GCAnalytics) GetLastSizeBefore() int64 { + if x != nil { + return x.LastSizeBefore + } + return 0 +} + +func (x *GCAnalytics) GetLastNumRecordsCleared() int64 { + if x != nil { + return x.LastNumRecordsCleared + } + return 0 +} + +func (x *GCAnalytics) GetLastSizeCleared() int64 { + if x != nil { + return x.LastSizeCleared + } + return 0 +} + +func (x *GCAnalytics) GetLastSuccess() bool { + if x != nil { + return x.LastSuccess + } + return false +} + type GCPolicy struct { state protoimpl.MessageState `protogen:"open.v1"` All bool `protobuf:"varint,1,opt,name=all,proto3" json:"all,omitempty"` @@ -121,7 +354,7 @@ type GCPolicy struct { func (x *GCPolicy) Reset() { *x = GCPolicy{} - mi := &file_github_com_moby_buildkit_api_types_worker_proto_msgTypes[1] + mi := &file_github_com_moby_buildkit_api_types_worker_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -133,7 +366,7 @@ func (x *GCPolicy) String() string { func (*GCPolicy) ProtoMessage() {} func (x *GCPolicy) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_api_types_worker_proto_msgTypes[1] + mi := &file_github_com_moby_buildkit_api_types_worker_proto_msgTypes[2] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -146,7 +379,7 @@ func (x *GCPolicy) ProtoReflect() protoreflect.Message { // Deprecated: Use GCPolicy.ProtoReflect.Descriptor instead. func (*GCPolicy) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_api_types_worker_proto_rawDescGZIP(), []int{1} + return file_github_com_moby_buildkit_api_types_worker_proto_rawDescGZIP(), []int{2} } func (x *GCPolicy) GetAll() bool { @@ -202,7 +435,7 @@ type BuildkitVersion struct { func (x *BuildkitVersion) Reset() { *x = BuildkitVersion{} - mi := &file_github_com_moby_buildkit_api_types_worker_proto_msgTypes[2] + mi := &file_github_com_moby_buildkit_api_types_worker_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -214,7 +447,7 @@ func (x *BuildkitVersion) String() string { func (*BuildkitVersion) ProtoMessage() {} func (x *BuildkitVersion) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_api_types_worker_proto_msgTypes[2] + mi := &file_github_com_moby_buildkit_api_types_worker_proto_msgTypes[3] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -227,7 +460,7 @@ func (x *BuildkitVersion) ProtoReflect() protoreflect.Message { // Deprecated: Use BuildkitVersion.ProtoReflect.Descriptor instead. func (*BuildkitVersion) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_api_types_worker_proto_rawDescGZIP(), []int{2} + return file_github_com_moby_buildkit_api_types_worker_proto_rawDescGZIP(), []int{3} } func (x *BuildkitVersion) GetPackage() string { @@ -263,7 +496,7 @@ type CDIDevice struct { func (x *CDIDevice) Reset() { *x = CDIDevice{} - mi := &file_github_com_moby_buildkit_api_types_worker_proto_msgTypes[3] + mi := &file_github_com_moby_buildkit_api_types_worker_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -275,7 +508,7 @@ func (x *CDIDevice) String() string { func (*CDIDevice) ProtoMessage() {} func (x *CDIDevice) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_api_types_worker_proto_msgTypes[3] + mi := &file_github_com_moby_buildkit_api_types_worker_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -288,7 +521,7 @@ func (x *CDIDevice) ProtoReflect() protoreflect.Message { // Deprecated: Use CDIDevice.ProtoReflect.Descriptor instead. func (*CDIDevice) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_api_types_worker_proto_rawDescGZIP(), []int{3} + return file_github_com_moby_buildkit_api_types_worker_proto_rawDescGZIP(), []int{4} } func (x *CDIDevice) GetName() string { @@ -323,7 +556,7 @@ var File_github_com_moby_buildkit_api_types_worker_proto protoreflect.FileDescri const file_github_com_moby_buildkit_api_types_worker_proto_rawDesc = "" + "\n" + - "/github.com/moby/buildkit/api/types/worker.proto\x12\x16moby.buildkit.v1.types\x1a,github.com/moby/buildkit/solver/pb/ops.proto\"\xa3\x03\n" + + "/github.com/moby/buildkit/api/types/worker.proto\x12\x16moby.buildkit.v1.types\x1a,github.com/moby/buildkit/solver/pb/ops.proto\"\xf2\x04\n" + "\fWorkerRecord\x12\x0e\n" + "\x02ID\x18\x01 \x01(\tR\x02ID\x12H\n" + "\x06Labels\x18\x02 \x03(\v20.moby.buildkit.v1.types.WorkerRecord.LabelsEntryR\x06Labels\x12*\n" + @@ -332,10 +565,36 @@ const file_github_com_moby_buildkit_api_types_worker_proto_rawDesc = "" + "\x0fBuildkitVersion\x18\x05 \x01(\v2'.moby.buildkit.v1.types.BuildkitVersionR\x0fBuildkitVersion\x12A\n" + "\n" + "CDIDevices\x18\x06 \x03(\v2!.moby.buildkit.v1.types.CDIDeviceR\n" + - "CDIDevices\x1a9\n" + + "CDIDevices\x12.\n" + + "\x12parallelismCurrent\x18e \x01(\x03R\x12parallelismCurrent\x12&\n" + + "\x0eparallelismMax\x18f \x01(\x03R\x0eparallelismMax\x12.\n" + + "\x12parallelismWaiting\x18g \x01(\x03R\x12parallelismWaiting\x12E\n" + + "\vGCAnalytics\x18h \x01(\v2#.moby.buildkit.v1.types.GCAnalyticsR\vGCAnalytics\x1a9\n" + "\vLabelsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xc8\x01\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x85\a\n" + + "\vGCAnalytics\x12\x18\n" + + "\anumRuns\x18\x01 \x01(\x03R\anumRuns\x12 \n" + + "\vnumFailures\x18\x02 \x01(\x03R\vnumFailures\x12$\n" + + "\ravgDurationMs\x18\x03 \x01(\x03R\ravgDurationMs\x12,\n" + + "\x11avgRecordsCleared\x18\x04 \x01(\x03R\x11avgRecordsCleared\x12&\n" + + "\x0eavgSizeCleared\x18\x05 \x01(\x03R\x0eavgSizeCleared\x12*\n" + + "\x10avgRecordsBefore\x18\x06 \x01(\x03R\x10avgRecordsBefore\x12$\n" + + "\ravgSizeBefore\x18\a \x01(\x03R\ravgSizeBefore\x12 \n" + + "\vallTimeRuns\x18\b \x01(\x03R\vallTimeRuns\x122\n" + + "\x14allTimeMaxDurationMs\x18\t \x01(\x03R\x14allTimeMaxDurationMs\x12,\n" + + "\x11allTimeDurationMs\x18\n" + + " \x01(\x03R\x11allTimeDurationMs\x12:\n" + + "\x18currentStartTimeSecEpoch\x18\v \x01(\x03R\x18currentStartTimeSecEpoch\x128\n" + + "\x17currentNumRecordsBefore\x18\f \x01(\x03R\x17currentNumRecordsBefore\x12,\n" + + "\x11currentSizeBefore\x18\r \x01(\x03R\x11currentSizeBefore\x124\n" + + "\x15lastStartTimeSecEpoch\x18\x0e \x01(\x03R\x15lastStartTimeSecEpoch\x120\n" + + "\x13lastEndTimeSecEpoch\x18\x0f \x01(\x03R\x13lastEndTimeSecEpoch\x122\n" + + "\x14lastNumRecordsBefore\x18\x10 \x01(\x03R\x14lastNumRecordsBefore\x12&\n" + + "\x0elastSizeBefore\x18\x11 \x01(\x03R\x0elastSizeBefore\x124\n" + + "\x15lastNumRecordsCleared\x18\x12 \x01(\x03R\x15lastNumRecordsCleared\x12(\n" + + "\x0flastSizeCleared\x18\x13 \x01(\x03R\x0flastSizeCleared\x12 \n" + + "\vlastSuccess\x18\x14 \x01(\bR\vlastSuccess\"\xc8\x01\n" + "\bGCPolicy\x12\x10\n" + "\x03all\x18\x01 \x01(\bR\x03all\x12\"\n" + "\fkeepDuration\x18\x02 \x01(\x03R\fkeepDuration\x12\x18\n" + @@ -368,28 +627,30 @@ func file_github_com_moby_buildkit_api_types_worker_proto_rawDescGZIP() []byte { return file_github_com_moby_buildkit_api_types_worker_proto_rawDescData } -var file_github_com_moby_buildkit_api_types_worker_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_github_com_moby_buildkit_api_types_worker_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_github_com_moby_buildkit_api_types_worker_proto_goTypes = []any{ (*WorkerRecord)(nil), // 0: moby.buildkit.v1.types.WorkerRecord - (*GCPolicy)(nil), // 1: moby.buildkit.v1.types.GCPolicy - (*BuildkitVersion)(nil), // 2: moby.buildkit.v1.types.BuildkitVersion - (*CDIDevice)(nil), // 3: moby.buildkit.v1.types.CDIDevice - nil, // 4: moby.buildkit.v1.types.WorkerRecord.LabelsEntry - nil, // 5: moby.buildkit.v1.types.CDIDevice.AnnotationsEntry - (*pb.Platform)(nil), // 6: pb.Platform + (*GCAnalytics)(nil), // 1: moby.buildkit.v1.types.GCAnalytics + (*GCPolicy)(nil), // 2: moby.buildkit.v1.types.GCPolicy + (*BuildkitVersion)(nil), // 3: moby.buildkit.v1.types.BuildkitVersion + (*CDIDevice)(nil), // 4: moby.buildkit.v1.types.CDIDevice + nil, // 5: moby.buildkit.v1.types.WorkerRecord.LabelsEntry + nil, // 6: moby.buildkit.v1.types.CDIDevice.AnnotationsEntry + (*pb.Platform)(nil), // 7: pb.Platform } var file_github_com_moby_buildkit_api_types_worker_proto_depIdxs = []int32{ - 4, // 0: moby.buildkit.v1.types.WorkerRecord.Labels:type_name -> moby.buildkit.v1.types.WorkerRecord.LabelsEntry - 6, // 1: moby.buildkit.v1.types.WorkerRecord.platforms:type_name -> pb.Platform - 1, // 2: moby.buildkit.v1.types.WorkerRecord.GCPolicy:type_name -> moby.buildkit.v1.types.GCPolicy - 2, // 3: moby.buildkit.v1.types.WorkerRecord.BuildkitVersion:type_name -> moby.buildkit.v1.types.BuildkitVersion - 3, // 4: moby.buildkit.v1.types.WorkerRecord.CDIDevices:type_name -> moby.buildkit.v1.types.CDIDevice - 5, // 5: moby.buildkit.v1.types.CDIDevice.Annotations:type_name -> moby.buildkit.v1.types.CDIDevice.AnnotationsEntry - 6, // [6:6] is the sub-list for method output_type - 6, // [6:6] is the sub-list for method input_type - 6, // [6:6] is the sub-list for extension type_name - 6, // [6:6] is the sub-list for extension extendee - 0, // [0:6] is the sub-list for field type_name + 5, // 0: moby.buildkit.v1.types.WorkerRecord.Labels:type_name -> moby.buildkit.v1.types.WorkerRecord.LabelsEntry + 7, // 1: moby.buildkit.v1.types.WorkerRecord.platforms:type_name -> pb.Platform + 2, // 2: moby.buildkit.v1.types.WorkerRecord.GCPolicy:type_name -> moby.buildkit.v1.types.GCPolicy + 3, // 3: moby.buildkit.v1.types.WorkerRecord.BuildkitVersion:type_name -> moby.buildkit.v1.types.BuildkitVersion + 4, // 4: moby.buildkit.v1.types.WorkerRecord.CDIDevices:type_name -> moby.buildkit.v1.types.CDIDevice + 1, // 5: moby.buildkit.v1.types.WorkerRecord.GCAnalytics:type_name -> moby.buildkit.v1.types.GCAnalytics + 6, // 6: moby.buildkit.v1.types.CDIDevice.Annotations:type_name -> moby.buildkit.v1.types.CDIDevice.AnnotationsEntry + 7, // [7:7] is the sub-list for method output_type + 7, // [7:7] is the sub-list for method input_type + 7, // [7:7] is the sub-list for extension type_name + 7, // [7:7] is the sub-list for extension extendee + 0, // [0:7] is the sub-list for field type_name } func init() { file_github_com_moby_buildkit_api_types_worker_proto_init() } @@ -403,7 +664,7 @@ func file_github_com_moby_buildkit_api_types_worker_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_github_com_moby_buildkit_api_types_worker_proto_rawDesc), len(file_github_com_moby_buildkit_api_types_worker_proto_rawDesc)), NumEnums: 0, - NumMessages: 6, + NumMessages: 7, NumExtensions: 0, NumServices: 0, }, diff --git a/api/types/worker.proto b/api/types/worker.proto index 8f5656646..20b5ae9c4 100644 --- a/api/types/worker.proto +++ b/api/types/worker.proto @@ -13,6 +13,39 @@ message WorkerRecord { repeated GCPolicy GCPolicy = 4; BuildkitVersion BuildkitVersion = 5; repeated CDIDevice CDIDevices = 6; + + // Earthly-specific. + int64 parallelismCurrent = 101; + int64 parallelismMax = 102; + int64 parallelismWaiting = 103; + GCAnalytics GCAnalytics = 104; +} + +message GCAnalytics { + // Summary of last numRuns. + int64 numRuns = 1; + int64 numFailures = 2; + int64 avgDurationMs = 3; + int64 avgRecordsCleared = 4; + int64 avgSizeCleared = 5; + int64 avgRecordsBefore = 6; + int64 avgSizeBefore = 7; + // All-time summary. + int64 allTimeRuns = 8; + int64 allTimeMaxDurationMs = 9; + int64 allTimeDurationMs = 10; + // Current run (if one is ongoing). + int64 currentStartTimeSecEpoch = 11; + int64 currentNumRecordsBefore = 12; + int64 currentSizeBefore = 13; + // Last run. + int64 lastStartTimeSecEpoch = 14; + int64 lastEndTimeSecEpoch = 15; + int64 lastNumRecordsBefore = 16; + int64 lastSizeBefore = 17; + int64 lastNumRecordsCleared = 18; + int64 lastSizeCleared = 19; + bool lastSuccess = 20; } message GCPolicy { diff --git a/api/types/worker_vtproto.pb.go b/api/types/worker_vtproto.pb.go index 6113a6048..5f0ed2833 100644 --- a/api/types/worker_vtproto.pb.go +++ b/api/types/worker_vtproto.pb.go @@ -27,6 +27,10 @@ func (m *WorkerRecord) CloneVT() *WorkerRecord { r := new(WorkerRecord) r.ID = m.ID r.BuildkitVersion = m.BuildkitVersion.CloneVT() + r.ParallelismCurrent = m.ParallelismCurrent + r.ParallelismMax = m.ParallelismMax + r.ParallelismWaiting = m.ParallelismWaiting + r.GCAnalytics = m.GCAnalytics.CloneVT() if rhs := m.Labels; rhs != nil { tmpContainer := make(map[string]string, len(rhs)) for k, v := range rhs { @@ -66,6 +70,42 @@ func (m *WorkerRecord) CloneMessageVT() proto.Message { return m.CloneVT() } +func (m *GCAnalytics) CloneVT() *GCAnalytics { + if m == nil { + return (*GCAnalytics)(nil) + } + r := new(GCAnalytics) + r.NumRuns = m.NumRuns + r.NumFailures = m.NumFailures + r.AvgDurationMs = m.AvgDurationMs + r.AvgRecordsCleared = m.AvgRecordsCleared + r.AvgSizeCleared = m.AvgSizeCleared + r.AvgRecordsBefore = m.AvgRecordsBefore + r.AvgSizeBefore = m.AvgSizeBefore + r.AllTimeRuns = m.AllTimeRuns + r.AllTimeMaxDurationMs = m.AllTimeMaxDurationMs + r.AllTimeDurationMs = m.AllTimeDurationMs + r.CurrentStartTimeSecEpoch = m.CurrentStartTimeSecEpoch + r.CurrentNumRecordsBefore = m.CurrentNumRecordsBefore + r.CurrentSizeBefore = m.CurrentSizeBefore + r.LastStartTimeSecEpoch = m.LastStartTimeSecEpoch + r.LastEndTimeSecEpoch = m.LastEndTimeSecEpoch + r.LastNumRecordsBefore = m.LastNumRecordsBefore + r.LastSizeBefore = m.LastSizeBefore + r.LastNumRecordsCleared = m.LastNumRecordsCleared + r.LastSizeCleared = m.LastSizeCleared + r.LastSuccess = m.LastSuccess + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *GCAnalytics) CloneMessageVT() proto.Message { + return m.CloneVT() +} + func (m *GCPolicy) CloneVT() *GCPolicy { if m == nil { return (*GCPolicy)(nil) @@ -212,6 +252,18 @@ func (this *WorkerRecord) EqualVT(that *WorkerRecord) bool { } } } + if this.ParallelismCurrent != that.ParallelismCurrent { + return false + } + if this.ParallelismMax != that.ParallelismMax { + return false + } + if this.ParallelismWaiting != that.ParallelismWaiting { + return false + } + if !this.GCAnalytics.EqualVT(that.GCAnalytics) { + return false + } return string(this.unknownFields) == string(that.unknownFields) } @@ -222,6 +274,82 @@ func (this *WorkerRecord) EqualMessageVT(thatMsg proto.Message) bool { } return this.EqualVT(that) } +func (this *GCAnalytics) EqualVT(that *GCAnalytics) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + if this.NumRuns != that.NumRuns { + return false + } + if this.NumFailures != that.NumFailures { + return false + } + if this.AvgDurationMs != that.AvgDurationMs { + return false + } + if this.AvgRecordsCleared != that.AvgRecordsCleared { + return false + } + if this.AvgSizeCleared != that.AvgSizeCleared { + return false + } + if this.AvgRecordsBefore != that.AvgRecordsBefore { + return false + } + if this.AvgSizeBefore != that.AvgSizeBefore { + return false + } + if this.AllTimeRuns != that.AllTimeRuns { + return false + } + if this.AllTimeMaxDurationMs != that.AllTimeMaxDurationMs { + return false + } + if this.AllTimeDurationMs != that.AllTimeDurationMs { + return false + } + if this.CurrentStartTimeSecEpoch != that.CurrentStartTimeSecEpoch { + return false + } + if this.CurrentNumRecordsBefore != that.CurrentNumRecordsBefore { + return false + } + if this.CurrentSizeBefore != that.CurrentSizeBefore { + return false + } + if this.LastStartTimeSecEpoch != that.LastStartTimeSecEpoch { + return false + } + if this.LastEndTimeSecEpoch != that.LastEndTimeSecEpoch { + return false + } + if this.LastNumRecordsBefore != that.LastNumRecordsBefore { + return false + } + if this.LastSizeBefore != that.LastSizeBefore { + return false + } + if this.LastNumRecordsCleared != that.LastNumRecordsCleared { + return false + } + if this.LastSizeCleared != that.LastSizeCleared { + return false + } + if this.LastSuccess != that.LastSuccess { + return false + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *GCAnalytics) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*GCAnalytics) + if !ok { + return false + } + return this.EqualVT(that) +} func (this *GCPolicy) EqualVT(that *GCPolicy) bool { if this == that { return true @@ -354,6 +482,39 @@ func (m *WorkerRecord) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.GCAnalytics != nil { + size, err := m.GCAnalytics.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x6 + i-- + dAtA[i] = 0xc2 + } + if m.ParallelismWaiting != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.ParallelismWaiting)) + i-- + dAtA[i] = 0x6 + i-- + dAtA[i] = 0xb8 + } + if m.ParallelismMax != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.ParallelismMax)) + i-- + dAtA[i] = 0x6 + i-- + dAtA[i] = 0xb0 + } + if m.ParallelismCurrent != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.ParallelismCurrent)) + i-- + dAtA[i] = 0x6 + i-- + dAtA[i] = 0xa8 + } if len(m.CDIDevices) > 0 { for iNdEx := len(m.CDIDevices) - 1; iNdEx >= 0; iNdEx-- { size, err := m.CDIDevices[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) @@ -429,6 +590,154 @@ func (m *WorkerRecord) MarshalToSizedBufferVT(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *GCAnalytics) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *GCAnalytics) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *GCAnalytics) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.LastSuccess { + i-- + if m.LastSuccess { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xa0 + } + if m.LastSizeCleared != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.LastSizeCleared)) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x98 + } + if m.LastNumRecordsCleared != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.LastNumRecordsCleared)) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x90 + } + if m.LastSizeBefore != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.LastSizeBefore)) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x88 + } + if m.LastNumRecordsBefore != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.LastNumRecordsBefore)) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x80 + } + if m.LastEndTimeSecEpoch != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.LastEndTimeSecEpoch)) + i-- + dAtA[i] = 0x78 + } + if m.LastStartTimeSecEpoch != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.LastStartTimeSecEpoch)) + i-- + dAtA[i] = 0x70 + } + if m.CurrentSizeBefore != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.CurrentSizeBefore)) + i-- + dAtA[i] = 0x68 + } + if m.CurrentNumRecordsBefore != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.CurrentNumRecordsBefore)) + i-- + dAtA[i] = 0x60 + } + if m.CurrentStartTimeSecEpoch != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.CurrentStartTimeSecEpoch)) + i-- + dAtA[i] = 0x58 + } + if m.AllTimeDurationMs != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.AllTimeDurationMs)) + i-- + dAtA[i] = 0x50 + } + if m.AllTimeMaxDurationMs != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.AllTimeMaxDurationMs)) + i-- + dAtA[i] = 0x48 + } + if m.AllTimeRuns != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.AllTimeRuns)) + i-- + dAtA[i] = 0x40 + } + if m.AvgSizeBefore != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.AvgSizeBefore)) + i-- + dAtA[i] = 0x38 + } + if m.AvgRecordsBefore != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.AvgRecordsBefore)) + i-- + dAtA[i] = 0x30 + } + if m.AvgSizeCleared != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.AvgSizeCleared)) + i-- + dAtA[i] = 0x28 + } + if m.AvgRecordsCleared != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.AvgRecordsCleared)) + i-- + dAtA[i] = 0x20 + } + if m.AvgDurationMs != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.AvgDurationMs)) + i-- + dAtA[i] = 0x18 + } + if m.NumFailures != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.NumFailures)) + i-- + dAtA[i] = 0x10 + } + if m.NumRuns != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.NumRuns)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + func (m *GCPolicy) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil @@ -674,6 +983,89 @@ func (m *WorkerRecord) SizeVT() (n int) { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } } + if m.ParallelismCurrent != 0 { + n += 2 + protohelpers.SizeOfVarint(uint64(m.ParallelismCurrent)) + } + if m.ParallelismMax != 0 { + n += 2 + protohelpers.SizeOfVarint(uint64(m.ParallelismMax)) + } + if m.ParallelismWaiting != 0 { + n += 2 + protohelpers.SizeOfVarint(uint64(m.ParallelismWaiting)) + } + if m.GCAnalytics != nil { + l = m.GCAnalytics.SizeVT() + n += 2 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *GCAnalytics) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.NumRuns != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.NumRuns)) + } + if m.NumFailures != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.NumFailures)) + } + if m.AvgDurationMs != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.AvgDurationMs)) + } + if m.AvgRecordsCleared != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.AvgRecordsCleared)) + } + if m.AvgSizeCleared != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.AvgSizeCleared)) + } + if m.AvgRecordsBefore != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.AvgRecordsBefore)) + } + if m.AvgSizeBefore != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.AvgSizeBefore)) + } + if m.AllTimeRuns != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.AllTimeRuns)) + } + if m.AllTimeMaxDurationMs != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.AllTimeMaxDurationMs)) + } + if m.AllTimeDurationMs != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.AllTimeDurationMs)) + } + if m.CurrentStartTimeSecEpoch != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.CurrentStartTimeSecEpoch)) + } + if m.CurrentNumRecordsBefore != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.CurrentNumRecordsBefore)) + } + if m.CurrentSizeBefore != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.CurrentSizeBefore)) + } + if m.LastStartTimeSecEpoch != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.LastStartTimeSecEpoch)) + } + if m.LastEndTimeSecEpoch != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.LastEndTimeSecEpoch)) + } + if m.LastNumRecordsBefore != 0 { + n += 2 + protohelpers.SizeOfVarint(uint64(m.LastNumRecordsBefore)) + } + if m.LastSizeBefore != 0 { + n += 2 + protohelpers.SizeOfVarint(uint64(m.LastSizeBefore)) + } + if m.LastNumRecordsCleared != 0 { + n += 2 + protohelpers.SizeOfVarint(uint64(m.LastNumRecordsCleared)) + } + if m.LastSizeCleared != 0 { + n += 2 + protohelpers.SizeOfVarint(uint64(m.LastSizeCleared)) + } + if m.LastSuccess { + n += 3 + } n += len(m.unknownFields) return n } @@ -1085,6 +1477,531 @@ func (m *WorkerRecord) UnmarshalVT(dAtA []byte) error { return err } iNdEx = postIndex + case 101: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ParallelismCurrent", wireType) + } + m.ParallelismCurrent = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ParallelismCurrent |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 102: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ParallelismMax", wireType) + } + m.ParallelismMax = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ParallelismMax |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 103: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ParallelismWaiting", wireType) + } + m.ParallelismWaiting = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ParallelismWaiting |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 104: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GCAnalytics", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.GCAnalytics == nil { + m.GCAnalytics = &GCAnalytics{} + } + if err := m.GCAnalytics.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *GCAnalytics) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: GCAnalytics: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: GCAnalytics: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field NumRuns", wireType) + } + m.NumRuns = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.NumRuns |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field NumFailures", wireType) + } + m.NumFailures = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.NumFailures |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AvgDurationMs", wireType) + } + m.AvgDurationMs = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AvgDurationMs |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AvgRecordsCleared", wireType) + } + m.AvgRecordsCleared = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AvgRecordsCleared |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AvgSizeCleared", wireType) + } + m.AvgSizeCleared = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AvgSizeCleared |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AvgRecordsBefore", wireType) + } + m.AvgRecordsBefore = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AvgRecordsBefore |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 7: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AvgSizeBefore", wireType) + } + m.AvgSizeBefore = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AvgSizeBefore |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 8: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AllTimeRuns", wireType) + } + m.AllTimeRuns = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AllTimeRuns |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 9: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AllTimeMaxDurationMs", wireType) + } + m.AllTimeMaxDurationMs = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AllTimeMaxDurationMs |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 10: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AllTimeDurationMs", wireType) + } + m.AllTimeDurationMs = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AllTimeDurationMs |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 11: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CurrentStartTimeSecEpoch", wireType) + } + m.CurrentStartTimeSecEpoch = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CurrentStartTimeSecEpoch |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 12: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CurrentNumRecordsBefore", wireType) + } + m.CurrentNumRecordsBefore = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CurrentNumRecordsBefore |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 13: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CurrentSizeBefore", wireType) + } + m.CurrentSizeBefore = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CurrentSizeBefore |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 14: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LastStartTimeSecEpoch", wireType) + } + m.LastStartTimeSecEpoch = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LastStartTimeSecEpoch |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 15: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LastEndTimeSecEpoch", wireType) + } + m.LastEndTimeSecEpoch = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LastEndTimeSecEpoch |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 16: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LastNumRecordsBefore", wireType) + } + m.LastNumRecordsBefore = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LastNumRecordsBefore |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 17: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LastSizeBefore", wireType) + } + m.LastSizeBefore = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LastSizeBefore |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 18: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LastNumRecordsCleared", wireType) + } + m.LastNumRecordsCleared = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LastNumRecordsCleared |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 19: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LastSizeCleared", wireType) + } + m.LastSizeCleared = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LastSizeCleared |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 20: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LastSuccess", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.LastSuccess = bool(v != 0) default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) diff --git a/cache/gcanalytics.go b/cache/gcanalytics.go new file mode 100644 index 000000000..a192c90bd --- /dev/null +++ b/cache/gcanalytics.go @@ -0,0 +1,114 @@ +package cache + +import ( + "sync" + "time" +) + +type GCRunAnalytics struct { + Start *time.Time + End *time.Time + Success bool + NumRecordsBefore int64 + ClearedRecords int64 + SizeBefore int64 + ClearedSize int64 +} + +const maxPastGCAnalytics = 10 + +type GCAnalytics struct { + mu sync.Mutex + current GCRunAnalytics + past []GCRunAnalytics + allTimeRuns int64 + allTimeMaxDuration time.Duration + allTimeDuration time.Duration +} + +type GCSummary struct { + NumRuns int + NumFailures int + AvgDuration time.Duration + AvgRecordsCleared int64 + AvgSizeCleared int64 + AvgRecordsBefore int64 + AvgSizeBefore int64 + AllTimeRuns int64 + AllTimeMaxDuration time.Duration + AllTimeDuration time.Duration +} + +func (g *GCAnalytics) Start(numRecordsBefore int64, sizeBefore int64) { + g.mu.Lock() + defer g.mu.Unlock() + now := time.Now() + g.current.Start = &now + g.current.NumRecordsBefore = numRecordsBefore + g.current.SizeBefore = sizeBefore +} + +func (g *GCAnalytics) End(success bool, clearedRecords int64, clearedSize int64) { + g.mu.Lock() + defer g.mu.Unlock() + now := time.Now() + duration := now.Sub(*g.current.Start) + g.current.End = &now + g.current.Success = success + g.current.ClearedRecords = clearedRecords + g.current.ClearedSize = clearedSize + g.past = append(g.past, g.current) + if len(g.past) > maxPastGCAnalytics { + prev := g.past + g.past = make([]GCRunAnalytics, 0, maxPastGCAnalytics) + g.past = append(g.past, prev[1:]...) + } + g.current = GCRunAnalytics{} + g.allTimeRuns++ + g.allTimeDuration += duration + if g.allTimeDuration > g.allTimeMaxDuration { + g.allTimeMaxDuration = g.allTimeDuration + } +} + +func (g *GCAnalytics) GetStats() (GCSummary, *GCRunAnalytics, *GCRunAnalytics) { + g.mu.Lock() + defer g.mu.Unlock() + sum := GCSummary{ + NumRuns: len(g.past), + AllTimeRuns: g.allTimeRuns, + AllTimeMaxDuration: g.allTimeMaxDuration, + AllTimeDuration: g.allTimeDuration, + } + var curRet *GCRunAnalytics + var lastRet *GCRunAnalytics + if g.current.Start != nil { + cur := g.current + curRet = &cur + } + if len(g.past) > 0 { + last := g.past[len(g.past)-1] + lastRet = &last + } + if len(g.past) == 0 { + return sum, curRet, lastRet + } + for _, run := range g.past { + if !run.Success { + sum.NumFailures++ + } + if run.Start != nil && run.End != nil { + sum.AvgDuration += run.End.Sub(*run.Start) + } + sum.AvgRecordsCleared += run.ClearedRecords + sum.AvgSizeCleared += run.ClearedSize + sum.AvgRecordsBefore += run.NumRecordsBefore + sum.AvgSizeBefore += run.SizeBefore + } + sum.AvgDuration /= time.Duration(len(g.past)) + sum.AvgRecordsCleared /= int64(len(g.past)) + sum.AvgSizeCleared /= int64(len(g.past)) + sum.AvgRecordsBefore /= int64(len(g.past)) + sum.AvgSizeBefore /= int64(len(g.past)) + return sum, curRet, lastRet +} diff --git a/cache/manager.go b/cache/manager.go index bd109c9eb..6f582de46 100644 --- a/cache/manager.go +++ b/cache/manager.go @@ -78,6 +78,7 @@ type Controller interface { type Manager interface { Accessor Controller + GetGCAnalytics() (GCSummary, *GCRunAnalytics, *GCRunAnalytics) Close() error } @@ -103,8 +104,9 @@ type cacheManager struct { mountPool sharableMountPool - muPrune sync.Mutex // make sure parallel prune is not allowed so there will not be inconsistent results - unlazyG flightcontrol.Group[struct{}] + muPrune sync.Mutex // make sure parallel prune is not allowed so there will not be inconsistent results + GCAnalytics GCAnalytics // earthly-specific. + unlazyG flightcontrol.Group[struct{}] } func NewManager(opt ManagerOpt) (Manager, error) { @@ -136,6 +138,10 @@ func NewManager(opt ManagerOpt) (Manager, error) { return cm, nil } +func (cm *cacheManager) GetGCAnalytics() (GCSummary, *GCRunAnalytics, *GCRunAnalytics) { + return cm.GCAnalytics.GetStats() +} + func (cm *cacheManager) GetByBlob(ctx context.Context, desc ocispecs.Descriptor, parent ImmutableRef, opts ...RefOption) (ir ImmutableRef, rerr error) { diffID, err := diffIDFromDescriptor(desc) if err != nil { @@ -1048,8 +1054,15 @@ func (cm *cacheManager) prune(ctx context.Context, ch chan client.UsageInfo, opt check = c } + // earthly-specific: create an intermediate channel which is used to + // collect GC-related metrics. + interCh := ch totalSize := int64(0) - if opt.MaxUsedSpace != 0 || opt.ReservedSpace != 0 || opt.MinFreeSpace != 0 { + success := true + gcMode := (opt.MaxUsedSpace != 0 || opt.ReservedSpace != 0 || opt.MinFreeSpace != 0) + doneCh := make(chan struct{}) + if gcMode { + totalRecords := int64(0) du, err := cm.DiskUsage(ctx, client.DiskUsageInfo{}) if err != nil { return err @@ -1060,6 +1073,31 @@ func (cm *cacheManager) prune(ctx context.Context, ch chan client.UsageInfo, opt } totalSize += ui.Size } + totalRecords = int64(len(du)) + + cm.GCAnalytics.Start(totalRecords, totalSize) + interCh = make(chan client.UsageInfo, 1) + go func() { + clearedRecords := int64(0) + clearedSize := int64(0) + defer func() { + cm.GCAnalytics.End(success, clearedRecords, clearedSize) + close(doneCh) + }() + for { + select { + case <-ctx.Done(): + return + case ui, ok := <-interCh: + if !ok { + return + } + ch <- ui + clearedRecords++ + clearedSize += ui.Size + } + } + }() } var dstat disk.DiskStat @@ -1078,13 +1116,23 @@ func (cm *cacheManager) prune(ctx context.Context, ch chan client.UsageInfo, opt keepBytes: calculateKeepBytes(totalSize, dstat, opt), totalSize: totalSize, } + var retErr error for { - releasedSize, releasedCount, err := cm.pruneOnce(ctx, ch, popt) + releasedSize, releasedCount, err := cm.pruneOnce(ctx, interCh, popt) if err != nil || releasedCount == 0 { - return err + retErr = err + break } popt.totalSize -= releasedSize } + if retErr != nil { + success = false + } + if gcMode { + close(interCh) + <-doneCh + } + return retErr } func calculateKeepBytes(totalSize int64, dstat disk.DiskStat, opt client.PruneInfo) int64 { diff --git a/client/build.go b/client/build.go index 65f6a0650..6cca638d2 100644 --- a/client/build.go +++ b/client/build.go @@ -113,6 +113,11 @@ func (g *gatewayClientForBuild) ReadDir(ctx context.Context, in *gatewayapi.Read return g.gateway.ReadDir(ctx, in, opts...) } +func (g *gatewayClientForBuild) Export(ctx context.Context, in *gatewayapi.ExportRequest, opts ...grpc.CallOption) (*gatewayapi.ExportResponse, error) { + ctx = buildid.AppendToOutgoingContext(ctx, g.buildID) + return g.gateway.Export(ctx, in, opts...) +} + func (g *gatewayClientForBuild) StatFile(ctx context.Context, in *gatewayapi.StatFileRequest, opts ...grpc.CallOption) (*gatewayapi.StatFileResponse, error) { if g.caps != nil { if err := g.caps.Supports(gatewayapi.CapStatFile); err != nil { diff --git a/client/client.go b/client/client.go index ff3b6669b..fd1e7a786 100644 --- a/client/client.go +++ b/client/client.go @@ -7,10 +7,12 @@ import ( "net" "net/url" "os" + "strings" "time" contentapi "github.com/containerd/containerd/api/services/content/v1" "github.com/containerd/containerd/v2/defaults" + grpc_retry "github.com/grpc-ecosystem/go-grpc-middleware/retry" controlapi "github.com/moby/buildkit/api/services/control" "github.com/moby/buildkit/client/connhelper" "github.com/moby/buildkit/session" @@ -43,15 +45,21 @@ type ClientOpt interface { // New returns a new buildkit client. Address can be empty for the system-default address. func New(ctx context.Context, address string, opts ...ClientOpt) (*Client, error) { gopts := []grpc.DialOption{ + grpc.WithInitialWindowSize(65535 * 32), // earthly + grpc.WithInitialConnWindowSize(65535 * 16), // earthly grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)), grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)), + grpc.WithDefaultCallOptions(grpc_retry.WithMax(8)), // earthly + grpc.WithDefaultCallOptions(grpc_retry.WithBackoff(grpc_retry.BackoffExponentialWithJitter(10*time.Millisecond, 0.1))), // earthly } needDialer := true + useDefaultDialer := false // earthly-specific var customTracer bool // allows manually setting disabling tracing even if tracer in context var tracerProvider trace.TracerProvider var tracerDelegate TracerDelegate var sessionDialer func(context.Context, string, map[string][]string) (net.Conn, error) + var headersKV []string // earthly-specific var customDialOptions []grpc.DialOption var creds *withCredentials @@ -76,6 +84,17 @@ func New(ctx context.Context, address string, opts ...ClientOpt) (*Client, error if sd, ok := o.(*withSessionDialer); ok { sessionDialer = sd.dialer } + + // earthly-specific + if h, ok := o.(*withAdditionalHeaders); ok { + headersKV = h.kv + } + + // earthly-specific + if _, ok := o.(*withDefaultGRPCDialer); ok { + useDefaultDialer = true + } + if opt, ok := o.(*withGRPCDialOption); ok { customDialOptions = append(customDialOptions, opt.opt) } @@ -108,7 +127,7 @@ func New(ctx context.Context, address string, opts ...ClientOpt) (*Client, error )) } - if needDialer { + if needDialer && !useDefaultDialer { dialFn, err := resolveDialer(address) if err != nil { return nil, err @@ -147,10 +166,25 @@ func New(ctx context.Context, address string, opts ...ClientOpt) (*Client, error } gopts = append(gopts, grpc.WithAuthority(authority)) + + // earthly-specific: chain headers interceptors when additional headers are configured + if len(headersKV) > 0 { + gopts = append(gopts, grpc.WithChainUnaryInterceptor(headersUnaryInterceptor(headersKV...))) + gopts = append(gopts, grpc.WithChainStreamInterceptor(headersStreamInterceptor(headersKV...))) + } + gopts = append(gopts, grpc.WithUnaryInterceptor(grpcerrors.UnaryClientInterceptor)) gopts = append(gopts, grpc.WithStreamInterceptor(grpcerrors.StreamClientInterceptor)) gopts = append(gopts, customDialOptions...) + // earthly-specific + if useDefaultDialer { + split := strings.Split(address, "://") + if len(split) > 0 { + address = split[1] + } + } + // ignore SA1019 NewClient has different behavior and needs to be tested //nolint:staticcheck conn, err := grpc.DialContext(ctx, address, gopts...) diff --git a/client/client_earthly.go b/client/client_earthly.go new file mode 100644 index 000000000..fe9c21c94 --- /dev/null +++ b/client/client_earthly.go @@ -0,0 +1,45 @@ +package client + +import ( + "context" + + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" +) + +// WithAdditionalHeaders adds metadata to all outgoing gRPC calls. It acts just like metadata.Pairs does, and calls *will* fail if +// you provide an odd number of items. Values should be in alternating key-value order - ("k1", "v1", "k1", "v2", "k2", "v3"). +func WithAdditionalMetadataContext(kv ...string) ClientOpt { + return &withAdditionalHeaders{kv} +} + +type withAdditionalHeaders struct { + kv []string +} + +func (*withAdditionalHeaders) isClientOpt() {} + +func headersUnaryInterceptor(kv ...string) grpc.UnaryClientInterceptor { + return func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + ctx = metadata.AppendToOutgoingContext(ctx, kv...) + return invoker(ctx, method, req, reply, cc, opts...) + } +} + +func headersStreamInterceptor(kv ...string) grpc.StreamClientInterceptor { + return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + ctx = metadata.AppendToOutgoingContext(ctx, kv...) + return streamer(ctx, desc, cc, method, opts...) + } +} + +// WithDefaultGRPCDialer triggers the internal gRPC dialer to be used instead of the buildkit default. +// This can be important when buildkit server is behind an HTTP connect proxy, +// since the default dialer in gRPC already knows how to use those. +func WithDefaultGRPCDialer() ClientOpt { + return &withDefaultGRPCDialer{} +} + +type withDefaultGRPCDialer struct{} + +func (*withDefaultGRPCDialer) isClientOpt() {} diff --git a/client/client_test.go b/client/client_test.go index 7c0f5b64a..ae4f0009f 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -13616,15 +13616,15 @@ func testListenBuildHistoryExcludesSoftDeletedRecords(t *testing.T, sb integrati } } -func parseFSMetadata(t *testing.T, dt []byte) []fsutiltypes.Stat { - var m []fsutiltypes.Stat +func parseFSMetadata(t *testing.T, dt []byte) []*fsutiltypes.Stat { + var m []*fsutiltypes.Stat for len(dt) > 0 { - var s fsutiltypes.Stat + s := &fsutiltypes.Stat{} n := binary.LittleEndian.Uint32(dt[:4]) dt = dt[4:] err := s.Unmarshal(dt[:n]) require.NoError(t, err) - m = append(m, *s.CloneVT()) + m = append(m, s) dt = dt[n:] } return m diff --git a/client/exporters.go b/client/exporters.go index 0f70d59c8..7198ba823 100644 --- a/client/exporters.go +++ b/client/exporters.go @@ -1,9 +1,10 @@ package client const ( - ExporterImage = "image" - ExporterLocal = "local" - ExporterTar = "tar" - ExporterOCI = "oci" - ExporterDocker = "docker" + ExporterImage = "image" + ExporterLocal = "local" + ExporterTar = "tar" + ExporterOCI = "oci" + ExporterDocker = "docker" + ExporterEarthly = "earthly" ) diff --git a/client/info.go b/client/info.go index a637ffae1..535f10da0 100644 --- a/client/info.go +++ b/client/info.go @@ -3,13 +3,18 @@ package client import ( "context" + "github.com/pkg/errors" + controlapi "github.com/moby/buildkit/api/services/control" apitypes "github.com/moby/buildkit/api/types" - "github.com/pkg/errors" ) type Info struct { BuildkitVersion BuildkitVersion `json:"buildkitVersion"` + + // Earthly-specific. + NumSessions int `json:"numSessions"` + SecondsIdle int `json:"secondsIdle"` } type BuildkitVersion struct { @@ -32,6 +37,8 @@ func (c *Client) Info(ctx context.Context) (*Info, error) { } return &Info{ BuildkitVersion: fromAPIBuildkitVersion(res.BuildkitVersion), + NumSessions: int(res.NumSessions), + SecondsIdle: int(res.SecondsIdle), }, nil } diff --git a/client/llb/exec.go b/client/llb/exec.go index cd86f6f7c..e591ca9c0 100644 --- a/client/llb/exec.go +++ b/client/llb/exec.go @@ -45,6 +45,7 @@ type mount struct { cacheID string tmpfs bool tmpfsOpt TmpfsInfo + hostBind bool // earthly-specific cacheSharing CacheMountSharingMode noOutput bool contentCache MountContentCache @@ -60,6 +61,7 @@ type ExecOp struct { isValidated bool secrets []SecretInfo ssh []SSHInfo + socket []SocketInfo // earthly-specific cdiDevices []CDIDeviceInfo } @@ -325,6 +327,9 @@ func (e *ExecOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, [] if len(e.ssh) > 0 { addCap(&e.constraints, pb.CapExecMountSSH) } + if len(e.socket) > 0 { + addCap(&e.constraints, pb.CapExecMountSock) + } if len(e.cdiDevices) > 0 { addCap(&e.constraints, pb.CapExecMetaCDI) @@ -421,6 +426,9 @@ func (e *ExecOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, [] Size: m.tmpfsOpt.Size, } } + if m.hostBind { // earthly + pm.MountType = pb.MountType_HOST_BIND + } peo.Mounts = append(peo.Mounts, pm) } @@ -465,6 +473,19 @@ func (e *ExecOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, [] peo.Mounts = append(peo.Mounts, pm) } + for _, s := range e.socket { + peo.Mounts = append(peo.Mounts, &pb.Mount{ + Dest: s.Target, + MountType: pb.MountType_SOCKET, + SockOpt: &pb.SockOpt{ + ID: s.ID, + Uid: uint32(s.UID), + Gid: uint32(s.GID), + Mode: uint32(s.Mode), + }, + }) + } + dt, err := deterministicMarshal(pop) if err != nil { return "", nil, nil, nil, err @@ -594,6 +615,12 @@ type TmpfsInfo struct { Size int64 } +func HostBind() MountOption { // earthly-specific + return func(m *mount) { + m.hostBind = true + } +} + type RunOption interface { SetRunOption(es *ExecInfo) } @@ -755,6 +782,27 @@ type SSHInfo struct { Optional bool } +// SocketTarget mounts a unix socket which is forwarded back to a socketpriver session attachable; earthly-specific +func SocketTarget(id, target string, mode, uid, gid int) RunOption { + return runOptionFunc(func(ei *ExecInfo) { + ei.Socket = append(ei.Socket, SocketInfo{ + ID: id, + Target: target, + Mode: mode, + UID: uid, + GID: gid, + }) + }) +} + +type SocketInfo struct { + ID string + Target string + Mode int + UID int + GID int +} + // AddSecret is a RunOption that adds a secret to the exec. func AddSecret(dest string, opts ...SecretOption) RunOption { return runOptionFunc(func(ei *ExecInfo) { @@ -866,6 +914,7 @@ type ExecInfo struct { ProxyEnv *ProxyEnv Secrets []SecretInfo SSH []SSHInfo + Socket []SocketInfo // earthly-specific CDIDevices []CDIDeviceInfo } diff --git a/client/llb/resolver.go b/client/llb/resolver.go index 037c310e6..6c8ef5efb 100644 --- a/client/llb/resolver.go +++ b/client/llb/resolver.go @@ -5,6 +5,10 @@ import ( digest "github.com/opencontainers/go-digest" ) +// ResolveImageConfigOpt is an earthly-specific type alias for backward compatibility. +// Upstream refactored this into sourceresolver.Opt. +type ResolveImageConfigOpt = sourceresolver.Opt + // WithMetaResolver adds a metadata resolver to an image func WithMetaResolver(mr ImageMetaResolver) ImageOption { return imageOptionFunc(func(ii *ImageInfo) { diff --git a/client/llb/source.go b/client/llb/source.go index ecc6734ff..fe6ad9389 100644 --- a/client/llb/source.go +++ b/client/llb/source.go @@ -4,6 +4,7 @@ import ( "context" _ "crypto/sha256" // for opencontainers/go-digest "encoding/json" + "fmt" "os" "path" "slices" @@ -431,6 +432,18 @@ func Git(url, fragment string, opts ...GitOption) State { attrs[pb.AttrKeepGitDir] = "true" addCap(&gi.Constraints, pb.CapSourceGitKeepDir) } + if gi.LFSInclude != "" { // earthly-specific + attrs[pb.AttrGitLFSInclude] = gi.LFSInclude + addCap(&gi.Constraints, pb.CapSourceGitLFSInclude) + } + if gi.LogLevel > 0 { // earthly-specific + attrs[pb.AttrGitLogLevel] = fmt.Sprintf("%d", gi.LogLevel) + addCap(&gi.Constraints, pb.CapSourceGitLogLevel) + } + if gi.SSHCommand != "" { // earthly-specific + attrs[pb.AttrGitSSHCommand] = gi.SSHCommand + addCap(&gi.Constraints, pb.CapSourceGitSSHCommand) + } if url != "" { attrs[pb.AttrFullRemoteURL] = url addCap(&gi.Constraints, pb.CapSourceGitFullURL) @@ -455,7 +468,11 @@ func Git(url, fragment string, opts ...GitOption) State { if err == nil { // best effort attrs[pb.AttrKnownSSHHosts] = keyscan + } else { + // earthly-specific: panic in order to raise errors (should be caught by earthly cli) + panic(fmt.Sprintf("Git(%s,%s) failed to scan ssh keys: %v", url, ref, err)) } + attrs[pb.AttrKnownSSHHosts] = keyscan } addCap(&gi.Constraints, pb.CapSourceGitKnownSSHHosts) @@ -506,6 +523,9 @@ type GitInfo struct { addAuthCap bool KnownSSHHosts string MountSSHSock string + SSHCommand string // earthly-specific + LFSInclude string // earthly-specific + LogLevel gitutil.GitLogLevel // earthly-specific Checksum string Ref string SubDir string @@ -551,6 +571,20 @@ func KeepGitDir() GitOption { }) } +// LFSInclude is earthly-specific and passes an include string to git lfs pull --include=path; multiple values can be separate by commas (as supported by git lfs) +func LFSInclude(path string) GitOption { + return gitOptionFunc(func(gi *GitInfo) { + gi.LFSInclude = path + }) +} + +// LogLevel is earthly-specific and dynamically controls git logging levels without restarting buildkit +func LogLevel(level gitutil.GitLogLevel) GitOption { + return gitOptionFunc(func(gi *GitInfo) { + gi.LogLevel = level + }) +} + func AuthTokenSecret(v string) GitOption { return gitOptionFunc(func(gi *GitInfo) { gi.AuthTokenSecret = v @@ -571,6 +605,12 @@ func MountSSHSock(sshID string) GitOption { }) } +func SSHCommand(sshCommand string) GitOption { + return gitOptionFunc(func(gi *GitInfo) { + gi.SSHCommand = sshCommand + }) +} + func GitChecksum(v string) GitOption { return gitOptionFunc(func(gi *GitInfo) { gi.Checksum = v diff --git a/client/llb/state.go b/client/llb/state.go index 1ae234f15..2adbd142f 100644 --- a/client/llb/state.go +++ b/client/llb/state.go @@ -296,6 +296,7 @@ func (s State) Run(ro ...RunOption) ExecState { } exec.secrets = ei.Secrets exec.ssh = ei.SSH + exec.socket = ei.Socket exec.cdiDevices = ei.CDIDevices return ExecState{ diff --git a/client/registry.go b/client/registry.go new file mode 100644 index 000000000..0687a88af --- /dev/null +++ b/client/registry.go @@ -0,0 +1,10 @@ +package client + +import registry "github.com/moby/buildkit/api/services/registry" + +// RegistryClient creates a new gRPC client for the embedded Docker +// registry. The client & server use a gRPC data stream to proxy image pull +// requests to the embedded registry. Earthly-specific. +func (c *Client) RegistryClient() registry.RegistryClient { + return registry.NewRegistryClient(c.conn) +} diff --git a/client/reserve.go b/client/reserve.go new file mode 100644 index 000000000..be77e5f9a --- /dev/null +++ b/client/reserve.go @@ -0,0 +1,16 @@ +package client + +import ( + "context" + + controlapi "github.com/moby/buildkit/api/services/control" + "github.com/pkg/errors" +) + +func (c *Client) Reserve(ctx context.Context) error { + _, err := c.ControlClient().Reserve(ctx, &controlapi.ReserveRequest{}) + if err != nil { + return errors.WithStack(err) + } + return nil +} diff --git a/client/sessions.go b/client/sessions.go new file mode 100644 index 000000000..0867c53f7 --- /dev/null +++ b/client/sessions.go @@ -0,0 +1,17 @@ +package client + +import ( + "context" + + "github.com/pkg/errors" + + controlapi "github.com/moby/buildkit/api/services/control" +) + +func (c *Client) SessionHistory(ctx context.Context) ([]*controlapi.SessionHistoryResponse_History, error) { + res, err := c.ControlClient().SessionHistory(ctx, &controlapi.SessionHistoryRequest{}) + if err != nil { + return nil, errors.WithStack(err) + } + return res.History, nil +} diff --git a/client/shutdown.go b/client/shutdown.go new file mode 100644 index 000000000..d4ff27b36 --- /dev/null +++ b/client/shutdown.go @@ -0,0 +1,17 @@ +package client + +import ( + "context" + + controlapi "github.com/moby/buildkit/api/services/control" + "github.com/pkg/errors" +) + +func (c *Client) ShutdownIfIdle(ctx context.Context) (bool, int, error) { + res, err := c.ControlClient().ShutdownIfIdle( + ctx, &controlapi.ShutdownIfIdleRequest{}) + if err != nil { + return false, 0, errors.Wrap(err, "failed to call shutdown if idle") + } + return res.GetWillShutdown(), int(res.GetNumSessions()), nil +} diff --git a/client/solve.go b/client/solve.go index 76fae3b7e..3c653429a 100644 --- a/client/solve.go +++ b/client/solve.go @@ -22,6 +22,7 @@ import ( sessioncontent "github.com/moby/buildkit/session/content" "github.com/moby/buildkit/session/filesync" "github.com/moby/buildkit/session/grpchijack" + "github.com/moby/buildkit/session/pullping" "github.com/moby/buildkit/solver/pb" spb "github.com/moby/buildkit/sourcepolicy/pb" "github.com/moby/buildkit/util/bklog" @@ -57,11 +58,14 @@ type SolveOpt struct { } type ExportEntry struct { - Type string - Attrs map[string]string - Output filesync.FileOutputFunc // for ExporterOCI and ExporterDocker - OutputDir string // for ExporterLocal - OutputStore content.Store + Type string + Attrs map[string]string + Output func(map[string]string) (io.WriteCloser, error) // for ExporterOCI, ExporterDocker and ExporterEarthly + OutputDir string // for ExporterLocal + OutputDirFunc func(map[string]string) (string, error) // for ExporterEarthly + OutputPullCallback pullping.PullCallback // for ExporterEarthly + OutputStore content.Store + OnReceiveFile fsutil.ChangeFunc // earthly-specific: per-file callback for received export files } type CacheOptionsEntry struct { @@ -169,6 +173,8 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG if supportFile && supportStore { return nil, errors.Errorf("both file and store output is not supported by %s exporter", ex.Type) } + case ExporterEarthly: // earthly-specific + supportFile = true } if !supportFile && ex.Output != nil { return nil, errors.Errorf("output file writer is not supported by %s exporter", ex.Type) @@ -183,7 +189,15 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG if ex.Output == nil { return nil, errors.Errorf("output file writer is required for %s exporter", ex.Type) } - syncTargets = append(syncTargets, filesync.WithFSSync(exID, ex.Output)) + // earthly-specific: ExporterEarthly uses multi-target with pull callback + if ex.Type == ExporterEarthly { + if ex.OutputPullCallback != nil { + s.Allow(pullping.NewPullPing(ex.OutputPullCallback)) + } + s.Allow(filesync.NewFSSyncMultiTarget(ex.Output, ex.OutputDirFunc, ex.OnReceiveFile)) + } else { + syncTargets = append(syncTargets, filesync.WithFSSync(exID, ex.Output)) + } } if supportDir { if ex.OutputDir == "" { @@ -235,6 +249,9 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG } frontendAttrs := maps.Clone(opt.FrontendAttrs) + if frontendAttrs == nil { + frontendAttrs = map[string]string{} + } maps.Copy(frontendAttrs, cacheOpt.frontendAttrs) const statusInactivityTimeout = 5 * time.Second @@ -351,7 +368,8 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG eg.Go(func() error { stream, err := c.ControlClient().Status(statusContext, &controlapi.StatusRequest{ - Ref: ref, + Ref: ref, + StatsStream: true, // earthly-specific request stats be streamed back }) if err != nil { return errors.Wrap(err, "failed to get status") @@ -591,6 +609,15 @@ func prepareMounts(opt *SolveOpt) (map[string]fsutil.FS, error) { // merge local mounts and fallback local directories together mounts := make(map[string]fsutil.FS) maps.Copy(mounts, opt.LocalMounts) + + // TODO: earthly-specific hack used to do: + // if localDirs == nil { + // // Earthly specific - skip resolving local dirs found in the definition. + // return nil, nil + // } + // inside prepareSyncedDirs, which has been renamed to prepareSyncedFiles, and that code + // was moved into prepareMounts, which now also has a LocalMounts above + for k, dir := range opt.LocalDirs { mount, err := fsutil.NewFS(dir) if err != nil { diff --git a/client/workers.go b/client/workers.go index a55581751..290944b16 100644 --- a/client/workers.go +++ b/client/workers.go @@ -19,6 +19,39 @@ type WorkerInfo struct { GCPolicy []PruneInfo `json:"gcPolicy"` BuildkitVersion BuildkitVersion `json:"buildkitVersion"` CDIDevices []CDIDevice `json:"cdiDevices"` + + // Earthly-specific. + ParallelismCurrent int `json:"parallelismCurrent"` + ParallelismMax int `json:"parallelismMax"` + ParallelismWaiting int `json:"parallelismWaiting"` + GCAnalytics GCAnalytics +} + +type GCAnalytics struct { + // Summary of last numRuns. + NumRuns int + NumFailures int + AvgDuration time.Duration + AvgRecordsCleared int64 + AvgSizeCleared int64 + AvgRecordsBefore int64 + AvgSizeBefore int64 + // All-time summary. + AllTimeRuns int64 + AllTimeMaxDuration time.Duration + AllTimeDuration time.Duration + // Current run (if one is ongoing). + CurrentStartTime *time.Time + CurrentNumRecordsBefore int64 + CurrentSizeBefore int64 + // Last run. + LastStartTime *time.Time + LastEndTime *time.Time + LastNumRecordsBefore int64 + LastSizeBefore int64 + LastNumRecordsCleared int64 + LastSizeCleared int64 + LastSuccess bool } // ListWorkers lists all active workers @@ -37,6 +70,21 @@ func (c *Client) ListWorkers(ctx context.Context, opts ...ListWorkersOption) ([] var wi []*WorkerInfo for _, w := range resp.Record { + var currentStartTime *time.Time + if w.GetGCAnalytics().GetCurrentStartTimeSecEpoch() != 0 { + t := time.Unix(w.GetGCAnalytics().GetCurrentStartTimeSecEpoch(), 0) + currentStartTime = &t + } + var lastStartTime *time.Time + if w.GetGCAnalytics().GetLastStartTimeSecEpoch() != 0 { + t := time.Unix(w.GetGCAnalytics().GetLastStartTimeSecEpoch(), 0) + lastStartTime = &t + } + var lastEndTime *time.Time + if w.GetGCAnalytics().GetLastEndTimeSecEpoch() != 0 { + t := time.Unix(w.GetGCAnalytics().GetLastEndTimeSecEpoch(), 0) + lastEndTime = &t + } wi = append(wi, &WorkerInfo{ ID: w.ID, Labels: w.Labels, @@ -44,6 +92,33 @@ func (c *Client) ListWorkers(ctx context.Context, opts ...ListWorkersOption) ([] GCPolicy: fromAPIGCPolicy(w.GCPolicy), BuildkitVersion: fromAPIBuildkitVersion(w.BuildkitVersion), CDIDevices: fromAPICDIDevices(w.CDIDevices), + + ParallelismCurrent: int(w.ParallelismCurrent), + ParallelismMax: int(w.ParallelismMax), + ParallelismWaiting: int(w.ParallelismWaiting), + + GCAnalytics: GCAnalytics{ + NumRuns: int(w.GetGCAnalytics().GetNumRuns()), + NumFailures: int(w.GetGCAnalytics().GetNumFailures()), + AvgDuration: time.Duration(w.GetGCAnalytics().GetAvgDurationMs()) * time.Millisecond, + AvgRecordsCleared: w.GetGCAnalytics().GetAvgRecordsCleared(), + AvgSizeCleared: w.GetGCAnalytics().GetAvgSizeCleared(), + AvgRecordsBefore: w.GetGCAnalytics().GetAvgRecordsBefore(), + AvgSizeBefore: w.GetGCAnalytics().GetAvgSizeBefore(), + AllTimeRuns: w.GetGCAnalytics().GetAllTimeRuns(), + AllTimeMaxDuration: time.Duration(w.GetGCAnalytics().GetAllTimeMaxDurationMs()) * time.Millisecond, + AllTimeDuration: time.Duration(w.GetGCAnalytics().GetAllTimeDurationMs()) * time.Millisecond, + CurrentStartTime: currentStartTime, + CurrentNumRecordsBefore: w.GetGCAnalytics().GetCurrentNumRecordsBefore(), + CurrentSizeBefore: w.GetGCAnalytics().GetCurrentSizeBefore(), + LastStartTime: lastStartTime, + LastEndTime: lastEndTime, + LastNumRecordsBefore: w.GetGCAnalytics().GetLastNumRecordsBefore(), + LastSizeBefore: w.GetGCAnalytics().GetLastSizeBefore(), + LastNumRecordsCleared: w.GetGCAnalytics().GetLastNumRecordsCleared(), + LastSizeCleared: w.GetGCAnalytics().GetLastSizeCleared(), + LastSuccess: w.GetGCAnalytics().GetLastSuccess(), + }, }) } diff --git a/cmd/buildkitd/config/config.go b/cmd/buildkitd/config/config.go index 65447eda0..b8144121b 100644 --- a/cmd/buildkitd/config/config.go +++ b/cmd/buildkitd/config/config.go @@ -1,6 +1,8 @@ package config import ( + "time" + "github.com/moby/buildkit/cache/remotecache/gha/ghatypes" resolverconfig "github.com/moby/buildkit/util/resolver/config" ) @@ -35,6 +37,7 @@ type Config struct { DNS *DNSConfig `toml:"dns"` + Health HealthConfig `toml:"health"` History *HistoryConfig `toml:"history"` Frontends struct { @@ -77,6 +80,12 @@ type GRPCConfig struct { // MaxSendMsgSize int `toml:"max_send_message_size"` } +type HealthConfig struct { + Frequency time.Duration `toml:"frequency"` + Timeout time.Duration `toml:"timeout"` + AllowedFailures int `toml:"allowedFailures"` +} + type TLSConfig struct { Cert string `toml:"cert"` Key string `toml:"key"` @@ -143,6 +152,12 @@ type OCIConfig struct { // MaxParallelism is the maximum number of parallel build steps that can be run at the same time. MaxParallelism int `toml:"max-parallelism"` + + // earthly-specific: Hooks are things you can run during any phase of the OCI container runtime lifecycle. + Hooks []Hook `toml:"hook"` + + // SampleFrequency is the frequency between sampling runc processes for stats + SampleFrequency time.Duration `toml:"sample-frequency"` } type ContainerdConfig struct { @@ -208,6 +223,14 @@ type DNSConfig struct { SearchDomains []string `toml:"searchDomains"` } +type Hook struct { + Phase string + Path string + Args []string + Env []string + Timeout *int +} + type HistoryConfig struct { MaxAge Duration `toml:"maxAge"` MaxEntries int64 `toml:"maxEntries"` diff --git a/cmd/buildkitd/main.go b/cmd/buildkitd/main.go index 7de17c050..a1efe2ac7 100644 --- a/cmd/buildkitd/main.go +++ b/cmd/buildkitd/main.go @@ -15,6 +15,7 @@ import ( "strconv" "strings" "sync" + "time" "github.com/containerd/containerd/v2/core/remotes/docker" "github.com/containerd/containerd/v2/defaults" @@ -22,6 +23,8 @@ import ( "github.com/containerd/platforms" sddaemon "github.com/coreos/go-systemd/v22/daemon" "github.com/gofrs/flock" + grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" + regproxy "github.com/moby/buildkit/api/services/registry" "github.com/moby/buildkit/cache/remotecache" "github.com/moby/buildkit/cache/remotecache/azblob" "github.com/moby/buildkit/cache/remotecache/gha" @@ -33,6 +36,7 @@ import ( "github.com/moby/buildkit/cmd/buildkitd/config" "github.com/moby/buildkit/control" "github.com/moby/buildkit/executor/oci" + "github.com/moby/buildkit/exporter/earthlyoutputs/registry" "github.com/moby/buildkit/frontend" dockerfile "github.com/moby/buildkit/frontend/dockerfile/builder" "github.com/moby/buildkit/frontend/gateway" @@ -117,6 +121,11 @@ func registerWorkerInitializer(wi workerInitializer, flags ...cli.Flag) { } func main() { + // Disable gRPC ALPN enforcement to allow mixed grpc-go versions + // between earthly client and buildkitd during the upgrade transition. + // TODO: remove once all released earthly binaries use grpc-go >= 1.67 + os.Setenv("GRPC_ENFORCE_ALPN_ENABLED", "false") + cli.VersionPrinter = func(c *cli.Context) { fmt.Println(c.App.Name, version.Package, c.App.Version, version.Revision) } @@ -273,6 +282,8 @@ func main() { logrus.SetLevel(logrus.TraceLevel) } + logrus.SetOutput(os.Stderr) // earthly-specific: force logs to show up under earthly-buildkitd container logs + if sc := cfg.System; sc != nil { if v := sc.PlatformsCacheMaxAge; v != nil { archutil.CacheMaxAge = v.Duration @@ -302,12 +313,21 @@ func main() { otelgrpc.WithMeterProvider(mp), otelgrpc.WithPropagators(propagators), ) + + unary := grpc_middleware.ChainUnaryServer(unaryInterceptor, grpcerrors.UnaryServerInterceptor, + unaryTimeoutInterceptor(), // earthly-specific + ) + stream := grpc_middleware.ChainStreamServer(grpcerrors.StreamServerInterceptor, + streamTimeoutInterceptor(), // earthly-specific + ) + opts := []grpc.ServerOption{ grpc.StatsHandler(statsHandler), - grpc.ChainUnaryInterceptor(unaryInterceptor, grpcerrors.UnaryServerInterceptor), - grpc.StreamInterceptor(grpcerrors.StreamServerInterceptor), + grpc.UnaryInterceptor(unary), grpc.StreamInterceptor(stream), grpc.MaxRecvMsgSize(defaults.DefaultMaxRecvMsgSize), grpc.MaxSendMsgSize(defaults.DefaultMaxSendMsgSize), + grpc.InitialWindowSize(65535 * 32), + grpc.InitialConnWindowSize(65535 * 16), } server := grpc.NewServer(opts...) @@ -345,6 +365,8 @@ func main() { os.RemoveAll(lockPath) }() + shutdownCh := make(chan struct{}) + // listeners have to be initialized before the controller // https://github.com/moby/buildkit/issues/4618 listeners, err := newGRPCListeners(cfg.GRPC) @@ -361,7 +383,7 @@ func main() { defer db.Close() } - controller, err := newController(ctx, c, &cfg) + controller, err := newController(ctx, c, &cfg, shutdownCh) if err != nil { return err } @@ -371,6 +393,35 @@ func main() { controller.Register(server) reflection.Register(server) + // Earthly specific. + ctxReg, cancelReg := context.WithCancelCause(ctx) + defer cancelReg(nil) + lrPort, ok := os.LookupEnv("BUILDKIT_LOCAL_REGISTRY_LISTEN_PORT") + lrAddr := fmt.Sprintf("0.0.0.0:%s", lrPort) + if ok { + bklog.G(ctx).Infof("Starting local registry for outputs on port %s", lrPort) + serveErr := registry.Serve(ctxReg, lrAddr) + go func() { + for { + select { + case <-shutdownCh: + cancelReg(nil) + case err := <-serveErr: + if err != nil { + bklog.G(ctx).Errorf("Registry serve error: %s\n", err.Error()) + } + return + case <-ctxReg.Done(): + return + } + } + }() + } + + // Earthly specific. + reg := regproxy.NewServer(lrAddr) + regproxy.RegisterRegistryServer(server, reg) + ents := c.GlobalStringSlice("allow-insecure-entitlement") if len(ents) > 0 { cfg.Entitlements = []string{} @@ -404,6 +455,9 @@ func main() { cancel(err) case <-ctx.Done(): err = context.Cause(ctx) + case <-shutdownCh: + cancelReg(nil) + err = nil } bklog.G(ctx).Infof("stopping server") @@ -542,6 +596,18 @@ func setDefaultConfig(cfg *config.Config) { cfg.GRPC.Address = []string{appdefaults.Address} } + if cfg.Health.Frequency == 0 { + cfg.Health.Frequency = appdefaults.HealthFrequency + } + + if cfg.Health.Timeout == 0 { + cfg.Health.Timeout = appdefaults.HealthTimeout + } + + if cfg.Health.AllowedFailures == 0 { + cfg.Health.AllowedFailures = appdefaults.HealthAllowedFailures + } + if cfg.Workers.OCI.Platforms == nil { cfg.Workers.OCI.Platforms = formatPlatforms(archutil.SupportedPlatforms(false)) } @@ -562,6 +628,10 @@ func setDefaultConfig(cfg *config.Config) { appdefaults.EnsureUserAddressDir() } + if cfg.Workers.OCI.SampleFrequency == 0 { + cfg.Workers.OCI.SampleFrequency = time.Second + } + if cfg.OTEL.SocketPath == "" { cfg.OTEL.SocketPath = appdefaults.TraceSocketPath(isRootlessConfig()) } @@ -779,8 +849,13 @@ func serverCredentials(cfg config.TLSConfig) (*tls.Config, error) { return tlsConf, nil } -func newController(ctx context.Context, c *cli.Context, cfg *config.Config) (*control.Controller, error) { - sessionManager, err := session.NewManager() +func newController(ctx context.Context, c *cli.Context, cfg *config.Config, shutdownCh chan struct{}) (*control.Controller, error) { + sessionManager, err := session.NewManager(&session.ManagerOpt{ + HealthFrequency: cfg.Health.Frequency, + HealthAllowedFailures: cfg.Health.AllowedFailures, + HealthTimeout: cfg.Health.Timeout, + ShutdownCh: shutdownCh, + }) if err != nil { return nil, err } diff --git a/cmd/buildkitd/main_containerd_worker.go b/cmd/buildkitd/main_containerd_worker.go index 0ee74393e..0018e9d53 100644 --- a/cmd/buildkitd/main_containerd_worker.go +++ b/cmd/buildkitd/main_containerd_worker.go @@ -16,6 +16,7 @@ import ( "github.com/moby/buildkit/util/disk" "github.com/moby/buildkit/util/network/cniprovider" "github.com/moby/buildkit/util/network/netproviders" + "github.com/moby/buildkit/util/semutil" "github.com/moby/buildkit/worker" "github.com/moby/buildkit/worker/base" "github.com/moby/buildkit/worker/containerd" @@ -23,7 +24,6 @@ import ( "github.com/pelletier/go-toml/v2" "github.com/pkg/errors" "github.com/urfave/cli" - "golang.org/x/sync/semaphore" ) const ( @@ -307,9 +307,9 @@ func containerdWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([ }, } - var parallelismSem *semaphore.Weighted + var parallelismSem *semutil.Weighted if cfg.MaxParallelism > 0 { - parallelismSem = semaphore.NewWeighted(int64(cfg.MaxParallelism)) + parallelismSem = semutil.NewWeighted(int64(cfg.MaxParallelism)) } snapshotter := defaults.DefaultSnapshotter diff --git a/cmd/buildkitd/main_oci_worker.go b/cmd/buildkitd/main_oci_worker.go index f221e0430..bfbe27720 100644 --- a/cmd/buildkitd/main_oci_worker.go +++ b/cmd/buildkitd/main_oci_worker.go @@ -37,6 +37,7 @@ import ( "github.com/moby/buildkit/util/network/cniprovider" "github.com/moby/buildkit/util/network/netproviders" "github.com/moby/buildkit/util/resolver" + "github.com/moby/buildkit/util/semutil" "github.com/moby/buildkit/worker" "github.com/moby/buildkit/worker/base" "github.com/moby/buildkit/worker/runc" @@ -45,7 +46,6 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/urfave/cli" - "golang.org/x/sync/semaphore" "google.golang.org/grpc" "google.golang.org/grpc/backoff" "google.golang.org/grpc/credentials/insecure" @@ -322,12 +322,23 @@ func ociWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([]worker }, } - var parallelismSem *semaphore.Weighted + var parallelismSem *semutil.Weighted if cfg.MaxParallelism > 0 { - parallelismSem = semaphore.NewWeighted(int64(cfg.MaxParallelism)) + parallelismSem = semutil.NewWeighted(int64(cfg.MaxParallelism)) } - opt, err := runc.NewWorkerOpt(common.config.Root, snFactory, cfg.Rootless, processMode, cfg.Labels, idmapping, nc, dns, cfg.Binary, cfg.ApparmorProfile, cfg.SELinux, parallelismSem, common.traceSocket, cfg.DefaultCgroupParent, cdiManager) + ociHooks := []oci.OciHook{} + for _, h := range cfg.Hooks { + ociHooks = append(ociHooks, oci.OciHook{ + Phase: h.Phase, + Path: h.Path, + Args: h.Args, + Env: h.Env, + Timeout: h.Timeout, + }) + } + + opt, err := runc.NewWorkerOpt(common.config.Root, snFactory, cfg.Rootless, processMode, cfg.Labels, idmapping, nc, dns, cfg.Binary, cfg.ApparmorProfile, cfg.SELinux, parallelismSem, common.traceSocket, cfg.DefaultCgroupParent, ociHooks, cfg.SampleFrequency, cdiManager) if err != nil { return nil, err } @@ -371,6 +382,8 @@ func snapshotterFactory(commonRoot string, cfg config.OCIConfig, sm *session.Man grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithConnectParams(connParams), grpc.WithContextDialer(dialer.ContextDialer), + grpc.WithInitialWindowSize(65535 * 32), + grpc.WithInitialConnWindowSize(65535 * 16), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)), grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)), } diff --git a/cmd/buildkitd/main_unix.go b/cmd/buildkitd/main_unix.go index c503de509..2d5477dba 100644 --- a/cmd/buildkitd/main_unix.go +++ b/cmd/buildkitd/main_unix.go @@ -4,6 +4,7 @@ package main import ( "crypto/tls" + "fmt" "net" "os" "syscall" @@ -17,6 +18,19 @@ const socketScheme = "unix://" func init() { syscall.Umask(0) + + // Docker 29+ (containerd v2) lowered the default open file limit from + // 1048576 to 1024, which starves buildkitd. Raise it back. + var lim syscall.Rlimit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &lim); err == nil && lim.Cur < 1048576 { + lim.Cur = 1048576 + if lim.Max < 1048576 { + lim.Max = 1048576 + } + if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &lim); err != nil { + fmt.Fprintf(os.Stderr, "warning: failed to raise RLIMIT_NOFILE: %v\n", err) + } + } } func listenFD(addr string, tlsConfig *tls.Config) (net.Listener, error) { diff --git a/cmd/buildkitd/timeout_interceptor.go b/cmd/buildkitd/timeout_interceptor.go new file mode 100644 index 000000000..cb38da62b --- /dev/null +++ b/cmd/buildkitd/timeout_interceptor.go @@ -0,0 +1,119 @@ +package main + +// The sessionTimeout interceptors in this file are earthly-specific. +// They are used to automatically cancel builds in CI or Satellites that have run for too long. + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/pkg/errors" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" +) + +var sessionTimeout time.Duration + +var errSessionTimeout = errors.New("session timeout") + +func init() { + env, ok := os.LookupEnv("BUILDKIT_SESSION_TIMEOUT") + if !ok { + return + } + var err error + sessionTimeout, err = time.ParseDuration(env) + if err != nil { + panic(fmt.Sprintf("invalid value for 'BUILDKIT_SESSION_TIMEOUT': %s", env)) + } +} + +func unaryTimeoutInterceptor() grpc.UnaryServerInterceptor { + return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + if sessionTimeout > 0 { + // TODO we should replace the following code with context.WithTimeoutCause + // when it is is released in a future version of Go + // https://github.com/golang/go/blob/master/src/context/context.go#L688-L693 + ctx, cancel := context.WithCancelCause(ctx) + defer cancel(nil) + done := make(chan bool) + defer close(done) + go handleTimeout(done, cancel) + // End of TODO + resp, err := handler(ctx, req) + if errors.Is(err, context.Canceled) && errors.Is(context.Cause(ctx), errSessionTimeout) { + return resp, errors.Errorf("build exceeded max duration of %s", sessionTimeout.String()) + } + return resp, err + } + return handler(ctx, req) + } +} + +func streamTimeoutInterceptor() grpc.StreamServerInterceptor { + return func(srv any, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + if sessionTimeout > 0 { + // TODO we should replace the following code with context.WithTimeoutCause + // when it is is released in a future version of Go + // https://github.com/golang/go/blob/master/src/context/context.go#L688-L693 + ctx, cancel := context.WithCancelCause(stream.Context()) + defer cancel(nil) + done := make(chan bool) + defer close(done) + go handleTimeout(done, cancel) + // End of TODO + err := handler(srv, newWrappedStream(ctx, stream)) + if errors.Is(err, context.Canceled) && errors.Is(context.Cause(ctx), errSessionTimeout) { + return errors.Errorf("build exceeded max duration of %s", sessionTimeout.String()) + } + return err + } + return handler(srv, stream) + } +} + +func handleTimeout(doneCh chan bool, cancelFn func(error)) { + sessionTimer := time.NewTimer(sessionTimeout) + defer sessionTimer.Stop() + select { + case <-doneCh: + return + case <-sessionTimer.C: + cancelFn(errSessionTimeout) + } +} + +type wrappedStream struct { + s grpc.ServerStream + ctx context.Context +} + +func (w *wrappedStream) RecvMsg(m any) error { + return w.s.RecvMsg(m) +} + +func (w *wrappedStream) SendMsg(m any) error { + return w.s.SendMsg(m) +} + +func (w *wrappedStream) Context() context.Context { + return w.ctx +} + +func (w *wrappedStream) SetHeader(m metadata.MD) error { + return w.s.SetHeader(m) +} + +func (w *wrappedStream) SendHeader(m metadata.MD) error { + return w.s.SendHeader(m) +} + +func (w *wrappedStream) SetTrailer(m metadata.MD) { + w.s.SetTrailer(m) +} + +func newWrappedStream(ctx context.Context, s grpc.ServerStream) grpc.ServerStream { + return &wrappedStream{s: s, ctx: ctx} +} diff --git a/control/control.go b/control/control.go index e91f566c7..2d11ea720 100644 --- a/control/control.go +++ b/control/control.go @@ -4,6 +4,7 @@ import ( "context" stderrors "errors" "fmt" + "math" "runtime/trace" "strconv" "sync" @@ -559,7 +560,7 @@ func (c *Controller) Status(req *controlapi.StatusRequest, stream controlapi.Con eg, ctx := errgroup.WithContext(stream.Context()) eg.Go(func() error { - return c.solver.Status(ctx, req.Ref, ch) + return c.solver.Status(ctx, req.Ref, req.StatsStream, ch) }) eg.Go(func() error { @@ -608,6 +609,34 @@ func (c *Controller) ListWorkers(ctx context.Context, r *controlapi.ListWorkersR return nil, err } for _, w := range workers { + pc, pm, pw := w.ParallelismStatus() + gcSummary, gcCurrent, gcLast := w.GCAnalytics() + gca := &apitypes.GCAnalytics{ + NumRuns: int64(gcSummary.NumRuns), + NumFailures: int64(gcSummary.NumFailures), + AvgDurationMs: gcSummary.AvgDuration.Milliseconds(), + AvgRecordsCleared: gcSummary.AvgRecordsCleared, + AvgRecordsBefore: gcSummary.AvgRecordsBefore, + AvgSizeCleared: gcSummary.AvgSizeCleared, + AvgSizeBefore: gcSummary.AvgSizeBefore, + AllTimeRuns: gcSummary.AllTimeRuns, + AllTimeMaxDurationMs: gcSummary.AllTimeMaxDuration.Milliseconds(), + AllTimeDurationMs: gcSummary.AllTimeDuration.Milliseconds(), + } + if gcCurrent != nil { + gca.CurrentStartTimeSecEpoch = gcCurrent.Start.Unix() + gca.CurrentNumRecordsBefore = int64(gcCurrent.NumRecordsBefore) + gca.CurrentSizeBefore = int64(gcCurrent.SizeBefore) + } + if gcLast != nil { + gca.LastStartTimeSecEpoch = gcLast.Start.Unix() + gca.LastEndTimeSecEpoch = gcLast.End.Unix() + gca.LastNumRecordsBefore = int64(gcLast.NumRecordsBefore) + gca.LastSizeBefore = int64(gcLast.SizeBefore) + gca.LastNumRecordsCleared = int64(gcLast.ClearedRecords) + gca.LastSizeCleared = int64(gcLast.ClearedSize) + gca.LastSuccess = gcLast.Success + } resp.Record = append(resp.Record, &apitypes.WorkerRecord{ ID: w.ID(), Labels: w.Labels(), @@ -615,21 +644,60 @@ func (c *Controller) ListWorkers(ctx context.Context, r *controlapi.ListWorkersR GCPolicy: toPBGCPolicy(w.GCPolicy()), BuildkitVersion: toPBBuildkitVersion(w.BuildkitVersion()), CDIDevices: toPBCDIDevices(w.CDIManager()), + + ParallelismCurrent: pc, + ParallelismMax: pm, + ParallelismWaiting: pw, + + GCAnalytics: gca, }) } return resp, nil } func (c *Controller) Info(ctx context.Context, r *controlapi.InfoRequest) (*controlapi.InfoResponse, error) { + numSessions, durationIdle := c.opt.SessionManager.NumSessions() + secondsIdle := math.Round(durationIdle.Seconds()) return &controlapi.InfoResponse{ BuildkitVersion: &apitypes.BuildkitVersion{ Package: version.Package, Version: version.Version, Revision: version.Revision, }, + NumSessions: uint64(numSessions), + SecondsIdle: uint64(secondsIdle), + }, nil +} + +func (c *Controller) ShutdownIfIdle(ctx context.Context, r *controlapi.ShutdownIfIdleRequest) (*controlapi.ShutdownIfIdleResponse, error) { + ok, numSessions := c.opt.SessionManager.StopIfIdle() + return &controlapi.ShutdownIfIdleResponse{ + WillShutdown: ok, + NumSessions: uint64(numSessions), }, nil } +func (c *Controller) Reserve(ctx context.Context, r *controlapi.ReserveRequest) (*controlapi.ReserveResponse, error) { + err := c.opt.SessionManager.Reserve() + if err != nil { + return nil, status.Error(codes.Unavailable, "buildkit is shutting down") + } + return &controlapi.ReserveResponse{}, nil +} + +func (c *Controller) SessionHistory(ctx context.Context, r *controlapi.SessionHistoryRequest) (*controlapi.SessionHistoryResponse, error) { + history := c.opt.SessionManager.GetSessionHistory() + var resp []*controlapi.SessionHistoryResponse_History + for id, h := range history { + resp = append(resp, &controlapi.SessionHistoryResponse_History{ + SessionID: id, + Start: timestamppb.New(h.Start), + End: timestamppb.New(h.End), + }) + } + return &controlapi.SessionHistoryResponse{History: resp}, nil +} + func (c *Controller) gc() { c.gcmu.Lock() defer c.gcmu.Unlock() diff --git a/control/gateway/gateway.go b/control/gateway/gateway.go index a285acf2d..554da078c 100644 --- a/control/gateway/gateway.go +++ b/control/gateway/gateway.go @@ -164,6 +164,14 @@ func (gwf *GatewayForwarder) ReadDir(ctx context.Context, req *gwapi.ReadDirRequ return fwd.ReadDir(ctx, req) } +func (gwf *GatewayForwarder) Export(ctx context.Context, req *gwapi.ExportRequest) (*gwapi.ExportResponse, error) { + fwd, err := gwf.lookupForwarder(ctx) + if err != nil { + return nil, errors.Wrap(err, "forwarding Export") + } + return fwd.Export(ctx, req) +} + func (gwf *GatewayForwarder) StatFile(ctx context.Context, req *gwapi.StatFileRequest) (*gwapi.StatFileResponse, error) { fwd, err := gwf.lookupForwarder(ctx) if err != nil { diff --git a/executor/executor.go b/executor/executor.go index aa011e27d..e1298af4c 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -56,6 +56,7 @@ type ProcessInfo struct { Meta Meta Stdin io.ReadCloser Stdout, Stderr io.WriteCloser + StatsStream io.WriteCloser // earthly-specific Resize <-chan WinSize Signal <-chan syscall.Signal } diff --git a/executor/oci/hooks.go b/executor/oci/hooks.go new file mode 100644 index 000000000..ed9f59b0e --- /dev/null +++ b/executor/oci/hooks.go @@ -0,0 +1,53 @@ +package oci + +import ( + "context" + + "github.com/containerd/containerd/v2/core/containers" + "github.com/containerd/containerd/v2/pkg/oci" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" +) + +type OciHook struct { + Phase string + Path string + Args []string + Env []string + Timeout *int +} + +func WithHook(hook OciHook) oci.SpecOpts { + return func(ctx context.Context, client oci.Client, c *containers.Container, s *oci.Spec) error { + if s.Hooks == nil { + s.Hooks = &specs.Hooks{} + } + + h := specs.Hook{ + Path: hook.Path, + Args: hook.Args, + Env: hook.Env, + Timeout: hook.Timeout, + } + + // Yes, its verbose... but it reads _so much better_ than the golang reflection version + switch hook.Phase { + case "prestart": + s.Hooks.Prestart = append(s.Hooks.Prestart, h) //nolint:staticcheck // intentional support for legacy prestart hooks + case "createRuntime": + s.Hooks.CreateRuntime = append(s.Hooks.CreateRuntime, h) + case "createContainer": + s.Hooks.CreateContainer = append(s.Hooks.CreateContainer, h) + case "startContainer": + s.Hooks.StartContainer = append(s.Hooks.StartContainer, h) + case "poststart": + s.Hooks.Poststart = append(s.Hooks.Poststart, h) + case "poststop": + s.Hooks.Poststop = append(s.Hooks.Poststop, h) + default: + return errors.Errorf("%s is not a valid lifecycle hook", hook.Phase) + } + + return nil + } +} diff --git a/executor/runcexecutor/executor.go b/executor/runcexecutor/executor.go index c4e5f8fd0..fee1142a2 100644 --- a/executor/runcexecutor/executor.go +++ b/executor/runcexecutor/executor.go @@ -57,7 +57,9 @@ type Opt struct { ApparmorProfile string SELinux bool TracingSocket string + Hooks []oci.OciHook // earthly-specific ResourceMonitor *resources.Monitor + SampleFrequency time.Duration // earthly-specific CDIManager *cdidevices.Manager } @@ -79,7 +81,9 @@ type runcExecutor struct { apparmorProfile string selinux bool tracingSocket string + hooks []oci.OciHook // earthly-specific resmon *resources.Monitor + sampleFrequency time.Duration // earthly-specific cdiManager *cdidevices.Manager } @@ -146,7 +150,9 @@ func New(opt Opt, networkProviders map[pb.NetMode]network.Provider) (executor.Ex apparmorProfile: opt.ApparmorProfile, selinux: opt.SELinux, tracingSocket: opt.TracingSocket, + hooks: opt.Hooks, // earthly-specific resmon: opt.ResourceMonitor, + sampleFrequency: opt.SampleFrequency, // earthly-specific cdiManager: opt.CDIManager, } return w, nil @@ -280,6 +286,13 @@ func (w *runcExecutor) Run(ctx context.Context, id string, root executor.Mount, } } + // earthly-specific + if len(w.hooks) > 0 { + for _, h := range w.hooks { + opts = append(opts, oci.WithHook(h)) + } + } + spec, cleanup, err := oci.GenerateSpec(ctx, meta, mounts, id, resolvConf, hostsFile, namespace, w.cgroupParent, w.processMode, w.idmap, w.apparmorProfile, w.selinux, w.tracingSocket, w.cdiManager, opts...) if err != nil { return nil, err @@ -332,6 +345,9 @@ func (w *runcExecutor) Run(ctx context.Context, id string, root executor.Mount, if started != nil { close(started) } + if process.StatsStream != nil { + go w.monitorContainerStats(ctx, id, w.sampleFrequency, process.StatsStream) // earthly-specific + } if rec != nil { rec.Start() } diff --git a/executor/runcexecutor/monitor_stats.go b/executor/runcexecutor/monitor_stats.go new file mode 100644 index 000000000..37b3df081 --- /dev/null +++ b/executor/runcexecutor/monitor_stats.go @@ -0,0 +1,80 @@ +//go:build linux + +package runcexecutor + +import ( + "context" + "encoding/binary" + "encoding/json" + "io" + "time" + + runc "github.com/containerd/go-runc" + "github.com/moby/buildkit/util/bklog" + "github.com/pkg/errors" +) + +// earthly-specific: This entire file is earthly-specific, it is used to collect runc Stats and sends them via a stream to the client +// +// the stats protocol is: +// loop of stats events: +// version (currently 1) +// length of payload (stored as n) +// n bytes (a json-encoded string of the go-runc Stats structure) + +// writeUint32PrefixedBytes writes a uint32 representing the length of the b byte array, followed by the actual +// byte array. +func writeUint32PrefixedBytes(w io.Writer, b []byte) error { + n := len(b) + err := binary.Write(w, binary.LittleEndian, uint32(n)) + if err != nil { + return err + } + _, err = w.Write(b) + return err +} + +func writeStatsToStream(w io.Writer, stats *runc.Stats) error { + statsJSON, err := json.Marshal(stats) + if err != nil { + return errors.Wrap(err, "failed to encode runc stats") + } + err = binary.Write(w, binary.LittleEndian, uint8(1)) // earthly stats stream protocol v1 + if err != nil { + return err + } + return writeUint32PrefixedBytes(w, statsJSON) +} + +func (w *runcExecutor) monitorContainerStats(ctx context.Context, id string, sampleFrequency time.Duration, statsWriter io.WriteCloser) { + numFailuresAllowed := 10 + for { + // sleep at the top of the loop to give it time to start + time.Sleep(sampleFrequency) + + stats, err := w.runc.Stats(ctx, id) + if err != nil { + if errors.Is(err, context.Canceled) { + return + } + if numFailuresAllowed > 0 { + // allow the initial calls to runc.Stats to fail, for cases where the program didn't start within the initial + // sampleFrequency; this should only occur under heavy workloads + bklog.G(ctx).Warnf("ignoring runc stats collection error: %s", err) + numFailuresAllowed-- + continue + } + bklog.G(ctx).Errorf("runc stats collection error: %s", err) + return + } + + // once runc.Stats has succeeded, don't ignore future errors + numFailuresAllowed = 0 + + err = writeStatsToStream(statsWriter, stats) + if err != nil { + bklog.G(ctx).Errorf("failed to send runc stats to client-stream: %s", err) + return + } + } +} diff --git a/exporter/containerimage/writer.go b/exporter/containerimage/writer.go index d5363c714..c0cc1a0f2 100644 --- a/exporter/containerimage/writer.go +++ b/exporter/containerimage/writer.go @@ -124,6 +124,8 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session p = &ps.Platforms[0] if r, ok := inp.FindRef(p.ID); ok { ref = r + } else { + return nil, errors.Errorf("failed to findRef for %s", p.ID) // earthly-specific to make failures easier to detect, TODO: submit PR upstream for this } if opts.Epoch == nil { expEpoch, err = epoch.ParseSource(inp, p) diff --git a/exporter/earthlyoutputs/export.go b/exporter/earthlyoutputs/export.go new file mode 100644 index 000000000..dd764f690 --- /dev/null +++ b/exporter/earthlyoutputs/export.go @@ -0,0 +1,823 @@ +package earthlyoutputs + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "maps" + "os" + "strconv" + "strings" + "time" + + archiveexporter "github.com/containerd/containerd/v2/core/images/archive" + "github.com/containerd/containerd/v2/core/leases" + "github.com/containerd/containerd/v2/core/remotes/docker" + remoteserrors "github.com/containerd/containerd/v2/core/remotes/errors" + "github.com/containerd/platforms" + "github.com/distribution/reference" + "github.com/moby/buildkit/cache" + cacheconfig "github.com/moby/buildkit/cache/config" + "github.com/moby/buildkit/exporter" + "github.com/moby/buildkit/exporter/containerimage" + "github.com/moby/buildkit/exporter/containerimage/exptypes" + "github.com/moby/buildkit/exporter/earthlyoutputs/registry/eodriver" + "github.com/moby/buildkit/session" + "github.com/moby/buildkit/session/filesync" + "github.com/moby/buildkit/session/pullping" + "github.com/moby/buildkit/snapshot" + "github.com/moby/buildkit/util/compression" + "github.com/moby/buildkit/util/contentutil" + "github.com/moby/buildkit/util/grpcerrors" + "github.com/moby/buildkit/util/leaseutil" + "github.com/moby/buildkit/util/progress" + "github.com/moby/buildkit/util/push" + "github.com/moby/sys/user" + digest "github.com/opencontainers/go-digest" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "github.com/tonistiigi/fsutil" + fstypes "github.com/tonistiigi/fsutil/types" + "golang.org/x/sync/errgroup" + "golang.org/x/time/rate" + "google.golang.org/grpc/codes" +) + +type ExporterVariant string + +const ( + keyPush = "push" + keyPushByDigest = "push-by-digest" + keyInsecure = "registry.insecure" + keyUnpack = "unpack" + keyDanglingPrefix = "dangling-name-prefix" + keyNameCanonical = "name-canonical" + keyStore = "store" + + // keyUnsafeInternalStoreAllowIncomplete should only be used for tests. This option allows exporting image to the image store + // as well as lacking some blobs in the content store. Some integration tests for lazyref behaviour depends on this option. + // Ignored when store=false. + keyUnsafeInternalStoreAllowIncomplete = "unsafe-internal-store-allow-incomplete" + + // earthlyInlineCacheKey is the metadata key for inline cache entries. + // This was previously exptypes.ExporterInlineCache in upstream but was removed. + earthlyInlineCacheKey = "containerimage.inlinecache" +) + +type Opt struct { + SessionManager *session.Manager + ImageWriter *containerimage.ImageWriter + Variant ExporterVariant + RegistryHosts docker.RegistryHosts + LeaseManager leases.Manager +} + +type imageExporter struct { + opt Opt +} + +func New(opt Opt) (exporter.Exporter, error) { + im := &imageExporter{opt: opt} + return im, nil +} + +func (e *imageExporter) Resolve(ctx context.Context, id int, opt map[string]string) (exporter.ExporterInstance, error) { + i := &imageExporterInstance{ + imageExporter: e, + id: id, + attrs: opt, + opts: containerimage.ImageCommitOpts{ + RefCfg: cacheconfig.RefConfig{ + Compression: compression.New(compression.Default), + }, + Annotations: make(containerimage.AnnotationsGroup), + }, + store: true, + } + + opt, err := i.opts.Load(ctx, opt) + if err != nil { + return nil, err + } + + for k, v := range opt { + switch k { + case keyPush: + if v == "" { + i.push = true + continue + } + b, err := strconv.ParseBool(v) + if err != nil { + return nil, errors.Wrapf(err, "non-bool value specified for %s", k) + } + i.push = b + case keyPushByDigest: + if v == "" { + i.pushByDigest = true + continue + } + b, err := strconv.ParseBool(v) + if err != nil { + return nil, errors.Wrapf(err, "non-bool value specified for %s", k) + } + i.pushByDigest = b + case keyInsecure: + if v == "" { + i.insecure = true + continue + } + b, err := strconv.ParseBool(v) + if err != nil { + return nil, errors.Wrapf(err, "non-bool value specified for %s", k) + } + i.insecure = b + case keyUnpack: + if v == "" { + i.unpack = true + continue + } + b, err := strconv.ParseBool(v) + if err != nil { + return nil, errors.Wrapf(err, "non-bool value specified for %s", k) + } + i.unpack = b + case keyStore: + if v == "" { + i.store = true + continue + } + b, err := strconv.ParseBool(v) + if err != nil { + return nil, errors.Wrapf(err, "non-bool value specified for %s", k) + } + i.store = b + case keyUnsafeInternalStoreAllowIncomplete: + if v == "" { + i.storeAllowIncomplete = true + continue + } + b, err := strconv.ParseBool(v) + if err != nil { + return nil, errors.Wrapf(err, "non-bool value specified for %s", k) + } + i.storeAllowIncomplete = b + case keyDanglingPrefix: + i.danglingPrefix = v + case keyNameCanonical: + if v == "" { + i.nameCanonical = true + continue + } + b, err := strconv.ParseBool(v) + if err != nil { + return nil, errors.Wrapf(err, "non-bool value specified for %s", k) + } + i.nameCanonical = b + default: + if i.meta == nil { + i.meta = make(map[string][]byte) + } + i.meta[k] = []byte(v) + } + } + return i, nil +} + +type imageExporterInstance struct { + *imageExporter + id int + attrs map[string]string + opts containerimage.ImageCommitOpts + push bool + pushByDigest bool + unpack bool + store bool + storeAllowIncomplete bool + insecure bool + nameCanonical bool + danglingPrefix string + meta map[string][]byte +} + +func (e *imageExporterInstance) ID() int { + return e.id +} + +func (e *imageExporterInstance) Name() string { + return "[output] exporting outputs" +} + +func (e *imageExporterInstance) Type() string { + return "earthlyoutputs" +} + +func (e *imageExporterInstance) Attrs() map[string]string { + return e.attrs +} + +type imgData struct { + // localExport represents whether the image should be exported locally as a tar. + localExport bool + // localRegExport is set when the image should be exported via local registry. The value + // represents the image name that can be used to pull the image from the local registry. + localRegExport string + + // shouldPush is set when the image should be pushed. + shouldPush bool + // insecurePush is set when the push should take place over an unencrypted connection. + insecurePush bool + + // mp is the MultiProvider for this image. Note that for local reg exports, the + // mmp is used instead. + mp *contentutil.MultiProvider + + // platforms is a list of platforms to build for. + platforms []exptypes.Platform + + // expSrc is the exporter source constructed in a way that makes it look like this + // image is the only one being exported. + expSrc *exporter.Source + + // mfstDesc is the image manifest descriptor. + mfstDesc *ocispecs.Descriptor + + // tarWriter is the image tar writer (set only if localExport is true). + tarWriter io.WriteCloser + + // localRegExportReport is the one-off progress reporter for the local reg export. + localRegExportReport func() + // localExportReport is the one-off progress reporter for the tar export. + localExportReport func() + + opts containerimage.ImageCommitOpts +} + +func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source, buildInfo exporter.ExportBuildInfo) (map[string]string, exporter.FinalizeFunc, exporter.DescriptorReference, error) { + sessionID := buildInfo.SessionID + if src.Ref != nil { + return nil, nil, nil, errors.Errorf("export with src.Ref not supported") + } + + if len(src.Refs) == 0 { + // nothing to do + return map[string]string{}, nil, nil, nil + } + if src.Metadata == nil { + return nil, nil, nil, errors.Errorf("metadata is missing") + } + + maps.Copy(src.Metadata, e.meta) + images := make(map[string]*imgData) + hasAnyTarExport := false + hasAnyLocalRegExport := false + var dirExpSrcs []*exporter.Source + for k, ref := range src.Refs { + simpleMd := make(map[string][]byte) + mdPrefix := fmt.Sprintf("ref/%s/", k) + for mdK, mdV := range src.Metadata { + if after, ok := strings.CutPrefix(mdK, mdPrefix); ok { + simpleMd[after] = mdV + } + } + inlineCacheK := fmt.Sprintf("%s/%s", earthlyInlineCacheKey, k) + inlineCache, ok := src.Metadata[inlineCacheK] + if ok { + simpleMd[earthlyInlineCacheKey] = inlineCache + } + + opts := e.opts + as, _, err := containerimage.ParseAnnotations(simpleMd) + if err != nil { + return nil, nil, nil, err + } + opts.Annotations = as.Merge(opts.Annotations) + + le := false + isImage := false + if string(simpleMd["export-image"]) == "true" { + isImage = true + le = true + hasAnyTarExport = true + } + eilr := "" + if string(simpleMd["export-image-local-registry"]) != "" { + isImage = true + eilr = string(simpleMd["export-image-local-registry"]) + hasAnyLocalRegExport = true + } + sp := false + if string(simpleMd["export-image-push"]) == "true" { + isImage = true + sp = true + } + ip := false + if string(simpleMd["insecure-push"]) == "true" { + ip = true + } + if string(simpleMd["export-dir"]) == "true" { + expSrc := &exporter.Source{} + expSrc.SetRef(ref) + for k, v := range simpleMd { + expSrc.AddMeta(k, v) + } + dirExpSrcs = append(dirExpSrcs, expSrc) + } + if isImage { + name := "" + if n, ok := simpleMd["image.name"]; ok { + name = string(n) + } + if name == "" { + return nil, nil, nil, errors.Errorf("exporting image with no name") + } + imgNames, err := normalizedNames(name) + if err != nil { + return nil, nil, nil, err + } + if len(imgNames) == 0 { + return nil, nil, nil, errors.Errorf("exporting image with no name") + } + delete(simpleMd, "image.name") + platStr := string(simpleMd["platform"]) + for _, imgName := range imgNames { + img, ok := images[imgName] + if !ok { + img = &imgData{ + expSrc: &exporter.Source{}, + } + images[imgName] = img + } + img.localExport = img.localExport || le + img.localRegExport = eilr + img.shouldPush = img.shouldPush || sp + img.insecurePush = img.insecurePush || ip + img.opts = opts + + img.expSrc.AddMeta("image.name", []byte(imgName)) + if eilr != "" { + img.expSrc.AddMeta("export-image-local-registry", []byte(eilr)) + } + if le { + img.expSrc.AddMeta("export-image", []byte("true")) + } + if sp { + img.expSrc.AddMeta("export-image-push", []byte("true")) + } + if ip { + img.expSrc.AddMeta("insecure-push", []byte("true")) + } + if platStr != "" { + img.expSrc.AddRef(platStr, ref) + + p, err := platforms.Parse(platStr) + if err != nil { + return nil, nil, nil, errors.Wrap(err, "parse platform") + } + plat := exptypes.Platform{ + ID: platStr, + Platform: p, + } + img.platforms = append(img.platforms, plat) + } else { + ps, err := exptypes.ParsePlatforms(img.expSrc.Metadata) + if err != nil { + return nil, nil, nil, err + } + img.expSrc.SetRef(ref) + + dt, err := json.Marshal(ps) + if err != nil { + return nil, nil, nil, err + } + img.expSrc.AddMeta(exptypes.ExporterPlatformsKey, dt) + } + + for mdK, mdV := range simpleMd { + if platStr != "" { + img.expSrc.AddMeta(fmt.Sprintf("%s/%s", mdK, platStr), mdV) + } else { + img.expSrc.AddMeta(mdK, mdV) + } + } + } + } + } + for _, img := range images { + if len(img.platforms) > 0 { + expPlats := &exptypes.Platforms{Platforms: img.platforms} + dt, err := json.Marshal(expPlats) + if err != nil { + return nil, nil, nil, err + } + img.expSrc.AddMeta(exptypes.ExporterPlatformsKey, dt) + } + } + + ctx, done, err := leaseutil.WithLease(ctx, e.opt.LeaseManager, leaseutil.MakeTemporary) + if err != nil { + return nil, nil, nil, err + } + defer done(context.TODO()) + + resp := make(map[string]string) + for imgName, img := range images { + desc, err := e.opt.ImageWriter.Commit(ctx, img.expSrc, sessionID, buildInfo.InlineCache, &img.opts) + if err != nil { + return nil, nil, nil, err + } + img.mfstDesc = desc + defer func() { + e.opt.ImageWriter.ContentStore().Delete(context.TODO(), desc.Digest) + }() + if desc.Annotations == nil { + desc.Annotations = map[string]string{} + } + desc.Annotations[ocispecs.AnnotationCreated] = time.Now().UTC().Format(time.RFC3339) + + if v, ok := desc.Annotations[exptypes.ExporterConfigDigestKey]; ok { + cfgDgstKey := fmt.Sprintf("%s|%s", imgName, exptypes.ExporterConfigDigestKey) + resp[cfgDgstKey] = v + delete(desc.Annotations, exptypes.ExporterConfigDigestKey) + } + dtDesc, err := json.Marshal(desc) + if err != nil { + return nil, nil, nil, err + } + descKey := fmt.Sprintf("%s|%s", imgName, exptypes.ExporterImageDescriptorKey) + resp[descKey] = base64.StdEncoding.EncodeToString(dtDesc) + } + + timeoutCtx, cancel := context.WithTimeoutCause(ctx, 5*time.Second, nil) + defer cancel() + caller, err := e.opt.SessionManager.Get(timeoutCtx, sessionID, false) + if err != nil { + return nil, nil, nil, err + } + + for _, img := range images { + if !img.localExport { + continue + } + md := make(map[string]string) + for mdK, mdV := range img.expSrc.Metadata { + md[safeGrpcMetaKey(mdK)] = string(mdV) + } + img.tarWriter, err = filesync.CopyFileWriter(ctx, md, e.id, caller) + if err != nil { + return nil, nil, nil, err + } + } + dirEG, egCtx := errgroup.WithContext(ctx) + for _, expSrc := range dirExpSrcs { + md := make(map[string]string) + for mdK, mdV := range expSrc.Metadata { + md[safeGrpcMetaKey(mdK)] = string(mdV) + } + if expSrc.Ref == nil { + return nil, nil, nil, errors.New("dirExpSrcs got nil ref") + } + dirEG.Go(exportDirFunc(egCtx, md, caller, expSrc.Ref, sessionID)) + } + + mmp := eodriver.MultiMultiProviderSingleton + annotations := map[digest.Digest]map[string]string{} + for imgName, img := range images { + if img.localExport { + img.localExportReport = oneOffProgress(ctx, fmt.Sprintf("transferring (via tar) %s", imgName)) + } + if img.localRegExport != "" { + img.localRegExportReport = oneOffProgress(ctx, fmt.Sprintf("transferring %s", imgName)) + + err := mmp.AddImg(ctx, img.localRegExport, e.opt.ImageWriter.ContentStore(), img.mfstDesc.Digest) + if err != nil { + return nil, nil, nil, err + } + } + if img.localExport || img.shouldPush { + img.mp = contentutil.NewMultiProvider(e.opt.ImageWriter.ContentStore()) + } + + for _, r := range img.expSrc.Refs { + remotes, err := r.GetRemotes(ctx, false, e.opts.RefCfg, false, session.NewGroup(sessionID)) + if err != nil { + return nil, nil, nil, err + } + remote := remotes[0] + // unlazy before export as some consumers do not handle + // layer blobs in parallel (whereas unlazy does) + if unlazier, ok := remote.Provider.(cache.Unlazier); ok { + if err := unlazier.Unlazy(ctx); err != nil { + return nil, nil, nil, err + } + } + for _, desc := range remote.Descriptors { + if img.localRegExport != "" { + err := mmp.AddImgSub(img.localRegExport, desc.Digest, remote.Provider) + if err != nil { + return nil, nil, nil, err + } + } + if img.localExport || img.shouldPush { + img.mp.Add(desc.Digest, remote.Provider) + } + addAnnotations(annotations, desc) + } + } + if img.expSrc.Ref != nil { // This is a copy and paste of the above code + if len(img.platforms) != 0 { + return nil, nil, nil, errors.New("img.platforms should not be set when a single ref is used") + } + + var ref cache.ImmutableRef + var p exptypes.Platform + if len(img.platforms) > 0 { + p = img.platforms[0] + if r, ok := img.expSrc.FindRef(p.ID); ok { + ref = r + } else { + return nil, nil, nil, errors.Errorf("img.expSrc.FindRef failed on %s", p.ID) + } + } else { + ref = img.expSrc.Ref + } + + remotes, err := ref.GetRemotes(ctx, false, e.opts.RefCfg, false, session.NewGroup(sessionID)) + if err != nil { + return nil, nil, nil, err + } + remote := remotes[0] + // unlazy before export as some consumers do not handle + // layer blobs in parallel (whereas unlazy does) + if unlazier, ok := remote.Provider.(cache.Unlazier); ok { + if err := unlazier.Unlazy(ctx); err != nil { + return nil, nil, nil, err + } + } + for _, desc := range remote.Descriptors { + if img.localRegExport != "" { + err := mmp.AddImgSub(img.localRegExport, desc.Digest, remote.Provider) + if err != nil { + return nil, nil, nil, err + } + } + if img.localExport || img.shouldPush { + img.mp.Add(desc.Digest, remote.Provider) + } + addAnnotations(annotations, desc) + } + } + + if img.shouldPush { + err := push.Push( + ctx, e.opt.SessionManager, sessionID, img.mp, + e.opt.ImageWriter.ContentStore(), img.mfstDesc.Digest, + imgName, img.insecurePush, e.opt.RegistryHosts, false, annotations) + if err != nil { + var errStatus remoteserrors.ErrUnexpectedStatus + if errors.As(err, &errStatus) { + // TODO body might be json, e.g. `{"errors":[{"code":"DENIED","message":"The repository with name 'my-cool-image' in registry with id '123456789' already has the maximum allowed number of images which is '10000'"}]}`, we should attempt to parse this + return nil, nil, nil, errors.Wrapf(err, "failed to push %s: body=%s", imgName, errStatus.Body) + } + return nil, nil, nil, err + } + } + } + + var pullPingChan chan error + if hasAnyLocalRegExport { + // inform the client that it's safe to perform pulls now + pullImgs := make([]string, 0, len(images)) + for _, img := range images { + if img.localRegExport != "" { + pullImgs = append(pullImgs, img.localRegExport) + } + } + pullPingChan = pullping.PullPingChannel(ctx, pullImgs, resp, caller) + // wait for the client to finish pulling + select { + case err := <-pullPingChan: + if err != nil { + return nil, nil, nil, errors.Wrap(err, "pull ping error") + } + case <-caller.Context().Done(): + return nil, nil, nil, errors.Wrap(caller.Context().Err(), "caller context done") + } + for _, img := range images { + if img.localRegExport != "" { + img.localRegExportReport() + } + } + } + + if hasAnyTarExport { + for imgName, img := range images { + if !img.localExport { + continue + } + expOpts := []archiveexporter.ExportOpt{archiveexporter.WithManifest(*img.mfstDesc, imgName)} + if err := archiveexporter.Export(ctx, img.mp, img.tarWriter, expOpts...); err != nil { + img.tarWriter.Close() + if grpcerrors.Code(err) == codes.AlreadyExists { + continue + } + if errors.Is(err, io.EOF) { + // TODO(vladaionescu): This sometimes happens when server responds with + // GRPC code codes.AlreadyExists and + // we continue to try to send data. + continue + } + return nil, nil, nil, err + } + err = img.tarWriter.Close() + if grpcerrors.Code(err) == codes.AlreadyExists { + continue + } + if errors.Is(err, io.EOF) { + // TODO(vladaionescu): This sometimes happens when server responds with + // GRPC code codes.AlreadyExists and + // we continue to try to send data. + continue + } + if err != nil { + return nil, nil, nil, err + } + img.localExportReport() + } + // just in case any report was skipped due to a continue + for _, img := range images { + if img.localExportReport != nil { + img.localExportReport() + } + } + } + + if err := dirEG.Wait(); err != nil { + return nil, nil, nil, err + } + + return resp, nil, nil, nil +} + +func (e *imageExporterInstance) Config() *exporter.Config { + return exporter.NewConfigWithCompression(e.opts.RefCfg.Compression) +} + +func oneOffProgress(ctx context.Context, id string) func() { + pw, _, _ := progress.FromContext(ctx)(ctx) + now := time.Now() + st := progress.Status{ + Started: &now, + } + pw.Write(id, st) + return func() { + if st.Completed != nil { + // Don't close twice. + return + } + now := time.Now() + st.Completed = &now + pw.Write(id, st) + pw.Close() + } +} + +func normalizedNames(name string) ([]string, error) { + if name == "" { + return nil, nil + } + names := strings.Split(name, ",") + var tagNames = make([]string, len(names)) + for i, name := range names { + parsed, err := reference.ParseNormalizedNamed(name) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse %s", name) + } + tagNames[i] = reference.TagNameOnly(parsed).String() + } + return tagNames, nil +} + +func newProgressHandler(ctx context.Context, id string) func(int, bool) { + limiter := rate.NewLimiter(rate.Every(100*time.Millisecond), 1) + pw, _, _ := progress.FromContext(ctx)(ctx) + now := time.Now() + st := progress.Status{ + Started: &now, + Action: "transferring", + } + pw.Write(id, st) + return func(s int, last bool) { + if last || limiter.Allow() { + st.Current = s + if last { + now := time.Now() + st.Completed = &now + } + pw.Write(id, st) + if last { + pw.Close() + } + } + } +} + +func exportDirFunc(ctx context.Context, md map[string]string, caller session.Caller, ref cache.ImmutableRef, sessionID string) func() error { + return func() error { + var src string + var err error + var idmap *user.IdentityMapping + if ref == nil { + src, err = os.MkdirTemp("", "buildkit") + if err != nil { + return err + } + defer os.RemoveAll(src) + } else { + mount, err := ref.Mount(ctx, true, session.NewGroup(sessionID)) + if err != nil { + return err + } + + lm := snapshot.LocalMounter(mount) + + src, err = lm.Mount() + if err != nil { + return err + } + + idmap = mount.IdentityMapping() + + defer lm.Unmount() + } + + fs, err := fsutil.NewFS(src) + if err != nil { + return err + } + + // wrap the output filesystem, applying appropriate filters + filterOpt := &fsutil.FilterOpt{} + var idMapFunc func(p string, st *fstypes.Stat) fsutil.MapResult + if idmap != nil { + idMapFunc = func(p string, st *fstypes.Stat) fsutil.MapResult { + uid, gid, err := idmap.ToContainer(int(st.Uid), int(st.Gid)) + if err != nil { + return fsutil.MapResultExclude + } + st.Uid = uint32(uid) + st.Gid = uint32(gid) + return fsutil.MapResultKeep + } + } + filterOpt.Map = func(p string, st *fstypes.Stat) fsutil.MapResult { + res := fsutil.MapResultKeep + if idMapFunc != nil { + // apply host uid/gid + res = idMapFunc(p, st) + } + // TODO if opt.Epoch != nil { + // TODO // apply used-specified epoch time + // TODO st.ModTime = opt.Epoch.UnixNano() + // TODO} + return res + } + fs, err = fsutil.NewFilterFS(fs, filterOpt) + if err != nil { + return err + } + + progress := newProgressHandler(ctx, "copying files") + if err := filesync.CopyToCallerWithMeta(ctx, md, fs, caller, progress); err != nil { + if grpcerrors.Code(err) == codes.AlreadyExists { + return nil + } + if errors.Is(err, io.EOF) { + // TODO(vladaionescu): This sometimes happens when server responds with + // GRPC code codes.AlreadyExists and + // fsutil continues to try to send data. + return nil + } + return err + } + return nil + } +} + +func addAnnotations(m map[digest.Digest]map[string]string, desc ocispecs.Descriptor) { + if desc.Annotations == nil { + return + } + a, ok := m[desc.Digest] + if !ok { + m[desc.Digest] = desc.Annotations + return + } + maps.Copy(a, desc.Annotations) +} + +func safeGrpcMetaKey(k string) string { + return strings.ReplaceAll(k, "/", "-") +} diff --git a/exporter/earthlyoutputs/registry/eodriver/eodriver.go b/exporter/earthlyoutputs/registry/eodriver/eodriver.go new file mode 100644 index 000000000..3ca4aea72 --- /dev/null +++ b/exporter/earthlyoutputs/registry/eodriver/eodriver.go @@ -0,0 +1,319 @@ +package eodriver + +import ( + "bytes" + "context" + "fmt" + "io" + "strings" + "time" + + "github.com/containerd/containerd/v2/core/content" + storagedriver "github.com/docker/distribution/registry/storage/driver" + "github.com/docker/distribution/registry/storage/driver/base" + "github.com/docker/distribution/registry/storage/driver/factory" + digest "github.com/opencontainers/go-digest" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +const ( + driverName = "eodriver" + defaultMaxThreads = uint64(100) + + // minThreads is the minimum value for the maxthreads configuration + // parameter. If the driver's parameters are less than this we set + // the parameters to minThreads + minThreads = uint64(25) +) + +// DriverParameters represents all configuration options available for the +// eodriver driver +type DriverParameters struct { + RootDirectory string + MaxThreads uint64 +} + +func init() { + factory.Register(driverName, &earthlyOutputDriverFactory{}) +} + +// earthlyOutputDriverFactory implements the factory.StorageDriverFactory interface +type earthlyOutputDriverFactory struct{} + +func (factory *earthlyOutputDriverFactory) Create(parameters map[string]any) (storagedriver.StorageDriver, error) { + return FromParameters(parameters) +} + +type driver struct { + mmp *MultiMultiProvider +} + +type baseEmbed struct { + base.Base +} + +// Driver is a storagedriver.StorageDriver implementation backed by a local +// a multi-multi-provider. +type Driver struct { + baseEmbed +} + +// FromParameters constructs a new Driver with a given parameters map +// Optional Parameters: +// - maxthreads +func FromParameters(parameters map[string]any) (*Driver, error) { + params, err := fromParametersImpl(parameters) + if err != nil || params == nil { + return nil, err + } + return New(*params), nil +} + +func fromParametersImpl(parameters map[string]any) (*DriverParameters, error) { + var ( + err error + maxThreads = defaultMaxThreads + ) + + if parameters != nil { + maxThreads, err = base.GetLimitFromParameter(parameters["maxthreads"], minThreads, defaultMaxThreads) + if err != nil { + return nil, errors.Errorf("maxthreads config error: %s", err.Error()) + } + } + + params := &DriverParameters{ + MaxThreads: maxThreads, + } + return params, nil +} + +// New constructs a new Driver with a given rootDirectory +func New(params DriverParameters) *Driver { + fsDriver := &driver{ + mmp: MultiMultiProviderSingleton, + } + + return &Driver{ + baseEmbed: baseEmbed{ + Base: base.Base{ + StorageDriver: base.NewRegulator(fsDriver, params.MaxThreads), + }, + }, + } +} + +// Implement the storagedriver.StorageDriver interface + +func (d *driver) Name() string { + return driverName +} + +// GetContent retrieves the content stored at "path" as a []byte. +func (d *driver) GetContent(ctx context.Context, path string) ([]byte, error) { + rc, err := d.Reader(ctx, path, 0) + if err != nil { + return nil, err + } + defer rc.Close() + + p, err := io.ReadAll(rc) + if err != nil { + return nil, err + } + + return p, nil +} + +// PutContent stores the []byte content at a location designated by "path". +func (d *driver) PutContent(ctx context.Context, subPath string, contents []byte) error { + return storagedriver.ErrUnsupportedMethod{} +} + +// Reader retrieves an io.ReadCloser for the content stored at "path" with a +// given byte offset. +func (d *driver) Reader(ctx context.Context, path string, offset int64) (io.ReadCloser, error) { + rc, _, err := d.get(ctx, path, offset) + return rc, err +} + +func (d *driver) get(ctx context.Context, path string, offset int64) (io.ReadCloser, int64, error) { + subPath, ok := strings.CutPrefix(path, "/docker/registry/v2/") + if !ok { + return nil, 0, storagedriver.PathNotFoundError{} + } + if subSubPath, ok := strings.CutPrefix(subPath, "repositories/"); ok { + subSubPathSplit := strings.Split(subSubPath, "/") + indexPostImgName := -1 + for index, part := range subSubPathSplit { + if strings.HasPrefix(part, "_") { + indexPostImgName = index + break + } + } + if indexPostImgName == -1 { + return nil, 0, storagedriver.PathNotFoundError{} + } + imgName := strings.Join(subSubPathSplit[0:indexPostImgName], "/") + switch subSubPathSplit[indexPostImgName] { + case "_manifests": + switch subSubPathSplit[indexPostImgName+1] { + case "tags": + postTagsPath := strings.Join(subSubPathSplit[indexPostImgName+2:], "/") + tag := strings.TrimSuffix(postTagsPath, "/current/link") + fullImgName := fmt.Sprintf("%s:%s", imgName, tag) + _, baseDigest, err := d.mmp.Get(ctx, fullImgName) + if err != nil { + return nil, 0, errors.Wrapf(err, "get %s", path) + } + return stringReadCloserOffset(baseDigest.String(), offset) + case "revisions": + postRevisionsPath := strings.Join(subSubPathSplit[indexPostImgName+2:], "/") + if !strings.HasPrefix(postRevisionsPath, "sha256/") { + return nil, 0, storagedriver.PathNotFoundError{} + } + sha := strings.TrimSuffix(strings.TrimPrefix(postRevisionsPath, "sha256/"), "/link") + return stringReadCloserOffset(fmt.Sprintf("sha256:%s", sha), offset) + default: + return nil, 0, storagedriver.PathNotFoundError{} + } + case "_layers": + postLayersPath := strings.Join(subSubPathSplit[indexPostImgName+1:], "/") + if !strings.HasPrefix(postLayersPath, "sha256/") { + return nil, 0, storagedriver.PathNotFoundError{} + } + sha := strings.TrimSuffix(strings.TrimPrefix(postLayersPath, "sha256/"), "/link") + return stringReadCloserOffset(fmt.Sprintf("sha256:%s", sha), offset) + default: + return nil, 0, storagedriver.PathNotFoundError{} + } + } else if subSubPath, ok := strings.CutPrefix(subPath, "blobs/sha256/"); ok { + subSubPathSplit := strings.Split(subSubPath, "/") + if len(subSubPathSplit) != 3 { + return nil, 0, storagedriver.PathNotFoundError{} + } + if subSubPathSplit[2] != "data" { + return nil, 0, storagedriver.PathNotFoundError{} + } + sha := subSubPathSplit[1] + desc := ocispecs.Descriptor{ + Digest: digest.Digest(fmt.Sprintf("sha256:%s", sha)), + } + ra, err := d.mmp.ReaderAt(ctx, desc) + if err != nil { + return nil, 0, errors.Wrapf(err, "blob %s", path) + } + return &readerAtReadCloser{ + ra: ra, + offset: offset, + }, ra.Size(), nil + } else { + return nil, 0, storagedriver.PathNotFoundError{} + } +} + +func (d *driver) Writer(ctx context.Context, subPath string, append bool) (storagedriver.FileWriter, error) { + return nil, storagedriver.ErrUnsupportedMethod{} +} + +// Stat retrieves the FileInfo for the given path, including the current size +// in bytes and the creation time. +func (d *driver) Stat(ctx context.Context, subPath string) (storagedriver.FileInfo, error) { + _, size, err := d.get(ctx, subPath, 0) + if err != nil { + return nil, err + } + + return fileInfo{ + path: subPath, + size: size, + crTime: time.Now(), + }, nil +} + +// List returns a list of the objects that are direct descendants of the given +// path. +func (d *driver) List(ctx context.Context, subPath string) ([]string, error) { + return nil, storagedriver.ErrUnsupportedMethod{} +} + +// Move moves an object stored at sourcePath to destPath, removing the original +// object. +func (d *driver) Move(ctx context.Context, sourcePath string, destPath string) error { + return storagedriver.ErrUnsupportedMethod{} +} + +// Delete recursively deletes all objects stored at "path" and its subpaths. +func (d *driver) Delete(ctx context.Context, subPath string) error { + return storagedriver.ErrUnsupportedMethod{} +} + +// URLFor returns a URL which may be used to retrieve the content stored at the given path. +// May return an UnsupportedMethodErr in certain StorageDriver implementations. +func (d *driver) URLFor(ctx context.Context, path string, options map[string]any) (string, error) { + return "", storagedriver.ErrUnsupportedMethod{} +} + +// Walk traverses a filesystem defined within driver, starting +// from the given path, calling f on each file +func (d *driver) Walk(ctx context.Context, path string, f storagedriver.WalkFn) error { + return storagedriver.WalkFallback(ctx, d, path, f) +} + +type fileInfo struct { + size int64 + path string + crTime time.Time +} + +var _ storagedriver.FileInfo = fileInfo{} + +// Path provides the full path of the target of this file info. +func (fi fileInfo) Path() string { + return fi.path +} + +// Size returns current length in bytes of the file. The return value can +// be used to write to the end of the file at path. The value is +// meaningless if IsDir returns true. +func (fi fileInfo) Size() int64 { + return fi.size +} + +// ModTime returns the modification time for the file. For backends that +// don't have a modification time, the creation time should be returned. +func (fi fileInfo) ModTime() time.Time { + return fi.crTime +} + +// IsDir returns true if the path is a directory. +func (fi fileInfo) IsDir() bool { + return false +} + +func stringReadCloserOffset(str string, offset int64) (io.ReadCloser, int64, error) { + dt := []byte(str) + if offset > int64(len(dt)) { + return nil, 0, io.EOF + } + size := len(dt) + dt = dt[offset:] + reader := io.NopCloser(bytes.NewReader(dt)) + return reader, int64(size), nil +} + +type readerAtReadCloser struct { + ra content.ReaderAt + offset int64 +} + +func (rarc *readerAtReadCloser) Read(p []byte) (int, error) { + n, err := rarc.ra.ReadAt(p, rarc.offset) + rarc.offset += int64(n) + return n, err +} + +func (rarc *readerAtReadCloser) Close() error { + return rarc.ra.Close() +} diff --git a/exporter/earthlyoutputs/registry/eodriver/multimultiprovider.go b/exporter/earthlyoutputs/registry/eodriver/multimultiprovider.go new file mode 100644 index 000000000..c75f25a1e --- /dev/null +++ b/exporter/earthlyoutputs/registry/eodriver/multimultiprovider.go @@ -0,0 +1,166 @@ +package eodriver + +import ( + "context" + "encoding/json" + "sync" + + "github.com/containerd/containerd/v2/core/content" + cerrdefs "github.com/containerd/errdefs" + "github.com/moby/buildkit/util/contentutil" + digest "github.com/opencontainers/go-digest" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +// MultiMultiProviderSingleton is a multi-multi-provider that can be shared across runs. +// The reason this is a singleton is that it can be used as a basis for a registry storage driver. +var MultiMultiProviderSingleton = NewMultiMultiProvider() + +// NewMultiMultiProvider creates a new multi multi provider. +func NewMultiMultiProvider() *MultiMultiProvider { + return &MultiMultiProvider{ + imgs: make(map[string]*imgData), + digests: make(map[digest.Digest]map[string]bool), + } +} + +type imgData struct { + base content.InfoReaderProvider + baseDgst digest.Digest + subs map[digest.Digest]content.InfoReaderProvider +} + +// MultiMultiProvider is a provider backed by a set of images, each of which is made out of +// a set of providers. +type MultiMultiProvider struct { + mu sync.RWMutex + imgs map[string]*imgData // imgName -> imgData + digests map[digest.Digest]map[string]bool // digest -> set of imgName +} + +func (mmp *MultiMultiProvider) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) (content.ReaderAt, error) { + mmp.mu.RLock() + defer mmp.mu.RUnlock() + imgs := mmp.digests[desc.Digest] + // take the first one if multiple are found. + for imgName := range imgs { + mp, _, err := mmp.getNoLock(ctx, imgName) + if err != nil { + continue + } + return mp.ReaderAt(ctx, desc) + } + return nil, errors.Wrapf(cerrdefs.ErrNotFound, "content %v", desc.Digest) +} + +// Get returns a read-only MultiProvider and the base digest for a given imgName. +func (mmp *MultiMultiProvider) Get(ctx context.Context, imgName string) (*contentutil.MultiProvider, digest.Digest, error) { + mmp.mu.RLock() + defer mmp.mu.RUnlock() + return mmp.getNoLock(ctx, imgName) +} + +func (mmp *MultiMultiProvider) getNoLock(_ context.Context, imgName string) (*contentutil.MultiProvider, digest.Digest, error) { + imgData, ok := mmp.imgs[imgName] + if !ok { + return nil, "", errors.Wrapf(cerrdefs.ErrNotFound, "img name %v", imgName) + } + mp := contentutil.NewMultiProvider(imgData.base) + for dgst, p := range imgData.subs { + mp.Add(dgst, p) + } + return mp, imgData.baseDgst, nil +} + +// AddImgSub adds a new child content provider for an image. +func (mmp *MultiMultiProvider) AddImgSub(imgName string, dgst digest.Digest, p content.InfoReaderProvider) error { + mmp.mu.Lock() + defer mmp.mu.Unlock() + imgData, ok := mmp.imgs[imgName] + if !ok { + return errors.Wrapf(cerrdefs.ErrNotFound, "img name %v", imgName) + } + imgData.subs[dgst] = p + mmp.addDigestEntry(dgst, imgName) + return nil +} + +// AddImg adds a new child image. The image is removed from the collection when the context is canceled. +func (mmp *MultiMultiProvider) AddImg(ctx context.Context, imgName string, base content.InfoReaderProvider, baseDigest digest.Digest) error { + if baseDigest == "" { + return errors.Errorf("baseDigest cant be empty") + } + + // The config digest needs to be mapped manually - read out the manifest + // and find the config digest in there. Do most of this outside of the lock. + mfstRa, err := base.ReaderAt(ctx, ocispecs.Descriptor{Digest: baseDigest}) + if err != nil { + return err + } + mfstDt := make([]byte, mfstRa.Size()) + _, err = mfstRa.ReadAt(mfstDt, 0) + if err != nil { + return err + } + var mfst ocispecs.Manifest + err = json.Unmarshal(mfstDt, &mfst) + if err != nil { + return err + } + if mfst.Config.Digest == "" { + return errors.Errorf("manifest config digest is missing") + } + + configDgst := digest.Digest(mfst.Config.Digest) + if configDgst == "" { + return errors.Errorf("configDgst cant be empty") + } + + mmp.mu.Lock() + defer mmp.mu.Unlock() + mmp.maybeDelete(imgName) + imgData := &imgData{ + base: base, + baseDgst: baseDigest, + subs: make(map[digest.Digest]content.InfoReaderProvider), + } + mmp.imgs[imgName] = imgData + imgData.subs[baseDigest] = base + mmp.addDigestEntry(baseDigest, imgName) + mmp.addDigestEntry(configDgst, imgName) + + go func() { + <-ctx.Done() + mmp.mu.Lock() + defer mmp.mu.Unlock() + mmp.maybeDelete(imgName) + }() + return nil +} + +func (mmp *MultiMultiProvider) addDigestEntry(dgst digest.Digest, imgName string) { + if dgst == "" { + panic("dgst can't be empty") + } + imgSet, ok := mmp.digests[dgst] + if !ok { + imgSet = make(map[string]bool) + mmp.digests[dgst] = imgSet + } + imgSet[imgName] = true +} + +func (mmp *MultiMultiProvider) maybeDelete(imgName string) { + imgData, ok := mmp.imgs[imgName] + if !ok { + return + } + for dgst := range imgData.subs { + delete(mmp.digests[dgst], imgName) + if len(mmp.digests[dgst]) == 0 { + delete(mmp.digests, dgst) + } + } + delete(mmp.imgs, imgName) +} diff --git a/exporter/earthlyoutputs/registry/registry.go b/exporter/earthlyoutputs/registry/registry.go new file mode 100644 index 000000000..5420ae95c --- /dev/null +++ b/exporter/earthlyoutputs/registry/registry.go @@ -0,0 +1,48 @@ +package registry + +import ( + "context" + "net/http" + "strings" + "time" + + "github.com/docker/distribution/configuration" + "github.com/docker/distribution/registry/handlers" + "github.com/docker/distribution/registry/listener" + _ "github.com/moby/buildkit/exporter/earthlyoutputs/registry/eodriver" // register the driver +) + +// Serve creates a registry service and starts listening for connections on listenAddr. +// Cancelling the context shuts down the server. +func Serve(ctx context.Context, listenAddr string) chan error { + serveErr := make(chan error, 1) + config, err := configuration.Parse(strings.NewReader(` +version: 0.1 +storage: + eodriver: + maxthreads: 100`)) + if err != nil { + serveErr <- err + return serveErr + } + ln, err := listener.NewListener("tcp", listenAddr) + if err != nil { + serveErr <- err + return serveErr + } + app := handlers.NewApp(ctx, config) + server := &http.Server{ + Handler: app, + ReadHeaderTimeout: 30 * time.Second, + } + ctx2, cancel := context.WithCancelCause(ctx) + go func() { + serveErr <- server.Serve(ln) + cancel(nil) + }() + go func() { + <-ctx2.Done() + server.Close() // this does not drain + }() + return serveErr +} diff --git a/exporter/verifier/platforms.go b/exporter/verifier/platforms.go index 069e2cb0e..bc6bd8cae 100644 --- a/exporter/verifier/platforms.go +++ b/exporter/verifier/platforms.go @@ -10,7 +10,6 @@ import ( "github.com/moby/buildkit/client" "github.com/moby/buildkit/exporter/containerimage/exptypes" "github.com/moby/buildkit/solver/result" - "github.com/pkg/errors" ) func CheckInvalidPlatforms[T comparable](ctx context.Context, res *result.Result[T]) ([]client.VertexWarning, error) { @@ -21,7 +20,11 @@ func CheckInvalidPlatforms[T comparable](ctx context.Context, res *result.Result if _, ok := res.Metadata[exptypes.ExporterPlatformsKey]; !ok { if len(res.Refs) > 0 { - return nil, errors.Errorf("build result contains multiple refs without platforms mapping") + // Earthly's multi-BUILD pattern legitimately produces multiple + // refs without platform mapping. Downgrade to warning. + return []client.VertexWarning{{ + Short: []byte("build result contains multiple refs without platforms mapping"), + }}, nil } else if res.IsEmpty() { // No results and no exporter key. Don't run this check. return nil, nil diff --git a/frontend/dockerui/config.go b/frontend/dockerui/config.go index 619b83457..90b80ab91 100644 --- a/frontend/dockerui/config.go +++ b/frontend/dockerui/config.go @@ -81,6 +81,7 @@ type Client struct { Config client client.Client ignoreCache []string + bctx *buildContext // earthly-specific g flightcontrol.CachedGroup[*buildContext] bopts client.BuildOpts localsSessionIDs map[string]string @@ -308,9 +309,24 @@ func (bc *Client) init() error { return nil } +// Earthly specific +func (bc *Client) SetBuildContext(context *llb.State, contextName string) { + bc.bctx = &buildContext{ + context: context, + contextLocalName: contextName, + } +} + func (bc *Client) buildContext(ctx context.Context) (*buildContext, error) { return bc.g.Do(ctx, "initcontext", func(ctx context.Context) (*buildContext, error) { - return bc.initContext(ctx) + if bc.bctx != nil { + return bc.bctx, nil + } + bctx, err := bc.initContext(ctx) + if err == nil { + bc.bctx = bctx + } + return bctx, err }) } diff --git a/frontend/frontend.go b/frontend/frontend.go index 0278a1e06..fa3e743fa 100644 --- a/frontend/frontend.go +++ b/frontend/frontend.go @@ -3,6 +3,8 @@ package frontend import ( "context" + "github.com/moby/buildkit/cache" // earthly-specific this import breaks the mips64 frontend tests due to incorrect types on inode + "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/client/llb/sourceresolver" "github.com/moby/buildkit/executor" gw "github.com/moby/buildkit/frontend/gateway/client" @@ -32,6 +34,8 @@ type Frontend interface { type FrontendLLBBridge interface { sourceresolver.MetaResolver Solve(ctx context.Context, req SolveRequest, sid string) (*Result, error) + Export(ctx context.Context, refs map[string]cache.ImmutableRef, metadata map[string][]byte) error // earthly-specific + ResolveImageConfig(ctx context.Context, ref string, opt llb.ResolveImageConfigOpt) (string, digest.Digest, []byte, error) Warn(ctx context.Context, dgst digest.Digest, msg string, opts WarnOpts) error } diff --git a/frontend/gateway/client/client.go b/frontend/gateway/client/client.go index 65f9f2f64..0a02cd86b 100644 --- a/frontend/gateway/client/client.go +++ b/frontend/gateway/client/client.go @@ -29,6 +29,7 @@ func NewResult() *Result { type Client interface { sourceresolver.MetaResolver Solve(ctx context.Context, req SolveRequest) (*Result, error) + Export(ctx context.Context, req ExportRequest) error // earthly-specific ResolveImageConfig(ctx context.Context, ref string, opt sourceresolver.Opt) (string, digest.Digest, []byte, error) BuildOpts() BuildOpts Inputs(ctx context.Context) (map[string]llb.State, error) @@ -159,6 +160,11 @@ type CacheOptionsEntry struct { Attrs map[string]string } +type ExportRequest struct { + Refs map[string]Reference + Metadata map[string][]byte +} + type WorkerInfo struct { ID string Labels map[string]string diff --git a/frontend/gateway/container/container.go b/frontend/gateway/container/container.go index 042b7e793..196b7005e 100644 --- a/frontend/gateway/container/container.go +++ b/frontend/gateway/container/container.go @@ -190,6 +190,9 @@ func PrepareMounts(ctx context.Context, mm *mounts.MountManager, cm cache.Manage } switch m.MountType { + case opspb.MountType_HOST_BIND: // earthly + mountable = mm.MountableHostBind(ctx, m) + case opspb.MountType_BIND: // if mount creates an output if m.Output != int64(opspb.SkipOutput) { @@ -262,6 +265,15 @@ func PrepareMounts(ctx context.Context, mm *mounts.MountManager, cm cache.Manage if mountable == nil { continue } + case opspb.MountType_SOCKET: // earthly-specific + var err error + mountable, err = mm.MountableSocket(ctx, m, g) + if err != nil { + return p, err + } + if mountable == nil { + continue + } default: return p, errors.Errorf("mount type %s not implemented", m.MountType) diff --git a/frontend/gateway/forwarder/forward.go b/frontend/gateway/forwarder/forward.go index e4bcb054d..dd3dfd848 100644 --- a/frontend/gateway/forwarder/forward.go +++ b/frontend/gateway/forwarder/forward.go @@ -92,6 +92,11 @@ func (c *BridgeClient) Solve(ctx context.Context, req client.SolveRequest) (*cli return cRes, nil } +// Export is only used by earthly via the grpcclient implementation +func (c *BridgeClient) Export(ctx context.Context, req client.ExportRequest) error { + return errors.Errorf("forwarder.bridgeClient does not support Export") +} + func (c *BridgeClient) ResolveImageConfig(ctx context.Context, ref string, opt sourceresolver.Opt) (string, digest.Digest, []byte, error) { imr := sourceresolver.NewImageMetaResolver(c) return imr.ResolveImageConfig(ctx, ref, opt) diff --git a/frontend/gateway/gateway.go b/frontend/gateway/gateway.go index 983942b1c..984b3fe04 100644 --- a/frontend/gateway/gateway.go +++ b/frontend/gateway/gateway.go @@ -458,6 +458,8 @@ func serveLLBBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLB grpc.StreamInterceptor(grpcerrors.StreamServerInterceptor), grpc.MaxRecvMsgSize(defaults.DefaultMaxRecvMsgSize), grpc.MaxSendMsgSize(defaults.DefaultMaxSendMsgSize), + grpc.InitialWindowSize(65535 * 32), // earthly-specific + grpc.InitialConnWindowSize(65535 * 16), // earthly-specific } server := grpc.NewServer(serverOpt...) grpc_health_v1.RegisterHealthServer(server, health.NewServer()) @@ -853,7 +855,7 @@ func (lbf *llbBridgeForwarder) Solve(ctx context.Context, req *pb.SolveRequest) return resp, nil } -func (lbf *llbBridgeForwarder) getImmutableRef(ctx context.Context, id string) (cache.ImmutableRef, error) { +func (lbf *llbBridgeForwarder) getImmutableRef(ctx context.Context, id, path string) (cache.ImmutableRef, error) { lbf.mu.Lock() ref, ok := lbf.refs[id] if !ok { @@ -869,7 +871,7 @@ func (lbf *llbBridgeForwarder) getImmutableRef(ctx context.Context, id string) ( } lbf.mu.Unlock() if ref == nil { - return nil, errors.Errorf("empty ref: %s", id) + return nil, errors.Wrap(os.ErrNotExist, path) } r, err := ref.Result(ctx) @@ -889,10 +891,11 @@ func (lbf *llbBridgeForwarder) getMounter(ctx context.Context, id string, ref ca lbf.mountsMu.Lock() defer lbf.mountsMu.Unlock() - mounter, ok := lbf.mounts[id] - if ok { - return mounter, nil - } + // Earthly: skip mount cache to ensure persistent cache mounts + // reflect latest writes between consecutive SAVE ARTIFACT calls. + // Upstream commit 7aaa7974d introduced this cache for performance, + // but it causes stale reads when the same cache mount is modified + // between operations within a single target. var mountable snapshot.Mountable if ref != nil { var err error @@ -902,7 +905,7 @@ func (lbf *llbBridgeForwarder) getMounter(ctx context.Context, id string, ref ca } } - mounter = snapshot.LocalMounter(mountable) + mounter := snapshot.LocalMounter(mountable) lbf.mounts[id] = mounter return mounter, nil } @@ -919,7 +922,7 @@ func (lbf *llbBridgeForwarder) getMount(ctx context.Context, id string, ref cach func (lbf *llbBridgeForwarder) ReadFile(ctx context.Context, req *pb.ReadFileRequest) (*pb.ReadFileResponse, error) { ctx = tracing.ContextWithSpanFromContext(ctx, lbf.callCtx) - ref, err := lbf.getImmutableRef(ctx, req.Ref) + ref, err := lbf.getImmutableRef(ctx, req.Ref, req.FilePath) if err != nil { return nil, err } @@ -949,7 +952,7 @@ func (lbf *llbBridgeForwarder) ReadFile(ctx context.Context, req *pb.ReadFileReq func (lbf *llbBridgeForwarder) ReadDir(ctx context.Context, req *pb.ReadDirRequest) (*pb.ReadDirResponse, error) { ctx = tracing.ContextWithSpanFromContext(ctx, lbf.callCtx) - ref, err := lbf.getImmutableRef(ctx, req.Ref) + ref, err := lbf.getImmutableRef(ctx, req.Ref, req.DirPath) if err != nil { return nil, err } @@ -971,10 +974,30 @@ func (lbf *llbBridgeForwarder) ReadDir(ctx context.Context, req *pb.ReadDirReque return &pb.ReadDirResponse{Entries: entries}, nil } +func (lbf *llbBridgeForwarder) Export(ctx context.Context, req *pb.ExportRequest) (*pb.ExportResponse, error) { + var err error + ctx = tracing.ContextWithSpanFromContext(ctx, lbf.callCtx) + + refs := map[string]cache.ImmutableRef{} + for k, ref := range req.Refs.Refs { + ref, err := lbf.getImmutableRef(ctx, ref.Id, "/") // the path is only used in error messages, in our case we want the full image + if err != nil { + return nil, err + } + refs[k] = ref + } + + err = lbf.llbBridge.Export(ctx, refs, req.Metadata) + if err != nil { + return nil, lbf.wrapSolveError(err) + } + return &pb.ExportResponse{}, nil +} + func (lbf *llbBridgeForwarder) StatFile(ctx context.Context, req *pb.StatFileRequest) (*pb.StatFileResponse, error) { ctx = tracing.ContextWithSpanFromContext(ctx, lbf.callCtx) - ref, err := lbf.getImmutableRef(ctx, req.Ref) + ref, err := lbf.getImmutableRef(ctx, req.Ref, req.Path) if err != nil { return nil, err } @@ -994,7 +1017,7 @@ func (lbf *llbBridgeForwarder) StatFile(ctx context.Context, req *pb.StatFileReq func (lbf *llbBridgeForwarder) Evaluate(ctx context.Context, req *pb.EvaluateRequest) (*pb.EvaluateResponse, error) { ctx = tracing.ContextWithSpanFromContext(ctx, lbf.callCtx) - _, err := lbf.getImmutableRef(ctx, req.Ref) + _, err := lbf.getImmutableRef(ctx, req.Ref, "/") if err != nil { return nil, err } diff --git a/frontend/gateway/grpcclient/client.go b/frontend/gateway/grpcclient/client.go index 205e260a6..ae6cf5e20 100644 --- a/frontend/gateway/grpcclient/client.go +++ b/frontend/gateway/grpcclient/client.go @@ -14,6 +14,7 @@ import ( "syscall" "time" + "github.com/containerd/containerd/v2/defaults" distreference "github.com/distribution/reference" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/client/llb/sourceresolver" @@ -471,6 +472,26 @@ func (c *grpcClient) Solve(ctx context.Context, creq client.SolveRequest) (res * return res, nil } +// Export is earthly-specific +func (c *grpcClient) Export(ctx context.Context, req client.ExportRequest) error { + m := map[string]*pb.Ref{} + for k, r := range req.Refs { + pbRef, err := convertRef(r) + if err != nil { + return err + } + m[k] = pbRef + } + _, err := c.client.Export(ctx, &pb.ExportRequest{ + Refs: &pb.RefMap{Refs: m}, + Metadata: req.Metadata, + }) + if err != nil { + return err + } + return nil +} + func (c *grpcClient) ResolveSourceMetadata(ctx context.Context, op *opspb.SourceOp, opt sourceresolver.Opt) (*sourceresolver.MetaResponse, error) { requiresImageAttestationResolve := opt.ImageOpt != nil && (opt.ImageOpt.AttestationChain || len(opt.ImageOpt.ResolveAttestations) > 0) requiresHTTPChecksumRequest := opt.HTTPOpt != nil && opt.HTTPOpt.ChecksumReq != nil @@ -1451,8 +1472,10 @@ func grpcClientConn(ctx context.Context) (context.Context, *grpc.ClientConn, err grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithUnaryInterceptor(grpcerrors.UnaryClientInterceptor), grpc.WithStreamInterceptor(grpcerrors.StreamClientInterceptor), - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(16 << 20)), - grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(16 << 20)), + grpc.WithInitialWindowSize(65535 * 32), // earthly-specific + grpc.WithInitialConnWindowSize(65535 * 16), // earthly-specific + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)), + grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)), } //nolint:staticcheck // ignore SA1019 NewClient has different behavior and needs to be tested diff --git a/frontend/gateway/pb/gateway.pb.go b/frontend/gateway/pb/gateway.pb.go index 18be05aec..b2cd0bcaf 100644 --- a/frontend/gateway/pb/gateway.pb.go +++ b/frontend/gateway/pb/gateway.pb.go @@ -2170,6 +2170,94 @@ func (x *ReadDirResponse) GetEntries() []*types.Stat { return nil } +type ExportRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Refs *RefMap `protobuf:"bytes,1,opt,name=refs,proto3" json:"refs,omitempty"` + Metadata map[string][]byte `protobuf:"bytes,2,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExportRequest) Reset() { + *x = ExportRequest{} + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[32] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExportRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExportRequest) ProtoMessage() {} + +func (x *ExportRequest) ProtoReflect() protoreflect.Message { + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[32] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExportRequest.ProtoReflect.Descriptor instead. +func (*ExportRequest) Descriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{32} +} + +func (x *ExportRequest) GetRefs() *RefMap { + if x != nil { + return x.Refs + } + return nil +} + +func (x *ExportRequest) GetMetadata() map[string][]byte { + if x != nil { + return x.Metadata + } + return nil +} + +type ExportResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExportResponse) Reset() { + *x = ExportResponse{} + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[33] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExportResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExportResponse) ProtoMessage() {} + +func (x *ExportResponse) ProtoReflect() protoreflect.Message { + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[33] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExportResponse.ProtoReflect.Descriptor instead. +func (*ExportResponse) Descriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{33} +} + type StatFileRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Ref string `protobuf:"bytes,1,opt,name=Ref,proto3" json:"Ref,omitempty"` @@ -2181,7 +2269,7 @@ type StatFileRequest struct { func (x *StatFileRequest) Reset() { *x = StatFileRequest{} - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[32] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2193,7 +2281,7 @@ func (x *StatFileRequest) String() string { func (*StatFileRequest) ProtoMessage() {} func (x *StatFileRequest) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[32] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[34] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2206,7 +2294,7 @@ func (x *StatFileRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use StatFileRequest.ProtoReflect.Descriptor instead. func (*StatFileRequest) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{32} + return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{34} } func (x *StatFileRequest) GetRef() string { @@ -2239,7 +2327,7 @@ type StatFileResponse struct { func (x *StatFileResponse) Reset() { *x = StatFileResponse{} - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[33] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2251,7 +2339,7 @@ func (x *StatFileResponse) String() string { func (*StatFileResponse) ProtoMessage() {} func (x *StatFileResponse) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[33] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[35] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2264,7 +2352,7 @@ func (x *StatFileResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use StatFileResponse.ProtoReflect.Descriptor instead. func (*StatFileResponse) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{33} + return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{35} } func (x *StatFileResponse) GetStat() *types.Stat { @@ -2283,7 +2371,7 @@ type EvaluateRequest struct { func (x *EvaluateRequest) Reset() { *x = EvaluateRequest{} - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[34] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2295,7 +2383,7 @@ func (x *EvaluateRequest) String() string { func (*EvaluateRequest) ProtoMessage() {} func (x *EvaluateRequest) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[34] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[36] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2308,7 +2396,7 @@ func (x *EvaluateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use EvaluateRequest.ProtoReflect.Descriptor instead. func (*EvaluateRequest) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{34} + return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{36} } func (x *EvaluateRequest) GetRef() string { @@ -2326,7 +2414,7 @@ type EvaluateResponse struct { func (x *EvaluateResponse) Reset() { *x = EvaluateResponse{} - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[35] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2338,7 +2426,7 @@ func (x *EvaluateResponse) String() string { func (*EvaluateResponse) ProtoMessage() {} func (x *EvaluateResponse) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[35] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[37] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2351,7 +2439,7 @@ func (x *EvaluateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use EvaluateResponse.ProtoReflect.Descriptor instead. func (*EvaluateResponse) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{35} + return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{37} } type PingRequest struct { @@ -2362,7 +2450,7 @@ type PingRequest struct { func (x *PingRequest) Reset() { *x = PingRequest{} - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[36] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2374,7 +2462,7 @@ func (x *PingRequest) String() string { func (*PingRequest) ProtoMessage() {} func (x *PingRequest) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[36] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[38] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2387,7 +2475,7 @@ func (x *PingRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use PingRequest.ProtoReflect.Descriptor instead. func (*PingRequest) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{36} + return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{38} } type PongResponse struct { @@ -2401,7 +2489,7 @@ type PongResponse struct { func (x *PongResponse) Reset() { *x = PongResponse{} - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[37] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2413,7 +2501,7 @@ func (x *PongResponse) String() string { func (*PongResponse) ProtoMessage() {} func (x *PongResponse) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[37] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[39] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2426,7 +2514,7 @@ func (x *PongResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use PongResponse.ProtoReflect.Descriptor instead. func (*PongResponse) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{37} + return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{39} } func (x *PongResponse) GetFrontendAPICaps() []*pb2.APICap { @@ -2465,7 +2553,7 @@ type WarnRequest struct { func (x *WarnRequest) Reset() { *x = WarnRequest{} - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[38] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2477,7 +2565,7 @@ func (x *WarnRequest) String() string { func (*WarnRequest) ProtoMessage() {} func (x *WarnRequest) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[38] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[40] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2490,7 +2578,7 @@ func (x *WarnRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use WarnRequest.ProtoReflect.Descriptor instead. func (*WarnRequest) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{38} + return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{40} } func (x *WarnRequest) GetDigest() string { @@ -2550,7 +2638,7 @@ type WarnResponse struct { func (x *WarnResponse) Reset() { *x = WarnResponse{} - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[39] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2562,7 +2650,7 @@ func (x *WarnResponse) String() string { func (*WarnResponse) ProtoMessage() {} func (x *WarnResponse) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[39] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[41] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2575,7 +2663,7 @@ func (x *WarnResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use WarnResponse.ProtoReflect.Descriptor instead. func (*WarnResponse) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{39} + return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{41} } type NewContainerRequest struct { @@ -2594,7 +2682,7 @@ type NewContainerRequest struct { func (x *NewContainerRequest) Reset() { *x = NewContainerRequest{} - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[40] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2606,7 +2694,7 @@ func (x *NewContainerRequest) String() string { func (*NewContainerRequest) ProtoMessage() {} func (x *NewContainerRequest) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[40] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[42] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2619,7 +2707,7 @@ func (x *NewContainerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use NewContainerRequest.ProtoReflect.Descriptor instead. func (*NewContainerRequest) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{40} + return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{42} } func (x *NewContainerRequest) GetContainerID() string { @@ -2679,7 +2767,7 @@ type NewContainerResponse struct { func (x *NewContainerResponse) Reset() { *x = NewContainerResponse{} - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[41] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2691,7 +2779,7 @@ func (x *NewContainerResponse) String() string { func (*NewContainerResponse) ProtoMessage() {} func (x *NewContainerResponse) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[41] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[43] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2704,7 +2792,7 @@ func (x *NewContainerResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use NewContainerResponse.ProtoReflect.Descriptor instead. func (*NewContainerResponse) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{41} + return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{43} } type ReleaseContainerRequest struct { @@ -2716,7 +2804,7 @@ type ReleaseContainerRequest struct { func (x *ReleaseContainerRequest) Reset() { *x = ReleaseContainerRequest{} - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[42] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2728,7 +2816,7 @@ func (x *ReleaseContainerRequest) String() string { func (*ReleaseContainerRequest) ProtoMessage() {} func (x *ReleaseContainerRequest) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[42] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[44] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2741,7 +2829,7 @@ func (x *ReleaseContainerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ReleaseContainerRequest.ProtoReflect.Descriptor instead. func (*ReleaseContainerRequest) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{42} + return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{44} } func (x *ReleaseContainerRequest) GetContainerID() string { @@ -2759,7 +2847,7 @@ type ReleaseContainerResponse struct { func (x *ReleaseContainerResponse) Reset() { *x = ReleaseContainerResponse{} - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[43] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2771,7 +2859,7 @@ func (x *ReleaseContainerResponse) String() string { func (*ReleaseContainerResponse) ProtoMessage() {} func (x *ReleaseContainerResponse) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[43] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[45] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2784,7 +2872,7 @@ func (x *ReleaseContainerResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReleaseContainerResponse.ProtoReflect.Descriptor instead. func (*ReleaseContainerResponse) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{43} + return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{45} } type ExecMessage struct { @@ -2806,7 +2894,7 @@ type ExecMessage struct { func (x *ExecMessage) Reset() { *x = ExecMessage{} - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[44] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2818,7 +2906,7 @@ func (x *ExecMessage) String() string { func (*ExecMessage) ProtoMessage() {} func (x *ExecMessage) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[44] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[46] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2831,7 +2919,7 @@ func (x *ExecMessage) ProtoReflect() protoreflect.Message { // Deprecated: Use ExecMessage.ProtoReflect.Descriptor instead. func (*ExecMessage) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{44} + return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{46} } func (x *ExecMessage) GetProcessID() string { @@ -2983,7 +3071,7 @@ type InitMessage struct { func (x *InitMessage) Reset() { *x = InitMessage{} - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[45] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[47] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2995,7 +3083,7 @@ func (x *InitMessage) String() string { func (*InitMessage) ProtoMessage() {} func (x *InitMessage) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[45] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[47] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3008,7 +3096,7 @@ func (x *InitMessage) ProtoReflect() protoreflect.Message { // Deprecated: Use InitMessage.ProtoReflect.Descriptor instead. func (*InitMessage) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{45} + return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{47} } func (x *InitMessage) GetContainerID() string { @@ -3063,7 +3151,7 @@ type ExitMessage struct { func (x *ExitMessage) Reset() { *x = ExitMessage{} - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[46] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[48] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3075,7 +3163,7 @@ func (x *ExitMessage) String() string { func (*ExitMessage) ProtoMessage() {} func (x *ExitMessage) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[46] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[48] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3088,7 +3176,7 @@ func (x *ExitMessage) ProtoReflect() protoreflect.Message { // Deprecated: Use ExitMessage.ProtoReflect.Descriptor instead. func (*ExitMessage) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{46} + return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{48} } func (x *ExitMessage) GetCode() uint32 { @@ -3113,7 +3201,7 @@ type StartedMessage struct { func (x *StartedMessage) Reset() { *x = StartedMessage{} - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[47] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[49] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3125,7 +3213,7 @@ func (x *StartedMessage) String() string { func (*StartedMessage) ProtoMessage() {} func (x *StartedMessage) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[47] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[49] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3138,7 +3226,7 @@ func (x *StartedMessage) ProtoReflect() protoreflect.Message { // Deprecated: Use StartedMessage.ProtoReflect.Descriptor instead. func (*StartedMessage) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{47} + return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{49} } type DoneMessage struct { @@ -3149,7 +3237,7 @@ type DoneMessage struct { func (x *DoneMessage) Reset() { *x = DoneMessage{} - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[48] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3161,7 +3249,7 @@ func (x *DoneMessage) String() string { func (*DoneMessage) ProtoMessage() {} func (x *DoneMessage) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[48] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[50] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3174,7 +3262,7 @@ func (x *DoneMessage) ProtoReflect() protoreflect.Message { // Deprecated: Use DoneMessage.ProtoReflect.Descriptor instead. func (*DoneMessage) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{48} + return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{50} } type FdMessage struct { @@ -3188,7 +3276,7 @@ type FdMessage struct { func (x *FdMessage) Reset() { *x = FdMessage{} - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[49] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3200,7 +3288,7 @@ func (x *FdMessage) String() string { func (*FdMessage) ProtoMessage() {} func (x *FdMessage) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[49] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[51] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3213,7 +3301,7 @@ func (x *FdMessage) ProtoReflect() protoreflect.Message { // Deprecated: Use FdMessage.ProtoReflect.Descriptor instead. func (*FdMessage) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{49} + return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{51} } func (x *FdMessage) GetFd() uint32 { @@ -3247,7 +3335,7 @@ type ResizeMessage struct { func (x *ResizeMessage) Reset() { *x = ResizeMessage{} - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[50] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[52] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3259,7 +3347,7 @@ func (x *ResizeMessage) String() string { func (*ResizeMessage) ProtoMessage() {} func (x *ResizeMessage) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[50] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[52] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3272,7 +3360,7 @@ func (x *ResizeMessage) ProtoReflect() protoreflect.Message { // Deprecated: Use ResizeMessage.ProtoReflect.Descriptor instead. func (*ResizeMessage) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{50} + return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{52} } func (x *ResizeMessage) GetRows() uint32 { @@ -3300,7 +3388,7 @@ type SignalMessage struct { func (x *SignalMessage) Reset() { *x = SignalMessage{} - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[51] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3312,7 +3400,7 @@ func (x *SignalMessage) String() string { func (*SignalMessage) ProtoMessage() {} func (x *SignalMessage) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[51] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[53] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3325,7 +3413,7 @@ func (x *SignalMessage) ProtoReflect() protoreflect.Message { // Deprecated: Use SignalMessage.ProtoReflect.Descriptor instead. func (*SignalMessage) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{51} + return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{53} } func (x *SignalMessage) GetName() string { @@ -3345,7 +3433,7 @@ type Blob struct { func (x *Blob) Reset() { *x = Blob{} - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[52] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[54] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3357,7 +3445,7 @@ func (x *Blob) String() string { func (*Blob) ProtoMessage() {} func (x *Blob) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[52] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[54] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3370,7 +3458,7 @@ func (x *Blob) ProtoReflect() protoreflect.Message { // Deprecated: Use Blob.ProtoReflect.Descriptor instead. func (*Blob) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{52} + return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{54} } func (x *Blob) GetDescriptor_() *Descriptor { @@ -3399,7 +3487,7 @@ type Descriptor struct { func (x *Descriptor) Reset() { *x = Descriptor{} - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[53] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[55] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3411,7 +3499,7 @@ func (x *Descriptor) String() string { func (*Descriptor) ProtoMessage() {} func (x *Descriptor) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[53] + mi := &file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[55] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3424,7 +3512,7 @@ func (x *Descriptor) ProtoReflect() protoreflect.Message { // Deprecated: Use Descriptor.ProtoReflect.Descriptor instead. func (*Descriptor) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{53} + return file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP(), []int{55} } func (x *Descriptor) GetMediaType() string { @@ -3635,7 +3723,14 @@ const file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDesc = "MountIndex\x18\x04 \x01(\x05R\n" + "MountIndex\"?\n" + "\x0fReadDirResponse\x12,\n" + - "\aentries\x18\x01 \x03(\v2\x12.fsutil.types.StatR\aentries\"W\n" + + "\aentries\x18\x01 \x03(\v2\x12.fsutil.types.StatR\aentries\"\xd7\x01\n" + + "\rExportRequest\x125\n" + + "\x04refs\x18\x01 \x01(\v2!.moby.buildkit.v1.frontend.RefMapR\x04refs\x12R\n" + + "\bmetadata\x18\x02 \x03(\v26.moby.buildkit.v1.frontend.ExportRequest.MetadataEntryR\bmetadata\x1a;\n" + + "\rMetadataEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\fR\x05value:\x028\x01\"\x10\n" + + "\x0eExportResponse\"W\n" + "\x0fStatFileRequest\x12\x10\n" + "\x03Ref\x18\x01 \x01(\tR\x03Ref\x12\x12\n" + "\x04Path\x18\x02 \x01(\tR\x04Path\x12\x1e\n" + @@ -3729,7 +3824,7 @@ const file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDesc = "\x06Bundle\x10\x01*&\n" + "\x11InTotoSubjectKind\x12\b\n" + "\x04Self\x10\x00\x12\a\n" + - "\x03Raw\x10\x012\x84\x0e\n" + + "\x03Raw\x10\x012\xe3\x0e\n" + "\tLLBBridge\x12\x81\x01\n" + "\x12ResolveImageConfig\x124.moby.buildkit.v1.frontend.ResolveImageConfigRequest\x1a5.moby.buildkit.v1.frontend.ResolveImageConfigResponse\x12~\n" + "\x11ResolveSourceMeta\x123.moby.buildkit.v1.frontend.ResolveSourceMetaRequest\x1a4.moby.buildkit.v1.frontend.ResolveSourceMetaResponse\x12Z\n" + @@ -3740,7 +3835,8 @@ const file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDesc = "\bEvaluate\x12*.moby.buildkit.v1.frontend.EvaluateRequest\x1a+.moby.buildkit.v1.frontend.EvaluateResponse\x12W\n" + "\x04Ping\x12&.moby.buildkit.v1.frontend.PingRequest\x1a'.moby.buildkit.v1.frontend.PongResponse\x12]\n" + "\x06Return\x12(.moby.buildkit.v1.frontend.ReturnRequest\x1a).moby.buildkit.v1.frontend.ReturnResponse\x12]\n" + - "\x06Inputs\x12(.moby.buildkit.v1.frontend.InputsRequest\x1a).moby.buildkit.v1.frontend.InputsResponse\x12o\n" + + "\x06Inputs\x12(.moby.buildkit.v1.frontend.InputsRequest\x1a).moby.buildkit.v1.frontend.InputsResponse\x12]\n" + + "\x06Export\x12(.moby.buildkit.v1.frontend.ExportRequest\x1a).moby.buildkit.v1.frontend.ExportResponse\x12o\n" + "\fNewContainer\x12..moby.buildkit.v1.frontend.NewContainerRequest\x1a/.moby.buildkit.v1.frontend.NewContainerResponse\x12{\n" + "\x10ReleaseContainer\x122.moby.buildkit.v1.frontend.ReleaseContainerRequest\x1a3.moby.buildkit.v1.frontend.ReleaseContainerResponse\x12a\n" + "\vExecProcess\x12&.moby.buildkit.v1.frontend.ExecMessage\x1a&.moby.buildkit.v1.frontend.ExecMessage(\x010\x01\x12l\n" + @@ -3762,7 +3858,7 @@ func file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDescGZIP } var file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_enumTypes = make([]protoimpl.EnumInfo, 3) -var file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes = make([]protoimpl.MessageInfo, 65) +var file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes = make([]protoimpl.MessageInfo, 68) var file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_goTypes = []any{ (AttestationKind)(0), // 0: moby.buildkit.v1.frontend.AttestationKind (InTotoSubjectKind)(0), // 1: moby.buildkit.v1.frontend.InTotoSubjectKind @@ -3799,171 +3895,178 @@ var file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_goTypes = [] (*ReadFileResponse)(nil), // 32: moby.buildkit.v1.frontend.ReadFileResponse (*ReadDirRequest)(nil), // 33: moby.buildkit.v1.frontend.ReadDirRequest (*ReadDirResponse)(nil), // 34: moby.buildkit.v1.frontend.ReadDirResponse - (*StatFileRequest)(nil), // 35: moby.buildkit.v1.frontend.StatFileRequest - (*StatFileResponse)(nil), // 36: moby.buildkit.v1.frontend.StatFileResponse - (*EvaluateRequest)(nil), // 37: moby.buildkit.v1.frontend.EvaluateRequest - (*EvaluateResponse)(nil), // 38: moby.buildkit.v1.frontend.EvaluateResponse - (*PingRequest)(nil), // 39: moby.buildkit.v1.frontend.PingRequest - (*PongResponse)(nil), // 40: moby.buildkit.v1.frontend.PongResponse - (*WarnRequest)(nil), // 41: moby.buildkit.v1.frontend.WarnRequest - (*WarnResponse)(nil), // 42: moby.buildkit.v1.frontend.WarnResponse - (*NewContainerRequest)(nil), // 43: moby.buildkit.v1.frontend.NewContainerRequest - (*NewContainerResponse)(nil), // 44: moby.buildkit.v1.frontend.NewContainerResponse - (*ReleaseContainerRequest)(nil), // 45: moby.buildkit.v1.frontend.ReleaseContainerRequest - (*ReleaseContainerResponse)(nil), // 46: moby.buildkit.v1.frontend.ReleaseContainerResponse - (*ExecMessage)(nil), // 47: moby.buildkit.v1.frontend.ExecMessage - (*InitMessage)(nil), // 48: moby.buildkit.v1.frontend.InitMessage - (*ExitMessage)(nil), // 49: moby.buildkit.v1.frontend.ExitMessage - (*StartedMessage)(nil), // 50: moby.buildkit.v1.frontend.StartedMessage - (*DoneMessage)(nil), // 51: moby.buildkit.v1.frontend.DoneMessage - (*FdMessage)(nil), // 52: moby.buildkit.v1.frontend.FdMessage - (*ResizeMessage)(nil), // 53: moby.buildkit.v1.frontend.ResizeMessage - (*SignalMessage)(nil), // 54: moby.buildkit.v1.frontend.SignalMessage - (*Blob)(nil), // 55: moby.buildkit.v1.frontend.Blob - (*Descriptor)(nil), // 56: moby.buildkit.v1.frontend.Descriptor - nil, // 57: moby.buildkit.v1.frontend.Result.MetadataEntry - nil, // 58: moby.buildkit.v1.frontend.Result.AttestationsEntry - nil, // 59: moby.buildkit.v1.frontend.RefMapDeprecated.RefsEntry - nil, // 60: moby.buildkit.v1.frontend.RefMap.RefsEntry - nil, // 61: moby.buildkit.v1.frontend.Attestation.MetadataEntry - nil, // 62: moby.buildkit.v1.frontend.InputsResponse.DefinitionsEntry - nil, // 63: moby.buildkit.v1.frontend.AttestationChain.BlobsEntry - nil, // 64: moby.buildkit.v1.frontend.SolveRequest.FrontendOptEntry - nil, // 65: moby.buildkit.v1.frontend.SolveRequest.FrontendInputsEntry - nil, // 66: moby.buildkit.v1.frontend.CacheOptionsEntry.AttrsEntry - nil, // 67: moby.buildkit.v1.frontend.Descriptor.AnnotationsEntry - (*pb.Definition)(nil), // 68: pb.Definition - (*status.Status)(nil), // 69: google.rpc.Status - (*pb.Platform)(nil), // 70: pb.Platform - (*pb1.Policy)(nil), // 71: moby.buildkit.v1.sourcepolicy.Policy - (*pb.SourceOp)(nil), // 72: pb.SourceOp - (*timestamp.Timestamp)(nil), // 73: google.protobuf.Timestamp - (*types.Stat)(nil), // 74: fsutil.types.Stat - (*pb2.APICap)(nil), // 75: moby.buildkit.v1.apicaps.APICap - (*types1.WorkerRecord)(nil), // 76: moby.buildkit.v1.types.WorkerRecord - (*pb.SourceInfo)(nil), // 77: pb.SourceInfo - (*pb.Range)(nil), // 78: pb.Range - (*pb.Mount)(nil), // 79: pb.Mount - (pb.NetMode)(0), // 80: pb.NetMode - (*pb.WorkerConstraints)(nil), // 81: pb.WorkerConstraints - (*pb.HostIP)(nil), // 82: pb.HostIP - (*pb.Meta)(nil), // 83: pb.Meta - (pb.SecurityMode)(0), // 84: pb.SecurityMode - (*pb.SecretEnv)(nil), // 85: pb.SecretEnv + (*ExportRequest)(nil), // 35: moby.buildkit.v1.frontend.ExportRequest + (*ExportResponse)(nil), // 36: moby.buildkit.v1.frontend.ExportResponse + (*StatFileRequest)(nil), // 37: moby.buildkit.v1.frontend.StatFileRequest + (*StatFileResponse)(nil), // 38: moby.buildkit.v1.frontend.StatFileResponse + (*EvaluateRequest)(nil), // 39: moby.buildkit.v1.frontend.EvaluateRequest + (*EvaluateResponse)(nil), // 40: moby.buildkit.v1.frontend.EvaluateResponse + (*PingRequest)(nil), // 41: moby.buildkit.v1.frontend.PingRequest + (*PongResponse)(nil), // 42: moby.buildkit.v1.frontend.PongResponse + (*WarnRequest)(nil), // 43: moby.buildkit.v1.frontend.WarnRequest + (*WarnResponse)(nil), // 44: moby.buildkit.v1.frontend.WarnResponse + (*NewContainerRequest)(nil), // 45: moby.buildkit.v1.frontend.NewContainerRequest + (*NewContainerResponse)(nil), // 46: moby.buildkit.v1.frontend.NewContainerResponse + (*ReleaseContainerRequest)(nil), // 47: moby.buildkit.v1.frontend.ReleaseContainerRequest + (*ReleaseContainerResponse)(nil), // 48: moby.buildkit.v1.frontend.ReleaseContainerResponse + (*ExecMessage)(nil), // 49: moby.buildkit.v1.frontend.ExecMessage + (*InitMessage)(nil), // 50: moby.buildkit.v1.frontend.InitMessage + (*ExitMessage)(nil), // 51: moby.buildkit.v1.frontend.ExitMessage + (*StartedMessage)(nil), // 52: moby.buildkit.v1.frontend.StartedMessage + (*DoneMessage)(nil), // 53: moby.buildkit.v1.frontend.DoneMessage + (*FdMessage)(nil), // 54: moby.buildkit.v1.frontend.FdMessage + (*ResizeMessage)(nil), // 55: moby.buildkit.v1.frontend.ResizeMessage + (*SignalMessage)(nil), // 56: moby.buildkit.v1.frontend.SignalMessage + (*Blob)(nil), // 57: moby.buildkit.v1.frontend.Blob + (*Descriptor)(nil), // 58: moby.buildkit.v1.frontend.Descriptor + nil, // 59: moby.buildkit.v1.frontend.Result.MetadataEntry + nil, // 60: moby.buildkit.v1.frontend.Result.AttestationsEntry + nil, // 61: moby.buildkit.v1.frontend.RefMapDeprecated.RefsEntry + nil, // 62: moby.buildkit.v1.frontend.RefMap.RefsEntry + nil, // 63: moby.buildkit.v1.frontend.Attestation.MetadataEntry + nil, // 64: moby.buildkit.v1.frontend.InputsResponse.DefinitionsEntry + nil, // 65: moby.buildkit.v1.frontend.AttestationChain.BlobsEntry + nil, // 66: moby.buildkit.v1.frontend.SolveRequest.FrontendOptEntry + nil, // 67: moby.buildkit.v1.frontend.SolveRequest.FrontendInputsEntry + nil, // 68: moby.buildkit.v1.frontend.CacheOptionsEntry.AttrsEntry + nil, // 69: moby.buildkit.v1.frontend.ExportRequest.MetadataEntry + nil, // 70: moby.buildkit.v1.frontend.Descriptor.AnnotationsEntry + (*pb.Definition)(nil), // 71: pb.Definition + (*status.Status)(nil), // 72: google.rpc.Status + (*pb.Platform)(nil), // 73: pb.Platform + (*pb1.Policy)(nil), // 74: moby.buildkit.v1.sourcepolicy.Policy + (*pb.SourceOp)(nil), // 75: pb.SourceOp + (*timestamp.Timestamp)(nil), // 76: google.protobuf.Timestamp + (*types.Stat)(nil), // 77: fsutil.types.Stat + (*pb2.APICap)(nil), // 78: moby.buildkit.v1.apicaps.APICap + (*types1.WorkerRecord)(nil), // 79: moby.buildkit.v1.types.WorkerRecord + (*pb.SourceInfo)(nil), // 80: pb.SourceInfo + (*pb.Range)(nil), // 81: pb.Range + (*pb.Mount)(nil), // 82: pb.Mount + (pb.NetMode)(0), // 83: pb.NetMode + (*pb.WorkerConstraints)(nil), // 84: pb.WorkerConstraints + (*pb.HostIP)(nil), // 85: pb.HostIP + (*pb.Meta)(nil), // 86: pb.Meta + (pb.SecurityMode)(0), // 87: pb.SecurityMode + (*pb.SecretEnv)(nil), // 88: pb.SecretEnv } var file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_depIdxs = []int32{ 4, // 0: moby.buildkit.v1.frontend.Result.refsDeprecated:type_name -> moby.buildkit.v1.frontend.RefMapDeprecated 5, // 1: moby.buildkit.v1.frontend.Result.ref:type_name -> moby.buildkit.v1.frontend.Ref 6, // 2: moby.buildkit.v1.frontend.Result.refs:type_name -> moby.buildkit.v1.frontend.RefMap - 57, // 3: moby.buildkit.v1.frontend.Result.metadata:type_name -> moby.buildkit.v1.frontend.Result.MetadataEntry - 58, // 4: moby.buildkit.v1.frontend.Result.attestations:type_name -> moby.buildkit.v1.frontend.Result.AttestationsEntry - 59, // 5: moby.buildkit.v1.frontend.RefMapDeprecated.refs:type_name -> moby.buildkit.v1.frontend.RefMapDeprecated.RefsEntry - 68, // 6: moby.buildkit.v1.frontend.Ref.def:type_name -> pb.Definition - 60, // 7: moby.buildkit.v1.frontend.RefMap.refs:type_name -> moby.buildkit.v1.frontend.RefMap.RefsEntry + 59, // 3: moby.buildkit.v1.frontend.Result.metadata:type_name -> moby.buildkit.v1.frontend.Result.MetadataEntry + 60, // 4: moby.buildkit.v1.frontend.Result.attestations:type_name -> moby.buildkit.v1.frontend.Result.AttestationsEntry + 61, // 5: moby.buildkit.v1.frontend.RefMapDeprecated.refs:type_name -> moby.buildkit.v1.frontend.RefMapDeprecated.RefsEntry + 71, // 6: moby.buildkit.v1.frontend.Ref.def:type_name -> pb.Definition + 62, // 7: moby.buildkit.v1.frontend.RefMap.refs:type_name -> moby.buildkit.v1.frontend.RefMap.RefsEntry 8, // 8: moby.buildkit.v1.frontend.Attestations.attestation:type_name -> moby.buildkit.v1.frontend.Attestation 0, // 9: moby.buildkit.v1.frontend.Attestation.kind:type_name -> moby.buildkit.v1.frontend.AttestationKind - 61, // 10: moby.buildkit.v1.frontend.Attestation.metadata:type_name -> moby.buildkit.v1.frontend.Attestation.MetadataEntry + 63, // 10: moby.buildkit.v1.frontend.Attestation.metadata:type_name -> moby.buildkit.v1.frontend.Attestation.MetadataEntry 5, // 11: moby.buildkit.v1.frontend.Attestation.ref:type_name -> moby.buildkit.v1.frontend.Ref 9, // 12: moby.buildkit.v1.frontend.Attestation.inTotoSubjects:type_name -> moby.buildkit.v1.frontend.InTotoSubject 1, // 13: moby.buildkit.v1.frontend.InTotoSubject.kind:type_name -> moby.buildkit.v1.frontend.InTotoSubjectKind 3, // 14: moby.buildkit.v1.frontend.ReturnRequest.result:type_name -> moby.buildkit.v1.frontend.Result - 69, // 15: moby.buildkit.v1.frontend.ReturnRequest.error:type_name -> google.rpc.Status - 62, // 16: moby.buildkit.v1.frontend.InputsResponse.Definitions:type_name -> moby.buildkit.v1.frontend.InputsResponse.DefinitionsEntry - 70, // 17: moby.buildkit.v1.frontend.ResolveImageConfigRequest.Platform:type_name -> pb.Platform - 71, // 18: moby.buildkit.v1.frontend.ResolveImageConfigRequest.SourcePolicies:type_name -> moby.buildkit.v1.sourcepolicy.Policy - 72, // 19: moby.buildkit.v1.frontend.ResolveSourceMetaRequest.Source:type_name -> pb.SourceOp - 70, // 20: moby.buildkit.v1.frontend.ResolveSourceMetaRequest.Platform:type_name -> pb.Platform + 72, // 15: moby.buildkit.v1.frontend.ReturnRequest.error:type_name -> google.rpc.Status + 64, // 16: moby.buildkit.v1.frontend.InputsResponse.Definitions:type_name -> moby.buildkit.v1.frontend.InputsResponse.DefinitionsEntry + 73, // 17: moby.buildkit.v1.frontend.ResolveImageConfigRequest.Platform:type_name -> pb.Platform + 74, // 18: moby.buildkit.v1.frontend.ResolveImageConfigRequest.SourcePolicies:type_name -> moby.buildkit.v1.sourcepolicy.Policy + 75, // 19: moby.buildkit.v1.frontend.ResolveSourceMetaRequest.Source:type_name -> pb.SourceOp + 73, // 20: moby.buildkit.v1.frontend.ResolveSourceMetaRequest.Platform:type_name -> pb.Platform 21, // 21: moby.buildkit.v1.frontend.ResolveSourceMetaRequest.Git:type_name -> moby.buildkit.v1.frontend.ResolveSourceGitRequest 18, // 22: moby.buildkit.v1.frontend.ResolveSourceMetaRequest.Image:type_name -> moby.buildkit.v1.frontend.ResolveSourceImageRequest 24, // 23: moby.buildkit.v1.frontend.ResolveSourceMetaRequest.HTTP:type_name -> moby.buildkit.v1.frontend.ResolveSourceHTTPRequest - 71, // 24: moby.buildkit.v1.frontend.ResolveSourceMetaRequest.SourcePolicies:type_name -> moby.buildkit.v1.sourcepolicy.Policy - 72, // 25: moby.buildkit.v1.frontend.ResolveSourceMetaResponse.Source:type_name -> pb.SourceOp + 74, // 24: moby.buildkit.v1.frontend.ResolveSourceMetaRequest.SourcePolicies:type_name -> moby.buildkit.v1.sourcepolicy.Policy + 75, // 25: moby.buildkit.v1.frontend.ResolveSourceMetaResponse.Source:type_name -> pb.SourceOp 20, // 26: moby.buildkit.v1.frontend.ResolveSourceMetaResponse.Image:type_name -> moby.buildkit.v1.frontend.ResolveSourceImageResponse 22, // 27: moby.buildkit.v1.frontend.ResolveSourceMetaResponse.Git:type_name -> moby.buildkit.v1.frontend.ResolveSourceGitResponse 23, // 28: moby.buildkit.v1.frontend.ResolveSourceMetaResponse.HTTP:type_name -> moby.buildkit.v1.frontend.ResolveSourceHTTPResponse - 63, // 29: moby.buildkit.v1.frontend.AttestationChain.Blobs:type_name -> moby.buildkit.v1.frontend.AttestationChain.BlobsEntry + 65, // 29: moby.buildkit.v1.frontend.AttestationChain.Blobs:type_name -> moby.buildkit.v1.frontend.AttestationChain.BlobsEntry 19, // 30: moby.buildkit.v1.frontend.ResolveSourceImageResponse.AttestationChain:type_name -> moby.buildkit.v1.frontend.AttestationChain - 73, // 31: moby.buildkit.v1.frontend.ResolveSourceHTTPResponse.LastModified:type_name -> google.protobuf.Timestamp + 76, // 31: moby.buildkit.v1.frontend.ResolveSourceHTTPResponse.LastModified:type_name -> google.protobuf.Timestamp 26, // 32: moby.buildkit.v1.frontend.ResolveSourceHTTPResponse.ChecksumResponse:type_name -> moby.buildkit.v1.frontend.ChecksumResponse 25, // 33: moby.buildkit.v1.frontend.ResolveSourceHTTPRequest.ChecksumRequest:type_name -> moby.buildkit.v1.frontend.ChecksumRequest 2, // 34: moby.buildkit.v1.frontend.ChecksumRequest.Algo:type_name -> moby.buildkit.v1.frontend.ChecksumRequest.ChecksumAlgo - 68, // 35: moby.buildkit.v1.frontend.SolveRequest.Definition:type_name -> pb.Definition - 64, // 36: moby.buildkit.v1.frontend.SolveRequest.FrontendOpt:type_name -> moby.buildkit.v1.frontend.SolveRequest.FrontendOptEntry + 71, // 35: moby.buildkit.v1.frontend.SolveRequest.Definition:type_name -> pb.Definition + 66, // 36: moby.buildkit.v1.frontend.SolveRequest.FrontendOpt:type_name -> moby.buildkit.v1.frontend.SolveRequest.FrontendOptEntry 28, // 37: moby.buildkit.v1.frontend.SolveRequest.CacheImports:type_name -> moby.buildkit.v1.frontend.CacheOptionsEntry - 65, // 38: moby.buildkit.v1.frontend.SolveRequest.FrontendInputs:type_name -> moby.buildkit.v1.frontend.SolveRequest.FrontendInputsEntry - 71, // 39: moby.buildkit.v1.frontend.SolveRequest.SourcePolicies:type_name -> moby.buildkit.v1.sourcepolicy.Policy - 66, // 40: moby.buildkit.v1.frontend.CacheOptionsEntry.Attrs:type_name -> moby.buildkit.v1.frontend.CacheOptionsEntry.AttrsEntry + 67, // 38: moby.buildkit.v1.frontend.SolveRequest.FrontendInputs:type_name -> moby.buildkit.v1.frontend.SolveRequest.FrontendInputsEntry + 74, // 39: moby.buildkit.v1.frontend.SolveRequest.SourcePolicies:type_name -> moby.buildkit.v1.sourcepolicy.Policy + 68, // 40: moby.buildkit.v1.frontend.CacheOptionsEntry.Attrs:type_name -> moby.buildkit.v1.frontend.CacheOptionsEntry.AttrsEntry 3, // 41: moby.buildkit.v1.frontend.SolveResponse.result:type_name -> moby.buildkit.v1.frontend.Result 31, // 42: moby.buildkit.v1.frontend.ReadFileRequest.Range:type_name -> moby.buildkit.v1.frontend.FileRange - 74, // 43: moby.buildkit.v1.frontend.ReadDirResponse.entries:type_name -> fsutil.types.Stat - 74, // 44: moby.buildkit.v1.frontend.StatFileResponse.stat:type_name -> fsutil.types.Stat - 75, // 45: moby.buildkit.v1.frontend.PongResponse.FrontendAPICaps:type_name -> moby.buildkit.v1.apicaps.APICap - 75, // 46: moby.buildkit.v1.frontend.PongResponse.LLBCaps:type_name -> moby.buildkit.v1.apicaps.APICap - 76, // 47: moby.buildkit.v1.frontend.PongResponse.Workers:type_name -> moby.buildkit.v1.types.WorkerRecord - 77, // 48: moby.buildkit.v1.frontend.WarnRequest.info:type_name -> pb.SourceInfo - 78, // 49: moby.buildkit.v1.frontend.WarnRequest.ranges:type_name -> pb.Range - 79, // 50: moby.buildkit.v1.frontend.NewContainerRequest.Mounts:type_name -> pb.Mount - 80, // 51: moby.buildkit.v1.frontend.NewContainerRequest.Network:type_name -> pb.NetMode - 70, // 52: moby.buildkit.v1.frontend.NewContainerRequest.platform:type_name -> pb.Platform - 81, // 53: moby.buildkit.v1.frontend.NewContainerRequest.constraints:type_name -> pb.WorkerConstraints - 82, // 54: moby.buildkit.v1.frontend.NewContainerRequest.extraHosts:type_name -> pb.HostIP - 48, // 55: moby.buildkit.v1.frontend.ExecMessage.Init:type_name -> moby.buildkit.v1.frontend.InitMessage - 52, // 56: moby.buildkit.v1.frontend.ExecMessage.File:type_name -> moby.buildkit.v1.frontend.FdMessage - 53, // 57: moby.buildkit.v1.frontend.ExecMessage.Resize:type_name -> moby.buildkit.v1.frontend.ResizeMessage - 50, // 58: moby.buildkit.v1.frontend.ExecMessage.Started:type_name -> moby.buildkit.v1.frontend.StartedMessage - 49, // 59: moby.buildkit.v1.frontend.ExecMessage.Exit:type_name -> moby.buildkit.v1.frontend.ExitMessage - 51, // 60: moby.buildkit.v1.frontend.ExecMessage.Done:type_name -> moby.buildkit.v1.frontend.DoneMessage - 54, // 61: moby.buildkit.v1.frontend.ExecMessage.Signal:type_name -> moby.buildkit.v1.frontend.SignalMessage - 83, // 62: moby.buildkit.v1.frontend.InitMessage.Meta:type_name -> pb.Meta - 84, // 63: moby.buildkit.v1.frontend.InitMessage.Security:type_name -> pb.SecurityMode - 85, // 64: moby.buildkit.v1.frontend.InitMessage.secretenv:type_name -> pb.SecretEnv - 69, // 65: moby.buildkit.v1.frontend.ExitMessage.Error:type_name -> google.rpc.Status - 56, // 66: moby.buildkit.v1.frontend.Blob.descriptor:type_name -> moby.buildkit.v1.frontend.Descriptor - 67, // 67: moby.buildkit.v1.frontend.Descriptor.annotations:type_name -> moby.buildkit.v1.frontend.Descriptor.AnnotationsEntry - 7, // 68: moby.buildkit.v1.frontend.Result.AttestationsEntry.value:type_name -> moby.buildkit.v1.frontend.Attestations - 5, // 69: moby.buildkit.v1.frontend.RefMap.RefsEntry.value:type_name -> moby.buildkit.v1.frontend.Ref - 68, // 70: moby.buildkit.v1.frontend.InputsResponse.DefinitionsEntry.value:type_name -> pb.Definition - 55, // 71: moby.buildkit.v1.frontend.AttestationChain.BlobsEntry.value:type_name -> moby.buildkit.v1.frontend.Blob - 68, // 72: moby.buildkit.v1.frontend.SolveRequest.FrontendInputsEntry.value:type_name -> pb.Definition - 14, // 73: moby.buildkit.v1.frontend.LLBBridge.ResolveImageConfig:input_type -> moby.buildkit.v1.frontend.ResolveImageConfigRequest - 16, // 74: moby.buildkit.v1.frontend.LLBBridge.ResolveSourceMeta:input_type -> moby.buildkit.v1.frontend.ResolveSourceMetaRequest - 27, // 75: moby.buildkit.v1.frontend.LLBBridge.Solve:input_type -> moby.buildkit.v1.frontend.SolveRequest - 30, // 76: moby.buildkit.v1.frontend.LLBBridge.ReadFile:input_type -> moby.buildkit.v1.frontend.ReadFileRequest - 33, // 77: moby.buildkit.v1.frontend.LLBBridge.ReadDir:input_type -> moby.buildkit.v1.frontend.ReadDirRequest - 35, // 78: moby.buildkit.v1.frontend.LLBBridge.StatFile:input_type -> moby.buildkit.v1.frontend.StatFileRequest - 37, // 79: moby.buildkit.v1.frontend.LLBBridge.Evaluate:input_type -> moby.buildkit.v1.frontend.EvaluateRequest - 39, // 80: moby.buildkit.v1.frontend.LLBBridge.Ping:input_type -> moby.buildkit.v1.frontend.PingRequest - 10, // 81: moby.buildkit.v1.frontend.LLBBridge.Return:input_type -> moby.buildkit.v1.frontend.ReturnRequest - 12, // 82: moby.buildkit.v1.frontend.LLBBridge.Inputs:input_type -> moby.buildkit.v1.frontend.InputsRequest - 43, // 83: moby.buildkit.v1.frontend.LLBBridge.NewContainer:input_type -> moby.buildkit.v1.frontend.NewContainerRequest - 45, // 84: moby.buildkit.v1.frontend.LLBBridge.ReleaseContainer:input_type -> moby.buildkit.v1.frontend.ReleaseContainerRequest - 47, // 85: moby.buildkit.v1.frontend.LLBBridge.ExecProcess:input_type -> moby.buildkit.v1.frontend.ExecMessage - 30, // 86: moby.buildkit.v1.frontend.LLBBridge.ReadFileContainer:input_type -> moby.buildkit.v1.frontend.ReadFileRequest - 33, // 87: moby.buildkit.v1.frontend.LLBBridge.ReadDirContainer:input_type -> moby.buildkit.v1.frontend.ReadDirRequest - 35, // 88: moby.buildkit.v1.frontend.LLBBridge.StatFileContainer:input_type -> moby.buildkit.v1.frontend.StatFileRequest - 41, // 89: moby.buildkit.v1.frontend.LLBBridge.Warn:input_type -> moby.buildkit.v1.frontend.WarnRequest - 15, // 90: moby.buildkit.v1.frontend.LLBBridge.ResolveImageConfig:output_type -> moby.buildkit.v1.frontend.ResolveImageConfigResponse - 17, // 91: moby.buildkit.v1.frontend.LLBBridge.ResolveSourceMeta:output_type -> moby.buildkit.v1.frontend.ResolveSourceMetaResponse - 29, // 92: moby.buildkit.v1.frontend.LLBBridge.Solve:output_type -> moby.buildkit.v1.frontend.SolveResponse - 32, // 93: moby.buildkit.v1.frontend.LLBBridge.ReadFile:output_type -> moby.buildkit.v1.frontend.ReadFileResponse - 34, // 94: moby.buildkit.v1.frontend.LLBBridge.ReadDir:output_type -> moby.buildkit.v1.frontend.ReadDirResponse - 36, // 95: moby.buildkit.v1.frontend.LLBBridge.StatFile:output_type -> moby.buildkit.v1.frontend.StatFileResponse - 38, // 96: moby.buildkit.v1.frontend.LLBBridge.Evaluate:output_type -> moby.buildkit.v1.frontend.EvaluateResponse - 40, // 97: moby.buildkit.v1.frontend.LLBBridge.Ping:output_type -> moby.buildkit.v1.frontend.PongResponse - 11, // 98: moby.buildkit.v1.frontend.LLBBridge.Return:output_type -> moby.buildkit.v1.frontend.ReturnResponse - 13, // 99: moby.buildkit.v1.frontend.LLBBridge.Inputs:output_type -> moby.buildkit.v1.frontend.InputsResponse - 44, // 100: moby.buildkit.v1.frontend.LLBBridge.NewContainer:output_type -> moby.buildkit.v1.frontend.NewContainerResponse - 46, // 101: moby.buildkit.v1.frontend.LLBBridge.ReleaseContainer:output_type -> moby.buildkit.v1.frontend.ReleaseContainerResponse - 47, // 102: moby.buildkit.v1.frontend.LLBBridge.ExecProcess:output_type -> moby.buildkit.v1.frontend.ExecMessage - 32, // 103: moby.buildkit.v1.frontend.LLBBridge.ReadFileContainer:output_type -> moby.buildkit.v1.frontend.ReadFileResponse - 34, // 104: moby.buildkit.v1.frontend.LLBBridge.ReadDirContainer:output_type -> moby.buildkit.v1.frontend.ReadDirResponse - 36, // 105: moby.buildkit.v1.frontend.LLBBridge.StatFileContainer:output_type -> moby.buildkit.v1.frontend.StatFileResponse - 42, // 106: moby.buildkit.v1.frontend.LLBBridge.Warn:output_type -> moby.buildkit.v1.frontend.WarnResponse - 90, // [90:107] is the sub-list for method output_type - 73, // [73:90] is the sub-list for method input_type - 73, // [73:73] is the sub-list for extension type_name - 73, // [73:73] is the sub-list for extension extendee - 0, // [0:73] is the sub-list for field type_name + 77, // 43: moby.buildkit.v1.frontend.ReadDirResponse.entries:type_name -> fsutil.types.Stat + 6, // 44: moby.buildkit.v1.frontend.ExportRequest.refs:type_name -> moby.buildkit.v1.frontend.RefMap + 69, // 45: moby.buildkit.v1.frontend.ExportRequest.metadata:type_name -> moby.buildkit.v1.frontend.ExportRequest.MetadataEntry + 77, // 46: moby.buildkit.v1.frontend.StatFileResponse.stat:type_name -> fsutil.types.Stat + 78, // 47: moby.buildkit.v1.frontend.PongResponse.FrontendAPICaps:type_name -> moby.buildkit.v1.apicaps.APICap + 78, // 48: moby.buildkit.v1.frontend.PongResponse.LLBCaps:type_name -> moby.buildkit.v1.apicaps.APICap + 79, // 49: moby.buildkit.v1.frontend.PongResponse.Workers:type_name -> moby.buildkit.v1.types.WorkerRecord + 80, // 50: moby.buildkit.v1.frontend.WarnRequest.info:type_name -> pb.SourceInfo + 81, // 51: moby.buildkit.v1.frontend.WarnRequest.ranges:type_name -> pb.Range + 82, // 52: moby.buildkit.v1.frontend.NewContainerRequest.Mounts:type_name -> pb.Mount + 83, // 53: moby.buildkit.v1.frontend.NewContainerRequest.Network:type_name -> pb.NetMode + 73, // 54: moby.buildkit.v1.frontend.NewContainerRequest.platform:type_name -> pb.Platform + 84, // 55: moby.buildkit.v1.frontend.NewContainerRequest.constraints:type_name -> pb.WorkerConstraints + 85, // 56: moby.buildkit.v1.frontend.NewContainerRequest.extraHosts:type_name -> pb.HostIP + 50, // 57: moby.buildkit.v1.frontend.ExecMessage.Init:type_name -> moby.buildkit.v1.frontend.InitMessage + 54, // 58: moby.buildkit.v1.frontend.ExecMessage.File:type_name -> moby.buildkit.v1.frontend.FdMessage + 55, // 59: moby.buildkit.v1.frontend.ExecMessage.Resize:type_name -> moby.buildkit.v1.frontend.ResizeMessage + 52, // 60: moby.buildkit.v1.frontend.ExecMessage.Started:type_name -> moby.buildkit.v1.frontend.StartedMessage + 51, // 61: moby.buildkit.v1.frontend.ExecMessage.Exit:type_name -> moby.buildkit.v1.frontend.ExitMessage + 53, // 62: moby.buildkit.v1.frontend.ExecMessage.Done:type_name -> moby.buildkit.v1.frontend.DoneMessage + 56, // 63: moby.buildkit.v1.frontend.ExecMessage.Signal:type_name -> moby.buildkit.v1.frontend.SignalMessage + 86, // 64: moby.buildkit.v1.frontend.InitMessage.Meta:type_name -> pb.Meta + 87, // 65: moby.buildkit.v1.frontend.InitMessage.Security:type_name -> pb.SecurityMode + 88, // 66: moby.buildkit.v1.frontend.InitMessage.secretenv:type_name -> pb.SecretEnv + 72, // 67: moby.buildkit.v1.frontend.ExitMessage.Error:type_name -> google.rpc.Status + 58, // 68: moby.buildkit.v1.frontend.Blob.descriptor:type_name -> moby.buildkit.v1.frontend.Descriptor + 70, // 69: moby.buildkit.v1.frontend.Descriptor.annotations:type_name -> moby.buildkit.v1.frontend.Descriptor.AnnotationsEntry + 7, // 70: moby.buildkit.v1.frontend.Result.AttestationsEntry.value:type_name -> moby.buildkit.v1.frontend.Attestations + 5, // 71: moby.buildkit.v1.frontend.RefMap.RefsEntry.value:type_name -> moby.buildkit.v1.frontend.Ref + 71, // 72: moby.buildkit.v1.frontend.InputsResponse.DefinitionsEntry.value:type_name -> pb.Definition + 57, // 73: moby.buildkit.v1.frontend.AttestationChain.BlobsEntry.value:type_name -> moby.buildkit.v1.frontend.Blob + 71, // 74: moby.buildkit.v1.frontend.SolveRequest.FrontendInputsEntry.value:type_name -> pb.Definition + 14, // 75: moby.buildkit.v1.frontend.LLBBridge.ResolveImageConfig:input_type -> moby.buildkit.v1.frontend.ResolveImageConfigRequest + 16, // 76: moby.buildkit.v1.frontend.LLBBridge.ResolveSourceMeta:input_type -> moby.buildkit.v1.frontend.ResolveSourceMetaRequest + 27, // 77: moby.buildkit.v1.frontend.LLBBridge.Solve:input_type -> moby.buildkit.v1.frontend.SolveRequest + 30, // 78: moby.buildkit.v1.frontend.LLBBridge.ReadFile:input_type -> moby.buildkit.v1.frontend.ReadFileRequest + 33, // 79: moby.buildkit.v1.frontend.LLBBridge.ReadDir:input_type -> moby.buildkit.v1.frontend.ReadDirRequest + 37, // 80: moby.buildkit.v1.frontend.LLBBridge.StatFile:input_type -> moby.buildkit.v1.frontend.StatFileRequest + 39, // 81: moby.buildkit.v1.frontend.LLBBridge.Evaluate:input_type -> moby.buildkit.v1.frontend.EvaluateRequest + 41, // 82: moby.buildkit.v1.frontend.LLBBridge.Ping:input_type -> moby.buildkit.v1.frontend.PingRequest + 10, // 83: moby.buildkit.v1.frontend.LLBBridge.Return:input_type -> moby.buildkit.v1.frontend.ReturnRequest + 12, // 84: moby.buildkit.v1.frontend.LLBBridge.Inputs:input_type -> moby.buildkit.v1.frontend.InputsRequest + 35, // 85: moby.buildkit.v1.frontend.LLBBridge.Export:input_type -> moby.buildkit.v1.frontend.ExportRequest + 45, // 86: moby.buildkit.v1.frontend.LLBBridge.NewContainer:input_type -> moby.buildkit.v1.frontend.NewContainerRequest + 47, // 87: moby.buildkit.v1.frontend.LLBBridge.ReleaseContainer:input_type -> moby.buildkit.v1.frontend.ReleaseContainerRequest + 49, // 88: moby.buildkit.v1.frontend.LLBBridge.ExecProcess:input_type -> moby.buildkit.v1.frontend.ExecMessage + 30, // 89: moby.buildkit.v1.frontend.LLBBridge.ReadFileContainer:input_type -> moby.buildkit.v1.frontend.ReadFileRequest + 33, // 90: moby.buildkit.v1.frontend.LLBBridge.ReadDirContainer:input_type -> moby.buildkit.v1.frontend.ReadDirRequest + 37, // 91: moby.buildkit.v1.frontend.LLBBridge.StatFileContainer:input_type -> moby.buildkit.v1.frontend.StatFileRequest + 43, // 92: moby.buildkit.v1.frontend.LLBBridge.Warn:input_type -> moby.buildkit.v1.frontend.WarnRequest + 15, // 93: moby.buildkit.v1.frontend.LLBBridge.ResolveImageConfig:output_type -> moby.buildkit.v1.frontend.ResolveImageConfigResponse + 17, // 94: moby.buildkit.v1.frontend.LLBBridge.ResolveSourceMeta:output_type -> moby.buildkit.v1.frontend.ResolveSourceMetaResponse + 29, // 95: moby.buildkit.v1.frontend.LLBBridge.Solve:output_type -> moby.buildkit.v1.frontend.SolveResponse + 32, // 96: moby.buildkit.v1.frontend.LLBBridge.ReadFile:output_type -> moby.buildkit.v1.frontend.ReadFileResponse + 34, // 97: moby.buildkit.v1.frontend.LLBBridge.ReadDir:output_type -> moby.buildkit.v1.frontend.ReadDirResponse + 38, // 98: moby.buildkit.v1.frontend.LLBBridge.StatFile:output_type -> moby.buildkit.v1.frontend.StatFileResponse + 40, // 99: moby.buildkit.v1.frontend.LLBBridge.Evaluate:output_type -> moby.buildkit.v1.frontend.EvaluateResponse + 42, // 100: moby.buildkit.v1.frontend.LLBBridge.Ping:output_type -> moby.buildkit.v1.frontend.PongResponse + 11, // 101: moby.buildkit.v1.frontend.LLBBridge.Return:output_type -> moby.buildkit.v1.frontend.ReturnResponse + 13, // 102: moby.buildkit.v1.frontend.LLBBridge.Inputs:output_type -> moby.buildkit.v1.frontend.InputsResponse + 36, // 103: moby.buildkit.v1.frontend.LLBBridge.Export:output_type -> moby.buildkit.v1.frontend.ExportResponse + 46, // 104: moby.buildkit.v1.frontend.LLBBridge.NewContainer:output_type -> moby.buildkit.v1.frontend.NewContainerResponse + 48, // 105: moby.buildkit.v1.frontend.LLBBridge.ReleaseContainer:output_type -> moby.buildkit.v1.frontend.ReleaseContainerResponse + 49, // 106: moby.buildkit.v1.frontend.LLBBridge.ExecProcess:output_type -> moby.buildkit.v1.frontend.ExecMessage + 32, // 107: moby.buildkit.v1.frontend.LLBBridge.ReadFileContainer:output_type -> moby.buildkit.v1.frontend.ReadFileResponse + 34, // 108: moby.buildkit.v1.frontend.LLBBridge.ReadDirContainer:output_type -> moby.buildkit.v1.frontend.ReadDirResponse + 38, // 109: moby.buildkit.v1.frontend.LLBBridge.StatFileContainer:output_type -> moby.buildkit.v1.frontend.StatFileResponse + 44, // 110: moby.buildkit.v1.frontend.LLBBridge.Warn:output_type -> moby.buildkit.v1.frontend.WarnResponse + 93, // [93:111] is the sub-list for method output_type + 75, // [75:93] is the sub-list for method input_type + 75, // [75:75] is the sub-list for extension type_name + 75, // [75:75] is the sub-list for extension extendee + 0, // [0:75] is the sub-list for field type_name } func init() { file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_init() } @@ -3977,7 +4080,7 @@ func file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_init() { (*Result_Ref)(nil), (*Result_Refs)(nil), } - file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[44].OneofWrappers = []any{ + file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_msgTypes[46].OneofWrappers = []any{ (*ExecMessage_Init)(nil), (*ExecMessage_File)(nil), (*ExecMessage_Resize)(nil), @@ -3992,7 +4095,7 @@ func file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDesc), len(file_github_com_moby_buildkit_frontend_gateway_pb_gateway_proto_rawDesc)), NumEnums: 3, - NumMessages: 65, + NumMessages: 68, NumExtensions: 0, NumServices: 1, }, diff --git a/frontend/gateway/pb/gateway.proto b/frontend/gateway/pb/gateway.proto index 648077857..6e64fea34 100644 --- a/frontend/gateway/pb/gateway.proto +++ b/frontend/gateway/pb/gateway.proto @@ -32,6 +32,8 @@ service LLBBridge { // apicaps:CapFrontendInputs rpc Inputs(InputsRequest) returns (InputsResponse); + rpc Export(ExportRequest) returns (ExportResponse); + rpc NewContainer(NewContainerRequest) returns (NewContainerResponse); rpc ReleaseContainer(ReleaseContainerRequest) returns (ReleaseContainerResponse); rpc ExecProcess(stream ExecMessage) returns (stream ExecMessage); @@ -278,6 +280,15 @@ message ReadDirResponse { repeated fsutil.types.Stat entries = 1; } +message ExportRequest { + RefMap refs = 1; + map metadata = 2; +} + +message ExportResponse { + // nothing yet +} + message StatFileRequest { string Ref = 1; string Path = 2; diff --git a/frontend/gateway/pb/gateway_grpc.pb.go b/frontend/gateway/pb/gateway_grpc.pb.go index 903046935..cb68b238d 100644 --- a/frontend/gateway/pb/gateway_grpc.pb.go +++ b/frontend/gateway/pb/gateway_grpc.pb.go @@ -29,6 +29,7 @@ const ( LLBBridge_Ping_FullMethodName = "/moby.buildkit.v1.frontend.LLBBridge/Ping" LLBBridge_Return_FullMethodName = "/moby.buildkit.v1.frontend.LLBBridge/Return" LLBBridge_Inputs_FullMethodName = "/moby.buildkit.v1.frontend.LLBBridge/Inputs" + LLBBridge_Export_FullMethodName = "/moby.buildkit.v1.frontend.LLBBridge/Export" LLBBridge_NewContainer_FullMethodName = "/moby.buildkit.v1.frontend.LLBBridge/NewContainer" LLBBridge_ReleaseContainer_FullMethodName = "/moby.buildkit.v1.frontend.LLBBridge/ReleaseContainer" LLBBridge_ExecProcess_FullMethodName = "/moby.buildkit.v1.frontend.LLBBridge/ExecProcess" @@ -60,6 +61,7 @@ type LLBBridgeClient interface { Return(ctx context.Context, in *ReturnRequest, opts ...grpc.CallOption) (*ReturnResponse, error) // apicaps:CapFrontendInputs Inputs(ctx context.Context, in *InputsRequest, opts ...grpc.CallOption) (*InputsResponse, error) + Export(ctx context.Context, in *ExportRequest, opts ...grpc.CallOption) (*ExportResponse, error) NewContainer(ctx context.Context, in *NewContainerRequest, opts ...grpc.CallOption) (*NewContainerResponse, error) ReleaseContainer(ctx context.Context, in *ReleaseContainerRequest, opts ...grpc.CallOption) (*ReleaseContainerResponse, error) ExecProcess(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ExecMessage, ExecMessage], error) @@ -179,6 +181,16 @@ func (c *lLBBridgeClient) Inputs(ctx context.Context, in *InputsRequest, opts .. return out, nil } +func (c *lLBBridgeClient) Export(ctx context.Context, in *ExportRequest, opts ...grpc.CallOption) (*ExportResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ExportResponse) + err := c.cc.Invoke(ctx, LLBBridge_Export_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *lLBBridgeClient) NewContainer(ctx context.Context, in *NewContainerRequest, opts ...grpc.CallOption) (*NewContainerResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(NewContainerResponse) @@ -274,6 +286,7 @@ type LLBBridgeServer interface { Return(context.Context, *ReturnRequest) (*ReturnResponse, error) // apicaps:CapFrontendInputs Inputs(context.Context, *InputsRequest) (*InputsResponse, error) + Export(context.Context, *ExportRequest) (*ExportResponse, error) NewContainer(context.Context, *NewContainerRequest) (*NewContainerResponse, error) ReleaseContainer(context.Context, *ReleaseContainerRequest) (*ReleaseContainerResponse, error) ExecProcess(grpc.BidiStreamingServer[ExecMessage, ExecMessage]) error @@ -322,6 +335,9 @@ func (UnimplementedLLBBridgeServer) Return(context.Context, *ReturnRequest) (*Re func (UnimplementedLLBBridgeServer) Inputs(context.Context, *InputsRequest) (*InputsResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Inputs not implemented") } +func (UnimplementedLLBBridgeServer) Export(context.Context, *ExportRequest) (*ExportResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Export not implemented") +} func (UnimplementedLLBBridgeServer) NewContainer(context.Context, *NewContainerRequest) (*NewContainerResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method NewContainer not implemented") } @@ -543,6 +559,24 @@ func _LLBBridge_Inputs_Handler(srv interface{}, ctx context.Context, dec func(in return interceptor(ctx, in, info, handler) } +func _LLBBridge_Export_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ExportRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(LLBBridgeServer).Export(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: LLBBridge_Export_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(LLBBridgeServer).Export(ctx, req.(*ExportRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _LLBBridge_NewContainer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(NewContainerRequest) if err := dec(in); err != nil { @@ -705,6 +739,10 @@ var LLBBridge_ServiceDesc = grpc.ServiceDesc{ MethodName: "Inputs", Handler: _LLBBridge_Inputs_Handler, }, + { + MethodName: "Export", + Handler: _LLBBridge_Export_Handler, + }, { MethodName: "NewContainer", Handler: _LLBBridge_NewContainer_Handler, diff --git a/frontend/gateway/pb/gateway_vtproto.pb.go b/frontend/gateway/pb/gateway_vtproto.pb.go index 1076b6ee2..1fab09fc8 100644 --- a/frontend/gateway/pb/gateway_vtproto.pb.go +++ b/frontend/gateway/pb/gateway_vtproto.pb.go @@ -833,6 +833,48 @@ func (m *ReadDirResponse) CloneMessageVT() proto.Message { return m.CloneVT() } +func (m *ExportRequest) CloneVT() *ExportRequest { + if m == nil { + return (*ExportRequest)(nil) + } + r := new(ExportRequest) + r.Refs = m.Refs.CloneVT() + if rhs := m.Metadata; rhs != nil { + tmpContainer := make(map[string][]byte, len(rhs)) + for k, v := range rhs { + tmpBytes := make([]byte, len(v)) + copy(tmpBytes, v) + tmpContainer[k] = tmpBytes + } + r.Metadata = tmpContainer + } + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *ExportRequest) CloneMessageVT() proto.Message { + return m.CloneVT() +} + +func (m *ExportResponse) CloneVT() *ExportResponse { + if m == nil { + return (*ExportResponse)(nil) + } + r := new(ExportResponse) + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *ExportResponse) CloneMessageVT() proto.Message { + return m.CloneVT() +} + func (m *StatFileRequest) CloneVT() *StatFileRequest { if m == nil { return (*StatFileRequest)(nil) @@ -2532,6 +2574,53 @@ func (this *ReadDirResponse) EqualMessageVT(thatMsg proto.Message) bool { } return this.EqualVT(that) } +func (this *ExportRequest) EqualVT(that *ExportRequest) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + if !this.Refs.EqualVT(that.Refs) { + return false + } + if len(this.Metadata) != len(that.Metadata) { + return false + } + for i, vx := range this.Metadata { + vy, ok := that.Metadata[i] + if !ok { + return false + } + if string(vx) != string(vy) { + return false + } + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *ExportRequest) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*ExportRequest) + if !ok { + return false + } + return this.EqualVT(that) +} +func (this *ExportResponse) EqualVT(that *ExportResponse) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *ExportResponse) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*ExportResponse) + if !ok { + return false + } + return this.EqualVT(that) +} func (this *StatFileRequest) EqualVT(that *StatFileRequest) bool { if this == that { return true @@ -5411,6 +5500,101 @@ func (m *ReadDirResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *ExportRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ExportRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *ExportRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Metadata) > 0 { + for k := range m.Metadata { + v := m.Metadata[k] + baseI := i + i -= len(v) + copy(dAtA[i:], v) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(v))) + i-- + dAtA[i] = 0x12 + i -= len(k) + copy(dAtA[i:], k) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(k))) + i-- + dAtA[i] = 0xa + i = protohelpers.EncodeVarint(dAtA, i, uint64(baseI-i)) + i-- + dAtA[i] = 0x12 + } + } + if m.Refs != nil { + size, err := m.Refs.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *ExportResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ExportResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *ExportResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + return len(dAtA) - i, nil +} + func (m *StatFileRequest) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil @@ -7549,6 +7733,39 @@ func (m *ReadDirResponse) SizeVT() (n int) { return n } +func (m *ExportRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Refs != nil { + l = m.Refs.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if len(m.Metadata) > 0 { + for k, v := range m.Metadata { + _ = k + _ = v + l = 1 + len(v) + protohelpers.SizeOfVarint(uint64(len(v))) + mapEntrySize := 1 + len(k) + protohelpers.SizeOfVarint(uint64(len(k))) + l + n += mapEntrySize + 1 + protohelpers.SizeOfVarint(uint64(mapEntrySize)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *ExportResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + n += len(m.unknownFields) + return n +} + func (m *StatFileRequest) SizeVT() (n int) { if m == nil { return 0 @@ -13801,6 +14018,272 @@ func (m *ReadDirResponse) UnmarshalVT(dAtA []byte) error { } return nil } +func (m *ExportRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ExportRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ExportRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Refs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Refs == nil { + m.Refs = &RefMap{} + } + if err := m.Refs.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Metadata", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Metadata == nil { + m.Metadata = make(map[string][]byte) + } + var mapkey string + var mapvalue []byte + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return protohelpers.ErrInvalidLength + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey < 0 { + return protohelpers.ErrInvalidLength + } + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + var mapbyteLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapbyteLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intMapbyteLen := int(mapbyteLen) + if intMapbyteLen < 0 { + return protohelpers.ErrInvalidLength + } + postbytesIndex := iNdEx + intMapbyteLen + if postbytesIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postbytesIndex > l { + return io.ErrUnexpectedEOF + } + mapvalue = make([]byte, mapbyteLen) + copy(mapvalue, dAtA[iNdEx:postbytesIndex]) + iNdEx = postbytesIndex + } else { + iNdEx = entryPreIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + m.Metadata[mapkey] = mapvalue + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ExportResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ExportResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ExportResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *StatFileRequest) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/go.mod b/go.mod index 167fdb506..d7a23b57a 100644 --- a/go.mod +++ b/go.mod @@ -116,6 +116,11 @@ require ( tags.cncf.io/container-device-interface v1.1.0 ) +require ( + github.com/docker/distribution v2.8.3+incompatible + github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 +) + require ( cyphar.com/go-pathrs v0.2.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect @@ -155,9 +160,11 @@ require ( github.com/dimchansky/utfbom v1.1.1 // indirect github.com/docker/docker-credential-helpers v0.9.5 // indirect github.com/docker/go-metrics v0.0.1 // indirect + github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/felixge/fgprof v0.9.3 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/garyburd/redigo v1.6.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/analysis v0.24.3 // indirect @@ -189,6 +196,8 @@ require ( github.com/google/go-containerregistry v0.20.7 // indirect github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/handlers v1.5.2 // indirect + github.com/gorilla/mux v1.8.1 // indirect github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect github.com/hanwen/go-fuse/v2 v2.9.0 // indirect @@ -230,6 +239,7 @@ require ( golang.org/x/text v0.34.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect kernel.org/pub/linux/libs/security/libcap/psx v1.2.77 // indirect sigs.k8s.io/yaml v1.6.0 // indirect diff --git a/go.sum b/go.sum index 7da0f484e..4f61d12fa 100644 --- a/go.sum +++ b/go.sum @@ -107,6 +107,7 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8 github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk= github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -201,12 +202,16 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/cli v29.3.1+incompatible h1:M04FDj2TRehDacrosh7Vlkgc7AuQoWloQkf1PA5hmoI= github.com/docker/cli v29.3.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= +github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker-credential-helpers v0.9.5 h1:EFNN8DHvaiK8zVqFA2DT6BjXE0GzfLOZ38ggPTKePkY= github.com/docker/docker-credential-helpers v0.9.5/go.mod h1:v1S+hepowrQXITkEfw6o4+BMbGot02wiKpzWhGUZK6c= github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -219,14 +224,18 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/garyburd/redigo v1.6.4 h1:LFu2R3+ZOPgSMWMOL+saa/zXRjw0ID2G8FepO53BGlg= +github.com/garyburd/redigo v1.6.4/go.mod h1:rTb6epsqigu3kYKBnaF028A7Tf/Aw5s0cqA47doKKqw= github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= github.com/go-chi/chi/v5 v5.2.4 h1:WtFKPHwlywe8Srng8j2BhOD9312j9cGUxG1SP4V2cR4= github.com/go-chi/chi/v5 v5.2.4/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0= github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -304,6 +313,7 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -342,6 +352,10 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.9 h1:TOpi/QG8iDcZlkQlGlF github.com/googleapis/enterprise-certificate-proxy v0.3.9/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= github.com/googleapis/gax-go/v2 v2.16.0 h1:iHbQmKLLZrexmb0OSsNGTeSTS0HO4YvFOG8g5E4Zd0Y= github.com/googleapis/gax-go/v2 v2.16.0/go.mod h1:o1vfQjjNZn4+dPnRdl/4ZD7S9414Y4xA+a/6Icj6l14= +github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyEE= +github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc h1:GN2Lv3MGO7AS6PrRoT6yV5+wkrOpcszoIsO4+4ds248= github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= @@ -414,8 +428,11 @@ github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBF github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= @@ -488,6 +505,7 @@ github.com/opencontainers/runtime-tools v0.9.1-0.20251114084447-edf4cb3d2116 h1: github.com/opencontainers/runtime-tools v0.9.1-0.20251114084447-edf4cb3d2116/go.mod h1:DKDEfzxvRkoQ6n9TGhxQgg2IM1lY4aM0eaQP4e3oElw= github.com/opencontainers/selinux v1.13.1 h1:A8nNeceYngH9Ow++M+VVEwJVpdFmrlxsN22F+ISDCJE= github.com/opencontainers/selinux v1.13.1/go.mod h1:S10WXZ/osk2kWOYKy1x2f/eXF5ZHJoUs8UU/2caNRbg= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/package-url/packageurl-go v0.1.1 h1:KTRE0bK3sKbFKAk3yy63DpeskU7Cvs/x/Da5l+RtzyU= github.com/package-url/packageurl-go v0.1.1/go.mod h1:uQd4a7Rh3ZsVg5j0lNyAfyxIeGde9yrlhjF78GzeW0c= github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= @@ -498,6 +516,7 @@ github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7/go.mod h1:pxMtw7c github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA= @@ -568,6 +587,7 @@ github.com/sigstore/sigstore/pkg/signature/kms/hashivault v1.10.3/go.mod h1:b2rV github.com/sigstore/timestamp-authority/v2 v2.0.3 h1:sRyYNtdED/ttLCMdaYnwpf0zre1A9chvjTnCmWWxN8Y= github.com/sigstore/timestamp-authority/v2 v2.0.3/go.mod h1:mDaHxkt3HmZYoIlwYj4QWo0RUr7VjYU52aVO5f5Qb3I= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/spdx/tools-golang v0.5.7 h1:+sWcKGnhwp3vLdMqPcLdA6QK679vd86cK9hQWH3AwCg= @@ -584,6 +604,8 @@ github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -681,12 +703,16 @@ go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjce go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.step.sm/crypto v0.74.0 h1:/APBEv45yYR4qQFg47HA8w1nesIGcxh44pGyQNw6JRA= go.step.sm/crypto v0.74.0/go.mod h1:UoXqCAJjjRgzPte0Llaqen7O9P7XjPmgjgTHQGkKCDk= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= @@ -705,6 +731,7 @@ golang.org/x/exp v0.0.0-20250911091902-df9299821621/go.mod h1:TwQYMMnGpvZyc+JpB/ golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= @@ -738,9 +765,11 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -759,6 +788,7 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -776,6 +806,7 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217 h1:GvESR9BIyHUahIb0NcTum6itIWtdoglGX+rnGxm2934= google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:yJ2HH4EHEDTd3JiLmhds6NkJ17ITVYOdV3m3VKOnws0= @@ -787,6 +818,7 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= @@ -805,11 +837,16 @@ google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBN google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= diff --git a/hack/images b/hack/images new file mode 100755 index 000000000..ed6ca995b --- /dev/null +++ b/hack/images @@ -0,0 +1,102 @@ +#!/usr/bin/env bash + +TAG=$1 +REPO=$2 +PUSH=$3 + +. $(dirname $0)/util +set -eu -o pipefail + +: "${RELEASE=false}" +: "${PLATFORMS=}" +: "${TARGET=}" +: "${BUILDKIT_DEBUG=}" + +versionTag=$(git describe --always --tags --match "v[0-9]*") + +if [[ ! "$versionTag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + versionTag="" +fi + +usage() { + echo "usage: $0 [push]" + exit 1 +} + +if [ -z "$TAG" ] || [ -z "$REPO" ]; then + usage +fi + +platformFlag="" +if [ -n "$PLATFORMS" ]; then + platformFlag="--platform=$PLATFORMS" +fi + +localmode="" +if [[ "$TAG" == "local" ]]; then + localmode="1" + if [ "$PUSH" = "push" ]; then + echo >&2 "local images cannot be pushed" + exit 1 + fi +fi + +attestFlags="$(buildAttestFlags)" + +outputFlag="--output=type=image,push=false" +if [ "$PUSH" = "push" ]; then + outputFlag="--output=type=image,push=true" +fi +if [ -n "$localmode" ]; then + outputFlag="--output=type=docker" + attestFlags="" +fi + +targetFlag="" +if [ -n "$TARGET" ]; then + targetFlag="--target=$TARGET" +fi + +tagNames="$REPO:$TAG" +if [ -n "$TARGET" ]; then + tagNames="$tagNames-$TARGET" +fi + +if [[ "$versionTag" == "$TAG" ]]; then + if [ -n "$TARGET" ]; then + tagNames="$tagNames $REPO:$TARGET" + else + tagNames="$tagNames $REPO:latest" + fi +fi + +importCacheFlags="" +for tagName in $tagNames; do + importCacheFlags="$importCacheFlags--cache-from=type=registry,ref=$tagName " +done +if [ -n "$cacheFromFlags" ]; then + importCacheFlags="$importCacheFlags$cacheFromFlags" +fi +if [ -n "$localmode" ]; then + importCacheFlags="" +fi + +exportCacheFlags="" +if [ -n "$cacheToFlags" ]; then + exportCacheFlags="$cacheToFlags" +elif [ "$PUSH" = "push" ]; then + exportCacheFlags="$exportCacheFlags--cache-to=type=inline " +fi + +tagFlags="" +for tagName in $tagNames; do + tagFlags="$tagFlags--tag=$tagName " +done + +nocacheFilterFlag="" +if [[ "$RELEASE" = "true" ]] && [[ "$GITHUB_ACTIONS" = "true" ]]; then + nocacheFilterFlag="--no-cache-filter=git,buildkit-export,gobuild-base" +fi + +buildxCmd build --build-arg BUILDKIT_DEBUG $platformFlag $targetFlag $importCacheFlags $exportCacheFlags $tagFlags $outputFlag $nocacheFilterFlag \ + $currentcontext diff --git a/hack/release b/hack/release new file mode 100755 index 000000000..0b0357c30 --- /dev/null +++ b/hack/release @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +: "${GITHUB_ACTIONS=}" +: "${GITHUB_REPOSITORY=}" +: "${GITHUB_RUN_ID=}" + +: "${BUILDX_CMD=docker buildx}" +: "${DESTDIR=./bin/release}" +: "${CACHE_FROM=}" +: "${CACHE_TO=}" +: "${RELEASE=false}" +: "${PLATFORMS=}" + +if [ -n "$CACHE_FROM" ]; then + for cfrom in $CACHE_FROM; do + setFlags+=(--set "*.cache-from=$cfrom") + done +fi +if [ -n "$CACHE_TO" ]; then + for cto in $CACHE_TO; do + setFlags+=(--set "*.cache-to=$cto") + done +fi +if [ -n "$PLATFORMS" ]; then + setFlags+=(--set "*.platform=$PLATFORMS") +fi +if ${BUILDX_CMD} build --help 2>&1 | grep -- '--attest' >/dev/null; then + prvattrs="false" + # if [ "$GITHUB_ACTIONS" = "true" ]; then + # prvattrs="$prvattrs,builder-id=https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + # fi + # setFlags+=(--set "*.attest=type=sbom") + # setFlags+=(--set "*.attest=type=provenance,$prvattrs") +fi +if [[ "$RELEASE" = "true" ]] && [[ "$GITHUB_ACTIONS" = "true" ]]; then + setFlags+=(--set "*.no-cache-filter=git,gobuild-base") +fi + +output=$(mktemp -d -t buildkit-output.XXXXXXXXXX) + +( + set -x + ${BUILDX_CMD} bake "${setFlags[@]}" --set "*.args.BUILDKIT_MULTI_PLATFORM=true" --set "*.output=$output" release +) + +for pdir in "${output}"/*/; do + ( + cd "$pdir" + releasetar=$(find . -name '*.tar.gz') + filename=$(basename "${releasetar%.tar.gz}") + # TODO(jhorsts): replace with upstream + # mv "provenance.json" "${filename}.provenance.json" + # mv "sbom-binaries.spdx.json" "${filename}.sbom.json" + find . -name 'sbom*.json' -exec rm {} \; + ) +done + +mkdir -p "$DESTDIR" +mv "$output"/**/* "$DESTDIR/" +rm -rf "$output" diff --git a/hack/util b/hack/util index 11669f362..305e95186 100755 --- a/hack/util +++ b/hack/util @@ -27,13 +27,14 @@ buildxCmd() { } buildAttestFlags() { - if ${BUILDX_CMD} build --help 2>&1 | grep -- '--attest' >/dev/null; then - prvattrs="mode=max" - if [ "$GITHUB_ACTIONS" = "true" ]; then - prvattrs="$prvattrs,builder-id=https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" - fi - echo "--attest=type=sbom --attest=type=provenance,version=v1,$prvattrs" - fi + : # no-op + # if ${BUILDX_CMD} build --help 2>&1 | grep -- '--attest' >/dev/null; then + # prvattrs="mode=max" + # if [ "$GITHUB_ACTIONS" = "true" ]; then + # prvattrs="$prvattrs,builder-id=https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + # fi + # echo "--attest=type=sbom --attest=type=provenance,version=v1,$prvattrs" + # fi } currentcontext="." diff --git a/session/auth/auth.pb.go b/session/auth/auth.pb.go index 231c247fd..ce3a1d8fc 100644 --- a/session/auth/auth.pb.go +++ b/session/auth/auth.pb.go @@ -196,8 +196,9 @@ func (x *FetchTokenRequest) GetScopes() []string { type FetchTokenResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Token string `protobuf:"bytes,1,opt,name=Token,proto3" json:"Token,omitempty"` - ExpiresIn int64 `protobuf:"varint,2,opt,name=ExpiresIn,proto3" json:"ExpiresIn,omitempty"` // seconds - IssuedAt int64 `protobuf:"varint,3,opt,name=IssuedAt,proto3" json:"IssuedAt,omitempty"` // timestamp + ExpiresIn int64 `protobuf:"varint,2,opt,name=ExpiresIn,proto3" json:"ExpiresIn,omitempty"` // seconds + IssuedAt int64 `protobuf:"varint,3,opt,name=IssuedAt,proto3" json:"IssuedAt,omitempty"` // timestamp + Anonymous bool `protobuf:"varint,99,opt,name=Anonymous,proto3" json:"Anonymous,omitempty"` // earthly-specific unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -253,6 +254,13 @@ func (x *FetchTokenResponse) GetIssuedAt() int64 { return 0 } +func (x *FetchTokenResponse) GetAnonymous() bool { + if x != nil { + return x.Anonymous + } + return false +} + type GetTokenAuthorityRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Host string `protobuf:"bytes,1,opt,name=Host,proto3" json:"Host,omitempty"` @@ -468,11 +476,12 @@ const file_github_com_moby_buildkit_session_auth_auth_proto_rawDesc = "" + "\x04Host\x18\x02 \x01(\tR\x04Host\x12\x14\n" + "\x05Realm\x18\x03 \x01(\tR\x05Realm\x12\x18\n" + "\aService\x18\x04 \x01(\tR\aService\x12\x16\n" + - "\x06Scopes\x18\x05 \x03(\tR\x06Scopes\"d\n" + + "\x06Scopes\x18\x05 \x03(\tR\x06Scopes\"\x82\x01\n" + "\x12FetchTokenResponse\x12\x14\n" + "\x05Token\x18\x01 \x01(\tR\x05Token\x12\x1c\n" + "\tExpiresIn\x18\x02 \x01(\x03R\tExpiresIn\x12\x1a\n" + - "\bIssuedAt\x18\x03 \x01(\x03R\bIssuedAt\"B\n" + + "\bIssuedAt\x18\x03 \x01(\x03R\bIssuedAt\x12\x1c\n" + + "\tAnonymous\x18c \x01(\bR\tAnonymous\"B\n" + "\x18GetTokenAuthorityRequest\x12\x12\n" + "\x04Host\x18\x01 \x01(\tR\x04Host\x12\x12\n" + "\x04Salt\x18\x02 \x01(\fR\x04Salt\"9\n" + diff --git a/session/auth/auth.proto b/session/auth/auth.proto index a4ed59962..52ba19e7b 100644 --- a/session/auth/auth.proto +++ b/session/auth/auth.proto @@ -4,16 +4,16 @@ package moby.filesync.v1; option go_package = "github.com/moby/buildkit/session/auth"; -service Auth{ +service Auth { rpc Credentials(CredentialsRequest) returns (CredentialsResponse); rpc FetchToken(FetchTokenRequest) returns (FetchTokenResponse); - rpc GetTokenAuthority(GetTokenAuthorityRequest) returns (GetTokenAuthorityResponse); - rpc VerifyTokenAuthority(VerifyTokenAuthorityRequest) returns (VerifyTokenAuthorityResponse); + rpc GetTokenAuthority(GetTokenAuthorityRequest) + returns (GetTokenAuthorityResponse); + rpc VerifyTokenAuthority(VerifyTokenAuthorityRequest) + returns (VerifyTokenAuthorityResponse); } -message CredentialsRequest { - string Host = 1; -} +message CredentialsRequest { string Host = 1; } message CredentialsResponse { string Username = 1; @@ -31,7 +31,9 @@ message FetchTokenRequest { message FetchTokenResponse { string Token = 1; int64 ExpiresIn = 2; // seconds - int64 IssuedAt = 3; // timestamp + int64 IssuedAt = 3; // timestamp + + bool Anonymous = 99; // earthly-specific } message GetTokenAuthorityRequest { @@ -39,9 +41,7 @@ message GetTokenAuthorityRequest { bytes Salt = 2; } -message GetTokenAuthorityResponse { - bytes PublicKey = 1; -} +message GetTokenAuthorityResponse { bytes PublicKey = 1; } message VerifyTokenAuthorityRequest { string Host = 1; @@ -49,6 +49,4 @@ message VerifyTokenAuthorityRequest { bytes Salt = 3; } -message VerifyTokenAuthorityResponse { - bytes Signed = 1; -} +message VerifyTokenAuthorityResponse { bytes Signed = 1; } diff --git a/session/auth/auth_vtproto.pb.go b/session/auth/auth_vtproto.pb.go index 0b114d869..cddc5ead6 100644 --- a/session/auth/auth_vtproto.pb.go +++ b/session/auth/auth_vtproto.pb.go @@ -87,6 +87,7 @@ func (m *FetchTokenResponse) CloneVT() *FetchTokenResponse { r.Token = m.Token r.ExpiresIn = m.ExpiresIn r.IssuedAt = m.IssuedAt + r.Anonymous = m.Anonymous if len(m.unknownFields) > 0 { r.unknownFields = make([]byte, len(m.unknownFields)) copy(r.unknownFields, m.unknownFields) @@ -282,6 +283,9 @@ func (this *FetchTokenResponse) EqualVT(that *FetchTokenResponse) bool { if this.IssuedAt != that.IssuedAt { return false } + if this.Anonymous != that.Anonymous { + return false + } return string(this.unknownFields) == string(that.unknownFields) } @@ -564,6 +568,18 @@ func (m *FetchTokenResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.Anonymous { + i-- + if m.Anonymous { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x6 + i-- + dAtA[i] = 0x98 + } if m.IssuedAt != 0 { i = protohelpers.EncodeVarint(dAtA, i, uint64(m.IssuedAt)) i-- @@ -845,6 +861,9 @@ func (m *FetchTokenResponse) SizeVT() (n int) { if m.IssuedAt != 0 { n += 1 + protohelpers.SizeOfVarint(uint64(m.IssuedAt)) } + if m.Anonymous { + n += 3 + } n += len(m.unknownFields) return n } @@ -1425,6 +1444,26 @@ func (m *FetchTokenResponse) UnmarshalVT(dAtA []byte) error { break } } + case 99: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Anonymous", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Anonymous = bool(v != 0) default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) diff --git a/session/auth/authprovider/authprovider.go b/session/auth/authprovider/authprovider.go index 4ca5a6457..ded68fc0d 100644 --- a/session/auth/authprovider/authprovider.go +++ b/session/auth/authprovider/authprovider.go @@ -105,7 +105,7 @@ func (ap *authProvider) FetchToken(ctx context.Context, req *auth.FetchTokenRequ // check for statically configured bearer token if ac.RegistryToken != "" { - return toTokenResponse(ac.RegistryToken, time.Time{}, 0), nil + return toTokenResponse(ac.RegistryToken, time.Time{}, 0, false), nil } creds := toCredentials(*ac) @@ -150,19 +150,20 @@ func (ap *authProvider) FetchToken(ctx context.Context, req *auth.FetchTokenRequ if err != nil { return nil, err } - return toTokenResponse(resp.Token, resp.IssuedAt, resp.ExpiresInSeconds), nil + return toTokenResponse(resp.Token, resp.IssuedAt, resp.ExpiresInSeconds, false), nil } } return nil, err } - return toTokenResponse(resp.AccessToken, resp.IssuedAt, resp.ExpiresInSeconds), nil + return toTokenResponse(resp.AccessToken, resp.IssuedAt, resp.ExpiresInSeconds, false), nil } // do request anonymously resp, err := authutil.FetchToken(ctx, httpClient, nil, to) if err != nil { return nil, errors.Wrap(err, "failed to fetch anonymous token") } - return toTokenResponse(resp.Token, resp.IssuedAt, resp.ExpiresInSeconds), nil + + return toTokenResponse(resp.Token, resp.IssuedAt, resp.ExpiresInSeconds, true), nil } func (ap *authProvider) tlsConfig(host string) (*tls.Config, error) { @@ -303,7 +304,7 @@ func (ap *authProvider) getAuthorityKey(ctx context.Context, host string, salt [ return ed25519.NewKeyFromSeed(sum[:ed25519.SeedSize]), nil } -func toTokenResponse(token string, issuedAt time.Time, expires int) *auth.FetchTokenResponse { +func toTokenResponse(token string, issuedAt time.Time, expires int, _ bool) *auth.FetchTokenResponse { if expires == 0 { expires = defaultExpiration } diff --git a/session/auth/authprovider/podman.go b/session/auth/authprovider/podman.go new file mode 100644 index 000000000..1959db1d7 --- /dev/null +++ b/session/auth/authprovider/podman.go @@ -0,0 +1,73 @@ +package authprovider + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/docker/cli/cli/config" + "github.com/docker/cli/cli/config/configfile" + "github.com/pkg/errors" + + "github.com/moby/buildkit/session" + "github.com/moby/buildkit/util/bklog" +) + +const ( + // PodmanConfigFileName is the name of config file + PodmanConfigFileName = "auth.json" + + // XDGSubPath is the subpath in the XDG_RUNTIME_DIR directory that contains the podman config file + XDGSubPath = "containers" +) + +// NewPodmanAuthProvider provides an attachable that loads the Podman config instead of the hardcoded docker ones. It is +// More or less a copy of the relevant pieces for loading Docker config files used by the DockerAuthProvider, just using +// Podmans default config paths & names. +func NewPodmanAuthProvider(stderr io.Writer) session.Attachable { + xdgRuntime, ok := os.LookupEnv("XDG_RUNTIME_DIR") + if !ok { + // Podman uses docker's default settings location when the XDG_RUNTIME_DIR is missing. + // See here for more details: https://docs.podman.io/en/latest/markdown/podman-login.1.html + bklog.G(context.TODO()).Debugf("WARNING: XDG_RUNTIME_DIR is not set, trying Docker config") + cfg := config.LoadDefaultConfigFile(stderr) + return NewDockerAuthProvider(DockerAuthProviderConfig{ + AuthConfigProvider: LoadAuthConfig(cfg), + }) + } + + filename := filepath.Join(xdgRuntime, XDGSubPath, PodmanConfigFileName) + configFile := configfile.New(filename) + + if file, err := os.Open(filename); err == nil { + defer file.Close() + + err = configFile.LoadFromReader(file) + if err != nil { + err = errors.Wrap(err, filename) + fmt.Fprintf(stderr, "WARNING: Error loading config file: %v\n", err) + } + + return NewDockerAuthProvider(DockerAuthProviderConfig{ + AuthConfigProvider: LoadAuthConfig(configFile), + }) + } else if !os.IsNotExist(err) { + // if file is there but we can't stat it for any reason other + // than it doesn't exist then stop + fmt.Fprintf(stderr, "WARNING: Error loading config file: %v\n", err) + return NewDockerAuthProvider(DockerAuthProviderConfig{ + AuthConfigProvider: LoadAuthConfig(configFile), + }) + } + + // No Podman config file, just use the docker one then to pick up docker credentials. + // Podman uses docker's default settings location when the XDG path does not exist. + // See here for more details: https://docs.podman.io/en/latest/markdown/podman-login.1.html + bklog.G(context.TODO()).Debugf("WARNING: %s did not exist, trying Docker config", filename) + cfg := config.LoadDefaultConfigFile(stderr) + return NewDockerAuthProvider(DockerAuthProviderConfig{ + AuthConfigProvider: LoadAuthConfig(cfg), + }) +} diff --git a/session/configurabletimeout.go b/session/configurabletimeout.go new file mode 100644 index 000000000..ead570bbb --- /dev/null +++ b/session/configurabletimeout.go @@ -0,0 +1,56 @@ +package session + +import ( + "context" + "time" + + "github.com/moby/buildkit/util/bklog" + "github.com/sirupsen/logrus" + "google.golang.org/grpc" + "google.golang.org/grpc/health/grpc_health_v1" +) + +func configurableMonitorHealth(ctx context.Context, cc *grpc.ClientConn, cancelConn func(), healthCfg ManagerHealthCfg) { + defer cancelConn() + defer cc.Close() + + ticker := time.NewTicker(healthCfg.frequency) + defer ticker.Stop() + healthClient := grpc_health_v1.NewHealthClient(cc) + + consecutiveFailures := 0 + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + timeoutStart := time.Now().UTC() + + ctx, cancel := context.WithTimeoutCause(ctx, healthCfg.timeout, nil) + _, err := healthClient.Check(ctx, &grpc_health_v1.HealthCheckRequest{}) + cancel() + + logFields := logrus.Fields{ + "timeout": healthCfg.timeout, + "actualDuration": time.Since(timeoutStart), + } + + if err != nil { + consecutiveFailures++ + + logFields["allowedFailures"] = healthCfg.allowedFailures + logFields["consecutiveFailures"] = consecutiveFailures + bklog.G(ctx).WithFields(logFields).Warn("healthcheck failed") + + if consecutiveFailures >= healthCfg.allowedFailures { + bklog.G(ctx).Error("healthcheck failed too many times") + return + } + } else { + bklog.G(ctx).WithFields(logFields).Debug("healthcheck completed") + consecutiveFailures = 0 + } + } + } +} diff --git a/session/content/content_test.go b/session/content/content_test.go index bc3e5fa84..c9976ffb3 100644 --- a/session/content/content_test.go +++ b/session/content/content_test.go @@ -3,6 +3,7 @@ package content import ( "context" "testing" + "time" "github.com/containerd/containerd/v2/core/content" "github.com/containerd/containerd/v2/plugins/content/local" @@ -43,7 +44,11 @@ func TestContentAttachable(t *testing.T) { s, err := session.NewSession(ctx, "bar") require.NoError(t, err) - m, err := session.NewManager() + m, err := session.NewManager(&session.ManagerOpt{ + HealthFrequency: 1 * time.Second, + HealthTimeout: 10 * time.Second, + HealthAllowedFailures: 1, + }) require.NoError(t, err) a := NewAttachable(attachableStores) diff --git a/session/filesync/diffcopy.go b/session/filesync/diffcopy.go index 5e8b40b93..53f8d9eb7 100644 --- a/session/filesync/diffcopy.go +++ b/session/filesync/diffcopy.go @@ -107,14 +107,14 @@ func recvDiffCopy(ds grpc.ClientStream, dest string, cu CacheUpdater, progress p ProgressCb: progress, Filter: fsutil.FilterFunc(filter), Differ: differ, - MetadataOnly: metadataOnlyFilter, })) } -func syncTargetDiffCopy(ds grpc.ServerStream, dest string) error { +func syncTargetDiffCopy(ds grpc.ServerStream, dest string, notifyFn fsutil.ChangeFunc) error { if err := os.MkdirAll(dest, 0700); err != nil { return errors.Wrapf(err, "failed to create synctarget dest dir %s", dest) } + modTime := time.Now().UnixNano() // earthly-specific return errors.WithStack(fsutil.Receive(ds.Context(), ds, dest, fsutil.ReceiveOpt{ Merge: true, Filter: func() func(string, *fstypes.Stat) bool { @@ -123,9 +123,13 @@ func syncTargetDiffCopy(ds grpc.ServerStream, dest string) error { return func(p string, st *fstypes.Stat) bool { st.Uid = uint32(uid) st.Gid = uint32(gid) + if st.ModTime == 0 { + st.ModTime = modTime + } return true } }(), + NotifyHashed: notifyFn, })) } diff --git a/session/filesync/filesync.go b/session/filesync/filesync.go index 230493f16..4c707c13e 100644 --- a/session/filesync/filesync.go +++ b/session/filesync/filesync.go @@ -247,6 +247,75 @@ func FSSync(ctx context.Context, c session.Caller, opt FSSendRequestOpt) error { return pr.recvFn(stream, opt.DestDir, opt.CacheUpdater, opt.ProgressCb, opt.Differ, opt.Filter, metadataOnlyFilter) } +// NewFSSyncTargetDir allows writing into a directory +// earthly-specific: notifyFn reports each received file (stock fsutil NotifyHashed) +func NewFSSyncTargetDir(outdir string, notifyFn fsutil.ChangeFunc) session.Attachable { + p := &earthlySyncTarget{ + outdir: outdir, + notifyFn: notifyFn, + } + return p +} + +// NewFSSyncMultiTarget allows writing into an io.WriteCloser; it is earthly-specific +func NewFSSyncMultiTarget(f func(map[string]string) (io.WriteCloser, error), outdirFunc func(map[string]string) (string, error), notifyFn fsutil.ChangeFunc) session.Attachable { + p := &earthlySyncTarget{ + f: f, + outdirFunc: outdirFunc, + notifyFn: notifyFn, + } + return p +} + +// earthlySyncTarget is earthly-specific, supports a per-file notify callback and outdirFunc +type earthlySyncTarget struct { + outdir string + outdirFunc func(map[string]string) (string, error) + f func(map[string]string) (io.WriteCloser, error) + notifyFn fsutil.ChangeFunc +} + +func (sp *earthlySyncTarget) Register(server *grpc.Server) { + RegisterFileSendServer(server, sp) +} + +func (sp *earthlySyncTarget) DiffCopy(stream FileSend_DiffCopyServer) (err error) { + if sp.outdir != "" { + return syncTargetDiffCopy(stream, sp.outdir, sp.notifyFn) + } + + opts, _ := metadata.FromIncomingContext(stream.Context()) + md := map[string]string{} + for k, v := range opts { + if after, ok0 := strings.CutPrefix(k, keyExporterMetaPrefix); ok0 { + md[after] = strings.Join(v, ",") + } + } + if sp.outdirFunc != nil { + outdir, err := sp.outdirFunc(md) + if err != nil { + return err + } + if outdir != "" { + return syncTargetDiffCopy(stream, outdir, sp.notifyFn) + } + } + wc, err := sp.f(md) + if err != nil { + return err + } + if wc == nil { + return status.Errorf(codes.AlreadyExists, "target already exists") + } + defer func() { + err1 := wc.Close() + if err == nil { + err = err1 + } + }() + return writeTargetFile(stream, wc) +} + type FSSyncTarget interface { target() *fsSyncTarget } @@ -326,7 +395,7 @@ func (sp *SyncTarget) chooser(ctx context.Context) int { func (sp *SyncTarget) DiffCopy(stream FileSend_DiffCopyServer) (err error) { id := sp.chooser(stream.Context()) if outdir, ok := sp.outdirs[id]; ok { - return syncTargetDiffCopy(stream, outdir) + return syncTargetDiffCopy(stream, outdir, nil) } f, ok := sp.fs[id] if !ok { @@ -382,6 +451,30 @@ func CopyToCaller(ctx context.Context, fs fsutil.FS, id int, c session.Caller, p return sendDiffCopy(cc, fs, progress) } +// CopyToCallerWithMeta is earthly specific +func CopyToCallerWithMeta(ctx context.Context, md map[string]string, fs fsutil.FS, c session.Caller, progress func(int, bool)) error { + method := session.MethodURL(FileSend_ServiceDesc.ServiceName, "diffcopy") + if !c.Supports(method) { + return errors.Errorf("method %s not supported by the client", method) + } + + client := NewFileSendClient(c.Conn()) + + opts := make(map[string][]string, len(md)) + for k, v := range md { + opts[keyExporterMetaPrefix+k] = []string{v} + } + + ctx = metadata.NewOutgoingContext(ctx, opts) + + cc, err := client.DiffCopy(ctx) + if err != nil { + return errors.WithStack(err) + } + + return sendDiffCopy(cc, fs, progress) +} + func CopyFileWriter(ctx context.Context, md map[string]string, id int, c session.Caller) (io.WriteCloser, error) { method := session.MethodURL(FileSend_ServiceDesc.ServiceName, "diffcopy") if !c.Supports(method) { diff --git a/session/filesync/filesync_test.go b/session/filesync/filesync_test.go index 25b909e2d..c8c562a90 100644 --- a/session/filesync/filesync_test.go +++ b/session/filesync/filesync_test.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/testutil" @@ -32,7 +33,11 @@ func TestFileSyncIncludePatterns(t *testing.T) { s, err := session.NewSession(ctx, "bar") require.NoError(t, err) - m, err := session.NewManager() + m, err := session.NewManager(&session.ManagerOpt{ + HealthFrequency: 1 * time.Second, + HealthTimeout: 10 * time.Second, + HealthAllowedFailures: 1, + }) require.NoError(t, err) fs := NewFSSyncProvider(StaticDirSource{"test0": tmpFS}) diff --git a/session/grpc.go b/session/grpc.go index d177fd54e..91232b8fa 100644 --- a/session/grpc.go +++ b/session/grpc.go @@ -2,23 +2,19 @@ package session import ( "context" - "math" "net" "sync/atomic" - "time" "github.com/containerd/containerd/v2/defaults" "github.com/moby/buildkit/util/bklog" "github.com/moby/buildkit/util/grpcerrors" "github.com/moby/buildkit/util/tracing" "github.com/pkg/errors" - "github.com/sirupsen/logrus" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.opentelemetry.io/otel/trace" "golang.org/x/net/http2" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/health/grpc_health_v1" ) func serve(ctx context.Context, grpcServer *grpc.Server, conn net.Conn) { @@ -30,7 +26,7 @@ func serve(ctx context.Context, grpcServer *grpc.Server, conn net.Conn) { (&http2.Server{}).ServeConn(conn, &http2.ServeConnOpts{Handler: grpcServer}) } -func grpcClientConn(ctx context.Context, conn net.Conn) (context.Context, *grpc.ClientConn, error) { +func grpcClientConn(ctx context.Context, conn net.Conn, healthCfg ManagerHealthCfg) (context.Context, *grpc.ClientConn, error) { var dialCount int64 dialer := grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { if c := atomic.AddInt64(&dialCount, 1); c > 1 { @@ -42,6 +38,8 @@ func grpcClientConn(ctx context.Context, conn net.Conn) (context.Context, *grpc. dialOpts := []grpc.DialOption{ dialer, grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithInitialWindowSize(65535 * 32), + grpc.WithInitialConnWindowSize(65535 * 16), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)), grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)), grpc.WithUnaryInterceptor(grpcerrors.UnaryClientInterceptor), @@ -63,71 +61,7 @@ func grpcClientConn(ctx context.Context, conn net.Conn) (context.Context, *grpc. } ctx, cancel := context.WithCancelCause(ctx) - go monitorHealth(ctx, cc, cancel) + go configurableMonitorHealth(ctx, cc, func() { cancel(nil) }, healthCfg) return ctx, cc, nil } - -func monitorHealth(ctx context.Context, cc *grpc.ClientConn, cancelConn func(error)) { - defer cancelConn(errors.WithStack(context.Canceled)) - defer cc.Close() - - ticker := time.NewTicker(5 * time.Second) - defer ticker.Stop() - healthClient := grpc_health_v1.NewHealthClient(cc) - - failedBefore := false - consecutiveSuccessful := 0 - defaultHealthcheckDuration := 30 * time.Second - lastHealthcheckDuration := time.Duration(0) - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - // This healthcheck can erroneously fail in some instances, such as receiving lots of data in a low-bandwidth scenario or too many concurrent builds. - // So, this healthcheck is purposely long, and can tolerate some failures on purpose. - - healthcheckStart := time.Now() - - timeout := time.Duration(math.Max(float64(defaultHealthcheckDuration), float64(lastHealthcheckDuration)*1.5)) - - ctx, cancel := context.WithCancelCause(ctx) - ctx, _ = context.WithTimeoutCause(ctx, timeout, errors.WithStack(context.DeadlineExceeded)) //nolint:govet - _, err := healthClient.Check(ctx, &grpc_health_v1.HealthCheckRequest{}) - cancel(errors.WithStack(context.Canceled)) - - lastHealthcheckDuration = time.Since(healthcheckStart) - logFields := logrus.Fields{ - "timeout": timeout, - "actualDuration": lastHealthcheckDuration, - } - - if err != nil { - select { - case <-ctx.Done(): - return - default: - } - if failedBefore { - bklog.G(ctx).Error("healthcheck failed fatally") - return - } - - failedBefore = true - consecutiveSuccessful = 0 - bklog.G(ctx).WithFields(logFields).Warn("healthcheck failed") - } else { - consecutiveSuccessful++ - - if consecutiveSuccessful >= 5 && failedBefore { - failedBefore = false - bklog.G(ctx).WithFields(logFields).Debug("reset healthcheck failure") - } - } - - bklog.G(ctx).WithFields(logFields).Trace("healthcheck completed") - } - } -} diff --git a/session/localhost/generate.go b/session/localhost/generate.go new file mode 100644 index 000000000..6678f8f0b --- /dev/null +++ b/session/localhost/generate.go @@ -0,0 +1,3 @@ +package localhost + +//go:generate protoc --gogoslick_out=plugins=grpc:. localhost.proto diff --git a/session/localhost/localhost.go b/session/localhost/localhost.go new file mode 100644 index 000000000..e2c13b63b --- /dev/null +++ b/session/localhost/localhost.go @@ -0,0 +1,330 @@ +package localhost + +import ( + "context" + "fmt" + "io" + "os" + "time" + + "github.com/moby/buildkit/session" + "github.com/moby/sys/user" + "github.com/pkg/errors" + "github.com/tonistiigi/fsutil" + "github.com/tonistiigi/fsutil/types" +) + +// RunOnLocalHostMagicStr is a magic mount path which is used to signal the RUN command +// should be run locally whenever this mount path exists; the contents of the mounted path is irrelevant. +const RunOnLocalHostMagicStr = "271c67a1-94d9-4241-8bca-cbae334622ae" + +// CopyFileMagicStr is a magic command that copies a file from the local system into a snapshot +// it's used as "CopyFileMagicStr src dest" +const CopyFileMagicStr = "39a51ba7-d8c6-43ac-b3aa-f987b2db1ced" + +// SendFileMagicStr is a magic command that copies a file from a snapshot to the localhost +// it's used as "SendFileMagicStr src dest" +const SendFileMagicStr = "98325231-d2e6-931c-b12a-84273bca21db" + +// Mountable is from buildkit/snapshot; however the snapshot package wont build on darwin +// so we must pull this in here to avoid pulling in linux-specific packages. +type Mountable interface { + IdentityMapping() *user.IdentityMapping +} + +// LocalhostExec is called by buildkitd; it connects to the user's client to request the client execute a command localy. +func LocalhostExec(ctx context.Context, c session.Caller, args []string, dir string, stdout, stderr io.Writer) error { + // stdout and stderr get closed in execOp.Exec() + + client := NewLocalhostClient(c.Conn()) + stream, err := client.Exec(ctx) + if err != nil { + return errors.WithStack(err) + } + defer stream.CloseSend() + + req := InputMessage{ + Command: args, + Dir: dir, + } + if err := stream.SendMsg(&req); err != nil { + return errors.WithStack(err) + } + + var exitCodeSet bool + var exitCode int + for { + msg, err := stream.Recv() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return errors.WithStack(err) + } + stdout.Write(msg.Stdout) + stderr.Write(msg.Stderr) + switch msg.Status { + case OutputMessage_RUNNING: + // ignore + case OutputMessage_DONE: + if exitCodeSet { + panic("received multiple DONE messages (shouldn't happen)") + } + exitCode = int(msg.ExitCode) + exitCodeSet = true + default: + return errors.Errorf("unhandled exit status: %d", msg.Status) + } + } + + if exitCode != 0 { + return errors.Errorf("exit code: %d", exitCode) + } + + return nil +} + +// LocalhostGet fetches a file or directory located at src on the localhost; and copies it into the mounted snapshot under dest +func LocalhostGet(ctx context.Context, c session.Caller, src, dest string, mount Mountable) error { + client := NewLocalhostClient(c.Conn()) + + stream, err := client.Get(ctx) + if err != nil { + return errors.WithStack(err) + } + defer stream.CloseSend() + + req := BytesMessage{ + Data: []byte(src), + } + if err := stream.SendMsg(&req); err != nil { + return errors.WithStack(err) + } + + msg, err := stream.Recv() + if err != nil { + return errors.WithStack(err) + } + if len(msg.Data) == 0 { + panic("received GetResponse contains no data; shouldn't happen") + } + mode := msg.Data[0] + switch mode { + case 'f': + version := msg.Data[1] + if version != 0 { + panic(fmt.Sprintf("unhandled file version %v", version)) + } + return receiveFile(stream, dest) + case 'd': + version := msg.Data[1] + if version != 0 { + panic(fmt.Sprintf("unhandled dir version %v", version)) + } + return receiveDir(stream, dest, mount) + default: + panic(fmt.Sprintf("unhandled mode %v", mode)) + } +} + +func receiveFile(stream Localhost_GetClient, dest string) (err error) { + msg, err := stream.Recv() + if err != nil { + return errors.WithStack(err) + } + + stat := types.Stat{} + err = stat.Unmarshal(msg.Data) + if err != nil { + return errors.WithStack(err) + } + + // change group and system-wide permissions to match user's permissions + mode := stat.Mode + umode := (mode & 0700) >> 6 + // ...???XXX?????? -> ...???000000000 -> ...???000000XXX -> ...???000XXXXXX -> ...???XXXXXXXXX + mode = (mode ^ (mode & 0777)) | umode | (umode << 3) | (umode << 6) + + f, err := os.OpenFile(dest, os.O_CREATE|os.O_WRONLY, os.FileMode(mode)) + if err != nil { + return errors.WithStack(err) + } + defer f.Close() + +outer: + for { + msg, err := stream.Recv() + if err == nil { + // continue + } else if errors.Is(err, io.EOF) { + break outer + } else { + return errors.WithStack(err) + } + _, err = f.Write(msg.Data) + if err != nil { + return errors.WithStack(err) + } + } + + mtime := time.Unix(0, stat.ModTime) + err = os.Chtimes(dest, mtime, mtime) + if err != nil { + return errors.Wrap(err, "failed to change file time") + } + return nil +} + +func receiveDir(stream Localhost_GetClient, dest string, mount Mountable) error { + err := os.MkdirAll(dest, 0775) + if err != nil { + return errors.WithStack(err) + } + + ctx := stream.Context() + return errors.WithStack(fsutil.Receive(ctx, stream, dest, fsutil.ReceiveOpt{ + Filter: func(p string, stat *types.Stat) bool { + if idmap := mount.IdentityMapping(); idmap != nil { + uid, gid, err := idmap.ToHost(int(stat.Uid), int(stat.Gid)) + if err != nil { + return false + } + stat.Uid = uint32(uid) + stat.Gid = uint32(gid) + } + // whatever permissions the user has, give them to group and others as well + // this matches behavior of gitsource, given that umask is 0 + // ...??XXX?????? -> ...0000000XXX + umode := (stat.Mode & 0700) >> 6 + // ...???XXX?????? -> ...???000000000 -> ...???000000XXX -> ...???000XXXXXX -> ...???XXXXXXXXX + stat.Mode = (stat.Mode ^ (stat.Mode & 0777)) | umode | (umode << 3) | (umode << 6) + return true + }, + })) +} + +// LocalhostPut is for uploading data from buildkit to the localhost +// src is a path on the buildkit host; dest is a path on the localhost +func LocalhostPut(ctx context.Context, c session.Caller, src, dst string) error { + client := NewLocalhostClient(c.Conn()) + + stream, err := client.Put(ctx) + if err != nil { + return errors.WithStack(err) + } + defer stream.CloseSend() + + fi, err := os.Stat(src) + if err != nil { + return err + } + switch mode := fi.Mode(); { + case mode.IsDir(): + return localhostPutSendDir(stream, src, dst) + case mode.IsRegular(): + return localhostPutSendFile(stream, src, dst) + default: + panic(fmt.Sprintf("unhandled mode file %v in localhostProvider.Get", mode)) + } +} + +func localhostPutSendFile(stream Localhost_PutClient, src, dst string) error { + // first tell localhost-provider server that we're sending a file + err := stream.Send(&BytesMessage{ + Data: []byte{'f', 0x00}, // 'f' denotes a file; 0 denotes version 0 of the send file protocol + }) + if err != nil { + return errors.WithStack(err) + } + + // next send a path of where we are uploading to + req := BytesMessage{ + Data: []byte(dst), + } + if err := stream.SendMsg(&req); err != nil { + return errors.WithStack(err) + } + + stat, err := fsutil.Stat(src) + if err != nil { + return errors.Wrapf(err, "failed to stat %s", src) + } + payload, err := stat.Marshal() + if err != nil { + return err + } + err = stream.Send(&BytesMessage{ + Data: payload, + }) + if err != nil { + return err + } + + f, err := os.Open(src) + if err != nil { + return errors.Wrapf(err, "failed to open %s", src) + } + defer f.Close() + + buf := make([]byte, 1024*1024) + for { + n, err := f.Read(buf) + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return errors.Wrapf(err, "failed to read from %s", src) + } + if n > 0 { + err = stream.Send(&BytesMessage{ + Data: buf[:n], + }) + if err != nil { + return err + } + } + } + + err = stream.CloseSend() + if err != nil { + return errors.Wrap(err, "stream.CloseSend failed") + } + + // wait for final message to confirm the client has finished receiving and writing the file + _, err = stream.Recv() + return err +} + +func localhostPutSendDir(stream Localhost_GetClient, src, dst string) error { + err := stream.Send(&BytesMessage{ + Data: []byte{'d', 0x00}, // 'd' denotes a file; 0 denotes version 0 of the send file protocol + }) + if err != nil { + return err + } + + err = stream.Send(&BytesMessage{ + Data: []byte(dst), + }) + if err != nil { + return err + } + + fs, err := fsutil.NewFS(src) + if err != nil { + return err + } + + fs, err = fsutil.NewFilterFS(fs, &fsutil.FilterOpt{ + IncludePatterns: []string{"*"}, + }) + if err != nil { + return err + } + + err = fsutil.Send(stream.Context(), stream, fs, nil) + if err != nil { + return errors.Wrap(err, "fsutil.Send failed") + } + return nil +} diff --git a/session/localhost/localhost.pb.go b/session/localhost/localhost.pb.go new file mode 100644 index 000000000..fc7234899 --- /dev/null +++ b/session/localhost/localhost.pb.go @@ -0,0 +1,332 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v3.11.4 +// source: github.com/moby/buildkit/session/localhost/localhost.proto + +package localhost + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type OutputMessage_Status int32 + +const ( + OutputMessage_RUNNING OutputMessage_Status = 0 + OutputMessage_DONE OutputMessage_Status = 1 + OutputMessage_KILLED OutputMessage_Status = 2 +) + +// Enum value maps for OutputMessage_Status. +var ( + OutputMessage_Status_name = map[int32]string{ + 0: "RUNNING", + 1: "DONE", + 2: "KILLED", + } + OutputMessage_Status_value = map[string]int32{ + "RUNNING": 0, + "DONE": 1, + "KILLED": 2, + } +) + +func (x OutputMessage_Status) Enum() *OutputMessage_Status { + p := new(OutputMessage_Status) + *p = x + return p +} + +func (x OutputMessage_Status) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (OutputMessage_Status) Descriptor() protoreflect.EnumDescriptor { + return file_github_com_moby_buildkit_session_localhost_localhost_proto_enumTypes[0].Descriptor() +} + +func (OutputMessage_Status) Type() protoreflect.EnumType { + return &file_github_com_moby_buildkit_session_localhost_localhost_proto_enumTypes[0] +} + +func (x OutputMessage_Status) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use OutputMessage_Status.Descriptor instead. +func (OutputMessage_Status) EnumDescriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_session_localhost_localhost_proto_rawDescGZIP(), []int{1, 0} +} + +// InputMessage contains stdin data +type InputMessage struct { + state protoimpl.MessageState `protogen:"open.v1"` + Command []string `protobuf:"bytes,1,rep,name=command,proto3" json:"command,omitempty"` + Stdin []byte `protobuf:"bytes,2,opt,name=stdin,proto3" json:"stdin,omitempty"` + Dir string `protobuf:"bytes,3,opt,name=dir,proto3" json:"dir,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InputMessage) Reset() { + *x = InputMessage{} + mi := &file_github_com_moby_buildkit_session_localhost_localhost_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InputMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InputMessage) ProtoMessage() {} + +func (x *InputMessage) ProtoReflect() protoreflect.Message { + mi := &file_github_com_moby_buildkit_session_localhost_localhost_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InputMessage.ProtoReflect.Descriptor instead. +func (*InputMessage) Descriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_session_localhost_localhost_proto_rawDescGZIP(), []int{0} +} + +func (x *InputMessage) GetCommand() []string { + if x != nil { + return x.Command + } + return nil +} + +func (x *InputMessage) GetStdin() []byte { + if x != nil { + return x.Stdin + } + return nil +} + +func (x *InputMessage) GetDir() string { + if x != nil { + return x.Dir + } + return "" +} + +// OutputMessage contains stdout and stderr data +type OutputMessage struct { + state protoimpl.MessageState `protogen:"open.v1"` + Stdout []byte `protobuf:"bytes,1,opt,name=stdout,proto3" json:"stdout,omitempty"` + Stderr []byte `protobuf:"bytes,2,opt,name=stderr,proto3" json:"stderr,omitempty"` + Status OutputMessage_Status `protobuf:"varint,3,opt,name=status,proto3,enum=moby.localhost.v1.OutputMessage_Status" json:"status,omitempty"` + ExitCode int32 `protobuf:"varint,4,opt,name=exit_code,json=exitCode,proto3" json:"exit_code,omitempty"` // only valid if status is done + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *OutputMessage) Reset() { + *x = OutputMessage{} + mi := &file_github_com_moby_buildkit_session_localhost_localhost_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *OutputMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*OutputMessage) ProtoMessage() {} + +func (x *OutputMessage) ProtoReflect() protoreflect.Message { + mi := &file_github_com_moby_buildkit_session_localhost_localhost_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use OutputMessage.ProtoReflect.Descriptor instead. +func (*OutputMessage) Descriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_session_localhost_localhost_proto_rawDescGZIP(), []int{1} +} + +func (x *OutputMessage) GetStdout() []byte { + if x != nil { + return x.Stdout + } + return nil +} + +func (x *OutputMessage) GetStderr() []byte { + if x != nil { + return x.Stderr + } + return nil +} + +func (x *OutputMessage) GetStatus() OutputMessage_Status { + if x != nil { + return x.Status + } + return OutputMessage_RUNNING +} + +func (x *OutputMessage) GetExitCode() int32 { + if x != nil { + return x.ExitCode + } + return 0 +} + +// BytesMessage contains a chunk of byte data +type BytesMessage struct { + state protoimpl.MessageState `protogen:"open.v1"` + Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BytesMessage) Reset() { + *x = BytesMessage{} + mi := &file_github_com_moby_buildkit_session_localhost_localhost_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BytesMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BytesMessage) ProtoMessage() {} + +func (x *BytesMessage) ProtoReflect() protoreflect.Message { + mi := &file_github_com_moby_buildkit_session_localhost_localhost_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BytesMessage.ProtoReflect.Descriptor instead. +func (*BytesMessage) Descriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_session_localhost_localhost_proto_rawDescGZIP(), []int{2} +} + +func (x *BytesMessage) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +var File_github_com_moby_buildkit_session_localhost_localhost_proto protoreflect.FileDescriptor + +const file_github_com_moby_buildkit_session_localhost_localhost_proto_rawDesc = "" + + "\n" + + ":github.com/moby/buildkit/session/localhost/localhost.proto\x12\x11moby.localhost.v1\"P\n" + + "\fInputMessage\x12\x18\n" + + "\acommand\x18\x01 \x03(\tR\acommand\x12\x14\n" + + "\x05stdin\x18\x02 \x01(\fR\x05stdin\x12\x10\n" + + "\x03dir\x18\x03 \x01(\tR\x03dir\"\xca\x01\n" + + "\rOutputMessage\x12\x16\n" + + "\x06stdout\x18\x01 \x01(\fR\x06stdout\x12\x16\n" + + "\x06stderr\x18\x02 \x01(\fR\x06stderr\x12?\n" + + "\x06status\x18\x03 \x01(\x0e2'.moby.localhost.v1.OutputMessage.StatusR\x06status\x12\x1b\n" + + "\texit_code\x18\x04 \x01(\x05R\bexitCode\"+\n" + + "\x06Status\x12\v\n" + + "\aRUNNING\x10\x00\x12\b\n" + + "\x04DONE\x10\x01\x12\n" + + "\n" + + "\x06KILLED\x10\x02\"\"\n" + + "\fBytesMessage\x12\x12\n" + + "\x04data\x18\x01 \x01(\fR\x04data2\xf4\x01\n" + + "\tLocalhost\x12M\n" + + "\x04Exec\x12\x1f.moby.localhost.v1.InputMessage\x1a .moby.localhost.v1.OutputMessage(\x010\x01\x12K\n" + + "\x03Get\x12\x1f.moby.localhost.v1.BytesMessage\x1a\x1f.moby.localhost.v1.BytesMessage(\x010\x01\x12K\n" + + "\x03Put\x12\x1f.moby.localhost.v1.BytesMessage\x1a\x1f.moby.localhost.v1.BytesMessage(\x010\x01B,Z*github.com/moby/buildkit/session/localhostb\x06proto3" + +var ( + file_github_com_moby_buildkit_session_localhost_localhost_proto_rawDescOnce sync.Once + file_github_com_moby_buildkit_session_localhost_localhost_proto_rawDescData []byte +) + +func file_github_com_moby_buildkit_session_localhost_localhost_proto_rawDescGZIP() []byte { + file_github_com_moby_buildkit_session_localhost_localhost_proto_rawDescOnce.Do(func() { + file_github_com_moby_buildkit_session_localhost_localhost_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_github_com_moby_buildkit_session_localhost_localhost_proto_rawDesc), len(file_github_com_moby_buildkit_session_localhost_localhost_proto_rawDesc))) + }) + return file_github_com_moby_buildkit_session_localhost_localhost_proto_rawDescData +} + +var file_github_com_moby_buildkit_session_localhost_localhost_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_github_com_moby_buildkit_session_localhost_localhost_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_github_com_moby_buildkit_session_localhost_localhost_proto_goTypes = []any{ + (OutputMessage_Status)(0), // 0: moby.localhost.v1.OutputMessage.Status + (*InputMessage)(nil), // 1: moby.localhost.v1.InputMessage + (*OutputMessage)(nil), // 2: moby.localhost.v1.OutputMessage + (*BytesMessage)(nil), // 3: moby.localhost.v1.BytesMessage +} +var file_github_com_moby_buildkit_session_localhost_localhost_proto_depIdxs = []int32{ + 0, // 0: moby.localhost.v1.OutputMessage.status:type_name -> moby.localhost.v1.OutputMessage.Status + 1, // 1: moby.localhost.v1.Localhost.Exec:input_type -> moby.localhost.v1.InputMessage + 3, // 2: moby.localhost.v1.Localhost.Get:input_type -> moby.localhost.v1.BytesMessage + 3, // 3: moby.localhost.v1.Localhost.Put:input_type -> moby.localhost.v1.BytesMessage + 2, // 4: moby.localhost.v1.Localhost.Exec:output_type -> moby.localhost.v1.OutputMessage + 3, // 5: moby.localhost.v1.Localhost.Get:output_type -> moby.localhost.v1.BytesMessage + 3, // 6: moby.localhost.v1.Localhost.Put:output_type -> moby.localhost.v1.BytesMessage + 4, // [4:7] is the sub-list for method output_type + 1, // [1:4] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_github_com_moby_buildkit_session_localhost_localhost_proto_init() } +func file_github_com_moby_buildkit_session_localhost_localhost_proto_init() { + if File_github_com_moby_buildkit_session_localhost_localhost_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_github_com_moby_buildkit_session_localhost_localhost_proto_rawDesc), len(file_github_com_moby_buildkit_session_localhost_localhost_proto_rawDesc)), + NumEnums: 1, + NumMessages: 3, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_github_com_moby_buildkit_session_localhost_localhost_proto_goTypes, + DependencyIndexes: file_github_com_moby_buildkit_session_localhost_localhost_proto_depIdxs, + EnumInfos: file_github_com_moby_buildkit_session_localhost_localhost_proto_enumTypes, + MessageInfos: file_github_com_moby_buildkit_session_localhost_localhost_proto_msgTypes, + }.Build() + File_github_com_moby_buildkit_session_localhost_localhost_proto = out.File + file_github_com_moby_buildkit_session_localhost_localhost_proto_goTypes = nil + file_github_com_moby_buildkit_session_localhost_localhost_proto_depIdxs = nil +} diff --git a/session/localhost/localhost.proto b/session/localhost/localhost.proto new file mode 100644 index 000000000..9bb8942bf --- /dev/null +++ b/session/localhost/localhost.proto @@ -0,0 +1,39 @@ +syntax = "proto3"; + +package moby.localhost.v1; + +option go_package = "github.com/moby/buildkit/session/localhost"; + +// Localhost is a service that allows for local execution on the client +service Localhost { + rpc Exec(stream InputMessage) returns (stream OutputMessage); + // Get fetches a file or dir from the localhost and copies it to buildkit + rpc Get(stream BytesMessage) returns (stream BytesMessage); + // Put sends a file or dir from buildkit and tells the localhost to save it locally + rpc Put(stream BytesMessage) returns (stream BytesMessage); +} + +// InputMessage contains stdin data +message InputMessage { + repeated string command = 1; + bytes stdin = 2; + string dir = 3; +} + +// OutputMessage contains stdout and stderr data +message OutputMessage { + bytes stdout = 1; + bytes stderr = 2; + enum Status { + RUNNING = 0; + DONE = 1; + KILLED = 2; + } + Status status = 3; + int32 exit_code = 4; // only valid if status is done +} + +// BytesMessage contains a chunk of byte data +message BytesMessage{ + bytes data = 1; +} diff --git a/session/localhost/localhost_grpc.pb.go b/session/localhost/localhost_grpc.pb.go new file mode 100644 index 000000000..52a3b6f13 --- /dev/null +++ b/session/localhost/localhost_grpc.pb.go @@ -0,0 +1,185 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.5.1 +// - protoc v3.11.4 +// source: github.com/moby/buildkit/session/localhost/localhost.proto + +package localhost + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + Localhost_Exec_FullMethodName = "/moby.localhost.v1.Localhost/Exec" + Localhost_Get_FullMethodName = "/moby.localhost.v1.Localhost/Get" + Localhost_Put_FullMethodName = "/moby.localhost.v1.Localhost/Put" +) + +// LocalhostClient is the client API for Localhost service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Localhost is a service that allows for local execution on the client +type LocalhostClient interface { + Exec(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[InputMessage, OutputMessage], error) + // Get fetches a file or dir from the localhost and copies it to buildkit + Get(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[BytesMessage, BytesMessage], error) + // Put sends a file or dir from buildkit and tells the localhost to save it locally + Put(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[BytesMessage, BytesMessage], error) +} + +type localhostClient struct { + cc grpc.ClientConnInterface +} + +func NewLocalhostClient(cc grpc.ClientConnInterface) LocalhostClient { + return &localhostClient{cc} +} + +func (c *localhostClient) Exec(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[InputMessage, OutputMessage], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &Localhost_ServiceDesc.Streams[0], Localhost_Exec_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[InputMessage, OutputMessage]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Localhost_ExecClient = grpc.BidiStreamingClient[InputMessage, OutputMessage] + +func (c *localhostClient) Get(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[BytesMessage, BytesMessage], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &Localhost_ServiceDesc.Streams[1], Localhost_Get_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[BytesMessage, BytesMessage]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Localhost_GetClient = grpc.BidiStreamingClient[BytesMessage, BytesMessage] + +func (c *localhostClient) Put(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[BytesMessage, BytesMessage], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &Localhost_ServiceDesc.Streams[2], Localhost_Put_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[BytesMessage, BytesMessage]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Localhost_PutClient = grpc.BidiStreamingClient[BytesMessage, BytesMessage] + +// LocalhostServer is the server API for Localhost service. +// All implementations should embed UnimplementedLocalhostServer +// for forward compatibility. +// +// Localhost is a service that allows for local execution on the client +type LocalhostServer interface { + Exec(grpc.BidiStreamingServer[InputMessage, OutputMessage]) error + // Get fetches a file or dir from the localhost and copies it to buildkit + Get(grpc.BidiStreamingServer[BytesMessage, BytesMessage]) error + // Put sends a file or dir from buildkit and tells the localhost to save it locally + Put(grpc.BidiStreamingServer[BytesMessage, BytesMessage]) error +} + +// UnimplementedLocalhostServer should be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedLocalhostServer struct{} + +func (UnimplementedLocalhostServer) Exec(grpc.BidiStreamingServer[InputMessage, OutputMessage]) error { + return status.Errorf(codes.Unimplemented, "method Exec not implemented") +} +func (UnimplementedLocalhostServer) Get(grpc.BidiStreamingServer[BytesMessage, BytesMessage]) error { + return status.Errorf(codes.Unimplemented, "method Get not implemented") +} +func (UnimplementedLocalhostServer) Put(grpc.BidiStreamingServer[BytesMessage, BytesMessage]) error { + return status.Errorf(codes.Unimplemented, "method Put not implemented") +} +func (UnimplementedLocalhostServer) testEmbeddedByValue() {} + +// UnsafeLocalhostServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to LocalhostServer will +// result in compilation errors. +type UnsafeLocalhostServer interface { + mustEmbedUnimplementedLocalhostServer() +} + +func RegisterLocalhostServer(s grpc.ServiceRegistrar, srv LocalhostServer) { + // If the following call pancis, it indicates UnimplementedLocalhostServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&Localhost_ServiceDesc, srv) +} + +func _Localhost_Exec_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(LocalhostServer).Exec(&grpc.GenericServerStream[InputMessage, OutputMessage]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Localhost_ExecServer = grpc.BidiStreamingServer[InputMessage, OutputMessage] + +func _Localhost_Get_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(LocalhostServer).Get(&grpc.GenericServerStream[BytesMessage, BytesMessage]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Localhost_GetServer = grpc.BidiStreamingServer[BytesMessage, BytesMessage] + +func _Localhost_Put_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(LocalhostServer).Put(&grpc.GenericServerStream[BytesMessage, BytesMessage]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Localhost_PutServer = grpc.BidiStreamingServer[BytesMessage, BytesMessage] + +// Localhost_ServiceDesc is the grpc.ServiceDesc for Localhost service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Localhost_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "moby.localhost.v1.Localhost", + HandlerType: (*LocalhostServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "Exec", + Handler: _Localhost_Exec_Handler, + ServerStreams: true, + ClientStreams: true, + }, + { + StreamName: "Get", + Handler: _Localhost_Get_Handler, + ServerStreams: true, + ClientStreams: true, + }, + { + StreamName: "Put", + Handler: _Localhost_Put_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "github.com/moby/buildkit/session/localhost/localhost.proto", +} diff --git a/session/localhost/localhost_vtproto.pb.go b/session/localhost/localhost_vtproto.pb.go new file mode 100644 index 000000000..3eae7f97d --- /dev/null +++ b/session/localhost/localhost_vtproto.pb.go @@ -0,0 +1,781 @@ +// Code generated by protoc-gen-go-vtproto. DO NOT EDIT. +// protoc-gen-go-vtproto version: v0.6.1-0.20240319094008-0393e58bdf10 +// source: github.com/moby/buildkit/session/localhost/localhost.proto + +package localhost + +import ( + fmt "fmt" + protohelpers "github.com/planetscale/vtprotobuf/protohelpers" + proto "google.golang.org/protobuf/proto" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + io "io" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +func (m *InputMessage) CloneVT() *InputMessage { + if m == nil { + return (*InputMessage)(nil) + } + r := new(InputMessage) + r.Dir = m.Dir + if rhs := m.Command; rhs != nil { + tmpContainer := make([]string, len(rhs)) + copy(tmpContainer, rhs) + r.Command = tmpContainer + } + if rhs := m.Stdin; rhs != nil { + tmpBytes := make([]byte, len(rhs)) + copy(tmpBytes, rhs) + r.Stdin = tmpBytes + } + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *InputMessage) CloneMessageVT() proto.Message { + return m.CloneVT() +} + +func (m *OutputMessage) CloneVT() *OutputMessage { + if m == nil { + return (*OutputMessage)(nil) + } + r := new(OutputMessage) + r.Status = m.Status + r.ExitCode = m.ExitCode + if rhs := m.Stdout; rhs != nil { + tmpBytes := make([]byte, len(rhs)) + copy(tmpBytes, rhs) + r.Stdout = tmpBytes + } + if rhs := m.Stderr; rhs != nil { + tmpBytes := make([]byte, len(rhs)) + copy(tmpBytes, rhs) + r.Stderr = tmpBytes + } + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *OutputMessage) CloneMessageVT() proto.Message { + return m.CloneVT() +} + +func (m *BytesMessage) CloneVT() *BytesMessage { + if m == nil { + return (*BytesMessage)(nil) + } + r := new(BytesMessage) + if rhs := m.Data; rhs != nil { + tmpBytes := make([]byte, len(rhs)) + copy(tmpBytes, rhs) + r.Data = tmpBytes + } + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *BytesMessage) CloneMessageVT() proto.Message { + return m.CloneVT() +} + +func (this *InputMessage) EqualVT(that *InputMessage) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + if len(this.Command) != len(that.Command) { + return false + } + for i, vx := range this.Command { + vy := that.Command[i] + if vx != vy { + return false + } + } + if string(this.Stdin) != string(that.Stdin) { + return false + } + if this.Dir != that.Dir { + return false + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *InputMessage) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*InputMessage) + if !ok { + return false + } + return this.EqualVT(that) +} +func (this *OutputMessage) EqualVT(that *OutputMessage) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + if string(this.Stdout) != string(that.Stdout) { + return false + } + if string(this.Stderr) != string(that.Stderr) { + return false + } + if this.Status != that.Status { + return false + } + if this.ExitCode != that.ExitCode { + return false + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *OutputMessage) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*OutputMessage) + if !ok { + return false + } + return this.EqualVT(that) +} +func (this *BytesMessage) EqualVT(that *BytesMessage) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + if string(this.Data) != string(that.Data) { + return false + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *BytesMessage) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*BytesMessage) + if !ok { + return false + } + return this.EqualVT(that) +} +func (m *InputMessage) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *InputMessage) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *InputMessage) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Dir) > 0 { + i -= len(m.Dir) + copy(dAtA[i:], m.Dir) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Dir))) + i-- + dAtA[i] = 0x1a + } + if len(m.Stdin) > 0 { + i -= len(m.Stdin) + copy(dAtA[i:], m.Stdin) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Stdin))) + i-- + dAtA[i] = 0x12 + } + if len(m.Command) > 0 { + for iNdEx := len(m.Command) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Command[iNdEx]) + copy(dAtA[i:], m.Command[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Command[iNdEx]))) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *OutputMessage) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *OutputMessage) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *OutputMessage) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.ExitCode != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.ExitCode)) + i-- + dAtA[i] = 0x20 + } + if m.Status != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Status)) + i-- + dAtA[i] = 0x18 + } + if len(m.Stderr) > 0 { + i -= len(m.Stderr) + copy(dAtA[i:], m.Stderr) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Stderr))) + i-- + dAtA[i] = 0x12 + } + if len(m.Stdout) > 0 { + i -= len(m.Stdout) + copy(dAtA[i:], m.Stdout) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Stdout))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *BytesMessage) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *BytesMessage) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *BytesMessage) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Data) > 0 { + i -= len(m.Data) + copy(dAtA[i:], m.Data) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Data))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *InputMessage) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Command) > 0 { + for _, s := range m.Command { + l = len(s) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + l = len(m.Stdin) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Dir) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *OutputMessage) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Stdout) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Stderr) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Status != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Status)) + } + if m.ExitCode != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.ExitCode)) + } + n += len(m.unknownFields) + return n +} + +func (m *BytesMessage) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Data) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *InputMessage) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: InputMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: InputMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Command", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Command = append(m.Command, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Stdin", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Stdin = append(m.Stdin[:0], dAtA[iNdEx:postIndex]...) + if m.Stdin == nil { + m.Stdin = []byte{} + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Dir", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Dir = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *OutputMessage) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: OutputMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: OutputMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Stdout", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Stdout = append(m.Stdout[:0], dAtA[iNdEx:postIndex]...) + if m.Stdout == nil { + m.Stdout = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Stderr", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Stderr = append(m.Stderr[:0], dAtA[iNdEx:postIndex]...) + if m.Stderr == nil { + m.Stderr = []byte{} + } + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) + } + m.Status = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Status |= OutputMessage_Status(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ExitCode", wireType) + } + m.ExitCode = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ExitCode |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *BytesMessage) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: BytesMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: BytesMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) + if m.Data == nil { + m.Data = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/session/localhost/localhostprovider/localhostprovider.go b/session/localhost/localhostprovider/localhostprovider.go new file mode 100644 index 000000000..87dc0f18d --- /dev/null +++ b/session/localhost/localhostprovider/localhostprovider.go @@ -0,0 +1,373 @@ +package localhostprovider + +import ( + "fmt" + "io" + "os" + "os/exec" + "path" + "sync" + "syscall" + "time" + + "github.com/moby/buildkit/session" + "github.com/moby/buildkit/session/localhost" + "github.com/pkg/errors" + "github.com/tonistiigi/fsutil" + "github.com/tonistiigi/fsutil/types" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" +) + +func NewLocalhostProvider() (session.Attachable, error) { + return &localhostProvider{ + m: &sync.Mutex{}, + }, nil +} + +type localhostProvider struct { + m *sync.Mutex +} + +func (lp *localhostProvider) Register(server *grpc.Server) { + localhost.RegisterLocalhostServer(server, lp) +} + +func (lp *localhostProvider) Exec(stream localhost.Localhost_ExecServer) error { + ctx := stream.Context() + opts, _ := metadata.FromIncomingContext(ctx) + _ = opts // opts aren't used for anything at the moment + + // first message must contain the command (and no stdin) + msg, err := stream.Recv() + if err != nil { + return err + } + + if len(msg.Command) == 0 { + return errors.Errorf("command is empty") + } + cmdStr := msg.Command[0] + args := msg.Command[1:] + workingDir := msg.Dir + + // it might be possible to run in parallel; but it hasn't been tested. + lp.m.Lock() + defer lp.m.Unlock() + + if workingDir != "" { + err = os.MkdirAll(workingDir, 0755) + if err != nil { + return errors.WithStack(err) + } + } + + cmd := exec.CommandContext(ctx, cmdStr, args...) + cmd.Dir = workingDir + + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + stderr, err := cmd.StderrPipe() + if err != nil { + return err + } + err = cmd.Start() + if err != nil { + return err + } + + m := sync.Mutex{} + var wg sync.WaitGroup + wg.Add(2) + const readSize = 8196 + var readErr []error + go func() { + defer wg.Done() + for buf := make([]byte, readSize); ; { + n, err := stdout.Read(buf) + if n > 0 { + m.Lock() + resp := localhost.OutputMessage{ + Stdout: buf[:n], + } + err := stream.Send(&resp) + if err != nil { + readErr = append(readErr, err) + m.Unlock() + return + } + m.Unlock() + } + if err != nil { + m.Lock() + if err != io.EOF { + readErr = append(readErr, err) + } + m.Unlock() + return + } + } + }() + go func() { + defer wg.Done() + for buf := make([]byte, readSize); ; { + n, err := stderr.Read(buf) + if n > 0 { + m.Lock() + resp := localhost.OutputMessage{ + Stderr: buf[:n], + } + err := stream.Send(&resp) + if err != nil { + readErr = append(readErr, err) + m.Unlock() + return + } + m.Unlock() + } + if err != nil { + m.Lock() + if err != io.EOF { + readErr = append(readErr, err) + } + m.Unlock() + return + } + } + }() + + wg.Wait() + if len(readErr) != 0 { + for _, err := range readErr { + fmt.Fprintf(os.Stderr, "got error while reading from locally-run process: %v\n", err) + } + return readErr[0] + } + + var exitCode int + status := localhost.OutputMessage_DONE + err = cmd.Wait() + if err != nil { + var exiterr *exec.ExitError + if errors.As(err, &exiterr) { + if waitStatus, ok := exiterr.Sys().(syscall.WaitStatus); ok { + exitCode = waitStatus.ExitStatus() + } else { + status = localhost.OutputMessage_KILLED + } + } else { + status = localhost.OutputMessage_KILLED + } + } + + resp := localhost.OutputMessage{ + ExitCode: int32(exitCode), + Status: status, + } + if err := stream.Send(&resp); err != nil { + return err + } + + return nil +} + +func (lp *localhostProvider) Get(stream localhost.Localhost_GetServer) error { + // first message must contain the path to fetch + msg, err := stream.Recv() + if err != nil { + return err + } + path := string(msg.Data) + + fi, err := os.Stat(path) + if err != nil { + return err + } + switch mode := fi.Mode(); { + case mode.IsDir(): + return sendDir(stream, path) + case mode.IsRegular(): + return sendFile(stream, path) + default: + panic(fmt.Sprintf("unhandled mode file %v in localhostProvider.Get", mode)) + } +} + +func sendFile(stream localhost.Localhost_GetServer, path string) error { + err := stream.Send(&localhost.BytesMessage{ + Data: []byte{'f', 0x00}, // 'f' denotes a file; 0 denotes version 0 of the send file protocol + }) + if err != nil { + return err + } + + stat, err := fsutil.Stat(path) + if err != nil { + return errors.Wrapf(err, "failed to stat %s", path) + } + payload, err := stat.Marshal() + if err != nil { + return err + } + err = stream.Send(&localhost.BytesMessage{ + Data: payload, + }) + if err != nil { + return err + } + + f, err := os.Open(path) + if err != nil { + return errors.Wrapf(err, "failed to open %s", path) + } + defer f.Close() + + buf := make([]byte, 1024*1024) + for { + n, err := f.Read(buf) + if n > 0 { + err = stream.Send(&localhost.BytesMessage{ + Data: buf[:n], + }) + } + if err == nil { + // continue + } else if errors.Is(err, io.EOF) { + return nil + } else { + return err + } + } +} + +func sendDir(stream localhost.Localhost_GetServer, path string) error { + err := stream.Send(&localhost.BytesMessage{ + Data: []byte{'d', 0x00}, // 'd' denotes a file; 0 denotes version 0 of the send file protocol + }) + if err != nil { + return err + } + + fs, err := fsutil.NewFS(path) + if err != nil { + return err + } + fs, err = fsutil.NewFilterFS(fs, &fsutil.FilterOpt{ + IncludePatterns: []string{"*"}, + }) + if err != nil { + return err + } + err = fsutil.Send(stream.Context(), stream, fs, nil) + if err != nil { + return errors.Wrap(err, "fsutil.Send failed") + } + return nil +} + +func (lp *localhostProvider) Put(stream localhost.Localhost_PutServer) error { + // first message must contain the path to fetch + msg, err := stream.Recv() + if err != nil { + return err + } + + mode := msg.Data[0] + switch mode { + case 'f': + version := msg.Data[1] + if version != 0 { + panic(fmt.Sprintf("unhandled file version %v", version)) + } + return receiveFile(stream) + case 'd': + version := msg.Data[1] + if version != 0 { + panic(fmt.Sprintf("unhandled dir version %v", version)) + } + return receiveDir(stream) + default: + panic(fmt.Sprintf("unhandled mode %v", mode)) + } +} + +func receiveFile(stream localhost.Localhost_PutServer) (err error) { + // first message contains the path + msg, err := stream.Recv() + if err != nil { + return errors.WithStack(err) + } + dstPath := string(msg.Data) + + dirPath := path.Dir(dstPath) + err = os.MkdirAll(dirPath, 0755) + if err != nil { + return err + } + + // second message contains the file stat + msg, err = stream.Recv() + if err != nil { + return err + } + stat := types.Stat{} + err = stat.Unmarshal(msg.Data) + if err != nil { + return errors.WithStack(err) + } + + f, err := os.OpenFile(dstPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(stat.Mode)) + if err != nil { + return errors.WithStack(err) + } + defer f.Close() + +outer: + for { + msg, err := stream.Recv() + if err == nil { + // continue + } else if errors.Is(err, io.EOF) { + break outer + } else { + return errors.WithStack(err) + } + _, err = f.Write(msg.Data) + if err != nil { + return errors.WithStack(err) + } + } + + mtime := time.Unix(0, stat.ModTime) + err = os.Chtimes(dstPath, mtime, mtime) + if err != nil { + return errors.Wrap(err, "failed to change file time") + } + + // send an empty message to confirm we're done processing + return stream.Send(&localhost.BytesMessage{}) +} + +func receiveDir(stream localhost.Localhost_PutServer) error { + msg, err := stream.Recv() + if err != nil { + return errors.WithStack(err) + } + dest := string(msg.Data) + err = os.MkdirAll(dest, 0775) + if err != nil { + return errors.WithStack(err) + } + uid := syscall.Getuid() + gid := syscall.Getgid() + ctx := stream.Context() + return errors.WithStack(fsutil.Receive(ctx, stream, dest, fsutil.ReceiveOpt{ + Filter: func(p string, stat *types.Stat) bool { + stat.Uid = uint32(uid) + stat.Gid = uint32(gid) + return true + }, + })) +} diff --git a/session/manager.go b/session/manager.go index f411f0a3b..59b785ab5 100644 --- a/session/manager.go +++ b/session/manager.go @@ -4,8 +4,10 @@ import ( "context" "net" "net/http" + "os" "strings" "sync" + "time" "github.com/pkg/errors" "google.golang.org/grpc" @@ -25,22 +27,140 @@ type client struct { supported map[string]struct{} } +type History struct { + Start time.Time + End time.Time +} + // Manager is a controller for accessing currently active sessions type Manager struct { sessions map[string]*client mu sync.Mutex updateCondition *sync.Cond + healthCfg ManagerHealthCfg + + stop bool // Earthly-specific. + shutdownCh chan struct{} + idleAt time.Time // Earthly-specific + history map[string]*History // Earthly-specific + historyDuration time.Duration // Earthly-specific +} + +// ManagerHealthCfg is the healthcheck configuration for gRPC healthchecks +type ManagerHealthCfg struct { + frequency time.Duration + timeout time.Duration + allowedFailures int +} + +// ManagerOpt is earthly-specific, and required for custom health-check overrides +type ManagerOpt struct { + HealthFrequency time.Duration + HealthTimeout time.Duration + HealthAllowedFailures int + ShutdownCh chan struct{} } // NewManager returns a new Manager -func NewManager() (*Manager, error) { +// earthly-specific: opt param is required for our custom health config +func NewManager(opt *ManagerOpt) (*Manager, error) { + var historyDuration time.Duration + if dur, ok := os.LookupEnv("BUILDKIT_SESSION_HISTORY_DURATION"); ok { + var err error + historyDuration, err = time.ParseDuration(dur) + if err != nil { + return nil, errors.Wrapf(err, "could not parse session history duration value of '%s'", dur) + } + } sm := &Manager{ sessions: make(map[string]*client), + healthCfg: ManagerHealthCfg{ + frequency: opt.HealthFrequency, + timeout: opt.HealthTimeout, + allowedFailures: opt.HealthAllowedFailures, + }, + shutdownCh: opt.ShutdownCh, + idleAt: time.Now(), + history: make(map[string]*History), + historyDuration: historyDuration, } sm.updateCondition = sync.NewCond(&sm.mu) return sm, nil } +// NumSessions returns the number of active sessions. +// earthly-specific +func (sm *Manager) NumSessions() (sessions int, durationIdle time.Duration) { + sm.mu.Lock() + defer sm.mu.Unlock() + sessions = len(sm.sessions) + if sessions == 0 { + durationIdle = time.Since(sm.idleAt) + } + return sessions, durationIdle +} + +// StopIfIdle stops the manager if there are no active sessions. +// earthly-specific +func (sm *Manager) StopIfIdle() (bool, int) { + sm.mu.Lock() + defer sm.mu.Unlock() + isIdle := sm.idleAt.Add(time.Minute).Before(time.Now()) + if isIdle && len(sm.sessions) == 0 { + close(sm.shutdownCh) + sm.stop = true + return true, 0 + } + return false, len(sm.sessions) +} + +// Reserve signals intent to start a build. +// It resets the idleAt counter so that the buildkit will not shutdown. +// earthly-specific +func (sm *Manager) Reserve() error { + sm.mu.Lock() + defer sm.mu.Unlock() + if sm.stop { + return errors.New("already shutting down") + } + sm.idleAt = time.Now() + return nil +} + +// earthly-specific +func (sm *Manager) recordSessionStart(sessionID string) { + sm.mu.Lock() + defer sm.mu.Unlock() + sm.history[sessionID] = &History{Start: time.Now()} +} + +// earthly-specific +func (sm *Manager) recordSessionEnd(sessionID string) { + sm.mu.Lock() + defer sm.mu.Unlock() + if h := sm.history[sessionID]; h != nil { + h.End = time.Now() + } + for id, history := range sm.history { + if time.Since(history.Start) > sm.historyDuration { + delete(sm.history, id) + } + } +} + +// GetSessionHistory returns a map of session ID to History entries +// earthly-specific +func (sm *Manager) GetSessionHistory() map[string]*History { + sm.mu.Lock() + defer sm.mu.Unlock() + historyCopy := make(map[string]*History) + for id, h := range sm.history { + history := *h + historyCopy[id] = &history + } + return historyCopy +} + // HandleHTTPRequest handles an incoming HTTP request func (sm *Manager) HandleHTTPRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error { hijacker, ok := w.(http.Hijacker) @@ -98,6 +218,11 @@ func (sm *Manager) HandleConn(ctx context.Context, conn net.Conn, opts map[strin // caller needs to take lock, this function will release it func (sm *Manager) handleConn(ctx context.Context, conn net.Conn, opts map[string][]string) error { + if sm.stop { + sm.mu.Unlock() + return errors.New("shutting down") + } + ctx, cancel := context.WithCancelCause(ctx) defer func() { cancel(errors.WithStack(context.Canceled)) }() @@ -107,7 +232,7 @@ func (sm *Manager) handleConn(ctx context.Context, conn net.Conn, opts map[strin id := h.Get(headerSessionID) sharedKey := h.Get(headerSessionSharedKey) - ctx, cc, err := grpcClientConn(ctx, conn) + ctx, cc, err := grpcClientConn(ctx, conn, sm.healthCfg) if err != nil { sm.mu.Unlock() return err @@ -131,11 +256,16 @@ func (sm *Manager) handleConn(ctx context.Context, conn net.Conn, opts map[strin sm.sessions[id] = c sm.updateCondition.Broadcast() sm.mu.Unlock() + sm.recordSessionStart(id) // earthly-specific defer func() { sm.mu.Lock() delete(sm.sessions, id) + if len(sm.sessions) == 0 { + sm.idleAt = time.Now() // earthly-specific + } sm.mu.Unlock() + sm.recordSessionEnd(id) // earthly-specific }() <-c.ctx.Done() diff --git a/session/pullping/generate.go b/session/pullping/generate.go new file mode 100644 index 000000000..7bdbe4120 --- /dev/null +++ b/session/pullping/generate.go @@ -0,0 +1,3 @@ +package pullping + +//go:generate protoc --gogoslick_out=plugins=grpc:. pullping.proto diff --git a/session/pullping/pullping.go b/session/pullping/pullping.go new file mode 100644 index 000000000..73ec04220 --- /dev/null +++ b/session/pullping/pullping.go @@ -0,0 +1,62 @@ +package pullping + +import ( + context "context" + + "github.com/moby/buildkit/session" + "github.com/pkg/errors" + "google.golang.org/grpc" +) + +type PullCallback func(ctx context.Context, images []string, resp map[string]string) error + +type pullPing struct { + callback PullCallback +} + +// NewPullPing creates a new PullPing attachable, given a callback. The callback will be issued +// when the remote registry is ready for pulls. The callback should return when the pulls +// have completed, in order to inform the remote exporter that the pulls have completed. +func NewPullPing(cb PullCallback) session.Attachable { + return &pullPing{callback: cb} +} + +// Register registers the object with a grpc server. +func (pp *pullPing) Register(server *grpc.Server) { + RegisterPullPingServer(server, pp) +} + +// Pull implements the gRPC Pull message. It calls the callback and sends a +// message to the client when the callback has completed. +func (pp *pullPing) Pull(pr *PullRequest, ps PullPing_PullServer) error { + err := pp.callback(ps.Context(), pr.GetImages(), pr.GetResp()) + if err != nil { + return err + } + err = ps.Send(&PullResponse{}) + if err != nil { + return err + } + return nil +} + +// PullPingChannel returns a channel which signals an error when the pull +// operation has completed from the client-side. The error is nil +// if the operation has been completed successfully. +func PullPingChannel(ctx context.Context, images []string, resp map[string]string, c session.Caller) chan error { + respChan := make(chan error, 1) + ppc := NewPullPingClient(c.Conn()) + pc, err := ppc.Pull(ctx, &PullRequest{ + Images: images, + Resp: resp, + }) + if err != nil { + respChan <- errors.Wrap(err, "pull ping request") + return respChan + } + go func() { + _, err := pc.Recv() + respChan <- errors.Wrap(err, "pull ping response") + }() + return respChan +} diff --git a/session/pullping/pullping.pb.go b/session/pullping/pullping.pb.go new file mode 100644 index 000000000..e258da580 --- /dev/null +++ b/session/pullping/pullping.pb.go @@ -0,0 +1,178 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v3.11.4 +// source: github.com/moby/buildkit/session/pullping/pullping.proto + +package pullping + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type PullRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Images []string `protobuf:"bytes,1,rep,name=images,proto3" json:"images,omitempty"` + Resp map[string]string `protobuf:"bytes,2,rep,name=resp,proto3" json:"resp,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PullRequest) Reset() { + *x = PullRequest{} + mi := &file_github_com_moby_buildkit_session_pullping_pullping_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PullRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PullRequest) ProtoMessage() {} + +func (x *PullRequest) ProtoReflect() protoreflect.Message { + mi := &file_github_com_moby_buildkit_session_pullping_pullping_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PullRequest.ProtoReflect.Descriptor instead. +func (*PullRequest) Descriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_session_pullping_pullping_proto_rawDescGZIP(), []int{0} +} + +func (x *PullRequest) GetImages() []string { + if x != nil { + return x.Images + } + return nil +} + +func (x *PullRequest) GetResp() map[string]string { + if x != nil { + return x.Resp + } + return nil +} + +type PullResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PullResponse) Reset() { + *x = PullResponse{} + mi := &file_github_com_moby_buildkit_session_pullping_pullping_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PullResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PullResponse) ProtoMessage() {} + +func (x *PullResponse) ProtoReflect() protoreflect.Message { + mi := &file_github_com_moby_buildkit_session_pullping_pullping_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PullResponse.ProtoReflect.Descriptor instead. +func (*PullResponse) Descriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_session_pullping_pullping_proto_rawDescGZIP(), []int{1} +} + +var File_github_com_moby_buildkit_session_pullping_pullping_proto protoreflect.FileDescriptor + +const file_github_com_moby_buildkit_session_pullping_pullping_proto_rawDesc = "" + + "\n" + + "8github.com/moby/buildkit/session/pullping/pullping.proto\x12\x10moby.pullping.v1\"\x9b\x01\n" + + "\vPullRequest\x12\x16\n" + + "\x06images\x18\x01 \x03(\tR\x06images\x12;\n" + + "\x04resp\x18\x02 \x03(\v2'.moby.pullping.v1.PullRequest.RespEntryR\x04resp\x1a7\n" + + "\tRespEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x0e\n" + + "\fPullResponse2S\n" + + "\bPullPing\x12G\n" + + "\x04Pull\x12\x1d.moby.pullping.v1.PullRequest\x1a\x1e.moby.pullping.v1.PullResponse0\x01B+Z)github.com/moby/buildkit/session/pullpingb\x06proto3" + +var ( + file_github_com_moby_buildkit_session_pullping_pullping_proto_rawDescOnce sync.Once + file_github_com_moby_buildkit_session_pullping_pullping_proto_rawDescData []byte +) + +func file_github_com_moby_buildkit_session_pullping_pullping_proto_rawDescGZIP() []byte { + file_github_com_moby_buildkit_session_pullping_pullping_proto_rawDescOnce.Do(func() { + file_github_com_moby_buildkit_session_pullping_pullping_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_github_com_moby_buildkit_session_pullping_pullping_proto_rawDesc), len(file_github_com_moby_buildkit_session_pullping_pullping_proto_rawDesc))) + }) + return file_github_com_moby_buildkit_session_pullping_pullping_proto_rawDescData +} + +var file_github_com_moby_buildkit_session_pullping_pullping_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_github_com_moby_buildkit_session_pullping_pullping_proto_goTypes = []any{ + (*PullRequest)(nil), // 0: moby.pullping.v1.PullRequest + (*PullResponse)(nil), // 1: moby.pullping.v1.PullResponse + nil, // 2: moby.pullping.v1.PullRequest.RespEntry +} +var file_github_com_moby_buildkit_session_pullping_pullping_proto_depIdxs = []int32{ + 2, // 0: moby.pullping.v1.PullRequest.resp:type_name -> moby.pullping.v1.PullRequest.RespEntry + 0, // 1: moby.pullping.v1.PullPing.Pull:input_type -> moby.pullping.v1.PullRequest + 1, // 2: moby.pullping.v1.PullPing.Pull:output_type -> moby.pullping.v1.PullResponse + 2, // [2:3] is the sub-list for method output_type + 1, // [1:2] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_github_com_moby_buildkit_session_pullping_pullping_proto_init() } +func file_github_com_moby_buildkit_session_pullping_pullping_proto_init() { + if File_github_com_moby_buildkit_session_pullping_pullping_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_github_com_moby_buildkit_session_pullping_pullping_proto_rawDesc), len(file_github_com_moby_buildkit_session_pullping_pullping_proto_rawDesc)), + NumEnums: 0, + NumMessages: 3, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_github_com_moby_buildkit_session_pullping_pullping_proto_goTypes, + DependencyIndexes: file_github_com_moby_buildkit_session_pullping_pullping_proto_depIdxs, + MessageInfos: file_github_com_moby_buildkit_session_pullping_pullping_proto_msgTypes, + }.Build() + File_github_com_moby_buildkit_session_pullping_pullping_proto = out.File + file_github_com_moby_buildkit_session_pullping_pullping_proto_goTypes = nil + file_github_com_moby_buildkit_session_pullping_pullping_proto_depIdxs = nil +} diff --git a/session/pullping/pullping.proto b/session/pullping/pullping.proto new file mode 100644 index 000000000..909ecd960 --- /dev/null +++ b/session/pullping/pullping.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package moby.pullping.v1; + +option go_package = "github.com/moby/buildkit/session/pullping"; + +service PullPing { + rpc Pull(PullRequest) returns (stream PullResponse); +} + +message PullRequest { + repeated string images = 1; + map resp = 2; +} + +message PullResponse { +} diff --git a/session/pullping/pullping_grpc.pb.go b/session/pullping/pullping_grpc.pb.go new file mode 100644 index 000000000..9191f0a14 --- /dev/null +++ b/session/pullping/pullping_grpc.pb.go @@ -0,0 +1,122 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.5.1 +// - protoc v3.11.4 +// source: github.com/moby/buildkit/session/pullping/pullping.proto + +package pullping + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + PullPing_Pull_FullMethodName = "/moby.pullping.v1.PullPing/Pull" +) + +// PullPingClient is the client API for PullPing service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type PullPingClient interface { + Pull(ctx context.Context, in *PullRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[PullResponse], error) +} + +type pullPingClient struct { + cc grpc.ClientConnInterface +} + +func NewPullPingClient(cc grpc.ClientConnInterface) PullPingClient { + return &pullPingClient{cc} +} + +func (c *pullPingClient) Pull(ctx context.Context, in *PullRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[PullResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &PullPing_ServiceDesc.Streams[0], PullPing_Pull_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[PullRequest, PullResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type PullPing_PullClient = grpc.ServerStreamingClient[PullResponse] + +// PullPingServer is the server API for PullPing service. +// All implementations should embed UnimplementedPullPingServer +// for forward compatibility. +type PullPingServer interface { + Pull(*PullRequest, grpc.ServerStreamingServer[PullResponse]) error +} + +// UnimplementedPullPingServer should be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedPullPingServer struct{} + +func (UnimplementedPullPingServer) Pull(*PullRequest, grpc.ServerStreamingServer[PullResponse]) error { + return status.Errorf(codes.Unimplemented, "method Pull not implemented") +} +func (UnimplementedPullPingServer) testEmbeddedByValue() {} + +// UnsafePullPingServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to PullPingServer will +// result in compilation errors. +type UnsafePullPingServer interface { + mustEmbedUnimplementedPullPingServer() +} + +func RegisterPullPingServer(s grpc.ServiceRegistrar, srv PullPingServer) { + // If the following call pancis, it indicates UnimplementedPullPingServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&PullPing_ServiceDesc, srv) +} + +func _PullPing_Pull_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(PullRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(PullPingServer).Pull(m, &grpc.GenericServerStream[PullRequest, PullResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type PullPing_PullServer = grpc.ServerStreamingServer[PullResponse] + +// PullPing_ServiceDesc is the grpc.ServiceDesc for PullPing service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var PullPing_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "moby.pullping.v1.PullPing", + HandlerType: (*PullPingServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "Pull", + Handler: _PullPing_Pull_Handler, + ServerStreams: true, + }, + }, + Metadata: "github.com/moby/buildkit/session/pullping/pullping.proto", +} diff --git a/session/pullping/pullping_vtproto.pb.go b/session/pullping/pullping_vtproto.pb.go new file mode 100644 index 000000000..9a48cee3a --- /dev/null +++ b/session/pullping/pullping_vtproto.pb.go @@ -0,0 +1,507 @@ +// Code generated by protoc-gen-go-vtproto. DO NOT EDIT. +// protoc-gen-go-vtproto version: v0.6.1-0.20240319094008-0393e58bdf10 +// source: github.com/moby/buildkit/session/pullping/pullping.proto + +package pullping + +import ( + fmt "fmt" + protohelpers "github.com/planetscale/vtprotobuf/protohelpers" + proto "google.golang.org/protobuf/proto" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + io "io" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +func (m *PullRequest) CloneVT() *PullRequest { + if m == nil { + return (*PullRequest)(nil) + } + r := new(PullRequest) + if rhs := m.Images; rhs != nil { + tmpContainer := make([]string, len(rhs)) + copy(tmpContainer, rhs) + r.Images = tmpContainer + } + if rhs := m.Resp; rhs != nil { + tmpContainer := make(map[string]string, len(rhs)) + for k, v := range rhs { + tmpContainer[k] = v + } + r.Resp = tmpContainer + } + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *PullRequest) CloneMessageVT() proto.Message { + return m.CloneVT() +} + +func (m *PullResponse) CloneVT() *PullResponse { + if m == nil { + return (*PullResponse)(nil) + } + r := new(PullResponse) + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *PullResponse) CloneMessageVT() proto.Message { + return m.CloneVT() +} + +func (this *PullRequest) EqualVT(that *PullRequest) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + if len(this.Images) != len(that.Images) { + return false + } + for i, vx := range this.Images { + vy := that.Images[i] + if vx != vy { + return false + } + } + if len(this.Resp) != len(that.Resp) { + return false + } + for i, vx := range this.Resp { + vy, ok := that.Resp[i] + if !ok { + return false + } + if vx != vy { + return false + } + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *PullRequest) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*PullRequest) + if !ok { + return false + } + return this.EqualVT(that) +} +func (this *PullResponse) EqualVT(that *PullResponse) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *PullResponse) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*PullResponse) + if !ok { + return false + } + return this.EqualVT(that) +} +func (m *PullRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PullRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *PullRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Resp) > 0 { + for k := range m.Resp { + v := m.Resp[k] + baseI := i + i -= len(v) + copy(dAtA[i:], v) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(v))) + i-- + dAtA[i] = 0x12 + i -= len(k) + copy(dAtA[i:], k) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(k))) + i-- + dAtA[i] = 0xa + i = protohelpers.EncodeVarint(dAtA, i, uint64(baseI-i)) + i-- + dAtA[i] = 0x12 + } + } + if len(m.Images) > 0 { + for iNdEx := len(m.Images) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Images[iNdEx]) + copy(dAtA[i:], m.Images[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Images[iNdEx]))) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *PullResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PullResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *PullResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + return len(dAtA) - i, nil +} + +func (m *PullRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Images) > 0 { + for _, s := range m.Images { + l = len(s) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + if len(m.Resp) > 0 { + for k, v := range m.Resp { + _ = k + _ = v + mapEntrySize := 1 + len(k) + protohelpers.SizeOfVarint(uint64(len(k))) + 1 + len(v) + protohelpers.SizeOfVarint(uint64(len(v))) + n += mapEntrySize + 1 + protohelpers.SizeOfVarint(uint64(mapEntrySize)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *PullResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + n += len(m.unknownFields) + return n +} + +func (m *PullRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PullRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PullRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Images", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Images = append(m.Images, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Resp", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Resp == nil { + m.Resp = make(map[string]string) + } + var mapkey string + var mapvalue string + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return protohelpers.ErrInvalidLength + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey < 0 { + return protohelpers.ErrInvalidLength + } + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + var stringLenmapvalue uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapvalue |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapvalue := int(stringLenmapvalue) + if intStringLenmapvalue < 0 { + return protohelpers.ErrInvalidLength + } + postStringIndexmapvalue := iNdEx + intStringLenmapvalue + if postStringIndexmapvalue < 0 { + return protohelpers.ErrInvalidLength + } + if postStringIndexmapvalue > l { + return io.ErrUnexpectedEOF + } + mapvalue = string(dAtA[iNdEx:postStringIndexmapvalue]) + iNdEx = postStringIndexmapvalue + } else { + iNdEx = entryPreIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + m.Resp[mapkey] = mapvalue + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *PullResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PullResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PullResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/session/session.go b/session/session.go index b67e25ece..f5779c9ed 100644 --- a/session/session.go +++ b/session/session.go @@ -51,7 +51,11 @@ type Session struct { func NewSession(ctx context.Context, sharedKey string) (*Session, error) { id := identity.NewID() + maxMsgSize := 67108864 // 64MB serverOpts := []grpc.ServerOption{ + grpc.MaxRecvMsgSize(maxMsgSize), grpc.MaxSendMsgSize(maxMsgSize), + grpc.InitialWindowSize(65535 * 32), + grpc.InitialConnWindowSize(65535 * 16), grpc.UnaryInterceptor(grpcerrors.UnaryServerInterceptor), grpc.StreamInterceptor(grpcerrors.StreamServerInterceptor), } diff --git a/session/socketforward/copy.go b/session/socketforward/copy.go new file mode 100644 index 000000000..2e661628a --- /dev/null +++ b/session/socketforward/copy.go @@ -0,0 +1,69 @@ +package socketforward + +import ( + "context" + "io" + + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" +) + +type Stream interface { + SendMsg(m any) error + RecvMsg(m any) error +} + +func Copy(ctx context.Context, conn io.ReadWriteCloser, stream Stream, closeStream func() error) error { + g, ctx := errgroup.WithContext(ctx) + + g.Go(func() (retErr error) { + p := &BytesMessage{} + for { + if err := stream.RecvMsg(p); err != nil { + conn.Close() + if errors.Is(err, io.EOF) { + return nil + } + return errors.WithStack(err) + } + select { + case <-ctx.Done(): + conn.Close() + return context.Cause(ctx) + default: + } + if _, err := conn.Write(p.Data); err != nil { + conn.Close() + return errors.WithStack(err) + } + p.Data = p.Data[:0] + } + }) + + g.Go(func() (retErr error) { + for { + buf := make([]byte, 32*1024) + n, err := conn.Read(buf) + switch { + case errors.Is(err, io.EOF): + if closeStream != nil { + closeStream() + } + return nil + case err != nil: + return errors.WithStack(err) + } + select { + case <-ctx.Done(): + return context.Cause(ctx) + default: + } + p := &BytesMessage{Data: buf[:n]} + if err := stream.SendMsg(p); err != nil { + return errors.WithStack(err) + } + } + }) + + return g.Wait() +} diff --git a/session/socketforward/generate.go b/session/socketforward/generate.go new file mode 100644 index 000000000..cfb5d1438 --- /dev/null +++ b/session/socketforward/generate.go @@ -0,0 +1,3 @@ +package socketforward + +//go:generate protoc --gogoslick_out=plugins=grpc:. socket.proto diff --git a/session/socketforward/socket.go b/session/socketforward/socket.go new file mode 100644 index 000000000..1ee0e95ad --- /dev/null +++ b/session/socketforward/socket.go @@ -0,0 +1,108 @@ +package socketforward + +import ( + "context" + "net" + "os" + "path/filepath" + + "github.com/moby/buildkit/session" + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/metadata" +) + +const SocketIDKey = "socket_id" + +type server struct { + caller session.Caller +} + +func (s *server) run(ctx context.Context, l net.Listener, id string) error { + eg, ctx := errgroup.WithContext(ctx) + + eg.Go(func() error { + <-ctx.Done() + return context.Cause(ctx) + }) + + eg.Go(func() error { + for { + conn, err := l.Accept() + if err != nil { + return err + } + + client := NewSocketClient(s.caller.Conn()) + + opts := make(map[string][]string) + opts[SocketIDKey] = []string{id} + reqCtx := metadata.NewOutgoingContext(ctx, opts) + + stream, err := client.Proxy(reqCtx) + if err != nil { + conn.Close() + return err + } + + go Copy(reqCtx, conn, stream, stream.CloseSend) + } + }) + + return eg.Wait() +} + +type SocketOpt struct { + ID string + UID int + GID int + Mode int +} + +func MountSocket(ctx context.Context, c session.Caller, opt SocketOpt) (sockPath string, closer func() error, err error) { + dir, err := os.MkdirTemp("", ".buildkit-socketforward") + if err != nil { + return "", nil, errors.WithStack(err) + } + + defer func() { + if err != nil { + os.RemoveAll(dir) + } + }() + + if err := os.Chmod(dir, 0711); err != nil { + return "", nil, errors.WithStack(err) + } + + sockPath = filepath.Join(dir, "sock") + + l, err := (&net.ListenConfig{}).Listen(ctx, "unix", sockPath) + if err != nil { + return "", nil, errors.WithStack(err) + } + + if err := os.Chown(sockPath, opt.UID, opt.GID); err != nil { + l.Close() + return "", nil, errors.WithStack(err) + } + if err := os.Chmod(sockPath, os.FileMode(opt.Mode)); err != nil { + l.Close() + return "", nil, errors.WithStack(err) + } + + s := &server{caller: c} + + id := opt.ID + if id == "" { + return "", nil, errors.Errorf("socket ID is missing") + } + + go s.run(ctx, l, id) // erroring per connection allowed + + return sockPath, func() error { + err := l.Close() + os.RemoveAll(sockPath) + return errors.WithStack(err) + }, nil +} diff --git a/session/socketforward/socket.pb.go b/session/socketforward/socket.pb.go new file mode 100644 index 000000000..445df3e28 --- /dev/null +++ b/session/socketforward/socket.pb.go @@ -0,0 +1,127 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v3.11.4 +// source: github.com/moby/buildkit/session/socketforward/socket.proto + +package socketforward + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// BytesMessage contains a chunk of byte data +type BytesMessage struct { + state protoimpl.MessageState `protogen:"open.v1"` + Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BytesMessage) Reset() { + *x = BytesMessage{} + mi := &file_github_com_moby_buildkit_session_socketforward_socket_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BytesMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BytesMessage) ProtoMessage() {} + +func (x *BytesMessage) ProtoReflect() protoreflect.Message { + mi := &file_github_com_moby_buildkit_session_socketforward_socket_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BytesMessage.ProtoReflect.Descriptor instead. +func (*BytesMessage) Descriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_session_socketforward_socket_proto_rawDescGZIP(), []int{0} +} + +func (x *BytesMessage) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +var File_github_com_moby_buildkit_session_socketforward_socket_proto protoreflect.FileDescriptor + +const file_github_com_moby_buildkit_session_socketforward_socket_proto_rawDesc = "" + + "\n" + + ";github.com/moby/buildkit/session/socketforward/socket.proto\x12\x15moby.socketforward.v1\"\"\n" + + "\fBytesMessage\x12\x12\n" + + "\x04data\x18\x01 \x01(\fR\x04data2_\n" + + "\x06Socket\x12U\n" + + "\x05Proxy\x12#.moby.socketforward.v1.BytesMessage\x1a#.moby.socketforward.v1.BytesMessage(\x010\x01B0Z.github.com/moby/buildkit/session/socketforwardb\x06proto3" + +var ( + file_github_com_moby_buildkit_session_socketforward_socket_proto_rawDescOnce sync.Once + file_github_com_moby_buildkit_session_socketforward_socket_proto_rawDescData []byte +) + +func file_github_com_moby_buildkit_session_socketforward_socket_proto_rawDescGZIP() []byte { + file_github_com_moby_buildkit_session_socketforward_socket_proto_rawDescOnce.Do(func() { + file_github_com_moby_buildkit_session_socketforward_socket_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_github_com_moby_buildkit_session_socketforward_socket_proto_rawDesc), len(file_github_com_moby_buildkit_session_socketforward_socket_proto_rawDesc))) + }) + return file_github_com_moby_buildkit_session_socketforward_socket_proto_rawDescData +} + +var file_github_com_moby_buildkit_session_socketforward_socket_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_github_com_moby_buildkit_session_socketforward_socket_proto_goTypes = []any{ + (*BytesMessage)(nil), // 0: moby.socketforward.v1.BytesMessage +} +var file_github_com_moby_buildkit_session_socketforward_socket_proto_depIdxs = []int32{ + 0, // 0: moby.socketforward.v1.Socket.Proxy:input_type -> moby.socketforward.v1.BytesMessage + 0, // 1: moby.socketforward.v1.Socket.Proxy:output_type -> moby.socketforward.v1.BytesMessage + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_github_com_moby_buildkit_session_socketforward_socket_proto_init() } +func file_github_com_moby_buildkit_session_socketforward_socket_proto_init() { + if File_github_com_moby_buildkit_session_socketforward_socket_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_github_com_moby_buildkit_session_socketforward_socket_proto_rawDesc), len(file_github_com_moby_buildkit_session_socketforward_socket_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_github_com_moby_buildkit_session_socketforward_socket_proto_goTypes, + DependencyIndexes: file_github_com_moby_buildkit_session_socketforward_socket_proto_depIdxs, + MessageInfos: file_github_com_moby_buildkit_session_socketforward_socket_proto_msgTypes, + }.Build() + File_github_com_moby_buildkit_session_socketforward_socket_proto = out.File + file_github_com_moby_buildkit_session_socketforward_socket_proto_goTypes = nil + file_github_com_moby_buildkit_session_socketforward_socket_proto_depIdxs = nil +} diff --git a/session/socketforward/socket.proto b/session/socketforward/socket.proto new file mode 100644 index 000000000..03e30d1f9 --- /dev/null +++ b/session/socketforward/socket.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +package moby.socketforward.v1; + +option go_package = "github.com/moby/buildkit/session/socketforward"; + +service Socket { + rpc Proxy(stream BytesMessage) returns (stream BytesMessage); +} + +// BytesMessage contains a chunk of byte data +message BytesMessage{ + bytes data = 1; +} diff --git a/session/socketforward/socket_grpc.pb.go b/session/socketforward/socket_grpc.pb.go new file mode 100644 index 000000000..0229b4efa --- /dev/null +++ b/session/socketforward/socket_grpc.pb.go @@ -0,0 +1,113 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.5.1 +// - protoc v3.11.4 +// source: github.com/moby/buildkit/session/socketforward/socket.proto + +package socketforward + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + Socket_Proxy_FullMethodName = "/moby.socketforward.v1.Socket/Proxy" +) + +// SocketClient is the client API for Socket service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type SocketClient interface { + Proxy(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[BytesMessage, BytesMessage], error) +} + +type socketClient struct { + cc grpc.ClientConnInterface +} + +func NewSocketClient(cc grpc.ClientConnInterface) SocketClient { + return &socketClient{cc} +} + +func (c *socketClient) Proxy(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[BytesMessage, BytesMessage], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &Socket_ServiceDesc.Streams[0], Socket_Proxy_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[BytesMessage, BytesMessage]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Socket_ProxyClient = grpc.BidiStreamingClient[BytesMessage, BytesMessage] + +// SocketServer is the server API for Socket service. +// All implementations should embed UnimplementedSocketServer +// for forward compatibility. +type SocketServer interface { + Proxy(grpc.BidiStreamingServer[BytesMessage, BytesMessage]) error +} + +// UnimplementedSocketServer should be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedSocketServer struct{} + +func (UnimplementedSocketServer) Proxy(grpc.BidiStreamingServer[BytesMessage, BytesMessage]) error { + return status.Errorf(codes.Unimplemented, "method Proxy not implemented") +} +func (UnimplementedSocketServer) testEmbeddedByValue() {} + +// UnsafeSocketServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to SocketServer will +// result in compilation errors. +type UnsafeSocketServer interface { + mustEmbedUnimplementedSocketServer() +} + +func RegisterSocketServer(s grpc.ServiceRegistrar, srv SocketServer) { + // If the following call pancis, it indicates UnimplementedSocketServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&Socket_ServiceDesc, srv) +} + +func _Socket_Proxy_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(SocketServer).Proxy(&grpc.GenericServerStream[BytesMessage, BytesMessage]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Socket_ProxyServer = grpc.BidiStreamingServer[BytesMessage, BytesMessage] + +// Socket_ServiceDesc is the grpc.ServiceDesc for Socket service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Socket_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "moby.socketforward.v1.Socket", + HandlerType: (*SocketServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "Proxy", + Handler: _Socket_Proxy_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "github.com/moby/buildkit/session/socketforward/socket.proto", +} diff --git a/session/socketforward/socket_vtproto.pb.go b/session/socketforward/socket_vtproto.pb.go new file mode 100644 index 000000000..111eace9d --- /dev/null +++ b/session/socketforward/socket_vtproto.pb.go @@ -0,0 +1,200 @@ +// Code generated by protoc-gen-go-vtproto. DO NOT EDIT. +// protoc-gen-go-vtproto version: v0.6.1-0.20240319094008-0393e58bdf10 +// source: github.com/moby/buildkit/session/socketforward/socket.proto + +package socketforward + +import ( + fmt "fmt" + protohelpers "github.com/planetscale/vtprotobuf/protohelpers" + proto "google.golang.org/protobuf/proto" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + io "io" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +func (m *BytesMessage) CloneVT() *BytesMessage { + if m == nil { + return (*BytesMessage)(nil) + } + r := new(BytesMessage) + if rhs := m.Data; rhs != nil { + tmpBytes := make([]byte, len(rhs)) + copy(tmpBytes, rhs) + r.Data = tmpBytes + } + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *BytesMessage) CloneMessageVT() proto.Message { + return m.CloneVT() +} + +func (this *BytesMessage) EqualVT(that *BytesMessage) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + if string(this.Data) != string(that.Data) { + return false + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *BytesMessage) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*BytesMessage) + if !ok { + return false + } + return this.EqualVT(that) +} +func (m *BytesMessage) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *BytesMessage) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *BytesMessage) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Data) > 0 { + i -= len(m.Data) + copy(dAtA[i:], m.Data) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Data))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *BytesMessage) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Data) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *BytesMessage) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: BytesMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: BytesMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) + if m.Data == nil { + m.Data = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/session/socketforward/socketprovider/socketprovider.go b/session/socketforward/socketprovider/socketprovider.go new file mode 100644 index 000000000..5d966f984 --- /dev/null +++ b/session/socketforward/socketprovider/socketprovider.go @@ -0,0 +1,65 @@ +package socketprovider + +import ( + "context" + "io" + + "github.com/moby/buildkit/session" + socketforward "github.com/moby/buildkit/session/socketforward" + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" +) + +// SocketAcceptCb is called when the unix socket is connected to +type SocketAcceptCb func(context.Context, io.ReadWriteCloser) error + +// NewSocketProvider creates a session provider that proxies a readwriter to a unix socket +func NewSocketProvider(m map[string]SocketAcceptCb) (session.Attachable, error) { + return &socketProvider{m: m}, nil +} + +type socketProvider struct { + m map[string]SocketAcceptCb +} + +func (sp *socketProvider) Register(server *grpc.Server) { + socketforward.RegisterSocketServer(server, sp) +} + +func (sp *socketProvider) Proxy(stream socketforward.Socket_ProxyServer) error { + opts, _ := metadata.FromIncomingContext(stream.Context()) // if no metadata continue with empty object + var id string + if v, ok := opts[socketforward.SocketIDKey]; ok && len(v) > 0 && v[0] != "" { + id = v[0] + } + + cb, ok := sp.m[id] + if !ok { + return errors.Errorf("no callback registered for socket ID: %s", id) + } + + s1, s2 := sockPair() + eg, ctx := errgroup.WithContext(stream.Context()) + eg.Go(func() error { + return cb(ctx, s1) + }) + eg.Go(func() error { + defer s1.Close() + return socketforward.Copy(ctx, s2, stream, nil) + }) + return eg.Wait() +} + +func sockPair() (io.ReadWriteCloser, io.ReadWriteCloser) { + pr1, pw1 := io.Pipe() + pr2, pw2 := io.Pipe() + return &sock{pr1, pw2, pw1}, &sock{pr2, pw1, pw2} +} + +type sock struct { + io.Reader + io.Writer + io.Closer +} diff --git a/solver/inconsistent_graph_state_error_tracker.go b/solver/inconsistent_graph_state_error_tracker.go new file mode 100644 index 000000000..8c8cd05a7 --- /dev/null +++ b/solver/inconsistent_graph_state_error_tracker.go @@ -0,0 +1,59 @@ +package solver + +// earthly-specific: this is used to collect information related to "inconsistent graph state" errors + +import ( + "fmt" + "strings" + "time" + + digest "github.com/opencontainers/go-digest" +) + +var dgstTrackerInst = newDgstTracker() + +type dgstTrackerItem struct { + dgst digest.Digest + action string + seen time.Time +} + +type dgstTracker struct { + head int + records []dgstTrackerItem +} + +func newDgstTracker() *dgstTracker { + n := 10000 + return &dgstTracker{ + records: make([]dgstTrackerItem, n), + } +} + +func (d *dgstTracker) add(dgst digest.Digest, action string) { + d.head++ + if d.head >= len(d.records) { + d.head = 0 + } + d.records[d.head].dgst = dgst + d.records[d.head].action = action + d.records[d.head].seen = time.Now() +} + +func (d *dgstTracker) String() string { + var sb strings.Builder + + for i := d.head; i >= 0; i-- { + if d.records[i].seen.IsZero() { + break + } + sb.WriteString(fmt.Sprintf("%s %s %s; ", d.records[i].dgst, d.records[i].action, d.records[i].seen)) + } + for i := len(d.records) - 1; i > d.head; i-- { + if d.records[i].seen.IsZero() { + break + } + sb.WriteString(fmt.Sprintf("%s %s %s; ", d.records[i].dgst, d.records[i].action, d.records[i].seen)) + } + return sb.String() +} diff --git a/solver/jobs.go b/solver/jobs.go index 840a038df..7b350fd07 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -461,8 +461,10 @@ func (jl *Solver) getEdge(e Edge) (redge *edge) { jl.mu.RLock() defer jl.mu.RUnlock() - st, ok := jl.actives[e.Vertex.Digest()] + dgst := e.Vertex.Digest() + st, ok := jl.actives[dgst] if !ok { + dgstTrackerInst.add(dgst, "get-edge-not-found") return nil } return st.getEdge(e.Index) @@ -518,11 +520,13 @@ func (jl *Solver) loadUnlocked(ctx context.Context, v, parent Vertex, j *Job, ca // existing active vertex, as otherwise the original vertex will use an // incorrect digest and can incorrectly delete it while it is still in use. v = st.vtx + dgstTrackerInst.add(dgst, "loadUnlocked-found-dgstWithoutCache") } if !ok { st, ok = jl.actives[dgst] + dgstTrackerInst.add(dgst, "loadUnlocked-not-found-dgstWithoutCache") // !ignorecache merges with ignorecache but ignorecache doesn't merge with !ignorecache if ok && !st.vtx.Options().IgnoreCache && v.Options().IgnoreCache { dgst = dgstWithoutCache @@ -556,6 +560,7 @@ func (jl *Solver) loadUnlocked(ctx context.Context, v, parent Vertex, j *Job, ca origDigest: origVtx.Digest(), } jl.actives[dgst] = st + dgstTrackerInst.add(dgst, "loadUnlocked-add") if debugScheduler { lg := bklog.G(ctx). @@ -574,15 +579,18 @@ func (jl *Solver) loadUnlocked(ctx context.Context, v, parent Vertex, j *Job, ca Debug("new active vertex input") } } - } else if debugScheduler { - lg := bklog.G(ctx). - WithField("vertex_name", v.Name()). - WithField("vertex_digest", v.Digest()). - WithField("actives_digest_key", dgst) - if j != nil { - lg = lg.WithField("job", j.id) + } else { + dgstTrackerInst.add(dgst, "loadUnlocked-exists") + if debugScheduler { + lg := bklog.G(ctx). + WithField("vertex_name", v.Name()). + WithField("vertex_digest", v.Digest()). + WithField("actives_digest_key", dgst) + if j != nil { + lg = lg.WithField("job", j.id) + } + lg.Debug("reusing active vertex") } - lg.Debug("reusing active vertex") } st.mu.Lock() @@ -598,6 +606,8 @@ func (jl *Solver) loadUnlocked(ctx context.Context, v, parent Vertex, j *Job, ca if _, ok := st.jobs[j]; !ok { st.jobs[j] = struct{}{} } + } else { + dgstTrackerInst.add(dgst, "loadUnlocked-nil-job") } st.mu.Unlock() @@ -725,6 +735,7 @@ func (jl *Solver) deleteIfUnreferenced(k digest.Digest, st *state) { } st.Release() delete(jl.actives, k) + dgstTrackerInst.add(k, "delete") } else if debugScheduler { var jobIDs []string for j := range st.jobs { diff --git a/solver/llbsolver/bridge.go b/solver/llbsolver/bridge.go index 47920a520..61dbf2958 100644 --- a/solver/llbsolver/bridge.go +++ b/solver/llbsolver/bridge.go @@ -3,16 +3,19 @@ package llbsolver import ( "context" "fmt" + "strings" "sync" "time" "github.com/containerd/platforms" "github.com/mitchellh/hashstructure/v2" + "github.com/moby/buildkit/cache" "github.com/moby/buildkit/cache/remotecache" "github.com/moby/buildkit/client" "github.com/moby/buildkit/client/llb/sourceresolver" "github.com/moby/buildkit/executor" resourcestypes "github.com/moby/buildkit/executor/resources/types" + "github.com/moby/buildkit/exporter" "github.com/moby/buildkit/frontend" gw "github.com/moby/buildkit/frontend/gateway/client" "github.com/moby/buildkit/identity" @@ -28,6 +31,7 @@ import ( digest "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + "golang.org/x/sync/errgroup" ) type llbBridge struct { @@ -77,6 +81,12 @@ func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImp if err != nil { return nil, err } + + // TODO FIXME earthly-specific wait group is required to ensure the remotecache/registry's ResolveCacheImporterFunc can run + // which requires the session to remain open in order to get dockerhub (or any other registry) credentials. + // It seems like the cleaner approach is to bake this in somewhere into the edge or Load + eg, _ := errgroup.WithContext(ctx) + srcPol, err := loadSourcePolicy(b.builder) if err != nil { return nil, err @@ -129,6 +139,14 @@ func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImp } return cmNew, nil }) + + cmInst := cm + eg.Go(func() error { + if lcm, ok := cmInst.(*lazyCacheManager); ok { + lcm.wait() + } + return nil + }) }(cmID, im) b.cms[cmID] = cm } else { @@ -137,6 +155,10 @@ func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImp cms = append(cms, cm) b.cmsMu.Unlock() } + err = eg.Wait() + if err != nil { + return nil, err + } dpc := &detectPrunedCacheID{} edge, err := Load(ctx, def, b.policy(polEngine), dpc.Load, ValidateEntitlements(ent, w.CDIManager()), WithCacheSources(cms), NormalizeRuntimePlatforms(), WithValidateCaps()) @@ -159,6 +181,55 @@ func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImp return res, nil } +// getExporter is earthly specific code which extracts the configured exporter +// from the job's metadata +func (b *llbBridge) getExporter(_ context.Context) (*ExporterRequest, error) { + var exp *ExporterRequest + numExporters := 0 + b.builder.EachValue(context.TODO(), keyEarthlyExporterInstance, func(v any) error { + numExporters++ + exp = v.(*ExporterRequest) + return nil + }) + if numExporters != 1 { + return nil, errors.Errorf("Export found %d exporters (should have been 1)", numExporters) // shouldn't happen + } + return exp, nil +} + +func (b *llbBridge) Export(ctx context.Context, refs map[string]cache.ImmutableRef, metadata map[string][]byte) error { + // generate an ID that's consistent for the refs + refKeys := []string{} + for k := range refs { + refKeys = append(refKeys, k) + } + id := strings.Join(refKeys, "-") + + inp := &exporter.Source{ + Refs: refs, + Metadata: metadata, + } + + exp, err := b.getExporter(ctx) + if err != nil { + return err + } + if len(exp.Exporters) == 0 { + return errors.Errorf("Export had no exporter configured") + } + + e := exp.Exporters[0] + return inBuilderContext(ctx, b.builder, e.Name(), id, func(ctx context.Context, jobCtx solver.JobContext) error { + sessionIDs := session.AllSessionIDs(jobCtx.Session()) + if len(sessionIDs) == 0 { + return errors.Errorf("group has no session IDs") // shouldnt happen + } + sessionID := sessionIDs[0] + _, _, _, err := e.Export(ctx, inp, exporter.ExportBuildInfo{SessionID: sessionID}) + return err + }) +} + func (b *llbBridge) policy(engine *sourcepolicy.Engine) SourcePolicyEvaluator { return &policyEvaluator{ llbBridge: b, @@ -212,6 +283,11 @@ func (b *llbBridge) loadExecutor() error { return b.executorErr } +func (b *llbBridge) ResolveImageConfig(ctx context.Context, ref string, opt sourceresolver.Opt) (string, digest.Digest, []byte, error) { + imr := sourceresolver.NewImageMetaResolver(b) + return imr.ResolveImageConfig(ctx, ref, opt) +} + func (b *llbBridge) ResolveSourceMetadata(ctx context.Context, op *pb.SourceOp, opt sourceresolver.Opt) (resp *sourceresolver.MetaResponse, err error) { return b.resolveSourceMetadata(ctx, op, opt, true) } diff --git a/solver/llbsolver/history.go b/solver/llbsolver/history.go index 4acfccf9c..bec75c72e 100644 --- a/solver/llbsolver/history.go +++ b/solver/llbsolver/history.go @@ -202,7 +202,7 @@ func (s *Solver) recordBuildHistory(ctx context.Context, id string, req frontend return nil }) eg.Go(func() error { - return j.Status(ctx2, ch) + return j.Status(ctx2, false, ch) }) setDeprecated := true diff --git a/solver/llbsolver/mounts/mount.go b/solver/llbsolver/mounts/mount.go index 58a962a80..a6951c9cc 100644 --- a/solver/llbsolver/mounts/mount.go +++ b/solver/llbsolver/mounts/mount.go @@ -15,6 +15,7 @@ import ( "github.com/moby/buildkit/identity" "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/secrets" + "github.com/moby/buildkit/session/socketforward" "github.com/moby/buildkit/session/sshforward" "github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/solver/pb" @@ -239,6 +240,85 @@ func (sm *sshMountInstance) Mount() ([]mount.Mount, func() error, error) { }}, release, nil } +// getMountableSocket is earthly-specific +func (mm *MountManager) getMountableSocket(ctx context.Context, m *pb.Mount, g session.Group) (cache.Mountable, error) { + var caller session.Caller + err := mm.sm.Any(ctx, g, func(ctx context.Context, _ string, c session.Caller) error { + caller = c + return nil + }) + if err != nil { + return nil, err + } + if caller == nil { + return nil, nil + } + // because forwarded socket remains active, to actually handle session disconnecting error + // should restart the whole exec with new session + return &sockMount{mount: m, caller: caller, idmap: mm.cm.IdentityMapping()}, nil +} + +type sockMount struct { + mount *pb.Mount + caller session.Caller + idmap *user.IdentityMapping +} + +func (sm *sockMount) Mount(ctx context.Context, readonly bool, g session.Group) (snapshot.Mountable, error) { + return &sockMountInstance{sm: sm, idmap: sm.idmap}, nil +} + +// sockMountInstance is earthly-specific +type sockMountInstance struct { + sm *sockMount + idmap *user.IdentityMapping +} + +func (sm *sockMountInstance) Mount() ([]mount.Mount, func() error, error) { + ctx, cancel := context.WithCancelCause(context.TODO()) + + uid := int(sm.sm.mount.SockOpt.Uid) + gid := int(sm.sm.mount.SockOpt.Gid) + + if sm.idmap != nil { + var err error + uid, gid, err = sm.idmap.ToHost(uid, gid) + if err != nil { + cancel(err) + return nil, nil, err + } + } + + sock, cleanup, err := socketforward.MountSocket(ctx, sm.sm.caller, socketforward.SocketOpt{ + ID: sm.sm.mount.SockOpt.ID, + UID: uid, + GID: gid, + Mode: int(sm.sm.mount.SockOpt.Mode & 0777), + }) + if err != nil { + cancel(err) + return nil, nil, err + } + release := func() error { + var err error + if cleanup != nil { + err = cleanup() + } + cancel(err) + return err + } + + return []mount.Mount{{ + Type: "bind", + Source: sock, + Options: []string{"rbind"}, + }}, release, nil +} + +func (sm *sockMountInstance) IdentityMapping() *user.IdentityMapping { + return sm.idmap +} + func (sm *sshMountInstance) IdentityMapping() *user.IdentityMapping { return sm.idmap } @@ -378,6 +458,15 @@ func (mm *MountManager) MountableTmpFS(m *pb.Mount) cache.Mountable { return newTmpfs(mm.cm.IdentityMapping(), m.TmpfsOpt) } +func (mm *MountManager) MountableHostBind(ctx context.Context, m *pb.Mount) cache.Mountable { // earthly-specific + // In the API, the SourcePath ends up as m.Selector. We want to use this as + // the source path instead, so that in unprivileged mode, FixUp[OCI] can + // correctly read the path. + source := m.Selector + m.Selector = "" + return newHostBind(source, mm.cm.IdentityMapping()) +} + func (mm *MountManager) MountableSecret(ctx context.Context, m *pb.Mount, g session.Group) (cache.Mountable, error) { return mm.getSecretMountable(ctx, m, g) } @@ -386,6 +475,11 @@ func (mm *MountManager) MountableSSH(ctx context.Context, m *pb.Mount, g session return mm.getSSHMountable(ctx, m, g) } +// earthly-specific +func (mm *MountManager) MountableSocket(ctx context.Context, m *pb.Mount, g session.Group) (cache.Mountable, error) { + return mm.getMountableSocket(ctx, m, g) +} + func newTmpfs(idmap *user.IdentityMapping, opt *pb.TmpfsOpt) cache.Mountable { return &tmpfs{idmap: idmap, opt: opt} } @@ -426,6 +520,44 @@ func (m *tmpfsMount) IdentityMapping() *user.IdentityMapping { return m.idmap } +// earthly-specific hostbind functions +func newHostBind(source string, idmap *user.IdentityMapping) cache.Mountable { + return &hostBind{source: source, idmap: idmap} +} + +type hostBind struct { + source string + idmap *user.IdentityMapping +} + +func (f *hostBind) Mount(ctx context.Context, readonly bool, g session.Group) (snapshot.Mountable, error) { + return &hostBindMount{source: f.source, readonly: readonly, idmap: f.idmap}, nil +} + +type hostBindMount struct { + source string + readonly bool + idmap *user.IdentityMapping +} + +func (m *hostBindMount) Mount() ([]mount.Mount, func() error, error) { + opt := []string{"rbind"} + if m.readonly { + opt = append(opt, "ro") + } else { + opt = append(opt, "rw") + } + return []mount.Mount{{ + Type: "bind", + Source: m.source, + Options: opt, + }}, func() error { return nil }, nil +} + +func (m *hostBindMount) IdentityMapping() *user.IdentityMapping { + return m.idmap +} + var ( cacheRefsLocker = locker.New() sharedCacheRefs = &cacheRefs{} diff --git a/solver/llbsolver/ops/exec.go b/solver/llbsolver/ops/exec.go index 09b37702c..0cc5ab445 100644 --- a/solver/llbsolver/ops/exec.go +++ b/solver/llbsolver/ops/exec.go @@ -5,11 +5,14 @@ import ( "context" "encoding/json" "fmt" + "io" "os" "path" + "path/filepath" "runtime" "slices" "strings" + "time" "github.com/containerd/platforms" "github.com/moby/buildkit/cache" @@ -17,7 +20,9 @@ import ( resourcestypes "github.com/moby/buildkit/executor/resources/types" "github.com/moby/buildkit/frontend/gateway/container" "github.com/moby/buildkit/session" + "github.com/moby/buildkit/session/localhost" "github.com/moby/buildkit/session/secrets" + "github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/solver" "github.com/moby/buildkit/solver/llbsolver/errdefs" "github.com/moby/buildkit/solver/llbsolver/mounts" @@ -25,13 +30,13 @@ import ( "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/cachedigest" "github.com/moby/buildkit/util/progress/logs" + "github.com/moby/buildkit/util/semutil" utilsystem "github.com/moby/buildkit/util/system" "github.com/moby/buildkit/worker" digest "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "go.opentelemetry.io/otel/trace" - "golang.org/x/sync/semaphore" ) const execCacheType = "buildkit.exec.v0" @@ -45,14 +50,31 @@ type ExecOp struct { w worker.Worker platform *pb.Platform numInputs int - parallelism *semaphore.Weighted + parallelism *semutil.Weighted // earthly-specific: use *semutil.Weighted instead of *semaphore.Weighted rec resourcestypes.Recorder digest digest.Digest } var _ solver.Op = &ExecOp{} -func NewExecOp(v solver.Vertex, op *pb.Op_Exec, platform *pb.Platform, cm cache.Manager, parallelism *semaphore.Weighted, sm *session.Manager, exec executor.Executor, w worker.Worker) (*ExecOp, error) { +// earthly-specific: a custom exec limit for certain customers. + +var errExecTimeoutExceeded = errors.New("max execution time exceeded") +var execTimeout time.Duration + +func init() { + env, ok := os.LookupEnv("BUILDKIT_EXEC_TIMEOUT") + if !ok { + return + } + var err error + execTimeout, err = time.ParseDuration(env) + if err != nil { + panic(fmt.Sprintf("invalid value for 'BUILDKIT_EXEC_TIMEOUT': %s", env)) + } +} + +func NewExecOp(v solver.Vertex, op *pb.Op_Exec, platform *pb.Platform, cm cache.Manager, parallelism *semutil.Weighted, sm *session.Manager, exec executor.Executor, w worker.Worker) (*ExecOp, error) { if err := opsutils.Validate(&pb.Op{Op: op}); err != nil { return nil, err } @@ -495,12 +517,38 @@ func (e *ExecOp) Exec(ctx context.Context, jobCtx solver.JobContext, inputs []so } }() - rec, execErr := e.exec.Run(ctx, "", p.Root, p.Mounts, executor.ProcessInfo{ - Meta: meta, - Stdin: nil, - Stdout: stdout, - Stderr: stderr, - }, nil) + // earthly-specific + statsStream, statsFlush := logs.NewStatsStreams(ctx, os.Getenv("BUILDKIT_DEBUG_EXEC_OUTPUT") == "1") + defer func() { + if err != nil { + statsFlush() + } + }() + + isLocal, err := e.doFromLocalHack(ctx, p.Root, p.Mounts, g, meta, stdout, stderr) + if err != nil { + return nil, err + } + // earthly-specific TODO: should the rec be set to a nopRecord, or can nil be safely used instead? + + // earthly-specific: enforce a time limit for certain customers. + if execTimeout > 0 { + var cancel func() + ctx, cancel = context.WithTimeoutCause(ctx, execTimeout, errExecTimeoutExceeded) + defer cancel() + } + + var execErr error + var rec resourcestypes.Recorder + if !isLocal { + rec, execErr = e.exec.Run(ctx, "", p.Root, p.Mounts, executor.ProcessInfo{ + Meta: meta, + Stdin: nil, + Stdout: stdout, + Stderr: stderr, + StatsStream: statsStream, // earthly-specific + }, nil) + } for i, out := range p.OutputRefs { if mutable, ok := out.Ref.(cache.MutableRef); ok { @@ -516,7 +564,184 @@ func (e *ExecOp) Exec(ctx context.Context, jobCtx solver.JobContext, inputs []so p.OutputRefs[i].Ref = nil } e.rec = rec - return results, errors.Wrapf(execErr, "process %q did not complete successfully", strings.Join(e.op.Meta.Args, " ")) + + // earthly-specific: customize error message on exec timeout. + retErr := errors.Wrapf(execErr, "process %q did not complete successfully", strings.Join(e.op.Meta.Args, " ")) + if cause := context.Cause(ctx); errors.Is(cause, errExecTimeoutExceeded) { + retErr = errors.Errorf("max execution time of %s exceeded", execTimeout) + } + + return results, retErr +} + +// earthly-specific +func (e *ExecOp) doFromLocalHack(ctx context.Context, root executor.Mount, mounts []executor.Mount, g session.Group, meta executor.Meta, stdout, stderr io.WriteCloser) (bool, error) { + var cmd string + if len(meta.Args) > 0 { + cmd = meta.Args[0] + } + switch cmd { + case localhost.CopyFileMagicStr: + return true, e.copyLocally(ctx, root, g, meta, stdout, stderr) + case localhost.RunOnLocalHostMagicStr: + return true, e.execLocally(ctx, root, g, meta, stdout, stderr) + case localhost.SendFileMagicStr: + return true, e.sendLocally(ctx, root, mounts, g, meta, stdout, stderr) + default: + return false, nil + } +} + +func (e *ExecOp) copyLocally(ctx context.Context, root executor.Mount, g session.Group, meta executor.Meta, _, _ io.WriteCloser) error { + if len(meta.Args) != 3 { + return errors.Errorf("CopyFileMagicStr takes exactly 2 args") + } + if meta.Args[0] != localhost.CopyFileMagicStr { + panic("arg[0] must be CopyFileMagicStr; this should not have happened") + } + src := meta.Args[1] + if !strings.HasPrefix(src, "/") && meta.Cwd != "" { + src = filepath.Join(meta.Cwd, src) + } + src = filepath.Clean(src) + dst := meta.Args[2] + + if src == "/" { + return errors.Errorf("copyLocally does not support copying the entire root filesystem") + } + + if strings.HasSuffix(dst, ".") || strings.HasSuffix(dst, "/") { + dst = filepath.Join(dst, filepath.Base(src)) + } + + return e.sm.Any(ctx, g, func(ctx context.Context, _ string, caller session.Caller) error { + mountable, err := root.Src.Mount(ctx, false) + if err != nil { + return err + } + + rootMounts, release, err := mountable.Mount() + if err != nil { + return err + } + if release != nil { + defer release() + } + + lm := snapshot.LocalMounterWithMounts(rootMounts) + rootfsPath, err := lm.Mount() + if err != nil { + return err + } + defer lm.Unmount() + + finalDest := rootfsPath + "/" + dst + err = localhost.LocalhostGet(ctx, caller, src, finalDest, mountable) + if err != nil { + return err + } + return nil + }) +} + +var errSendFileMagicStrMissingArgs = errors.Errorf("SendFileMagicStr args missing; should be SendFileMagicStr [--dir] [--] [ ...] ") + +func (e *ExecOp) sendLocally(ctx context.Context, _ executor.Mount, mounts []executor.Mount, g session.Group, meta executor.Meta, _, _ io.WriteCloser) error { + i := 0 + nArgs := len(meta.Args) + + if i >= nArgs || meta.Args[i] != localhost.SendFileMagicStr { + return errSendFileMagicStrMissingArgs + } + i++ + + // check for --dir + copyDir := false + if i >= nArgs { + return errSendFileMagicStrMissingArgs + } + if meta.Args[i] == "--dir" { + copyDir = true + i++ + } + + // check for - + if i >= nArgs { + return errSendFileMagicStrMissingArgs + } + if meta.Args[i] == "-" { + i++ + } + + dstIndex := len(meta.Args) - 1 + numFiles := dstIndex - i + if numFiles <= 0 { + return errors.Errorf("SendFileMagicStr args missing; should be SendFileMagicStr [--dir] [--] [ ...] ") + } + files := meta.Args[i:dstIndex] + dst := meta.Args[dstIndex] + + if len(mounts) != 1 { + return errors.Errorf("SendFileMagicStr must be given a mount with the artifacts to copy from") + } + + return e.sm.Any(ctx, g, func(ctx context.Context, _ string, caller session.Caller) error { + mnt := mounts[0] + + mountable2, err := mnt.Src.Mount(ctx, false) + if err != nil { + return err + } + + mounts, release, err := mountable2.Mount() + if err != nil { + return err + } + if release != nil { + defer release() + } + + lm := snapshot.LocalMounterWithMounts(mounts) + hackfsPath, err := lm.Mount() + if err != nil { + return err + } + defer lm.Unmount() + + for _, f := range files { + finalSrc := hackfsPath + "/" + f + var finalDst string + if dst == "." || strings.HasSuffix(dst, "/") || strings.HasSuffix(dst, "/.") || copyDir { + finalDst = path.Join(dst, path.Base(f)) + } else { + finalDst = dst + } + if !strings.HasPrefix(dst, "/") && meta.Cwd != "" { + finalDst = path.Join(meta.Cwd, finalDst) + } + err = localhost.LocalhostPut(ctx, caller, finalSrc, finalDst) + if err != nil { + return errors.Wrap(err, "error calling LocalhostExec") + } + } + return nil + }) +} + +func (e *ExecOp) execLocally(ctx context.Context, _ executor.Mount, g session.Group, meta executor.Meta, stdout, stderr io.WriteCloser) error { + if len(meta.Args) == 0 || meta.Args[0] != localhost.RunOnLocalHostMagicStr { + panic("first arg should be RunOnLocalHostMagicStr; this should not happen") + } + args := meta.Args[1:] // remove magic uuid from command prefix; the rest that follows is the actual command to run + cwd := meta.Cwd + + return e.sm.Any(ctx, g, func(ctx context.Context, _ string, caller session.Caller) error { + err := localhost.LocalhostExec(ctx, caller, args, cwd, stdout, stderr) + if err != nil { + return errors.Wrap(err, "error calling LocalhostExec") + } + return nil + }) } func proxyEnvList(p *pb.ProxyEnv) []string { diff --git a/solver/llbsolver/ops/file.go b/solver/llbsolver/ops/file.go index 232949868..6a3696ea5 100644 --- a/solver/llbsolver/ops/file.go +++ b/solver/llbsolver/ops/file.go @@ -21,11 +21,11 @@ import ( "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/cachedigest" "github.com/moby/buildkit/util/flightcontrol" + "github.com/moby/buildkit/util/semutil" "github.com/moby/buildkit/worker" digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" "golang.org/x/sync/errgroup" - "golang.org/x/sync/semaphore" ) const fileCacheType = "buildkit.file.v0" @@ -35,10 +35,10 @@ type fileOp struct { w worker.Worker refManager *file.RefManager numInputs int - parallelism *semaphore.Weighted + parallelism *semutil.Weighted } -func NewFileOp(v solver.Vertex, op *pb.Op_File, cm cache.Manager, parallelism *semaphore.Weighted, w worker.Worker) (solver.Op, error) { +func NewFileOp(v solver.Vertex, op *pb.Op_File, cm cache.Manager, parallelism *semutil.Weighted, w worker.Worker) (solver.Op, error) { if err := opsutils.Validate(&pb.Op{Op: op}); err != nil { return nil, err } diff --git a/solver/llbsolver/ops/source.go b/solver/llbsolver/ops/source.go index c9a2a4e92..ccbb5698b 100644 --- a/solver/llbsolver/ops/source.go +++ b/solver/llbsolver/ops/source.go @@ -11,9 +11,9 @@ import ( "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/source" "github.com/moby/buildkit/util/cachedigest" + "github.com/moby/buildkit/util/semutil" "github.com/moby/buildkit/worker" digest "github.com/opencontainers/go-digest" - "golang.org/x/sync/semaphore" ) const sourceCacheType = "buildkit.source.v0" @@ -27,14 +27,14 @@ type SourceOp struct { sessM *session.Manager w worker.Worker vtx solver.Vertex - parallelism *semaphore.Weighted + parallelism *semutil.Weighted pin string id source.Identifier } var _ solver.Op = &SourceOp{} -func NewSourceOp(vtx solver.Vertex, op *pb.Op_Source, platform *pb.Platform, sm *source.Manager, parallelism *semaphore.Weighted, sessM *session.Manager, w worker.Worker) (*SourceOp, error) { +func NewSourceOp(vtx solver.Vertex, op *pb.Op_Source, platform *pb.Platform, sm *source.Manager, parallelism *semutil.Weighted, sessM *session.Manager, w worker.Worker) (*SourceOp, error) { if err := opsutils.Validate(&pb.Op{Op: op}); err != nil { return nil, err } diff --git a/solver/llbsolver/provenance.go b/solver/llbsolver/provenance.go index 66b3337bd..fe010d17e 100644 --- a/solver/llbsolver/provenance.go +++ b/solver/llbsolver/provenance.go @@ -137,6 +137,11 @@ func (b *provenanceBridge) findByResult(rp solver.ResultProxy) (*resultWithBridg return nil, false } +func (b *provenanceBridge) ResolveImageConfig(ctx context.Context, ref string, opt sourceresolver.Opt) (string, digest.Digest, []byte, error) { + imr := sourceresolver.NewImageMetaResolver(b) + return imr.ResolveImageConfig(ctx, ref, opt) +} + func (b *provenanceBridge) ResolveSourceMetadata(ctx context.Context, op *pb.SourceOp, opt sourceresolver.Opt) (*sourceresolver.MetaResponse, error) { resp, err := b.llbBridge.ResolveSourceMetadata(ctx, op, opt) if err != nil { diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index 630f5d6f7..094ea6a9a 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -31,6 +31,8 @@ import ( "golang.org/x/sync/errgroup" ) +const keyEarthlyExporterInstance = "earthly-hack-exporter-instance" + type ExporterRequest struct { Exporters []exporter.ExporterInstance CacheExporters []RemoteCacheExporter @@ -191,6 +193,7 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro return nil, err } j.SetValue(keyEntitlements, set) + j.SetValue(keyEarthlyExporterInstance, &exp) if srcPol != nil { if err := validateSourcePolicy(srcPol); err != nil { @@ -395,7 +398,7 @@ func (s *Solver) leaseManager() (*leaseutil.Manager, error) { return w.LeaseManager(), nil } -func (s *Solver) Status(ctx context.Context, id string, statusChan chan *client.SolveStatus) error { +func (s *Solver) Status(ctx context.Context, id string, statsStream bool, statusChan chan *client.SolveStatus) error { if err := s.history.Status(ctx, id, statusChan); err != nil { if !errors.Is(err, os.ErrNotExist) { close(statusChan) @@ -410,7 +413,7 @@ func (s *Solver) Status(ctx context.Context, id string, statusChan chan *client. close(statusChan) return err } - return j.Status(ctx, statusChan) + return j.Status(ctx, statsStream, statusChan) } func defaultResolver(wc *worker.Controller) ResolveWorkerFunc { diff --git a/solver/pb/attr.go b/solver/pb/attr.go index 53aa4eafc..f2d6fb384 100644 --- a/solver/pb/attr.go +++ b/solver/pb/attr.go @@ -6,6 +6,9 @@ const AttrAuthHeaderSecret = "git.authheadersecret" const AttrAuthTokenSecret = "git.authtokensecret" const AttrKnownSSHHosts = "git.knownsshhosts" const AttrMountSSHSock = "git.mountsshsock" +const AttrGitLFSInclude = "git.lfsinclude" // earthly-specific +const AttrGitLogLevel = "git.loglevel" // earthly-specific +const AttrGitSSHCommand = "git.sshCommand" // earthly-specific const AttrGitChecksum = "git.checksum" const AttrGitSkipSubmodules = "git.skipsubmodules" const AttrGitMTime = "git.mtime" diff --git a/solver/pb/caps.go b/solver/pb/caps.go index d62c5e2c2..7d03d69d6 100644 --- a/solver/pb/caps.go +++ b/solver/pb/caps.go @@ -31,6 +31,9 @@ const ( CapSourceGitKnownSSHHosts apicaps.CapID = "source.git.knownsshhosts" CapSourceGitMountSSHSock apicaps.CapID = "source.git.mountsshsock" CapSourceGitSubdir apicaps.CapID = "source.git.subdir" + CapSourceGitLFSInclude apicaps.CapID = "source.git.lfsinclude" // earthly-specific + CapSourceGitLogLevel apicaps.CapID = "source.git.logLevel" // earthly-specific + CapSourceGitSSHCommand apicaps.CapID = "source.git.sshCommand" // earthly-specific CapSourceGitChecksum apicaps.CapID = "source.git.checksum" CapSourceGitSkipSubmodules apicaps.CapID = "source.git.skipsubmodules" CapSourceGitSignatureVerify apicaps.CapID = "source.git.signatureverify" @@ -70,6 +73,7 @@ const ( CapExecMountTmpfsSize apicaps.CapID = "exec.mount.tmpfs.size" CapExecMountSecret apicaps.CapID = "exec.mount.secret" CapExecMountSSH apicaps.CapID = "exec.mount.ssh" + CapExecMountSock apicaps.CapID = "exec.mount.sock" CapExecMountContentCache apicaps.CapID = "exec.mount.cache.content" CapExecCgroupsMounted apicaps.CapID = "exec.cgroup" CapExecSecretEnv apicaps.CapID = "exec.secretenv" @@ -115,6 +119,8 @@ const ( // ListenBuildHistory requests support server-side filters CapHistoryFilters apicaps.CapID = "history.filter" + + CapEarthlyRegistryProxy apicaps.CapID = "earthly.regproxy" // earthly-specific ) func init() { @@ -238,6 +244,24 @@ func init() { Status: apicaps.CapStatusExperimental, }) + Caps.Init(apicaps.Cap{ + ID: CapSourceGitLFSInclude, + Enabled: true, + Status: apicaps.CapStatusExperimental, + }) + + Caps.Init(apicaps.Cap{ + ID: CapSourceGitLogLevel, + Enabled: true, + Status: apicaps.CapStatusExperimental, + }) + + Caps.Init(apicaps.Cap{ + ID: CapSourceGitSSHCommand, + Enabled: true, + Status: apicaps.CapStatusExperimental, + }) + Caps.Init(apicaps.Cap{ ID: CapSourceGitChecksum, Enabled: true, @@ -430,6 +454,12 @@ func init() { Status: apicaps.CapStatusExperimental, }) + Caps.Init(apicaps.Cap{ + ID: CapExecMountSock, + Enabled: true, + Status: apicaps.CapStatusExperimental, + }) + Caps.Init(apicaps.Cap{ ID: CapExecMountContentCache, Enabled: true, @@ -614,4 +644,10 @@ func init() { Enabled: true, Status: apicaps.CapStatusExperimental, }) + + Caps.Init(apicaps.Cap{ + ID: CapEarthlyRegistryProxy, + Enabled: true, + Status: apicaps.CapStatusExperimental, + }) } diff --git a/solver/pb/ops.pb.go b/solver/pb/ops.pb.go index a64211acd..a427054bf 100644 --- a/solver/pb/ops.pb.go +++ b/solver/pb/ops.pb.go @@ -123,28 +123,34 @@ func (SecurityMode) EnumDescriptor() ([]byte, []int) { type MountType int32 const ( - MountType_BIND MountType = 0 - MountType_SECRET MountType = 1 - MountType_SSH MountType = 2 - MountType_CACHE MountType = 3 - MountType_TMPFS MountType = 4 + MountType_BIND MountType = 0 + MountType_SECRET MountType = 1 + MountType_SSH MountType = 2 + MountType_CACHE MountType = 3 + MountType_TMPFS MountType = 4 + MountType_HOST_BIND MountType = 100 // Earthly specific. + MountType_SOCKET MountType = 101 // Earthly specific. ) // Enum value maps for MountType. var ( MountType_name = map[int32]string{ - 0: "BIND", - 1: "SECRET", - 2: "SSH", - 3: "CACHE", - 4: "TMPFS", + 0: "BIND", + 1: "SECRET", + 2: "SSH", + 3: "CACHE", + 4: "TMPFS", + 100: "HOST_BIND", + 101: "SOCKET", } MountType_value = map[string]int32{ - "BIND": 0, - "SECRET": 1, - "SSH": 2, - "CACHE": 3, - "TMPFS": 4, + "BIND": 0, + "SECRET": 1, + "SSH": 2, + "CACHE": 3, + "TMPFS": 4, + "HOST_BIND": 100, + "SOCKET": 101, } ) @@ -1039,6 +1045,7 @@ type Mount struct { SSHOpt *SSHOpt `protobuf:"bytes,22,opt,name=SSHOpt,proto3" json:"SSHOpt,omitempty"` ResultID string `protobuf:"bytes,23,opt,name=resultID,proto3" json:"resultID,omitempty"` ContentCache MountContentCache `protobuf:"varint,24,opt,name=contentCache,proto3,enum=pb.MountContentCache" json:"contentCache,omitempty"` + SockOpt *SockOpt `protobuf:"bytes,100,opt,name=SockOpt,proto3" json:"SockOpt,omitempty"` // earthly unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1157,6 +1164,13 @@ func (x *Mount) GetContentCache() MountContentCache { return MountContentCache_DEFAULT } +func (x *Mount) GetSockOpt() *SockOpt { + if x != nil { + return x.SockOpt + } + return nil +} + // TmpfsOpt defines options describing tpmfs mounts type TmpfsOpt struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1424,6 +1438,79 @@ func (x *SSHOpt) GetOptional() bool { return false } +// SockOpt defines options describing sock mounts; earthly-specific +type SockOpt struct { + state protoimpl.MessageState `protogen:"open.v1"` + // ID of proxied socket. Used for mapping socket to user-supplied callback + ID string `protobuf:"bytes,1,opt,name=ID,proto3" json:"ID,omitempty"` + // UID of agent socket + Uid uint32 `protobuf:"varint,2,opt,name=uid,proto3" json:"uid,omitempty"` + // GID of agent socket + Gid uint32 `protobuf:"varint,3,opt,name=gid,proto3" json:"gid,omitempty"` + // Mode is the filesystem mode of agent socket + Mode uint32 `protobuf:"varint,4,opt,name=mode,proto3" json:"mode,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SockOpt) Reset() { + *x = SockOpt{} + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SockOpt) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SockOpt) ProtoMessage() {} + +func (x *SockOpt) ProtoReflect() protoreflect.Message { + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SockOpt.ProtoReflect.Descriptor instead. +func (*SockOpt) Descriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{14} +} + +func (x *SockOpt) GetID() string { + if x != nil { + return x.ID + } + return "" +} + +func (x *SockOpt) GetUid() uint32 { + if x != nil { + return x.Uid + } + return 0 +} + +func (x *SockOpt) GetGid() uint32 { + if x != nil { + return x.Gid + } + return 0 +} + +func (x *SockOpt) GetMode() uint32 { + if x != nil { + return x.Mode + } + return 0 +} + // SourceOp specifies a source such as build contexts and images. type SourceOp struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1438,7 +1525,7 @@ type SourceOp struct { func (x *SourceOp) Reset() { *x = SourceOp{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[14] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1450,7 +1537,7 @@ func (x *SourceOp) String() string { func (*SourceOp) ProtoMessage() {} func (x *SourceOp) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[14] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1463,7 +1550,7 @@ func (x *SourceOp) ProtoReflect() protoreflect.Message { // Deprecated: Use SourceOp.ProtoReflect.Descriptor instead. func (*SourceOp) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{14} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{15} } func (x *SourceOp) GetIdentifier() string { @@ -1494,7 +1581,7 @@ type BuildOp struct { func (x *BuildOp) Reset() { *x = BuildOp{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[15] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1506,7 +1593,7 @@ func (x *BuildOp) String() string { func (*BuildOp) ProtoMessage() {} func (x *BuildOp) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[15] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1519,7 +1606,7 @@ func (x *BuildOp) ProtoReflect() protoreflect.Message { // Deprecated: Use BuildOp.ProtoReflect.Descriptor instead. func (*BuildOp) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{15} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{16} } func (x *BuildOp) GetBuilder() int64 { @@ -1560,7 +1647,7 @@ type BuildInput struct { func (x *BuildInput) Reset() { *x = BuildInput{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[16] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1572,7 +1659,7 @@ func (x *BuildInput) String() string { func (*BuildInput) ProtoMessage() {} func (x *BuildInput) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[16] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1585,7 +1672,7 @@ func (x *BuildInput) ProtoReflect() protoreflect.Message { // Deprecated: Use BuildInput.ProtoReflect.Descriptor instead. func (*BuildInput) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{16} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{17} } func (x *BuildInput) GetInput() int64 { @@ -1613,7 +1700,7 @@ type OpMetadata struct { func (x *OpMetadata) Reset() { *x = OpMetadata{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[17] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1625,7 +1712,7 @@ func (x *OpMetadata) String() string { func (*OpMetadata) ProtoMessage() {} func (x *OpMetadata) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[17] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1638,7 +1725,7 @@ func (x *OpMetadata) ProtoReflect() protoreflect.Message { // Deprecated: Use OpMetadata.ProtoReflect.Descriptor instead. func (*OpMetadata) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{17} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{18} } func (x *OpMetadata) GetIgnoreCache() bool { @@ -1687,7 +1774,7 @@ type Source struct { func (x *Source) Reset() { *x = Source{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[18] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1699,7 +1786,7 @@ func (x *Source) String() string { func (*Source) ProtoMessage() {} func (x *Source) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[18] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1712,7 +1799,7 @@ func (x *Source) ProtoReflect() protoreflect.Message { // Deprecated: Use Source.ProtoReflect.Descriptor instead. func (*Source) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{18} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{19} } func (x *Source) GetLocations() map[string]*Locations { @@ -1739,7 +1826,7 @@ type Locations struct { func (x *Locations) Reset() { *x = Locations{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[19] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1751,7 +1838,7 @@ func (x *Locations) String() string { func (*Locations) ProtoMessage() {} func (x *Locations) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[19] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1764,7 +1851,7 @@ func (x *Locations) ProtoReflect() protoreflect.Message { // Deprecated: Use Locations.ProtoReflect.Descriptor instead. func (*Locations) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{19} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{20} } func (x *Locations) GetLocations() []*Location { @@ -1787,7 +1874,7 @@ type SourceInfo struct { func (x *SourceInfo) Reset() { *x = SourceInfo{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[20] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1799,7 +1886,7 @@ func (x *SourceInfo) String() string { func (*SourceInfo) ProtoMessage() {} func (x *SourceInfo) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[20] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1812,7 +1899,7 @@ func (x *SourceInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use SourceInfo.ProtoReflect.Descriptor instead. func (*SourceInfo) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{20} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{21} } func (x *SourceInfo) GetFilename() string { @@ -1854,7 +1941,7 @@ type Location struct { func (x *Location) Reset() { *x = Location{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[21] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1866,7 +1953,7 @@ func (x *Location) String() string { func (*Location) ProtoMessage() {} func (x *Location) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[21] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1879,7 +1966,7 @@ func (x *Location) ProtoReflect() protoreflect.Message { // Deprecated: Use Location.ProtoReflect.Descriptor instead. func (*Location) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{21} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{22} } func (x *Location) GetSourceIndex() int32 { @@ -1907,7 +1994,7 @@ type Range struct { func (x *Range) Reset() { *x = Range{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[22] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1919,7 +2006,7 @@ func (x *Range) String() string { func (*Range) ProtoMessage() {} func (x *Range) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[22] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1932,7 +2019,7 @@ func (x *Range) ProtoReflect() protoreflect.Message { // Deprecated: Use Range.ProtoReflect.Descriptor instead. func (*Range) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{22} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{23} } func (x *Range) GetStart() *Position { @@ -1960,7 +2047,7 @@ type Position struct { func (x *Position) Reset() { *x = Position{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[23] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1972,7 +2059,7 @@ func (x *Position) String() string { func (*Position) ProtoMessage() {} func (x *Position) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[23] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1985,7 +2072,7 @@ func (x *Position) ProtoReflect() protoreflect.Message { // Deprecated: Use Position.ProtoReflect.Descriptor instead. func (*Position) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{23} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{24} } func (x *Position) GetLine() int32 { @@ -2011,7 +2098,7 @@ type ExportCache struct { func (x *ExportCache) Reset() { *x = ExportCache{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[24] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2023,7 +2110,7 @@ func (x *ExportCache) String() string { func (*ExportCache) ProtoMessage() {} func (x *ExportCache) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[24] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2036,7 +2123,7 @@ func (x *ExportCache) ProtoReflect() protoreflect.Message { // Deprecated: Use ExportCache.ProtoReflect.Descriptor instead. func (*ExportCache) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{24} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{25} } func (x *ExportCache) GetValue() bool { @@ -2057,7 +2144,7 @@ type ProgressGroup struct { func (x *ProgressGroup) Reset() { *x = ProgressGroup{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[25] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2069,7 +2156,7 @@ func (x *ProgressGroup) String() string { func (*ProgressGroup) ProtoMessage() {} func (x *ProgressGroup) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[25] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2082,7 +2169,7 @@ func (x *ProgressGroup) ProtoReflect() protoreflect.Message { // Deprecated: Use ProgressGroup.ProtoReflect.Descriptor instead. func (*ProgressGroup) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{25} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{26} } func (x *ProgressGroup) GetId() string { @@ -2119,7 +2206,7 @@ type ProxyEnv struct { func (x *ProxyEnv) Reset() { *x = ProxyEnv{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[26] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2131,7 +2218,7 @@ func (x *ProxyEnv) String() string { func (*ProxyEnv) ProtoMessage() {} func (x *ProxyEnv) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[26] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[27] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2144,7 +2231,7 @@ func (x *ProxyEnv) ProtoReflect() protoreflect.Message { // Deprecated: Use ProxyEnv.ProtoReflect.Descriptor instead. func (*ProxyEnv) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{26} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{27} } func (x *ProxyEnv) GetHttpProxy() string { @@ -2192,7 +2279,7 @@ type WorkerConstraints struct { func (x *WorkerConstraints) Reset() { *x = WorkerConstraints{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[27] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2204,7 +2291,7 @@ func (x *WorkerConstraints) String() string { func (*WorkerConstraints) ProtoMessage() {} func (x *WorkerConstraints) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[27] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[28] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2217,7 +2304,7 @@ func (x *WorkerConstraints) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerConstraints.ProtoReflect.Descriptor instead. func (*WorkerConstraints) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{27} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{28} } func (x *WorkerConstraints) GetFilter() []string { @@ -2243,7 +2330,7 @@ type Definition struct { func (x *Definition) Reset() { *x = Definition{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[28] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2255,7 +2342,7 @@ func (x *Definition) String() string { func (*Definition) ProtoMessage() {} func (x *Definition) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[28] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2268,7 +2355,7 @@ func (x *Definition) ProtoReflect() protoreflect.Message { // Deprecated: Use Definition.ProtoReflect.Descriptor instead. func (*Definition) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{28} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{29} } func (x *Definition) GetDef() [][]byte { @@ -2301,7 +2388,7 @@ type FileOp struct { func (x *FileOp) Reset() { *x = FileOp{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[29] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2313,7 +2400,7 @@ func (x *FileOp) String() string { func (*FileOp) ProtoMessage() {} func (x *FileOp) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[29] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[30] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2326,7 +2413,7 @@ func (x *FileOp) ProtoReflect() protoreflect.Message { // Deprecated: Use FileOp.ProtoReflect.Descriptor instead. func (*FileOp) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{29} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{30} } func (x *FileOp) GetActions() []*FileAction { @@ -2356,7 +2443,7 @@ type FileAction struct { func (x *FileAction) Reset() { *x = FileAction{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[30] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2368,7 +2455,7 @@ func (x *FileAction) String() string { func (*FileAction) ProtoMessage() {} func (x *FileAction) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[30] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[31] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2381,7 +2468,7 @@ func (x *FileAction) ProtoReflect() protoreflect.Message { // Deprecated: Use FileAction.ProtoReflect.Descriptor instead. func (*FileAction) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{30} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{31} } func (x *FileAction) GetInput() int64 { @@ -2537,7 +2624,7 @@ type FileActionCopy struct { func (x *FileActionCopy) Reset() { *x = FileActionCopy{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[31] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2549,7 +2636,7 @@ func (x *FileActionCopy) String() string { func (*FileActionCopy) ProtoMessage() {} func (x *FileActionCopy) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[31] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[32] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2562,7 +2649,7 @@ func (x *FileActionCopy) ProtoReflect() protoreflect.Message { // Deprecated: Use FileActionCopy.ProtoReflect.Descriptor instead. func (*FileActionCopy) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{31} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{32} } func (x *FileActionCopy) GetSrc() string { @@ -2695,7 +2782,7 @@ type FileActionMkFile struct { func (x *FileActionMkFile) Reset() { *x = FileActionMkFile{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[32] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2707,7 +2794,7 @@ func (x *FileActionMkFile) String() string { func (*FileActionMkFile) ProtoMessage() {} func (x *FileActionMkFile) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[32] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[33] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2720,7 +2807,7 @@ func (x *FileActionMkFile) ProtoReflect() protoreflect.Message { // Deprecated: Use FileActionMkFile.ProtoReflect.Descriptor instead. func (*FileActionMkFile) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{32} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{33} } func (x *FileActionMkFile) GetPath() string { @@ -2774,7 +2861,7 @@ type FileActionSymlink struct { func (x *FileActionSymlink) Reset() { *x = FileActionSymlink{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[33] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2786,7 +2873,7 @@ func (x *FileActionSymlink) String() string { func (*FileActionSymlink) ProtoMessage() {} func (x *FileActionSymlink) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[33] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[34] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2799,7 +2886,7 @@ func (x *FileActionSymlink) ProtoReflect() protoreflect.Message { // Deprecated: Use FileActionSymlink.ProtoReflect.Descriptor instead. func (*FileActionSymlink) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{33} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{34} } func (x *FileActionSymlink) GetOldpath() string { @@ -2848,7 +2935,7 @@ type FileActionMkDir struct { func (x *FileActionMkDir) Reset() { *x = FileActionMkDir{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[34] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2860,7 +2947,7 @@ func (x *FileActionMkDir) String() string { func (*FileActionMkDir) ProtoMessage() {} func (x *FileActionMkDir) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[34] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[35] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2873,7 +2960,7 @@ func (x *FileActionMkDir) ProtoReflect() protoreflect.Message { // Deprecated: Use FileActionMkDir.ProtoReflect.Descriptor instead. func (*FileActionMkDir) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{34} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{35} } func (x *FileActionMkDir) GetPath() string { @@ -2925,7 +3012,7 @@ type FileActionRm struct { func (x *FileActionRm) Reset() { *x = FileActionRm{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[35] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2937,7 +3024,7 @@ func (x *FileActionRm) String() string { func (*FileActionRm) ProtoMessage() {} func (x *FileActionRm) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[35] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[36] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2950,7 +3037,7 @@ func (x *FileActionRm) ProtoReflect() protoreflect.Message { // Deprecated: Use FileActionRm.ProtoReflect.Descriptor instead. func (*FileActionRm) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{35} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{36} } func (x *FileActionRm) GetPath() string { @@ -2984,7 +3071,7 @@ type ChownOpt struct { func (x *ChownOpt) Reset() { *x = ChownOpt{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[36] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2996,7 +3083,7 @@ func (x *ChownOpt) String() string { func (*ChownOpt) ProtoMessage() {} func (x *ChownOpt) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[36] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[37] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3009,7 +3096,7 @@ func (x *ChownOpt) ProtoReflect() protoreflect.Message { // Deprecated: Use ChownOpt.ProtoReflect.Descriptor instead. func (*ChownOpt) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{36} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{37} } func (x *ChownOpt) GetUser() *UserOpt { @@ -3041,7 +3128,7 @@ type UserOpt struct { func (x *UserOpt) Reset() { *x = UserOpt{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[37] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3053,7 +3140,7 @@ func (x *UserOpt) String() string { func (*UserOpt) ProtoMessage() {} func (x *UserOpt) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[37] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[38] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3066,7 +3153,7 @@ func (x *UserOpt) ProtoReflect() protoreflect.Message { // Deprecated: Use UserOpt.ProtoReflect.Descriptor instead. func (*UserOpt) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{37} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{38} } func (x *UserOpt) GetUser() isUserOpt_User { @@ -3120,7 +3207,7 @@ type NamedUserOpt struct { func (x *NamedUserOpt) Reset() { *x = NamedUserOpt{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[38] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3132,7 +3219,7 @@ func (x *NamedUserOpt) String() string { func (*NamedUserOpt) ProtoMessage() {} func (x *NamedUserOpt) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[38] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[39] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3145,7 +3232,7 @@ func (x *NamedUserOpt) ProtoReflect() protoreflect.Message { // Deprecated: Use NamedUserOpt.ProtoReflect.Descriptor instead. func (*NamedUserOpt) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{38} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{39} } func (x *NamedUserOpt) GetName() string { @@ -3171,7 +3258,7 @@ type MergeInput struct { func (x *MergeInput) Reset() { *x = MergeInput{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[39] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3183,7 +3270,7 @@ func (x *MergeInput) String() string { func (*MergeInput) ProtoMessage() {} func (x *MergeInput) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[39] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[40] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3196,7 +3283,7 @@ func (x *MergeInput) ProtoReflect() protoreflect.Message { // Deprecated: Use MergeInput.ProtoReflect.Descriptor instead. func (*MergeInput) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{39} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{40} } func (x *MergeInput) GetInput() int64 { @@ -3215,7 +3302,7 @@ type MergeOp struct { func (x *MergeOp) Reset() { *x = MergeOp{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[40] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3227,7 +3314,7 @@ func (x *MergeOp) String() string { func (*MergeOp) ProtoMessage() {} func (x *MergeOp) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[40] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[41] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3240,7 +3327,7 @@ func (x *MergeOp) ProtoReflect() protoreflect.Message { // Deprecated: Use MergeOp.ProtoReflect.Descriptor instead. func (*MergeOp) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{40} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{41} } func (x *MergeOp) GetInputs() []*MergeInput { @@ -3259,7 +3346,7 @@ type LowerDiffInput struct { func (x *LowerDiffInput) Reset() { *x = LowerDiffInput{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[41] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3271,7 +3358,7 @@ func (x *LowerDiffInput) String() string { func (*LowerDiffInput) ProtoMessage() {} func (x *LowerDiffInput) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[41] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[42] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3284,7 +3371,7 @@ func (x *LowerDiffInput) ProtoReflect() protoreflect.Message { // Deprecated: Use LowerDiffInput.ProtoReflect.Descriptor instead. func (*LowerDiffInput) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{41} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{42} } func (x *LowerDiffInput) GetInput() int64 { @@ -3303,7 +3390,7 @@ type UpperDiffInput struct { func (x *UpperDiffInput) Reset() { *x = UpperDiffInput{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[42] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3315,7 +3402,7 @@ func (x *UpperDiffInput) String() string { func (*UpperDiffInput) ProtoMessage() {} func (x *UpperDiffInput) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[42] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[43] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3328,7 +3415,7 @@ func (x *UpperDiffInput) ProtoReflect() protoreflect.Message { // Deprecated: Use UpperDiffInput.ProtoReflect.Descriptor instead. func (*UpperDiffInput) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{42} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{43} } func (x *UpperDiffInput) GetInput() int64 { @@ -3348,7 +3435,7 @@ type DiffOp struct { func (x *DiffOp) Reset() { *x = DiffOp{} - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[43] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3360,7 +3447,7 @@ func (x *DiffOp) String() string { func (*DiffOp) ProtoMessage() {} func (x *DiffOp) ProtoReflect() protoreflect.Message { - mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[43] + mi := &file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[44] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3373,7 +3460,7 @@ func (x *DiffOp) ProtoReflect() protoreflect.Message { // Deprecated: Use DiffOp.ProtoReflect.Descriptor instead. func (*DiffOp) Descriptor() ([]byte, []int) { - return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{43} + return file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP(), []int{44} } func (x *DiffOp) GetLower() *LowerDiffInput { @@ -3460,7 +3547,7 @@ const file_github_com_moby_buildkit_solver_pb_ops_proto_rawDesc = "" + "\boptional\x18\x03 \x01(\bR\boptional\";\n" + "\tCDIDevice\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x1a\n" + - "\boptional\x18\x02 \x01(\bR\boptional\"\xaa\x03\n" + + "\boptional\x18\x02 \x01(\bR\boptional\"\xd1\x03\n" + "\x05Mount\x12\x14\n" + "\x05input\x18\x01 \x01(\x03R\x05input\x12\x1a\n" + "\bselector\x18\x02 \x01(\tR\bselector\x12\x12\n" + @@ -3474,7 +3561,8 @@ const file_github_com_moby_buildkit_solver_pb_ops_proto_rawDesc = "" + "\x06SSHOpt\x18\x16 \x01(\v2\n" + ".pb.SSHOptR\x06SSHOpt\x12\x1a\n" + "\bresultID\x18\x17 \x01(\tR\bresultID\x129\n" + - "\fcontentCache\x18\x18 \x01(\x0e2\x15.pb.MountContentCacheR\fcontentCache\"\x1e\n" + + "\fcontentCache\x18\x18 \x01(\x0e2\x15.pb.MountContentCacheR\fcontentCache\x12%\n" + + "\aSockOpt\x18d \x01(\v2\v.pb.SockOptR\aSockOpt\"\x1e\n" + "\bTmpfsOpt\x12\x12\n" + "\x04size\x18\x01 \x01(\x03R\x04size\"I\n" + "\bCacheOpt\x12\x0e\n" + @@ -3491,7 +3579,12 @@ const file_github_com_moby_buildkit_solver_pb_ops_proto_rawDesc = "" + "\x03uid\x18\x02 \x01(\rR\x03uid\x12\x10\n" + "\x03gid\x18\x03 \x01(\rR\x03gid\x12\x12\n" + "\x04mode\x18\x04 \x01(\rR\x04mode\x12\x1a\n" + - "\boptional\x18\x05 \x01(\bR\boptional\"\x93\x01\n" + + "\boptional\x18\x05 \x01(\bR\boptional\"Q\n" + + "\aSockOpt\x12\x0e\n" + + "\x02ID\x18\x01 \x01(\tR\x02ID\x12\x10\n" + + "\x03uid\x18\x02 \x01(\rR\x03uid\x12\x10\n" + + "\x03gid\x18\x03 \x01(\rR\x03gid\x12\x12\n" + + "\x04mode\x18\x04 \x01(\rR\x04mode\"\x93\x01\n" + "\bSourceOp\x12\x1e\n" + "\n" + "identifier\x18\x01 \x01(\tR\n" + @@ -3659,14 +3752,17 @@ const file_github_com_moby_buildkit_solver_pb_ops_proto_rawDesc = "" + "\x04NONE\x10\x02*)\n" + "\fSecurityMode\x12\v\n" + "\aSANDBOX\x10\x00\x12\f\n" + - "\bINSECURE\x10\x01*@\n" + + "\bINSECURE\x10\x01*[\n" + "\tMountType\x12\b\n" + "\x04BIND\x10\x00\x12\n" + "\n" + "\x06SECRET\x10\x01\x12\a\n" + "\x03SSH\x10\x02\x12\t\n" + "\x05CACHE\x10\x03\x12\t\n" + - "\x05TMPFS\x10\x04*1\n" + + "\x05TMPFS\x10\x04\x12\r\n" + + "\tHOST_BIND\x10d\x12\n" + + "\n" + + "\x06SOCKET\x10e*1\n" + "\x11MountContentCache\x12\v\n" + "\aDEFAULT\x10\x00\x12\x06\n" + "\x02ON\x10\x01\x12\a\n" + @@ -3691,7 +3787,7 @@ func file_github_com_moby_buildkit_solver_pb_ops_proto_rawDescGZIP() []byte { } var file_github_com_moby_buildkit_solver_pb_ops_proto_enumTypes = make([]protoimpl.EnumInfo, 5) -var file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes = make([]protoimpl.MessageInfo, 51) +var file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes = make([]protoimpl.MessageInfo, 52) var file_github_com_moby_buildkit_solver_pb_ops_proto_goTypes = []any{ (NetMode)(0), // 0: pb.NetMode (SecurityMode)(0), // 1: pb.SecurityMode @@ -3712,61 +3808,62 @@ var file_github_com_moby_buildkit_solver_pb_ops_proto_goTypes = []any{ (*CacheOpt)(nil), // 16: pb.CacheOpt (*SecretOpt)(nil), // 17: pb.SecretOpt (*SSHOpt)(nil), // 18: pb.SSHOpt - (*SourceOp)(nil), // 19: pb.SourceOp - (*BuildOp)(nil), // 20: pb.BuildOp - (*BuildInput)(nil), // 21: pb.BuildInput - (*OpMetadata)(nil), // 22: pb.OpMetadata - (*Source)(nil), // 23: pb.Source - (*Locations)(nil), // 24: pb.Locations - (*SourceInfo)(nil), // 25: pb.SourceInfo - (*Location)(nil), // 26: pb.Location - (*Range)(nil), // 27: pb.Range - (*Position)(nil), // 28: pb.Position - (*ExportCache)(nil), // 29: pb.ExportCache - (*ProgressGroup)(nil), // 30: pb.ProgressGroup - (*ProxyEnv)(nil), // 31: pb.ProxyEnv - (*WorkerConstraints)(nil), // 32: pb.WorkerConstraints - (*Definition)(nil), // 33: pb.Definition - (*FileOp)(nil), // 34: pb.FileOp - (*FileAction)(nil), // 35: pb.FileAction - (*FileActionCopy)(nil), // 36: pb.FileActionCopy - (*FileActionMkFile)(nil), // 37: pb.FileActionMkFile - (*FileActionSymlink)(nil), // 38: pb.FileActionSymlink - (*FileActionMkDir)(nil), // 39: pb.FileActionMkDir - (*FileActionRm)(nil), // 40: pb.FileActionRm - (*ChownOpt)(nil), // 41: pb.ChownOpt - (*UserOpt)(nil), // 42: pb.UserOpt - (*NamedUserOpt)(nil), // 43: pb.NamedUserOpt - (*MergeInput)(nil), // 44: pb.MergeInput - (*MergeOp)(nil), // 45: pb.MergeOp - (*LowerDiffInput)(nil), // 46: pb.LowerDiffInput - (*UpperDiffInput)(nil), // 47: pb.UpperDiffInput - (*DiffOp)(nil), // 48: pb.DiffOp - nil, // 49: pb.SourceOp.AttrsEntry - nil, // 50: pb.BuildOp.InputsEntry - nil, // 51: pb.BuildOp.AttrsEntry - nil, // 52: pb.OpMetadata.DescriptionEntry - nil, // 53: pb.OpMetadata.CapsEntry - nil, // 54: pb.Source.LocationsEntry - nil, // 55: pb.Definition.MetadataEntry + (*SockOpt)(nil), // 19: pb.SockOpt + (*SourceOp)(nil), // 20: pb.SourceOp + (*BuildOp)(nil), // 21: pb.BuildOp + (*BuildInput)(nil), // 22: pb.BuildInput + (*OpMetadata)(nil), // 23: pb.OpMetadata + (*Source)(nil), // 24: pb.Source + (*Locations)(nil), // 25: pb.Locations + (*SourceInfo)(nil), // 26: pb.SourceInfo + (*Location)(nil), // 27: pb.Location + (*Range)(nil), // 28: pb.Range + (*Position)(nil), // 29: pb.Position + (*ExportCache)(nil), // 30: pb.ExportCache + (*ProgressGroup)(nil), // 31: pb.ProgressGroup + (*ProxyEnv)(nil), // 32: pb.ProxyEnv + (*WorkerConstraints)(nil), // 33: pb.WorkerConstraints + (*Definition)(nil), // 34: pb.Definition + (*FileOp)(nil), // 35: pb.FileOp + (*FileAction)(nil), // 36: pb.FileAction + (*FileActionCopy)(nil), // 37: pb.FileActionCopy + (*FileActionMkFile)(nil), // 38: pb.FileActionMkFile + (*FileActionSymlink)(nil), // 39: pb.FileActionSymlink + (*FileActionMkDir)(nil), // 40: pb.FileActionMkDir + (*FileActionRm)(nil), // 41: pb.FileActionRm + (*ChownOpt)(nil), // 42: pb.ChownOpt + (*UserOpt)(nil), // 43: pb.UserOpt + (*NamedUserOpt)(nil), // 44: pb.NamedUserOpt + (*MergeInput)(nil), // 45: pb.MergeInput + (*MergeOp)(nil), // 46: pb.MergeOp + (*LowerDiffInput)(nil), // 47: pb.LowerDiffInput + (*UpperDiffInput)(nil), // 48: pb.UpperDiffInput + (*DiffOp)(nil), // 49: pb.DiffOp + nil, // 50: pb.SourceOp.AttrsEntry + nil, // 51: pb.BuildOp.InputsEntry + nil, // 52: pb.BuildOp.AttrsEntry + nil, // 53: pb.OpMetadata.DescriptionEntry + nil, // 54: pb.OpMetadata.CapsEntry + nil, // 55: pb.Source.LocationsEntry + nil, // 56: pb.Definition.MetadataEntry } var file_github_com_moby_buildkit_solver_pb_ops_proto_depIdxs = []int32{ 7, // 0: pb.Op.inputs:type_name -> pb.Input 8, // 1: pb.Op.exec:type_name -> pb.ExecOp - 19, // 2: pb.Op.source:type_name -> pb.SourceOp - 34, // 3: pb.Op.file:type_name -> pb.FileOp - 20, // 4: pb.Op.build:type_name -> pb.BuildOp - 45, // 5: pb.Op.merge:type_name -> pb.MergeOp - 48, // 6: pb.Op.diff:type_name -> pb.DiffOp + 20, // 2: pb.Op.source:type_name -> pb.SourceOp + 35, // 3: pb.Op.file:type_name -> pb.FileOp + 21, // 4: pb.Op.build:type_name -> pb.BuildOp + 46, // 5: pb.Op.merge:type_name -> pb.MergeOp + 49, // 6: pb.Op.diff:type_name -> pb.DiffOp 6, // 7: pb.Op.platform:type_name -> pb.Platform - 32, // 8: pb.Op.constraints:type_name -> pb.WorkerConstraints + 33, // 8: pb.Op.constraints:type_name -> pb.WorkerConstraints 9, // 9: pb.ExecOp.meta:type_name -> pb.Meta 14, // 10: pb.ExecOp.mounts:type_name -> pb.Mount 0, // 11: pb.ExecOp.network:type_name -> pb.NetMode 1, // 12: pb.ExecOp.security:type_name -> pb.SecurityMode 12, // 13: pb.ExecOp.secretenv:type_name -> pb.SecretEnv 13, // 14: pb.ExecOp.cdiDevices:type_name -> pb.CDIDevice - 31, // 15: pb.Meta.proxy_env:type_name -> pb.ProxyEnv + 32, // 15: pb.Meta.proxy_env:type_name -> pb.ProxyEnv 10, // 16: pb.Meta.extraHosts:type_name -> pb.HostIP 11, // 17: pb.Meta.ulimit:type_name -> pb.Ulimit 2, // 18: pb.Mount.mountType:type_name -> pb.MountType @@ -3775,48 +3872,49 @@ var file_github_com_moby_buildkit_solver_pb_ops_proto_depIdxs = []int32{ 17, // 21: pb.Mount.secretOpt:type_name -> pb.SecretOpt 18, // 22: pb.Mount.SSHOpt:type_name -> pb.SSHOpt 3, // 23: pb.Mount.contentCache:type_name -> pb.MountContentCache - 4, // 24: pb.CacheOpt.sharing:type_name -> pb.CacheSharingOpt - 49, // 25: pb.SourceOp.attrs:type_name -> pb.SourceOp.AttrsEntry - 50, // 26: pb.BuildOp.inputs:type_name -> pb.BuildOp.InputsEntry - 33, // 27: pb.BuildOp.def:type_name -> pb.Definition - 51, // 28: pb.BuildOp.attrs:type_name -> pb.BuildOp.AttrsEntry - 52, // 29: pb.OpMetadata.description:type_name -> pb.OpMetadata.DescriptionEntry - 29, // 30: pb.OpMetadata.export_cache:type_name -> pb.ExportCache - 53, // 31: pb.OpMetadata.caps:type_name -> pb.OpMetadata.CapsEntry - 30, // 32: pb.OpMetadata.progress_group:type_name -> pb.ProgressGroup - 54, // 33: pb.Source.locations:type_name -> pb.Source.LocationsEntry - 25, // 34: pb.Source.infos:type_name -> pb.SourceInfo - 26, // 35: pb.Locations.locations:type_name -> pb.Location - 33, // 36: pb.SourceInfo.definition:type_name -> pb.Definition - 27, // 37: pb.Location.ranges:type_name -> pb.Range - 28, // 38: pb.Range.start:type_name -> pb.Position - 28, // 39: pb.Range.end:type_name -> pb.Position - 55, // 40: pb.Definition.metadata:type_name -> pb.Definition.MetadataEntry - 23, // 41: pb.Definition.Source:type_name -> pb.Source - 35, // 42: pb.FileOp.actions:type_name -> pb.FileAction - 36, // 43: pb.FileAction.copy:type_name -> pb.FileActionCopy - 37, // 44: pb.FileAction.mkfile:type_name -> pb.FileActionMkFile - 39, // 45: pb.FileAction.mkdir:type_name -> pb.FileActionMkDir - 40, // 46: pb.FileAction.rm:type_name -> pb.FileActionRm - 38, // 47: pb.FileAction.symlink:type_name -> pb.FileActionSymlink - 41, // 48: pb.FileActionCopy.owner:type_name -> pb.ChownOpt - 41, // 49: pb.FileActionMkFile.owner:type_name -> pb.ChownOpt - 41, // 50: pb.FileActionSymlink.owner:type_name -> pb.ChownOpt - 41, // 51: pb.FileActionMkDir.owner:type_name -> pb.ChownOpt - 42, // 52: pb.ChownOpt.user:type_name -> pb.UserOpt - 42, // 53: pb.ChownOpt.group:type_name -> pb.UserOpt - 43, // 54: pb.UserOpt.byName:type_name -> pb.NamedUserOpt - 44, // 55: pb.MergeOp.inputs:type_name -> pb.MergeInput - 46, // 56: pb.DiffOp.lower:type_name -> pb.LowerDiffInput - 47, // 57: pb.DiffOp.upper:type_name -> pb.UpperDiffInput - 21, // 58: pb.BuildOp.InputsEntry.value:type_name -> pb.BuildInput - 24, // 59: pb.Source.LocationsEntry.value:type_name -> pb.Locations - 22, // 60: pb.Definition.MetadataEntry.value:type_name -> pb.OpMetadata - 61, // [61:61] is the sub-list for method output_type - 61, // [61:61] is the sub-list for method input_type - 61, // [61:61] is the sub-list for extension type_name - 61, // [61:61] is the sub-list for extension extendee - 0, // [0:61] is the sub-list for field type_name + 19, // 24: pb.Mount.SockOpt:type_name -> pb.SockOpt + 4, // 25: pb.CacheOpt.sharing:type_name -> pb.CacheSharingOpt + 50, // 26: pb.SourceOp.attrs:type_name -> pb.SourceOp.AttrsEntry + 51, // 27: pb.BuildOp.inputs:type_name -> pb.BuildOp.InputsEntry + 34, // 28: pb.BuildOp.def:type_name -> pb.Definition + 52, // 29: pb.BuildOp.attrs:type_name -> pb.BuildOp.AttrsEntry + 53, // 30: pb.OpMetadata.description:type_name -> pb.OpMetadata.DescriptionEntry + 30, // 31: pb.OpMetadata.export_cache:type_name -> pb.ExportCache + 54, // 32: pb.OpMetadata.caps:type_name -> pb.OpMetadata.CapsEntry + 31, // 33: pb.OpMetadata.progress_group:type_name -> pb.ProgressGroup + 55, // 34: pb.Source.locations:type_name -> pb.Source.LocationsEntry + 26, // 35: pb.Source.infos:type_name -> pb.SourceInfo + 27, // 36: pb.Locations.locations:type_name -> pb.Location + 34, // 37: pb.SourceInfo.definition:type_name -> pb.Definition + 28, // 38: pb.Location.ranges:type_name -> pb.Range + 29, // 39: pb.Range.start:type_name -> pb.Position + 29, // 40: pb.Range.end:type_name -> pb.Position + 56, // 41: pb.Definition.metadata:type_name -> pb.Definition.MetadataEntry + 24, // 42: pb.Definition.Source:type_name -> pb.Source + 36, // 43: pb.FileOp.actions:type_name -> pb.FileAction + 37, // 44: pb.FileAction.copy:type_name -> pb.FileActionCopy + 38, // 45: pb.FileAction.mkfile:type_name -> pb.FileActionMkFile + 40, // 46: pb.FileAction.mkdir:type_name -> pb.FileActionMkDir + 41, // 47: pb.FileAction.rm:type_name -> pb.FileActionRm + 39, // 48: pb.FileAction.symlink:type_name -> pb.FileActionSymlink + 42, // 49: pb.FileActionCopy.owner:type_name -> pb.ChownOpt + 42, // 50: pb.FileActionMkFile.owner:type_name -> pb.ChownOpt + 42, // 51: pb.FileActionSymlink.owner:type_name -> pb.ChownOpt + 42, // 52: pb.FileActionMkDir.owner:type_name -> pb.ChownOpt + 43, // 53: pb.ChownOpt.user:type_name -> pb.UserOpt + 43, // 54: pb.ChownOpt.group:type_name -> pb.UserOpt + 44, // 55: pb.UserOpt.byName:type_name -> pb.NamedUserOpt + 45, // 56: pb.MergeOp.inputs:type_name -> pb.MergeInput + 47, // 57: pb.DiffOp.lower:type_name -> pb.LowerDiffInput + 48, // 58: pb.DiffOp.upper:type_name -> pb.UpperDiffInput + 22, // 59: pb.BuildOp.InputsEntry.value:type_name -> pb.BuildInput + 25, // 60: pb.Source.LocationsEntry.value:type_name -> pb.Locations + 23, // 61: pb.Definition.MetadataEntry.value:type_name -> pb.OpMetadata + 62, // [62:62] is the sub-list for method output_type + 62, // [62:62] is the sub-list for method input_type + 62, // [62:62] is the sub-list for extension type_name + 62, // [62:62] is the sub-list for extension extendee + 0, // [0:62] is the sub-list for field type_name } func init() { file_github_com_moby_buildkit_solver_pb_ops_proto_init() } @@ -3832,14 +3930,14 @@ func file_github_com_moby_buildkit_solver_pb_ops_proto_init() { (*Op_Merge)(nil), (*Op_Diff)(nil), } - file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[30].OneofWrappers = []any{ + file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[31].OneofWrappers = []any{ (*FileAction_Copy)(nil), (*FileAction_Mkfile)(nil), (*FileAction_Mkdir)(nil), (*FileAction_Rm)(nil), (*FileAction_Symlink)(nil), } - file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[37].OneofWrappers = []any{ + file_github_com_moby_buildkit_solver_pb_ops_proto_msgTypes[38].OneofWrappers = []any{ (*UserOpt_ByName)(nil), (*UserOpt_ByID)(nil), } @@ -3849,7 +3947,7 @@ func file_github_com_moby_buildkit_solver_pb_ops_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_github_com_moby_buildkit_solver_pb_ops_proto_rawDesc), len(file_github_com_moby_buildkit_solver_pb_ops_proto_rawDesc)), NumEnums: 5, - NumMessages: 51, + NumMessages: 52, NumExtensions: 0, NumServices: 0, }, diff --git a/solver/pb/ops.proto b/solver/pb/ops.proto index 731123b6c..a11855b25 100644 --- a/solver/pb/ops.proto +++ b/solver/pb/ops.proto @@ -119,6 +119,8 @@ message Mount { SSHOpt SSHOpt = 22; string resultID = 23; MountContentCache contentCache = 24; + + SockOpt SockOpt = 100; // earthly } // MountType defines a type of a mount from a supported set @@ -128,6 +130,8 @@ enum MountType { SSH = 2; CACHE = 3; TMPFS = 4; + HOST_BIND = 100; // Earthly specific. + SOCKET = 101; // Earthly specific. } // MountContentCache ... @@ -191,6 +195,18 @@ message SSHOpt { bool optional = 5; } +// SockOpt defines options describing sock mounts; earthly-specific +message SockOpt { + // ID of proxied socket. Used for mapping socket to user-supplied callback + string ID = 1; + // UID of agent socket + uint32 uid = 2; + // GID of agent socket + uint32 gid = 3; + // Mode is the filesystem mode of agent socket + uint32 mode = 4; +} + // SourceOp specifies a source such as build contexts and images. message SourceOp { // TODO: use source type or any type instead of URL protocol. diff --git a/solver/pb/ops_vtproto.pb.go b/solver/pb/ops_vtproto.pb.go index 9ab8e766c..cd4055132 100644 --- a/solver/pb/ops_vtproto.pb.go +++ b/solver/pb/ops_vtproto.pb.go @@ -326,6 +326,7 @@ func (m *Mount) CloneVT() *Mount { r.SSHOpt = m.SSHOpt.CloneVT() r.ResultID = m.ResultID r.ContentCache = m.ContentCache + r.SockOpt = m.SockOpt.CloneVT() if len(m.unknownFields) > 0 { r.unknownFields = make([]byte, len(m.unknownFields)) copy(r.unknownFields, m.unknownFields) @@ -414,6 +415,26 @@ func (m *SSHOpt) CloneMessageVT() proto.Message { return m.CloneVT() } +func (m *SockOpt) CloneVT() *SockOpt { + if m == nil { + return (*SockOpt)(nil) + } + r := new(SockOpt) + r.ID = m.ID + r.Uid = m.Uid + r.Gid = m.Gid + r.Mode = m.Mode + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *SockOpt) CloneMessageVT() proto.Message { + return m.CloneVT() +} + func (m *SourceOp) CloneVT() *SourceOp { if m == nil { return (*SourceOp)(nil) @@ -1717,6 +1738,9 @@ func (this *Mount) EqualVT(that *Mount) bool { if this.ContentCache != that.ContentCache { return false } + if !this.SockOpt.EqualVT(that.SockOpt) { + return false + } return string(this.unknownFields) == string(that.unknownFields) } @@ -1830,6 +1854,34 @@ func (this *SSHOpt) EqualMessageVT(thatMsg proto.Message) bool { } return this.EqualVT(that) } +func (this *SockOpt) EqualVT(that *SockOpt) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + if this.ID != that.ID { + return false + } + if this.Uid != that.Uid { + return false + } + if this.Gid != that.Gid { + return false + } + if this.Mode != that.Mode { + return false + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *SockOpt) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*SockOpt) + if !ok { + return false + } + return this.EqualVT(that) +} func (this *SourceOp) EqualVT(that *SourceOp) bool { if this == that { return true @@ -3735,6 +3787,18 @@ func (m *Mount) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.SockOpt != nil { + size, err := m.SockOpt.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x6 + i-- + dAtA[i] = 0xa2 + } if m.ContentCache != 0 { i = protohelpers.EncodeVarint(dAtA, i, uint64(m.ContentCache)) i-- @@ -4054,6 +4118,61 @@ func (m *SSHOpt) MarshalToSizedBufferVT(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *SockOpt) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SockOpt) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *SockOpt) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.Mode != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Mode)) + i-- + dAtA[i] = 0x20 + } + if m.Gid != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Gid)) + i-- + dAtA[i] = 0x18 + } + if m.Uid != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Uid)) + i-- + dAtA[i] = 0x10 + } + if len(m.ID) > 0 { + i -= len(m.ID) + copy(dAtA[i:], m.ID) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.ID))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func (m *SourceOp) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil @@ -6373,6 +6492,10 @@ func (m *Mount) SizeVT() (n int) { if m.ContentCache != 0 { n += 2 + protohelpers.SizeOfVarint(uint64(m.ContentCache)) } + if m.SockOpt != nil { + l = m.SockOpt.SizeVT() + n += 2 + l + protohelpers.SizeOfVarint(uint64(l)) + } n += len(m.unknownFields) return n } @@ -6459,6 +6582,29 @@ func (m *SSHOpt) SizeVT() (n int) { return n } +func (m *SockOpt) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.ID) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Uid != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Uid)) + } + if m.Gid != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Gid)) + } + if m.Mode != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Mode)) + } + n += len(m.unknownFields) + return n +} + func (m *SourceOp) SizeVT() (n int) { if m == nil { return 0 @@ -9454,6 +9600,42 @@ func (m *Mount) UnmarshalVT(dAtA []byte) error { break } } + case 100: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SockOpt", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.SockOpt == nil { + m.SockOpt = &SockOpt{} + } + if err := m.SockOpt.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) @@ -9968,6 +10150,146 @@ func (m *SSHOpt) UnmarshalVT(dAtA []byte) error { } return nil } +func (m *SockOpt) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SockOpt: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SockOpt: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ID = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Uid", wireType) + } + m.Uid = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Uid |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Gid", wireType) + } + m.Gid = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Gid |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Mode", wireType) + } + m.Mode = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Mode |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *SourceOp) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/solver/progress.go b/solver/progress.go index 65d180234..6be93b69a 100644 --- a/solver/progress.go +++ b/solver/progress.go @@ -8,13 +8,14 @@ import ( "time" "github.com/moby/buildkit/util/bklog" + "github.com/moby/buildkit/util/progress/logs" "github.com/moby/buildkit/client" "github.com/moby/buildkit/util/progress" digest "github.com/opencontainers/go-digest" ) -func (j *Job) Status(ctx context.Context, ch chan *client.SolveStatus) error { +func (j *Job) Status(ctx context.Context, statsStream bool, ch chan *client.SolveStatus) error { vs := &vertexStream{cache: map[digest.Digest]*client.Vertex{}, wasCached: make(map[digest.Digest]struct{})} pr := j.pr.Reader(ctx) defer func() { @@ -24,6 +25,17 @@ func (j *Job) Status(ctx context.Context, ch chan *client.SolveStatus) error { close(ch) }() + if !statsStream { // earthly-specific: don't stream stats back to old clients (which will cause them to print binary data to stderr) + pr = progress.NewFilteredReader(pr, func(ctx context.Context, p *progress.Progress) (bool, error) { + if vl, ok := p.Sys.(client.VertexLog); ok { + if vl.Stream == logs.StatsStream { + return true, nil + } + } + return false, nil + }) + } + for { p, err := pr.Read(ctx) if err != nil { diff --git a/solver/scheduler.go b/solver/scheduler.go index 825542647..70f8fbdfc 100644 --- a/solver/scheduler.go +++ b/solver/scheduler.go @@ -2,10 +2,13 @@ package solver import ( "context" + "fmt" + "strings" "sync" "github.com/moby/buildkit/errdefs" "github.com/moby/buildkit/solver/internal/pipe" + "github.com/moby/buildkit/util/bklog" "github.com/moby/buildkit/util/cond" "github.com/pkg/errors" ) @@ -98,6 +101,11 @@ func (s *scheduler) loop() { // dispatch schedules an edge to be processed func (s *scheduler) dispatch(e *edge) { + helpMe := []string{"v1"} // earthly specific + + helpMe = append(helpMe, fmt.Sprintf("%s: %+v", e.edge.Vertex.Name(), e)) + + helpMe = append(helpMe, fmt.Sprintf("%d %d %d %d", len(s.incoming), len(s.outgoing), len(s.incoming[e]), len(s.outgoing[e]))) inc := make([]pipeSender, len(s.incoming[e])) for i, p := range s.incoming[e] { inc[i] = p.Sender @@ -117,6 +125,7 @@ func (s *scheduler) dispatch(e *edge) { e.hasActiveOutgoing = true } } + helpMe = append(helpMe, fmt.Sprintf("hasActiveOutgoing %v", e.hasActiveOutgoing)) pf := &pipeFactory{s: s, e: e} @@ -126,9 +135,12 @@ func (s *scheduler) dispatch(e *edge) { debugSchedulerPostUnpark(e, inc) // set up new requests that didn't complete/were added by this run + helpMe = append(helpMe, fmt.Sprintf("make openIncoming %d", len(inc))) openIncoming := make([]*edgePipe, 0, len(inc)) for _, r := range s.incoming[e] { - if !r.Sender.Status().Completed { + status := r.Sender.Status() + helpMe = append(helpMe, fmt.Sprintf("%+v", status)) + if !status.Completed { openIncoming = append(openIncoming, r) } } @@ -138,9 +150,12 @@ func (s *scheduler) dispatch(e *edge) { delete(s.incoming, e) } + helpMe = append(helpMe, fmt.Sprintf("make openOutgoing %d", len(out))) openOutgoing := make([]*edgePipe, 0, len(out)) for _, r := range s.outgoing[e] { - if !r.Receiver.Status().Completed { + status := r.Receiver.Status() + helpMe = append(helpMe, fmt.Sprintf("%+v", status)) + if !status.Completed { openOutgoing = append(openOutgoing, r) } } @@ -165,6 +180,7 @@ func (s *scheduler) dispatch(e *edge) { dest, src = src, dest } + bklog.G(context.TODO()).Debugf("merging edge %s[%d] to %s[%d]\n", src.edge.Vertex.Name(), src.edge.Index, dest.edge.Vertex.Name(), dest.edge.Index) debugSchedulerMergingEdges(src, dest) if s.mergeTo(dest, src) { s.ef.setEdge(src.edge, dest) @@ -177,14 +193,18 @@ func (s *scheduler) dispatch(e *edge) { e.keysDidChange = false } + helpMe = append(helpMe, fmt.Sprintf("in: %d out: %d", len(openIncoming), len(openOutgoing))) + // validation to avoid deadlocks/resource leaks: // TODO: if these start showing up in error reports they can be changed // to error the edge instead. They can only appear from algorithm bugs in // unpark(), not for any external input. if len(openIncoming) > 0 && len(openOutgoing) == 0 { + bklog.G(context.TODO()).Errorf("return leaving incoming open help me: %s\n", strings.Join(helpMe, ";")) e.markFailed(pf, errors.New("buildkit scheduler error: return leaving incoming open. Please report this with BUILDKIT_SCHEDULER_DEBUG=1")) } if len(openIncoming) == 0 && len(openOutgoing) > 0 { + bklog.G(context.TODO()).Errorf("return leaving outgoing open help me: %s\n", strings.Join(helpMe, ";")) e.markFailed(pf, errors.New("buildkit scheduler error: return leaving outgoing open. Please report this with BUILDKIT_SCHEDULER_DEBUG=1")) } } @@ -343,6 +363,8 @@ type pipeFactory struct { func (pf *pipeFactory) NewInputRequest(ee Edge, req *edgeRequest) pipeReceiver { target := pf.s.ef.getEdge(ee) if target == nil { + dgst := ee.Vertex.Digest() + bklog.G(context.TODO()).Errorf("failed to get edge dgst=%s name=%s desiredState=%s; actives history: %s", dgst, ee.Vertex.Name(), req.desiredState, dgstTrackerInst.String()) // earthly-specific debugSchedulerInconsistentGraphState(ee) return pf.NewFuncRequest(func(_ context.Context) (any, error) { return nil, errdefs.Internal(errors.Errorf("failed to get edge: inconsistent graph state in edge %s %s %d", ee.Vertex.Name(), ee.Vertex.Digest(), ee.Index)) diff --git a/source/git/identifier.go b/source/git/identifier.go index 7dda7f476..a4c7a91bd 100644 --- a/source/git/identifier.go +++ b/source/git/identifier.go @@ -18,6 +18,9 @@ type GitIdentifier struct { AuthHeaderSecret string MountSSHSock string KnownSSHHosts string + SSHCommand string // earthly-specific + LFSInclude string // earthly-specific + LogLevel gitutil.GitLogLevel // earthly-specific SkipSubmodules bool MTime string // "checkout" (default) or "commit" diff --git a/source/git/source.go b/source/git/source.go index c3a20ca60..e4375e5fe 100644 --- a/source/git/source.go +++ b/source/git/source.go @@ -109,6 +109,16 @@ func (gs *Source) Identifier(scheme, ref string, attrs map[string]string, platfo id.KnownSSHHosts = v case pb.AttrMountSSHSock: id.MountSSHSock = v + case pb.AttrGitLFSInclude: // earthly-specific + id.LFSInclude = v + case pb.AttrGitLogLevel: // earthly-specific + l, err := strconv.Atoi(v) + if err != nil { + return nil, errors.Wrapf(err, "invalid git log level %s", v) + } + id.LogLevel = gitutil.GitLogLevel(l) + case pb.AttrGitSSHCommand: // earthly-specific + id.SSHCommand = v case pb.AttrGitChecksum: id.Checksum = v case pb.AttrGitSkipSubmodules: @@ -267,6 +277,9 @@ func (gs *gitSourceHandler) shaToCacheKey(sha, ref string) string { if gs.src.Subdir != "" { key += ":" + gs.src.Subdir } + if gs.src.LFSInclude != "" { // earthly-specific + key += ";lfsinclude:" + gs.src.LFSInclude + } if gs.src.SkipSubmodules { key += "(skip-submodules)" } @@ -487,6 +500,7 @@ func (gs *gitSourceHandler) remoteKey() string { } func (gs *gitSourceHandler) resolveMetadata(ctx context.Context, jobCtx solver.JobContext) (md *Metadata, retErr error) { + ctx = context.WithValue(ctx, gitutil.EarthlyCtxDebugLevelKey, gs.src.LogLevel) // earthly-specific remote := gs.src.Remote gs.locker.Lock(remote) defer gs.locker.Unlock(remote) @@ -711,6 +725,7 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, jobCtx solver.JobConte } func (gs *gitSourceHandler) remoteFetch(ctx context.Context, jobCtx solver.JobContext) (_ *gitRepo, retErr error) { + ctx = context.WithValue(ctx, gitutil.EarthlyCtxDebugLevelKey, gs.src.LogLevel) // earthly-specific gs.locker.Lock(gs.src.Remote) cleanup := func() error { return gs.locker.Unlock(gs.src.Remote) } @@ -775,6 +790,7 @@ func (gs *gitSourceHandler) remoteFetch(ctx context.Context, jobCtx solver.JobCo } func (gs *gitSourceHandler) Snapshot(ctx context.Context, jobCtx solver.JobContext) (cache.ImmutableRef, error) { + ctx = context.WithValue(ctx, gitutil.EarthlyCtxDebugLevelKey, gs.src.LogLevel) // earthly-specific cacheKey := gs.cacheKey if cacheKey == "" { var err error @@ -1148,6 +1164,13 @@ func (gs *gitSourceHandler) checkout(ctx context.Context, repo *gitRepo, g sessi } } + if gs.src.LFSInclude != "" { // earthly-specific + _, err = git.Run(ctx, "lfs", "pull", "--include", gs.src.LFSInclude) + if err != nil { + return nil, errors.Wrapf(err, "failed to perform lfs pull %s", urlutil.RedactCredentials(gs.src.Remote)) + } + } + if idmap := mount.IdentityMapping(); idmap != nil { uid, gid := idmap.RootPair() err := filepath.WalkDir(gitDir, func(p string, _ os.DirEntry, _ error) error { @@ -1291,10 +1314,16 @@ func (gs *gitSourceHandler) emptyGitCli(ctx context.Context, g session.Group, op cleanups = append(cleanups, unmountKnownHosts) } + var sshCommand string + if gs.src.SSHCommand != "" { + sshCommand = gs.src.SSHCommand + } + opts = append([]gitutil.Option{ gitutil.WithArgs(gs.authArgs...), gitutil.WithSSHAuthSock(sock), gitutil.WithSSHKnownHosts(knownHosts), + gitutil.WithSSHCommand(sshCommand), }, opts...) return gitCLI(opts...), cleanup, err } diff --git a/source/local/source.go b/source/local/source.go index 71af585a9..c1f98d65d 100644 --- a/source/local/source.go +++ b/source/local/source.go @@ -297,6 +297,11 @@ func (ls *localSourceHandler) snapshot(ctx context.Context, caller session.Calle } stat.Uid = uint32(uid) stat.Gid = uint32(gid) + // earthly-specific + // whatever permissions the user has, give them to group and others as well + // this matches behavior of gitsource, given that umask is 0 + umode := (stat.Mode & 0700) >> 6 + stat.Mode = (stat.Mode ^ (stat.Mode & 0777)) | umode | (umode << 3) | (umode << 6) return true } } diff --git a/util/appdefaults/appdefaults_all.go b/util/appdefaults/appdefaults_all.go new file mode 100644 index 000000000..54e3874b2 --- /dev/null +++ b/util/appdefaults/appdefaults_all.go @@ -0,0 +1,9 @@ +package appdefaults + +import "time" + +const ( + HealthAllowedFailures = 1 + HealthFrequency = 1 * time.Second + HealthTimeout = 10 * time.Second +) diff --git a/util/archutil/check_nolinux.go b/util/archutil/check_nolinux.go new file mode 100644 index 000000000..a8fa69c42 --- /dev/null +++ b/util/archutil/check_nolinux.go @@ -0,0 +1,7 @@ +//go:build !linux && !windows + +package archutil + +func check(_, _ string) (string, error) { + return "", nil +} diff --git a/util/archutil/check_unix.go b/util/archutil/check_unix.go index ccb4e819f..7ab5c9a68 100644 --- a/util/archutil/check_unix.go +++ b/util/archutil/check_unix.go @@ -1,4 +1,4 @@ -//go:build !windows +//go:build linux package archutil @@ -13,6 +13,7 @@ import ( "syscall" "github.com/pkg/errors" + "golang.org/x/sys/unix" ) func withChroot(cmd *exec.Cmd, dir string) { @@ -21,6 +22,17 @@ func withChroot(cmd *exec.Cmd, dir string) { } } +// Earthly-specific. +func mountProc(target string) (func() error, error) { + err := unix.Mount("proc", target, "proc", 0, "") + if err != nil { + return nil, err + } + return func() error { + return unix.Unmount(target, 0) + }, nil +} + func check(arch, bin string) (string, error) { tmpdir, err := os.MkdirTemp("", "qemu-check") if err != nil { @@ -49,6 +61,20 @@ func check(arch, bin string) (string, error) { cmd := exec.CommandContext(context.TODO(), "/check") withChroot(cmd, tmpdir) + + // Earthly-specific. + // In case rosetta is used, /proc needs to be mounted since rosetta tries to access /proc/self/exe (unavailable in a chroot environment) + tmpProcDir := filepath.Join(tmpdir, "proc") + err = os.Mkdir(tmpProcDir, 0700) + if err != nil { + return "", err + } + umount, err := mountProc(tmpProcDir) + if err != nil { + return "", err + } + defer umount() + err = cmd.Run() if arch != "amd64" { return "", err diff --git a/util/bklog/log.go b/util/bklog/log.go index 387a2fa16..4846129e8 100644 --- a/util/bklog/log.go +++ b/util/bklog/log.go @@ -5,6 +5,7 @@ import ( "runtime/debug" "github.com/containerd/log" + "github.com/moby/buildkit/version" "github.com/sirupsen/logrus" "go.opentelemetry.io/otel/trace" ) @@ -20,6 +21,14 @@ var ( L = logrus.NewEntry(logrus.StandardLogger()) ) +var ( + logWithTraceID = false +) + +func EnableLogWithTraceID(b bool) { + logWithTraceID = b +} + type ( loggerKey struct{} ) @@ -43,11 +52,19 @@ func GetLogger(ctx context.Context) (l *logrus.Entry) { l = L } - if spanContext := trace.SpanFromContext(ctx).SpanContext(); spanContext.IsValid() { - return l.WithFields(logrus.Fields{ - "traceID": spanContext.TraceID(), - "spanID": spanContext.SpanID(), - }) + // earthly-specific + l = l.WithFields(logrus.Fields{ + "version": version.Version, + "revision": version.Revision, + }) + + if logWithTraceID { + if spanContext := trace.SpanFromContext(ctx).SpanContext(); spanContext.IsValid() { + return l.WithFields(logrus.Fields{ + "traceID": spanContext.TraceID(), + "spanID": spanContext.SpanID(), + }) + } } return l diff --git a/util/entitlements/security/security_linux.go b/util/entitlements/security/security_linux.go index 05ecde98a..ef304b587 100644 --- a/util/entitlements/security/security_linux.go +++ b/util/entitlements/security/security_linux.go @@ -37,12 +37,6 @@ func WithInsecureSpec() oci.SpecOpts { s.Linux.Resources.Devices = []specs.LinuxDeviceCgroup{ { Allow: true, - Type: "c", - Access: "rwm", - }, - { - Allow: true, - Type: "b", Access: "rwm", }, } diff --git a/util/gitutil/git_cli.go b/util/gitutil/git_cli.go index ebfe5fd7b..6fa075a1d 100644 --- a/util/gitutil/git_cli.go +++ b/util/gitutil/git_cli.go @@ -3,15 +3,33 @@ package gitutil import ( "bytes" "context" + "encoding/base64" + "fmt" "io" "os" "os/exec" "slices" "strings" + "github.com/moby/buildkit/util/bklog" + "github.com/moby/buildkit/util/urlutil" "github.com/pkg/errors" ) +type ctxEarthlyDebugLevelKey string // earthly-specific + +// earthlyCtxDebugLevelKey is earthly-specific and is used to pass along the debug level +const EarthlyCtxDebugLevelKey ctxEarthlyDebugLevelKey = "EARTHLY_DEBUG_LEVEL" // earthly-specific + +// GitLogLevel is earthly-specific +type GitLogLevel int + +const ( + GitLogLevelDefault GitLogLevel = iota + GitLogLevelDebug + GitLogLevelTrace +) + // GitCLI carries config to pass to the git cli to make running multiple // commands less repetitive. type GitCLI struct { @@ -27,6 +45,7 @@ type GitCLI struct { sshAuthSock string sshKnownHosts string + sshCommand string hostGitConfig bool } @@ -98,6 +117,13 @@ func WithSSHKnownHosts(sshKnownHosts string) Option { } } +// WithSSHCommand sets the GIT_SSH_COMMAND environment variable +func WithSSHCommand(sshCommand string) Option { + return func(b *GitCLI) { + b.sshCommand = sshCommand + } +} + // WithHostGitConfig allows git to read the host system and user git config. // This is intended for client-side local git inspection. The default remains // isolated so daemon-side callers do not leak host configuration into git. @@ -139,6 +165,10 @@ func (cli *GitCLI) New(opts ...Option) *GitCLI { return &clone } +func gitDebug() bool { + return os.Getenv("BUILDKIT_DEBUG_GIT") == "1" +} + // Run executes a git command with the given args. func (cli *GitCLI) Run(ctx context.Context, args ...string) (_ []byte, err error) { gitBinary := "git" @@ -150,6 +180,15 @@ func (cli *GitCLI) Run(ctx context.Context, args ...string) (_ []byte, err error "http_proxy", "https_proxy", "no_proxy", "all_proxy", } + // earthly-specific: smuggle in the logLevel + logLevel, ok := ctx.Value(EarthlyCtxDebugLevelKey).(GitLogLevel) + if !ok { + bklog.G(ctx).Warnf("failed to extract %s", EarthlyCtxDebugLevelKey) + } + if gitDebug() || logLevel >= GitLogLevelDebug { // earthly-specific + bklog.G(ctx).Infof("GitCLI.Run called with %v", args) + } + for { var cmd *exec.Cmd if cli.exec == nil { @@ -198,37 +237,34 @@ func (cli *GitCLI) Run(ctx context.Context, args ...string) (_ []byte, err error cmd.Env = []string{ "PATH=" + os.Getenv("PATH"), + + // earthly-specific settings + "HOME=" + os.Getenv("HOME"), // earthly-specific: we need this for git to read /root/.gitconfig + "GIT_LFS_SKIP_SMUDGE=1", // earthly-specific: dont automatically pull large files + "GIT_TERMINAL_PROMPT=0", - "GIT_SSH_COMMAND=" + getGitSSHCommand(cli.sshKnownHosts), - // "GIT_TRACE=1", + "GIT_SSH_COMMAND=" + getGitSSHCommand(cli.sshKnownHosts, logLevel, cli.sshCommand), + // "GIT_TRACE=1", + // earthly-specific: Commented out. We do not want to disable reading from gitconfig. + // "GIT_CONFIG_NOSYSTEM=1", // Disable reading from system gitconfig. + // "HOME=/dev/null", // Disable reading from user gitconfig. + "LC_ALL=C", // Ensure consistent output. } - if cli.hostGitConfig { - for _, ev := range [...]string{ - "HOME", - "XDG_CONFIG_HOME", - "USERPROFILE", - "HOMEDRIVE", - "HOMEPATH", - "GIT_CONFIG_GLOBAL", - "GIT_CONFIG_SYSTEM", - } { - if v, ok := os.LookupEnv(ev); ok { - cmd.Env = append(cmd.Env, ev+"="+v) - } - } - } else { - cmd.Env = append(cmd.Env, - "GIT_CONFIG_NOSYSTEM=1", // Disable reading from system gitconfig. - "HOME="+os.DevNull, // Disable reading from user gitconfig. - "GIT_CONFIG_GLOBAL="+os.DevNull, // Disable reading from global gitconfig. - ) - } + // Note: earthly-specific - we skip the hostGitConfig isolation logic + // because earthly always needs HOME set for git to read /root/.gitconfig. + // The upstream hostGitConfig option is still available for callers that need it. for _, ev := range proxyEnvVars { if v, ok := os.LookupEnv(ev); ok { cmd.Env = append(cmd.Env, ev+"="+v) } } + + // earthly-specific + if logLevel >= GitLogLevelTrace { + cmd.Env = append(cmd.Env, "GIT_TRACE=1") + } + if cli.sshAuthSock != "" { cmd.Env = append(cmd.Env, "SSH_AUTH_SOCK="+cli.sshAuthSock) } @@ -266,19 +302,36 @@ func (cli *GitCLI) Run(ctx context.Context, args ...string) (_ []byte, err error } } - return buf.Bytes(), errors.Wrapf(err, "git stderr:\n%s", errbuf.String()) + // earthly-specific + if gitDebug() { + bklog.G(ctx).Infof("knownHosts: %s", cli.sshKnownHosts) + bklog.G(ctx).Infof("git stdout: %s", buf.String()) + bklog.G(ctx).Infof("git stderr: %s", errbuf.String()) + } + redactedStderr := urlutil.RedactAllCredentials(fmt.Sprintf("git %s\n%s", strings.Join(args, " "), errbuf.String())) + err = errors.Wrapf(err, "git stderr:\n%s\nEARTHLY_GIT_STDERR: %s", redactedStderr, base64.StdEncoding.EncodeToString([]byte(redactedStderr))) // earthly-specific + return buf.Bytes(), err } + return buf.Bytes(), nil } } -func getGitSSHCommand(knownHosts string) string { +func getGitSSHCommand(knownHosts string, logLevel GitLogLevel, existingSSHCommand string) string { gitSSHCommand := "ssh -F " + os.DevNull + if existingSSHCommand != "" { + gitSSHCommand = existingSSHCommand // earthly-specific + } if knownHosts != "" { gitSSHCommand += " -o UserKnownHostsFile=" + knownHosts } else { gitSSHCommand += " -o StrictHostKeyChecking=no" } + if gitDebug() || logLevel >= GitLogLevelTrace { + gitSSHCommand += " -vvvv" + } else if logLevel >= GitLogLevelDebug { + gitSSHCommand += " -v" + } return gitSSHCommand } diff --git a/util/gitutil/git_cli_test.go b/util/gitutil/git_cli_test.go index 23d3b98b7..5ec5b2ee7 100644 --- a/util/gitutil/git_cli_test.go +++ b/util/gitutil/git_cli_test.go @@ -10,10 +10,10 @@ import ( ) func TestGetGitSSHCommandUsesConfigPath(t *testing.T) { - cmd := getGitSSHCommand("") + cmd := getGitSSHCommand("", GitLogLevelDefault, "") require.Equal(t, "ssh -F "+os.DevNull+" -o StrictHostKeyChecking=no", cmd) - cmd = getGitSSHCommand("/known-hosts") + cmd = getGitSSHCommand("/known-hosts", GitLogLevelDefault, "") require.Equal(t, "ssh -F "+os.DevNull+" -o UserKnownHostsFile=/known-hosts", cmd) } @@ -26,6 +26,9 @@ func TestGitCLIConfigEnv(t *testing.T) { t.Setenv("GIT_CONFIG_GLOBAL", "/tmp/global-gitconfig") t.Setenv("GIT_CONFIG_SYSTEM", "/tmp/system-gitconfig") + // earthly-specific: earthly's fork does NOT isolate git config by default. + // HOME is always passed through so git can read /root/.gitconfig, + // and GIT_CONFIG_NOSYSTEM is not set. t.Run("isolated by default", func(t *testing.T) { var got []string cli := NewGitCLI(WithExec(func(ctx context.Context, cmd *exec.Cmd) error { @@ -34,13 +37,9 @@ func TestGitCLIConfigEnv(t *testing.T) { })) _, err := cli.Run(context.Background(), "status") require.NoError(t, err) - require.Contains(t, got, "GIT_CONFIG_NOSYSTEM=1") - require.Contains(t, got, "HOME="+os.DevNull) - require.Contains(t, got, "GIT_CONFIG_GLOBAL="+os.DevNull) - require.NotContains(t, got, "HOME=/tmp/home") - require.NotContains(t, got, "XDG_CONFIG_HOME=/tmp/xdg") - require.NotContains(t, got, "GIT_CONFIG_GLOBAL=/tmp/global-gitconfig") - require.NotContains(t, got, "GIT_CONFIG_SYSTEM=/tmp/system-gitconfig") + // earthly-specific: no isolation — HOME is real, no GIT_CONFIG_NOSYSTEM + require.NotContains(t, got, "GIT_CONFIG_NOSYSTEM=1") + require.Contains(t, got, "HOME=/tmp/home") }) t.Run("host git config opt-in", func(t *testing.T) { @@ -54,15 +53,8 @@ func TestGitCLIConfigEnv(t *testing.T) { ) _, err := cli.Run(context.Background(), "status") require.NoError(t, err) + // earthly-specific: same as default — HOME is always passed through require.NotContains(t, got, "GIT_CONFIG_NOSYSTEM=1") - require.NotContains(t, got, "HOME="+os.DevNull) - require.NotContains(t, got, "GIT_CONFIG_GLOBAL="+os.DevNull) require.Contains(t, got, "HOME=/tmp/home") - require.Contains(t, got, "XDG_CONFIG_HOME=/tmp/xdg") - require.Contains(t, got, `USERPROFILE=C:\Users\tester`) - require.Contains(t, got, "HOMEDRIVE=C:") - require.Contains(t, got, `HOMEPATH=\Users\tester`) - require.Contains(t, got, "GIT_CONFIG_GLOBAL=/tmp/global-gitconfig") - require.Contains(t, got, "GIT_CONFIG_SYSTEM=/tmp/system-gitconfig") }) } diff --git a/util/progress/filtered.go b/util/progress/filtered.go new file mode 100644 index 000000000..a7fe2942d --- /dev/null +++ b/util/progress/filtered.go @@ -0,0 +1,40 @@ +package progress + +import ( + "context" +) + +// this entire file is earthly-specific, it implements a filtered Reader which can be used to skip statsStream data + +type FilteredReaderSkipFn func(ctx context.Context, p *Progress) (bool, error) + +type filteredReader struct { + r Reader + skipFn FilteredReaderSkipFn +} + +func NewFilteredReader(r Reader, skipFn FilteredReaderSkipFn) Reader { + return &filteredReader{ + r: r, + skipFn: skipFn, + } +} + +func (fr *filteredReader) Read(ctx context.Context) ([]*Progress, error) { + progress, err := fr.r.Read(ctx) + if err != nil { + return nil, err + } + filteredProgress := []*Progress{} + for _, p := range progress { + shouldSkip, err := fr.skipFn(ctx, p) + if err != nil { + return nil, err + } + if shouldSkip { + continue + } + filteredProgress = append(filteredProgress, p) + } + return filteredProgress, nil +} diff --git a/util/progress/logs/logs.go b/util/progress/logs/logs.go index 36bf32bf9..dc020ae53 100644 --- a/util/progress/logs/logs.go +++ b/util/progress/logs/logs.go @@ -23,12 +23,21 @@ var defaultMaxLogSize = 2 * 1024 * 1024 var defaultMaxLogSpeed = 200 * 1024 // per second const ( - stdout = 1 - stderr = 2 + stdout = 1 + stderr = 2 + StatsStream = 99 // earthly-specific ) var configCheckOnce sync.Once +// NewStatsStreams is earthly-specific +func NewStatsStreams(ctx context.Context, printOutput bool) (io.WriteCloser, func()) { + statsStream := newStreamWriter(ctx, StatsStream, printOutput) + return statsStream, func() { + statsStream.flushBuffer() + } +} + func NewLogStreams(ctx context.Context, printOutput bool) (io.WriteCloser, io.WriteCloser, func()) { stdout := newStreamWriter(ctx, stdout, printOutput) stderr := newStreamWriter(ctx, stderr, printOutput) @@ -148,6 +157,8 @@ func (sw *streamWriter) write(dt []byte) (int, error) { return os.Stdout.Write(dt) case 2: return os.Stderr.Write(dt) + case StatsStream: // earthly-specific + return os.Stderr.Write(dt) default: return 0, errors.Errorf("invalid stream %d", sw.stream) } diff --git a/util/resolver/authorizer.go b/util/resolver/authorizer.go index 4a62e5b01..bafb9e2c0 100644 --- a/util/resolver/authorizer.go +++ b/util/resolver/authorizer.go @@ -334,6 +334,12 @@ func (ah *authFetcher) fetchToken(ctx context.Context, sm *session.Manager, g se if exp := issuedAt.Add(time.Duration(float64(expires)*0.9) * time.Second); time.Now().Before(exp) { r.expires = exp } + } else if errors.Is(err, context.Canceled) { + // earthly-specific prevent context canceled errors from being permanent + r = &authResult{ + expires: time.Now().Add(-time.Hour), + } + err = nil } }() diff --git a/util/resolver/resolver.go b/util/resolver/resolver.go index a8c1c30db..9fb2bc268 100644 --- a/util/resolver/resolver.go +++ b/util/resolver/resolver.go @@ -205,10 +205,11 @@ func newDefaultTransport() *http.Transport { KeepAlive: 60 * time.Second, }).DialContext, MaxIdleConns: 30, - IdleConnTimeout: 120 * time.Second, + IdleConnTimeout: 5 * time.Second, // earthly needs this to stay low MaxIdleConnsPerHost: 4, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 5 * time.Second, + ResponseHeaderTimeout: 30 * time.Second, // earthly needs this; it was reverted by https://github.com/moby/buildkit/pull/3995 but if it's removed we will regress on reusing a connection that's become invalid/corrupt/bad due to hibernation. If we find a better way to detect those dead connections, we can remove that, however the IdleConnTimeout doesn't invalid it. TLSNextProto: make(map[string]func(authority string, c *tls.Conn) http.RoundTripper), } } diff --git a/util/semutil/weighted.go b/util/semutil/weighted.go new file mode 100644 index 000000000..c9baec7ea --- /dev/null +++ b/util/semutil/weighted.go @@ -0,0 +1,150 @@ +// Package semutil provides a weighted semaphore implementation. This is based +// on the Golang library at https://cs.opensource.google/go/x/sync/+/0de741cf:semaphore/semaphore.go;l=28 +// with the exception that this implementation exposes the number of workers and +// the number of waiters. +package semutil + +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +import ( + "container/list" + "context" + "sync" +) + +type waiter struct { + n int64 + ready chan<- struct{} // Closed when semaphore acquired. +} + +// NewWeighted creates a new weighted semaphore with the given +// maximum combined weight for concurrent access. +func NewWeighted(n int64) *Weighted { + w := &Weighted{size: n} + return w +} + +// Weighted provides a way to bound concurrent access to a resource. +// The callers can request access with a given weight. +type Weighted struct { + size int64 + cur int64 + mu sync.Mutex + waiters list.List +} + +// Status returns the status of the semaphore as a triplet of +// weight taken (in-use/blocked), total weight available, and number of waiters. +func (s *Weighted) Status() (taken int64, size int64, waiters int64) { + if s == nil { + return 0, 0, 0 + } + s.mu.Lock() + defer s.mu.Unlock() + return s.cur, s.size, int64(s.waiters.Len()) +} + +// Acquire acquires the semaphore with a weight of n, blocking until resources +// are available or ctx is done. On success, returns nil. On failure, returns +// ctx.Err() and leaves the semaphore unchanged. +// +// If ctx is already done, Acquire may still succeed without blocking. +func (s *Weighted) Acquire(ctx context.Context, n int64) error { + s.mu.Lock() + if s.size-s.cur >= n && s.waiters.Len() == 0 { + s.cur += n + s.mu.Unlock() + return nil + } + + if n > s.size { + // Don't make other Acquire calls block on one that's doomed to fail. + s.mu.Unlock() + <-ctx.Done() + return context.Cause(ctx) + } + + ready := make(chan struct{}) + w := waiter{n: n, ready: ready} + elem := s.waiters.PushBack(w) + s.mu.Unlock() + + select { + case <-ctx.Done(): + err := context.Cause(ctx) + s.mu.Lock() + select { + case <-ready: + // Acquired the semaphore after we were canceled. Rather than trying to + // fix up the queue, just pretend we didn't notice the cancelation. + err = nil + default: + isFront := s.waiters.Front() == elem + s.waiters.Remove(elem) + // If we're at the front and there're extra tokens left, notify other waiters. + if isFront && s.size > s.cur { + s.notifyWaiters() + } + } + s.mu.Unlock() + return err + + case <-ready: + return nil + } +} + +// TryAcquire acquires the semaphore with a weight of n without blocking. +// On success, returns true. On failure, returns false and leaves the semaphore unchanged. +func (s *Weighted) TryAcquire(n int64) bool { + s.mu.Lock() + success := s.size-s.cur >= n && s.waiters.Len() == 0 + if success { + s.cur += n + } + s.mu.Unlock() + return success +} + +// Release releases the semaphore with a weight of n. +func (s *Weighted) Release(n int64) { + s.mu.Lock() + s.cur -= n + if s.cur < 0 { + s.mu.Unlock() + panic("semaphore: released more than held") + } + s.notifyWaiters() + s.mu.Unlock() +} + +func (s *Weighted) notifyWaiters() { + for { + next := s.waiters.Front() + if next == nil { + break // No more waiters blocked. + } + + w := next.Value.(waiter) + if s.size-s.cur < w.n { + // Not enough tokens for the next waiter. We could keep going (to try to + // find a waiter with a smaller request), but under load that could cause + // starvation for large requests; instead, we leave all remaining waiters + // blocked. + // + // Consider a semaphore used as a read-write lock, with N tokens, N + // readers, and one writer. Each reader can Acquire(1) to obtain a read + // lock. The writer can Acquire(N) to obtain a write lock, excluding all + // of the readers. If we allow the readers to jump ahead in the queue, + // the writer will starve — there is always one token available for every + // reader. + break + } + + s.cur += w.n + s.waiters.Remove(next) + close(w.ready) + } +} diff --git a/util/sshutil/keyscan.go b/util/sshutil/keyscan.go index 163efee80..0c834b312 100644 --- a/util/sshutil/keyscan.go +++ b/util/sshutil/keyscan.go @@ -12,6 +12,8 @@ import ( const defaultPort = 22 +var defaultPortStr = fmt.Sprintf("%d", defaultPort) + var errCallbackDone = errors.New("callback failed on purpose") // addDefaultPort appends a default port if hostport doesn't contain one @@ -25,14 +27,21 @@ func addDefaultPort(hostport string, defaultPort int) string { } // SSHKeyScan scans a ssh server for the hostkey; server should be in the form hostname, or hostname:port +// the returned keyscan should be , or []: for non-standard ports func SSHKeyScan(server string) (string, error) { var key string KeyScanCallback := func(hostport string, remote net.Addr, pubKey ssh.PublicKey) error { - hostname, _, err := net.SplitHostPort(hostport) + hostname, port, err := net.SplitHostPort(hostport) if err != nil { return err } - key = strings.TrimSpace(fmt.Sprintf("%s %s", hostname, string(ssh.MarshalAuthorizedKey(pubKey)))) + var serverID string + if port == defaultPortStr { + serverID = hostname + } else { + serverID = fmt.Sprintf("[%s]:%s", hostname, port) + } + key = strings.TrimSpace(fmt.Sprintf("%s %s", serverID, string(ssh.MarshalAuthorizedKey(pubKey)))) return errCallbackDone } config := &ssh.ClientConfig{ diff --git a/util/urlutil/redact.go b/util/urlutil/redact.go index 385e6aef8..9f4b4931b 100644 --- a/util/urlutil/redact.go +++ b/util/urlutil/redact.go @@ -2,6 +2,8 @@ package urlutil import ( "net/url" + "regexp" + "strings" ) const mask = "xxxxx" @@ -31,3 +33,19 @@ func RedactCredentials(s string) string { } return ru.String() } + +var urlRegexp = regexp.MustCompile(`(https?://\S+)`) + +// RedactAllCredentials is earthly-specific +func RedactAllCredentials(s string) string { + var sb strings.Builder + matches := urlRegexp.FindAllStringIndex(s, -1) + i := 0 + for _, m := range matches { + sb.WriteString(s[i:m[0]]) + sb.WriteString(RedactCredentials(s[m[0]:m[1]])) + i = m[1] + } + sb.WriteString(s[i:]) + return sb.String() +} diff --git a/util/urlutil/redact_test.go b/util/urlutil/redact_test.go index 1b3143c96..e2853e17c 100644 --- a/util/urlutil/redact_test.go +++ b/util/urlutil/redact_test.go @@ -42,3 +42,35 @@ func TestRedactCredentials(t *testing.T) { }) } } + +// TestRedactAllCredentials is earthly specific +func TestRedactAllCredentials(t *testing.T) { + cases := []struct { + name string + url string + want string + }{ + { + name: "zero urls", + url: "one two three", + want: "one two three", + }, + { + name: "two urls", + url: "one two https://user:password@host.tld/this:that three http://abc@earthly.dev four", + want: "one two https://xxxxx:xxxxx@host.tld/this:that three http://xxxxx@earthly.dev four", + }, + { + name: "two urls at edges", + url: "https://user:password@host.tld/this:that middle http://abc@earthly.dev", + want: "https://xxxxx:xxxxx@host.tld/this:that middle http://xxxxx@earthly.dev", + }, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + if g, w := RedactAllCredentials(tt.url), tt.want; g != w { + t.Fatalf("got: %q\nwant: %q", g, w) + } + }) + } +} diff --git a/vendor/github.com/docker/distribution/.dockerignore b/vendor/github.com/docker/distribution/.dockerignore new file mode 100644 index 000000000..e660fd93d --- /dev/null +++ b/vendor/github.com/docker/distribution/.dockerignore @@ -0,0 +1 @@ +bin/ diff --git a/vendor/github.com/docker/distribution/.gitignore b/vendor/github.com/docker/distribution/.gitignore new file mode 100644 index 000000000..4cf7888e9 --- /dev/null +++ b/vendor/github.com/docker/distribution/.gitignore @@ -0,0 +1,38 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof + +# never checkin from the bin file (for now) +bin/* + +# Test key files +*.pem + +# Cover profiles +*.out + +# Editor/IDE specific files. +*.sublime-project +*.sublime-workspace +.idea/* diff --git a/vendor/github.com/docker/distribution/.golangci.yml b/vendor/github.com/docker/distribution/.golangci.yml new file mode 100644 index 000000000..61dd0e00e --- /dev/null +++ b/vendor/github.com/docker/distribution/.golangci.yml @@ -0,0 +1,33 @@ +linters: + enable: + - staticcheck + - unconvert + - gofmt + - goimports + - golint + - ineffassign + - vet + - unused + - misspell + disable: + - errcheck + +linters-settings: + revive: + rules: + # TODO(thaJeztah): temporarily disabled the "unused-parameter" check. + # It produces many warnings, and some of those may need to be looked at. + - name: unused-parameter + disabled: true + +run: + deadline: 2m + skip-dirs: + - vendor + +issues: + exclude-rules: + # io/ioutil is deprecated, but won't be removed until Go v2. It's safe to ignore for the release/2.8 branch. + - text: "SA1019: \"io/ioutil\" has been deprecated since Go 1.16" + linters: + - staticcheck diff --git a/vendor/github.com/docker/distribution/.mailmap b/vendor/github.com/docker/distribution/.mailmap new file mode 100644 index 000000000..d7b832d9e --- /dev/null +++ b/vendor/github.com/docker/distribution/.mailmap @@ -0,0 +1,54 @@ +Stephen J Day Stephen Day +Stephen J Day Stephen Day +Olivier Gambier Olivier Gambier +Brian Bland Brian Bland +Brian Bland Brian Bland +Josh Hawn Josh Hawn +Richard Scothern Richard +Richard Scothern Richard Scothern +Andrew Meredith Andrew Meredith +harche harche +Jessie Frazelle +Sharif Nassar Sharif Nassar +Sven Dowideit Sven Dowideit +Vincent Giersch Vincent Giersch +davidli davidli +Omer Cohen Omer Cohen +Eric Yang Eric Yang +Nikita Tarasov Nikita +Yu Wang yuwaMSFT2 +Yu Wang Yu Wang (UC) +Olivier Gambier dmp +Olivier Gambier Olivier +Olivier Gambier Olivier +Elsan Li 李楠 elsanli(李楠) +Rui Cao ruicao +Gwendolynne Barr gbarr01 +Haibing Zhou 周海兵 zhouhaibing089 +Feng Honglin tifayuki +Helen Xie Helen-xie +Mike Brown Mike Brown +Manish Tomar Manish Tomar +Sakeven Jiang sakeven +Milos Gajdos Milos Gajdos +Derek McGowan Derek McGowa +Adrian Plata Adrian Plata <@users.noreply.github.com> +Sebastiaan van Stijn Sebastiaan van Stijn +Vishesh Jindal Vishesh Jindal +Wang Yan Wang Yan +Chris Patterson Chris Patterson +Eohyung Lee Eohyung Lee +João Pereira <484633+joaodrp@users.noreply.github.com> +Smasherr Smasherr +Thomas Berger Thomas Berger +Samuel Karp Samuel Karp +Justin Cormack +sayboras +CrazyMax <1951866+crazy-max@users.noreply.github.com> +Hayley Swimelar +Jose D. Gomez R +Shengjing Zhu +Silvin Lubecki <31478878+silvin-lubecki@users.noreply.github.com> +James Hewitt +Marcus Pettersen Irgens +Ben Manuel diff --git a/vendor/github.com/docker/distribution/BUILDING.md b/vendor/github.com/docker/distribution/BUILDING.md new file mode 100644 index 000000000..4c43b03cb --- /dev/null +++ b/vendor/github.com/docker/distribution/BUILDING.md @@ -0,0 +1,117 @@ + +# Building the registry source + +## Use-case + +This is useful if you intend to actively work on the registry. + +### Alternatives + +Most people should use the [official Registry docker image](https://hub.docker.com/r/library/registry/). + +People looking for advanced operational use cases might consider rolling their own image with a custom Dockerfile inheriting `FROM registry:2`. + +OS X users who want to run natively can do so following [the instructions here](https://github.com/docker/docker.github.io/blob/master/registry/recipes/osx-setup-guide.md). + +### Gotchas + +You are expected to know your way around with go & git. + +If you are a casual user with no development experience, and no preliminary knowledge of go, building from source is probably not a good solution for you. + +## Build the development environment + +The first prerequisite of properly building distribution targets is to have a Go +development environment setup. Please follow [How to Write Go Code](https://golang.org/doc/code.html) +for proper setup. If done correctly, you should have a GOROOT and GOPATH set in the +environment. + +If a Go development environment is setup, one can use `go get` to install the +`registry` command from the current latest: + + go get github.com/docker/distribution/cmd/registry + +The above will install the source repository into the `GOPATH`. + +Now create the directory for the registry data (this might require you to set permissions properly) + + mkdir -p /var/lib/registry + +... or alternatively `export REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY=/somewhere` if you want to store data into another location. + +The `registry` +binary can then be run with the following: + + $ $GOPATH/bin/registry --version + $GOPATH/bin/registry github.com/docker/distribution v2.0.0-alpha.1+unknown + +> __NOTE:__ While you do not need to use `go get` to checkout the distribution +> project, for these build instructions to work, the project must be checked +> out in the correct location in the `GOPATH`. This should almost always be +> `$GOPATH/src/github.com/docker/distribution`. + +The registry can be run with the default config using the following +incantation: + + $ $GOPATH/bin/registry serve $GOPATH/src/github.com/docker/distribution/cmd/registry/config-example.yml + INFO[0000] endpoint local-5003 disabled, skipping app.id=34bbec38-a91a-494a-9a3f-b72f9010081f version=v2.0.0-alpha.1+unknown + INFO[0000] endpoint local-8083 disabled, skipping app.id=34bbec38-a91a-494a-9a3f-b72f9010081f version=v2.0.0-alpha.1+unknown + INFO[0000] listening on :5000 app.id=34bbec38-a91a-494a-9a3f-b72f9010081f version=v2.0.0-alpha.1+unknown + INFO[0000] debug server listening localhost:5001 + +If it is working, one should see the above log messages. + +### Repeatable Builds + +For the full development experience, one should `cd` into +`$GOPATH/src/github.com/docker/distribution`. From there, the regular `go` +commands, such as `go test`, should work per package (please see +[Developing](#developing) if they don't work). + +A `Makefile` has been provided as a convenience to support repeatable builds. +Please install the following into `GOPATH` for it to work: + + go get github.com/golang/lint/golint + +Once these commands are available in the `GOPATH`, run `make` to get a full +build: + + $ make + + clean + + fmt + + vet + + lint + + build + github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar + github.com/sirupsen/logrus + github.com/docker/libtrust + ... + github.com/yvasiyarov/gorelic + github.com/docker/distribution/registry/handlers + github.com/docker/distribution/cmd/registry + + test + ... + ok github.com/docker/distribution/digest 7.875s + ok github.com/docker/distribution/manifest 0.028s + ok github.com/docker/distribution/notifications 17.322s + ? github.com/docker/distribution/registry [no test files] + ok github.com/docker/distribution/registry/api/v2 0.101s + ? github.com/docker/distribution/registry/auth [no test files] + ok github.com/docker/distribution/registry/auth/silly 0.011s + ... + + /Users/sday/go/src/github.com/docker/distribution/bin/registry + + /Users/sday/go/src/github.com/docker/distribution/bin/registry-api-descriptor-template + + binaries + +The above provides a repeatable build using the contents of the vendor +directory. This includes formatting, vetting, linting, building, +testing and generating tagged binaries. We can verify this worked by running +the registry binary generated in the "./bin" directory: + + $ ./bin/registry --version + ./bin/registry github.com/docker/distribution v2.0.0-alpha.2-80-g16d8b2c.m + +### Optional build tags + +Optional [build tags](http://golang.org/pkg/go/build/) can be provided using +the environment variable `BUILDTAGS`. diff --git a/vendor/github.com/docker/distribution/CONTRIBUTING.md b/vendor/github.com/docker/distribution/CONTRIBUTING.md new file mode 100644 index 000000000..4c067d9e7 --- /dev/null +++ b/vendor/github.com/docker/distribution/CONTRIBUTING.md @@ -0,0 +1,148 @@ +# Contributing to the registry + +## Before reporting an issue... + +### If your problem is with... + + - automated builds + - your account on the [Docker Hub](https://hub.docker.com/) + - any other [Docker Hub](https://hub.docker.com/) issue + +Then please do not report your issue here - you should instead report it to [https://support.docker.com](https://support.docker.com) + +### If you... + + - need help setting up your registry + - can't figure out something + - are not sure what's going on or what your problem is + +Then please do not open an issue here yet - you should first try one of the following support forums: + + - irc: #docker-distribution on freenode + - mailing-list: or https://groups.google.com/a/dockerproject.org/forum/#!forum/distribution + +### Reporting security issues + +The Docker maintainers take security seriously. If you discover a security +issue, please bring it to their attention right away! + +Please **DO NOT** file a public issue, instead send your report privately to +[security@docker.com](mailto:security@docker.com). + +## Reporting an issue properly + +By following these simple rules you will get better and faster feedback on your issue. + + - search the bugtracker for an already reported issue + +### If you found an issue that describes your problem: + + - please read other user comments first, and confirm this is the same issue: a given error condition might be indicative of different problems - you may also find a workaround in the comments + - please refrain from adding "same thing here" or "+1" comments + - you don't need to comment on an issue to get notified of updates: just hit the "subscribe" button + - comment if you have some new, technical and relevant information to add to the case + - __DO NOT__ comment on closed issues or merged PRs. If you think you have a related problem, open up a new issue and reference the PR or issue. + +### If you have not found an existing issue that describes your problem: + + 1. create a new issue, with a succinct title that describes your issue: + - bad title: "It doesn't work with my docker" + - good title: "Private registry push fail: 400 error with E_INVALID_DIGEST" + 2. copy the output of: + - `docker version` + - `docker info` + - `docker exec registry --version` + 3. copy the command line you used to launch your Registry + 4. restart your docker daemon in debug mode (add `-D` to the daemon launch arguments) + 5. reproduce your problem and get your docker daemon logs showing the error + 6. if relevant, copy your registry logs that show the error + 7. provide any relevant detail about your specific Registry configuration (e.g., storage backend used) + 8. indicate if you are using an enterprise proxy, Nginx, or anything else between you and your Registry + +## Contributing a patch for a known bug, or a small correction + +You should follow the basic GitHub workflow: + + 1. fork + 2. commit a change + 3. make sure the tests pass + 4. PR + +Additionally, you must [sign your commits](https://github.com/docker/docker/blob/master/CONTRIBUTING.md#sign-your-work). It's very simple: + + - configure your name with git: `git config user.name "Real Name" && git config user.email mail@example.com` + - sign your commits using `-s`: `git commit -s -m "My commit"` + +Some simple rules to ensure quick merge: + + - clearly point to the issue(s) you want to fix in your PR comment (e.g., `closes #12345`) + - prefer multiple (smaller) PRs addressing individual issues over a big one trying to address multiple issues at once + - if you need to amend your PR following comments, please squash instead of adding more commits + +## Contributing new features + +You are heavily encouraged to first discuss what you want to do. You can do so on the irc channel, or by opening an issue that clearly describes the use case you want to fulfill, or the problem you are trying to solve. + +If this is a major new feature, you should then submit a proposal that describes your technical solution and reasoning. +If you did discuss it first, this will likely be greenlighted very fast. It's advisable to address all feedback on this proposal before starting actual work. + +Then you should submit your implementation, clearly linking to the issue (and possible proposal). + +Your PR will be reviewed by the community, then ultimately by the project maintainers, before being merged. + +It's mandatory to: + + - interact respectfully with other community members and maintainers - more generally, you are expected to abide by the [Docker community rules](https://github.com/docker/docker/blob/master/CONTRIBUTING.md#docker-community-guidelines) + - address maintainers' comments and modify your submission accordingly + - write tests for any new code + +Complying to these simple rules will greatly accelerate the review process, and will ensure you have a pleasant experience in contributing code to the Registry. + +Have a look at a great, successful contribution: the [Swift driver PR](https://github.com/docker/distribution/pull/493) + +## Coding Style + +Unless explicitly stated, we follow all coding guidelines from the Go +community. While some of these standards may seem arbitrary, they somehow seem +to result in a solid, consistent codebase. + +It is possible that the code base does not currently comply with these +guidelines. We are not looking for a massive PR that fixes this, since that +goes against the spirit of the guidelines. All new contributions should make a +best effort to clean up and make the code base better than they left it. +Obviously, apply your best judgement. Remember, the goal here is to make the +code base easier for humans to navigate and understand. Always keep that in +mind when nudging others to comply. + +The rules: + +1. All code should be formatted with `gofmt -s`. +2. All code should pass the default levels of + [`golint`](https://github.com/golang/lint). +3. All code should follow the guidelines covered in [Effective + Go](http://golang.org/doc/effective_go.html) and [Go Code Review + Comments](https://github.com/golang/go/wiki/CodeReviewComments). +4. Comment the code. Tell us the why, the history and the context. +5. Document _all_ declarations and methods, even private ones. Declare + expectations, caveats and anything else that may be important. If a type + gets exported, having the comments already there will ensure it's ready. +6. Variable name length should be proportional to its context and no longer. + `noCommaALongVariableNameLikeThisIsNotMoreClearWhenASimpleCommentWouldDo`. + In practice, short methods will have short variable names and globals will + have longer names. +7. No underscores in package names. If you need a compound name, step back, + and re-examine why you need a compound name. If you still think you need a + compound name, lose the underscore. +8. No utils or helpers packages. If a function is not general enough to + warrant its own package, it has not been written generally enough to be a + part of a util package. Just leave it unexported and well-documented. +9. All tests should run with `go test` and outside tooling should not be + required. No, we don't need another unit testing framework. Assertion + packages are acceptable if they provide _real_ incremental value. +10. Even though we call these "rules" above, they are actually just + guidelines. Since you've read all the rules, you now know that. + +If you are having trouble getting into the mood of idiomatic Go, we recommend +reading through [Effective Go](http://golang.org/doc/effective_go.html). The +[Go Blog](http://blog.golang.org/) is also a great resource. Drinking the +kool-aid is a lot easier than going thirsty. diff --git a/vendor/github.com/docker/distribution/Dockerfile b/vendor/github.com/docker/distribution/Dockerfile new file mode 100644 index 000000000..ebd42c242 --- /dev/null +++ b/vendor/github.com/docker/distribution/Dockerfile @@ -0,0 +1,60 @@ +# syntax=docker/dockerfile:1 + +ARG GO_VERSION=1.20.8 +ARG ALPINE_VERSION=3.18 +ARG XX_VERSION=1.2.1 + +FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx +FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS base +COPY --from=xx / / +RUN apk add --no-cache bash coreutils file git +ENV GO111MODULE=auto +ENV CGO_ENABLED=0 +WORKDIR /go/src/github.com/docker/distribution + +FROM base AS version +ARG PKG="github.com/docker/distribution" +RUN --mount=target=. \ + VERSION=$(git describe --match 'v[0-9]*' --dirty='.m' --always --tags) REVISION=$(git rev-parse HEAD)$(if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi); \ + echo "-X ${PKG}/version.Version=${VERSION#v} -X ${PKG}/version.Revision=${REVISION} -X ${PKG}/version.Package=${PKG}" | tee /tmp/.ldflags; \ + echo -n "${VERSION}" | tee /tmp/.version; + +FROM base AS build +ARG TARGETPLATFORM +ARG LDFLAGS="-s -w" +ARG BUILDTAGS="include_oss,include_gcs" +RUN --mount=type=bind,target=/go/src/github.com/docker/distribution,rw \ + --mount=type=cache,target=/root/.cache/go-build \ + --mount=target=/go/pkg/mod,type=cache \ + --mount=type=bind,source=/tmp/.ldflags,target=/tmp/.ldflags,from=version \ + set -x ; xx-go build -tags "${BUILDTAGS}" -trimpath -ldflags "$(cat /tmp/.ldflags) ${LDFLAGS}" -o /usr/bin/registry ./cmd/registry \ + && xx-verify --static /usr/bin/registry + +FROM scratch AS binary +COPY --from=build /usr/bin/registry / + +FROM base AS releaser +ARG TARGETOS +ARG TARGETARCH +ARG TARGETVARIANT +WORKDIR /work +RUN --mount=from=binary,target=/build \ + --mount=type=bind,target=/src \ + --mount=type=bind,source=/tmp/.version,target=/tmp/.version,from=version \ + VERSION=$(cat /tmp/.version) \ + && mkdir -p /out \ + && cp /build/registry /src/README.md /src/LICENSE . \ + && tar -czvf "/out/registry_${VERSION#v}_${TARGETOS}_${TARGETARCH}${TARGETVARIANT}.tar.gz" * \ + && sha256sum -z "/out/registry_${VERSION#v}_${TARGETOS}_${TARGETARCH}${TARGETVARIANT}.tar.gz" | awk '{ print $1 }' > "/out/registry_${VERSION#v}_${TARGETOS}_${TARGETARCH}${TARGETVARIANT}.tar.gz.sha256" + +FROM scratch AS artifact +COPY --from=releaser /out / + +FROM alpine:${ALPINE_VERSION} +RUN apk add --no-cache ca-certificates +COPY cmd/registry/config-dev.yml /etc/docker/registry/config.yml +COPY --from=binary /registry /bin/registry +VOLUME ["/var/lib/registry"] +EXPOSE 5000 +ENTRYPOINT ["registry"] +CMD ["serve", "/etc/docker/registry/config.yml"] diff --git a/vendor/github.com/docker/distribution/LICENSE b/vendor/github.com/docker/distribution/LICENSE new file mode 100644 index 000000000..e06d20818 --- /dev/null +++ b/vendor/github.com/docker/distribution/LICENSE @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/vendor/github.com/docker/distribution/MAINTAINERS b/vendor/github.com/docker/distribution/MAINTAINERS new file mode 100644 index 000000000..3183620c5 --- /dev/null +++ b/vendor/github.com/docker/distribution/MAINTAINERS @@ -0,0 +1,243 @@ +# Distribution maintainers file +# +# This file describes who runs the docker/distribution project and how. +# This is a living document - if you see something out of date or missing, speak up! +# +# It is structured to be consumable by both humans and programs. +# To extract its contents programmatically, use any TOML-compliant parser. +# + +[Rules] + + [Rules.maintainers] + + title = "What is a maintainer?" + + text = """ +There are different types of maintainers, with different responsibilities, but +all maintainers have 3 things in common: + +1) They share responsibility in the project's success. +2) They have made a long-term, recurring time investment to improve the project. +3) They spend that time doing whatever needs to be done, not necessarily what +is the most interesting or fun. + +Maintainers are often under-appreciated, because their work is harder to appreciate. +It's easy to appreciate a really cool and technically advanced feature. It's harder +to appreciate the absence of bugs, the slow but steady improvement in stability, +or the reliability of a release process. But those things distinguish a good +project from a great one. +""" + + [Rules.reviewer] + + title = "What is a reviewer?" + + text = """ +A reviewer is a core role within the project. +They share in reviewing issues and pull requests and their LGTM count towards the +required LGTM count to merge a code change into the project. + +Reviewers are part of the organization but do not have write access. +Becoming a reviewer is a core aspect in the journey to becoming a maintainer. +""" + + [Rules.adding-maintainers] + + title = "How are maintainers added?" + + text = """ +Maintainers are first and foremost contributors that have shown they are +committed to the long term success of a project. Contributors wanting to become +maintainers are expected to be deeply involved in contributing code, pull +request review, and triage of issues in the project for more than three months. + +Just contributing does not make you a maintainer, it is about building trust +with the current maintainers of the project and being a person that they can +depend on and trust to make decisions in the best interest of the project. + +Periodically, the existing maintainers curate a list of contributors that have +shown regular activity on the project over the prior months. From this list, +maintainer candidates are selected and proposed on the maintainers mailing list. + +After a candidate has been announced on the maintainers mailing list, the +existing maintainers are given five business days to discuss the candidate, +raise objections and cast their vote. Candidates must be approved by at least 66% of the current maintainers by adding their vote on the mailing +list. Only maintainers of the repository that the candidate is proposed for are +allowed to vote. + +If a candidate is approved, a maintainer will contact the candidate to invite +the candidate to open a pull request that adds the contributor to the +MAINTAINERS file. The candidate becomes a maintainer once the pull request is +merged. +""" + + [Rules.stepping-down-policy] + + title = "Stepping down policy" + + text = """ +Life priorities, interests, and passions can change. If you're a maintainer but +feel you must remove yourself from the list, inform other maintainers that you +intend to step down, and if possible, help find someone to pick up your work. +At the very least, ensure your work can be continued where you left off. + +After you've informed other maintainers, create a pull request to remove +yourself from the MAINTAINERS file. +""" + + [Rules.inactive-maintainers] + + title = "Removal of inactive maintainers" + + text = """ +Similar to the procedure for adding new maintainers, existing maintainers can +be removed from the list if they do not show significant activity on the +project. Periodically, the maintainers review the list of maintainers and their +activity over the last three months. + +If a maintainer has shown insufficient activity over this period, a neutral +person will contact the maintainer to ask if they want to continue being +a maintainer. If the maintainer decides to step down as a maintainer, they +open a pull request to be removed from the MAINTAINERS file. + +If the maintainer wants to remain a maintainer, but is unable to perform the +required duties they can be removed with a vote of at least 66% of +the current maintainers. An e-mail is sent to the +mailing list, inviting maintainers of the project to vote. The voting period is +five business days. Issues related to a maintainer's performance should be +discussed with them among the other maintainers so that they are not surprised +by a pull request removing them. +""" + + [Rules.decisions] + + title = "How are decisions made?" + + text = """ +Short answer: EVERYTHING IS A PULL REQUEST. + +distribution is an open-source project with an open design philosophy. This means +that the repository is the source of truth for EVERY aspect of the project, +including its philosophy, design, road map, and APIs. *If it's part of the +project, it's in the repo. If it's in the repo, it's part of the project.* + +As a result, all decisions can be expressed as changes to the repository. An +implementation change is a change to the source code. An API change is a change +to the API specification. A philosophy change is a change to the philosophy +manifesto, and so on. + +All decisions affecting distribution, big and small, follow the same 3 steps: + +* Step 1: Open a pull request. Anyone can do this. + +* Step 2: Discuss the pull request. Anyone can do this. + +* Step 3: Merge or refuse the pull request. Who does this depends on the nature +of the pull request and which areas of the project it affects. +""" + + [Rules.DCO] + + title = "Helping contributors with the DCO" + + text = """ +The [DCO or `Sign your work`]( +https://github.com/moby/moby/blob/master/CONTRIBUTING.md#sign-your-work) +requirement is not intended as a roadblock or speed bump. + +Some distribution contributors are not as familiar with `git`, or have used a web +based editor, and thus asking them to `git commit --amend -s` is not the best +way forward. + +In this case, maintainers can update the commits based on clause (c) of the DCO. +The most trivial way for a contributor to allow the maintainer to do this, is to +add a DCO signature in a pull requests's comment, or a maintainer can simply +note that the change is sufficiently trivial that it does not substantially +change the existing contribution - i.e., a spelling change. + +When you add someone's DCO, please also add your own to keep a log. +""" + + [Rules."no direct push"] + + title = "I'm a maintainer. Should I make pull requests too?" + + text = """ +Yes. Nobody should ever push to master directly. All changes should be +made through a pull request. +""" + + [Rules.tsc] + + title = "Conflict Resolution and technical disputes" + + text = """ +distribution defers to the [Technical Steering Committee](https://github.com/moby/tsc) for escalations and resolution on disputes for technical matters." + """ + + [Rules.meta] + + title = "How is this process changed?" + + text = "Just like everything else: by making a pull request :)" + +# Current project organization +[Org] + + [Org.Maintainers] + people = [ + "dmcgowan", + "dmp42", + "stevvooe", + ] + [Org.Reviewers] + people = [ + "manishtomar", + "caervs", + "davidswu", + "RobbKistler" + ] + +[people] + +# A reference list of all people associated with the project. +# All other sections should refer to people by their canonical key +# in the people section. + + # ADD YOURSELF HERE IN ALPHABETICAL ORDER + + [people.caervs] + Name = "Ryan Abrams" + Email = "rdabrams@gmail.com" + GitHub = "caervs" + + [people.davidswu] + Name = "David Wu" + Email = "dwu7401@gmail.com" + GitHub = "davidswu" + + [people.dmcgowan] + Name = "Derek McGowan" + Email = "derek@mcgstyle.net" + GitHub = "dmcgowan" + + [people.dmp42] + Name = "Olivier Gambier" + Email = "olivier@docker.com" + GitHub = "dmp42" + + [people.manishtomar] + Name = "Manish Tomar" + Email = "manish.tomar@docker.com" + GitHub = "manishtomar" + + [people.RobbKistler] + Name = "Robb Kistler" + Email = "robb.kistler@docker.com" + GitHub = "RobbKistler" + + [people.stevvooe] + Name = "Stephen Day" + Email = "stephen.day@docker.com" + GitHub = "stevvooe" diff --git a/vendor/github.com/docker/distribution/Makefile b/vendor/github.com/docker/distribution/Makefile new file mode 100644 index 000000000..dcdbcb547 --- /dev/null +++ b/vendor/github.com/docker/distribution/Makefile @@ -0,0 +1,102 @@ +# Root directory of the project (absolute path). +ROOTDIR=$(dir $(abspath $(lastword $(MAKEFILE_LIST)))) + +# Used to populate version variable in main package. +VERSION=$(shell git describe --match 'v[0-9]*' --dirty='.m' --always) +REVISION=$(shell git rev-parse HEAD)$(shell if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi) + + +PKG=github.com/docker/distribution + +# Project packages. +PACKAGES=$(shell go list -tags "${BUILDTAGS}" ./... | grep -v /vendor/) +INTEGRATION_PACKAGE=${PKG} +COVERAGE_PACKAGES=$(filter-out ${PKG}/registry/storage/driver/%,${PACKAGES}) + + +# Project binaries. +COMMANDS=registry digest registry-api-descriptor-template + +# Allow turning off function inlining and variable registerization +ifeq (${DISABLE_OPTIMIZATION},true) + GO_GCFLAGS=-gcflags "-N -l" + VERSION:="$(VERSION)-noopt" +endif + +WHALE = "+" + +# Go files +# +TESTFLAGS_RACE= +GOFILES=$(shell find . -type f -name '*.go') +GO_TAGS=$(if $(BUILDTAGS),-tags "$(BUILDTAGS)",) +GO_LDFLAGS=-ldflags '-s -w -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -X $(PKG)/version.Package=$(PKG) $(EXTRA_LDFLAGS)' + +BINARIES=$(addprefix bin/,$(COMMANDS)) + +# Flags passed to `go test` +TESTFLAGS ?= -v $(TESTFLAGS_RACE) +TESTFLAGS_PARALLEL ?= 8 + +.PHONY: all build binaries check clean test test-race test-full integration coverage +.DEFAULT: all + +all: binaries + +# This only needs to be generated by hand when cutting full releases. +version/version.go: + @echo "$(WHALE) $@" + ./version/version.sh > $@ + +check: ## run all linters (TODO: enable "unused", "varcheck", "ineffassign", "unconvert", "staticheck", "goimports", "structcheck") + @echo "$(WHALE) $@" + @GO111MODULE=off golangci-lint --build-tags "${BUILDTAGS}" run + +test: ## run tests, except integration test with test.short + @echo "$(WHALE) $@" + @go test ${GO_TAGS} -test.short ${TESTFLAGS} $(filter-out ${INTEGRATION_PACKAGE},${PACKAGES}) + +test-race: ## run tests, except integration test with test.short and race + @echo "$(WHALE) $@" + @go test ${GO_TAGS} -race -test.short ${TESTFLAGS} $(filter-out ${INTEGRATION_PACKAGE},${PACKAGES}) + +test-full: ## run tests, except integration tests + @echo "$(WHALE) $@" + @go test ${GO_TAGS} ${TESTFLAGS} $(filter-out ${INTEGRATION_PACKAGE},${PACKAGES}) + +integration: ## run integration tests + @echo "$(WHALE) $@" + @go test ${TESTFLAGS} -parallel ${TESTFLAGS_PARALLEL} ${INTEGRATION_PACKAGE} + +coverage: ## generate coverprofiles from the unit tests + @echo "$(WHALE) $@" + @rm -f coverage.txt + @go test ${GO_TAGS} -i ${TESTFLAGS} $(filter-out ${INTEGRATION_PACKAGE},${COVERAGE_PACKAGES}) 2> /dev/null + @( for pkg in $(filter-out ${INTEGRATION_PACKAGE},${COVERAGE_PACKAGES}); do \ + go test ${GO_TAGS} ${TESTFLAGS} \ + -cover \ + -coverprofile=profile.out \ + -covermode=atomic $$pkg || exit; \ + if [ -f profile.out ]; then \ + cat profile.out >> coverage.txt; \ + rm profile.out; \ + fi; \ + done ) + +FORCE: + +# Build a binary from a cmd. +bin/%: cmd/% FORCE + @echo "$(WHALE) $@${BINARY_SUFFIX}" + @go build ${GO_GCFLAGS} ${GO_BUILD_FLAGS} -o $@${BINARY_SUFFIX} ${GO_LDFLAGS} ${GO_TAGS} ./$< + +binaries: $(BINARIES) ## build binaries + @echo "$(WHALE) $@" + +build: + @echo "$(WHALE) $@" + @go build ${GO_GCFLAGS} ${GO_BUILD_FLAGS} ${GO_LDFLAGS} ${GO_TAGS} $(PACKAGES) + +clean: ## clean up binaries + @echo "$(WHALE) $@" + @rm -f $(BINARIES) diff --git a/vendor/github.com/docker/distribution/README.md b/vendor/github.com/docker/distribution/README.md new file mode 100644 index 000000000..e513c18e9 --- /dev/null +++ b/vendor/github.com/docker/distribution/README.md @@ -0,0 +1,130 @@ +# Distribution + +The Docker toolset to pack, ship, store, and deliver content. + +This repository provides the Docker Registry 2.0 implementation +for storing and distributing Docker images. It supersedes the +[docker/docker-registry](https://github.com/docker/docker-registry) +project with a new API design, focused around security and performance. + + + +[![Circle CI](https://circleci.com/gh/docker/distribution/tree/master.svg?style=svg)](https://circleci.com/gh/docker/distribution/tree/master) +[![GoDoc](https://godoc.org/github.com/docker/distribution?status.svg)](https://godoc.org/github.com/docker/distribution) + +This repository contains the following components: + +|**Component** |Description | +|--------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **registry** | An implementation of the [Docker Registry HTTP API V2](docs/spec/api.md) for use with docker 1.6+. | +| **libraries** | A rich set of libraries for interacting with distribution components. Please see [godoc](https://godoc.org/github.com/docker/distribution) for details. **Note**: These libraries are **unstable**. | +| **specifications** | _Distribution_ related specifications are available in [docs/spec](docs/spec) | +| **documentation** | Docker's full documentation set is available at [docs.docker.com](https://docs.docker.com). This repository [contains the subset](docs/) related just to the registry. | + +### How does this integrate with Docker engine? + +This project should provide an implementation to a V2 API for use in the [Docker +core project](https://github.com/docker/docker). The API should be embeddable +and simplify the process of securely pulling and pushing content from `docker` +daemons. + +### What are the long term goals of the Distribution project? + +The _Distribution_ project has the further long term goal of providing a +secure tool chain for distributing content. The specifications, APIs and tools +should be as useful with Docker as they are without. + +Our goal is to design a professional grade and extensible content distribution +system that allow users to: + +* Enjoy an efficient, secured and reliable way to store, manage, package and + exchange content +* Hack/roll their own on top of healthy open-source components +* Implement their own home made solution through good specs, and solid + extensions mechanism. + +## More about Registry 2.0 + +The new registry implementation provides the following benefits: + +- faster push and pull +- new, more efficient implementation +- simplified deployment +- pluggable storage backend +- webhook notifications + +For information on upcoming functionality, please see [ROADMAP.md](ROADMAP.md). + +### Who needs to deploy a registry? + +By default, Docker users pull images from Docker's public registry instance. +[Installing Docker](https://docs.docker.com/engine/installation/) gives users this +ability. Users can also push images to a repository on Docker's public registry, +if they have a [Docker Hub](https://hub.docker.com/) account. + +For some users and even companies, this default behavior is sufficient. For +others, it is not. + +For example, users with their own software products may want to maintain a +registry for private, company images. Also, you may wish to deploy your own +image repository for images used to test or in continuous integration. For these +use cases and others, [deploying your own registry instance](https://github.com/docker/docker.github.io/blob/master/registry/deploying.md) +may be the better choice. + +### Migration to Registry 2.0 + +For those who have previously deployed their own registry based on the Registry +1.0 implementation and wish to deploy a Registry 2.0 while retaining images, +data migration is required. A tool to assist with migration efforts has been +created. For more information see [docker/migrator](https://github.com/docker/migrator). + +## Contribute + +Please see [CONTRIBUTING.md](CONTRIBUTING.md) for details on how to contribute +issues, fixes, and patches to this project. If you are contributing code, see +the instructions for [building a development environment](BUILDING.md). + +## Support + +If any issues are encountered while using the _Distribution_ project, several +avenues are available for support: + + + + + + + + + + + + + + + + + + +
+ IRC + + #docker-distribution on FreeNode +
+ Issue Tracker + + github.com/docker/distribution/issues +
+ Google Groups + + https://groups.google.com/a/dockerproject.org/forum/#!forum/distribution +
+ Mailing List + + docker@dockerproject.org +
+ + +## License + +This project is distributed under [Apache License, Version 2.0](LICENSE). diff --git a/vendor/github.com/docker/distribution/ROADMAP.md b/vendor/github.com/docker/distribution/ROADMAP.md new file mode 100644 index 000000000..701127afe --- /dev/null +++ b/vendor/github.com/docker/distribution/ROADMAP.md @@ -0,0 +1,267 @@ +# Roadmap + +The Distribution Project consists of several components, some of which are +still being defined. This document defines the high-level goals of the +project, identifies the current components, and defines the release- +relationship to the Docker Platform. + +* [Distribution Goals](#distribution-goals) +* [Distribution Components](#distribution-components) +* [Project Planning](#project-planning): release-relationship to the Docker Platform. + +This road map is a living document, providing an overview of the goals and +considerations made in respect of the future of the project. + +## Distribution Goals + +- Replace the existing [docker registry](github.com/docker/docker-registry) + implementation as the primary implementation. +- Replace the existing push and pull code in the docker engine with the + distribution package. +- Define a strong data model for distributing docker images +- Provide a flexible distribution tool kit for use in the docker platform +- Unlock new distribution models + +## Distribution Components + +Components of the Distribution Project are managed via github [milestones](https://github.com/docker/distribution/milestones). Upcoming +features and bugfixes for a component will be added to the relevant milestone. If a feature or +bugfix is not part of a milestone, it is currently unscheduled for +implementation. + +* [Registry](#registry) +* [Distribution Package](#distribution-package) + +*** + +### Registry + +The new Docker registry is the main portion of the distribution repository. +Registry 2.0 is the first release of the next-generation registry. This was +primarily focused on implementing the [new registry +API](https://github.com/docker/distribution/blob/master/docs/spec/api.md), +with a focus on security and performance. + +Following from the Distribution project goals above, we have a set of goals +for registry v2 that we would like to follow in the design. New features +should be compared against these goals. + +#### Data Storage and Distribution First + +The registry's first goal is to provide a reliable, consistent storage +location for Docker images. The registry should only provide the minimal +amount of indexing required to fetch image data and no more. + +This means we should be selective in new features and API additions, including +those that may require expensive, ever growing indexes. Requests should be +servable in "constant time". + +#### Content Addressability + +All data objects used in the registry API should be content addressable. +Content identifiers should be secure and verifiable. This provides a secure, +reliable base from which to build more advanced content distribution systems. + +#### Content Agnostic + +In the past, changes to the image format would require large changes in Docker +and the Registry. By decoupling the distribution and image format, we can +allow the formats to progress without having to coordinate between the two. +This means that we should be focused on decoupling Docker from the registry +just as much as decoupling the registry from Docker. Such an approach will +allow us to unlock new distribution models that haven't been possible before. + +We can take this further by saying that the new registry should be content +agnostic. The registry provides a model of names, tags, manifests and content +addresses and that model can be used to work with content. + +#### Simplicity + +The new registry should be closer to a microservice component than its +predecessor. This means it should have a narrower API and a low number of +service dependencies. It should be easy to deploy. + +This means that other solutions should be explored before changing the API or +adding extra dependencies. If functionality is required, can it be added as an +extension or companion service. + +#### Extensibility + +The registry should provide extension points to add functionality. By keeping +the scope narrow, but providing the ability to add functionality. + +Features like search, indexing, synchronization and registry explorers fall +into this category. No such feature should be added unless we've found it +impossible to do through an extension. + +#### Active Feature Discussions + +The following are feature discussions that are currently active. + +If you don't see your favorite, unimplemented feature, feel free to contact us +via IRC or the mailing list and we can talk about adding it. The goal here is +to make sure that new features go through a rigid design process before +landing in the registry. + +##### Proxying to other Registries + +A _pull-through caching_ mode exists for the registry, but is restricted from +within the docker client to only mirror the official Docker Hub. This functionality +can be expanded when image provenance has been specified and implemented in the +distribution project. + +##### Metadata storage + +Metadata for the registry is currently stored with the manifest and layer data on +the storage backend. While this is a big win for simplicity and reliably maintaining +state, it comes with the cost of consistency and high latency. The mutable registry +metadata operations should be abstracted behind an API which will allow ACID compliant +storage systems to handle metadata. + +##### Peer to Peer transfer + +Discussion has started here: https://docs.google.com/document/d/1rYDpSpJiQWmCQy8Cuiaa3NH-Co33oK_SC9HeXYo87QA/edit + +##### Indexing, Search and Discovery + +The original registry provided some implementation of search for use with +private registries. Support has been elided from V2 since we'd like to both +decouple search functionality from the registry. The makes the registry +simpler to deploy, especially in use cases where search is not needed, and +let's us decouple the image format from the registry. + +There are explorations into using the catalog API and notification system to +build external indexes. The current line of thought is that we will define a +common search API to index and query docker images. Such a system could be run +as a companion to a registry or set of registries to power discovery. + +The main issue with search and discovery is that there are so many ways to +accomplish it. There are two aspects to this project. The first is deciding on +how it will be done, including an API definition that can work with changing +data formats. The second is the process of integrating with `docker search`. +We expect that someone attempts to address the problem with the existing tools +and propose it as a standard search API or uses it to inform a standardization +process. Once this has been explored, we integrate with the docker client. + +Please see the following for more detail: + +- https://github.com/docker/distribution/issues/206 + +##### Deletes + +> __NOTE:__ Deletes are a much asked for feature. Before requesting this +feature or participating in discussion, we ask that you read this section in +full and understand the problems behind deletes. + +While, at first glance, implementing deleting seems simple, there are a number +mitigating factors that make many solutions not ideal or even pathological in +the context of a registry. The following paragraph discuss the background and +approaches that could be applied to arrive at a solution. + +The goal of deletes in any system is to remove unused or unneeded data. Only +data requested for deletion should be removed and no other data. Removing +unintended data is worse than _not_ removing data that was requested for +removal but ideally, both are supported. Generally, according to this rule, we +err on holding data longer than needed, ensuring that it is only removed when +we can be certain that it can be removed. With the current behavior, we opt to +hold onto the data forever, ensuring that data cannot be incorrectly removed. + +To understand the problems with implementing deletes, one must understand the +data model. All registry data is stored in a filesystem layout, implemented on +a "storage driver", effectively a _virtual file system_ (VFS). The storage +system must assume that this VFS layer will be eventually consistent and has +poor read- after-write consistency, since this is the lower common denominator +among the storage drivers. This is mitigated by writing values in reverse- +dependent order, but makes wider transactional operations unsafe. + +Layered on the VFS model is a content-addressable _directed, acyclic graph_ +(DAG) made up of blobs. Manifests reference layers. Tags reference manifests. +Since the same data can be referenced by multiple manifests, we only store +data once, even if it is in different repositories. Thus, we have a set of +blobs, referenced by tags and manifests. If we want to delete a blob we need +to be certain that it is no longer referenced by another manifest or tag. When +we delete a manifest, we also can try to delete the referenced blobs. Deciding +whether or not a blob has an active reference is the crux of the problem. + +Conceptually, deleting a manifest and its resources is quite simple. Just find +all the manifests, enumerate the referenced blobs and delete the blobs not in +that set. An astute observer will recognize this as a garbage collection +problem. As with garbage collection in programming languages, this is very +simple when one always has a consistent view. When one adds parallelism and an +inconsistent view of data, it becomes very challenging. + +A simple example can demonstrate this. Let's say we are deleting a manifest +_A_ in one process. We scan the manifest and decide that all the blobs are +ready for deletion. Concurrently, we have another process accepting a new +manifest _B_ referencing one or more blobs from the manifest _A_. Manifest _B_ +is accepted and all the blobs are considered present, so the operation +proceeds. The original process then deletes the referenced blobs, assuming +they were unreferenced. The manifest _B_, which we thought had all of its data +present, can no longer be served by the registry, since the dependent data has +been deleted. + +Deleting data from the registry safely requires some way to coordinate this +operation. The following approaches are being considered: + +- _Reference Counting_ - Maintain a count of references to each blob. This is + challenging for a number of reasons: 1. maintaining a consistent consensus + of reference counts across a set of Registries and 2. Building the initial + list of reference counts for an existing registry. These challenges can be + met with a consensus protocol like Paxos or Raft in the first case and a + necessary but simple scan in the second.. +- _Lock the World GC_ - Halt all writes to the data store. Walk the data store + and find all blob references. Delete all unreferenced blobs. This approach + is very simple but requires disabling writes for a period of time while the + service reads all data. This is slow and expensive but very accurate and + effective. +- _Generational GC_ - Do something similar to above but instead of blocking + writes, writes are sent to another storage backend while reads are broadcast + to the new and old backends. GC is then performed on the read-only portion. + Because writes land in the new backend, the data in the read-only section + can be safely deleted. The main drawbacks of this approach are complexity + and coordination. +- _Centralized Oracle_ - Using a centralized, transactional database, we can + know exactly which data is referenced at any given time. This avoids + coordination problem by managing this data in a single location. We trade + off metadata scalability for simplicity and performance. This is a very good + option for most registry deployments. This would create a bottleneck for + registry metadata. However, metadata is generally not the main bottleneck + when serving images. + +Please let us know if other solutions exist that we have yet to enumerate. +Note that for any approach, implementation is a massive consideration. For +example, a mark-sweep based solution may seem simple but the amount of work in +coordination offset the extra work it might take to build a _Centralized +Oracle_. We'll accept proposals for any solution but please coordinate with us +before dropping code. + +At this time, we have traded off simplicity and ease of deployment for disk +space. Simplicity and ease of deployment tend to reduce developer involvement, +which is currently the most expensive resource in software engineering. Taking +on any solution for deletes will greatly effect these factors, trading off +very cheap disk space for a complex deployment and operational story. + +Please see the following issues for more detail: + +- https://github.com/docker/distribution/issues/422 +- https://github.com/docker/distribution/issues/461 +- https://github.com/docker/distribution/issues/462 + +### Distribution Package + +At its core, the Distribution Project is a set of Go packages that make up +Distribution Components. At this time, most of these packages make up the +Registry implementation. + +The package itself is considered unstable. If you're using it, please take care to vendor the dependent version. + +For feature additions, please see the Registry section. In the future, we may break out a +separate Roadmap for distribution-specific features that apply to more than +just the registry. + +*** + +### Project Planning + +An [Open-Source Planning Process](https://github.com/docker/distribution/wiki/Open-Source-Planning-Process) is used to define the Roadmap. [Project Pages](https://github.com/docker/distribution/wiki) define the goals for each Milestone and identify current progress. + diff --git a/vendor/github.com/docker/distribution/blobs.go b/vendor/github.com/docker/distribution/blobs.go new file mode 100644 index 000000000..671372abf --- /dev/null +++ b/vendor/github.com/docker/distribution/blobs.go @@ -0,0 +1,265 @@ +package distribution + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "time" + + "github.com/distribution/reference" + "github.com/opencontainers/go-digest" + v1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +var ( + // ErrBlobExists returned when blob already exists + ErrBlobExists = errors.New("blob exists") + + // ErrBlobDigestUnsupported when blob digest is an unsupported version. + ErrBlobDigestUnsupported = errors.New("unsupported blob digest") + + // ErrBlobUnknown when blob is not found. + ErrBlobUnknown = errors.New("unknown blob") + + // ErrBlobUploadUnknown returned when upload is not found. + ErrBlobUploadUnknown = errors.New("blob upload unknown") + + // ErrBlobInvalidLength returned when the blob has an expected length on + // commit, meaning mismatched with the descriptor or an invalid value. + ErrBlobInvalidLength = errors.New("blob invalid length") +) + +// ErrBlobInvalidDigest returned when digest check fails. +type ErrBlobInvalidDigest struct { + Digest digest.Digest + Reason error +} + +func (err ErrBlobInvalidDigest) Error() string { + return fmt.Sprintf("invalid digest for referenced layer: %v, %v", + err.Digest, err.Reason) +} + +// ErrBlobMounted returned when a blob is mounted from another repository +// instead of initiating an upload session. +type ErrBlobMounted struct { + From reference.Canonical + Descriptor Descriptor +} + +func (err ErrBlobMounted) Error() string { + return fmt.Sprintf("blob mounted from: %v to: %v", + err.From, err.Descriptor) +} + +// Descriptor describes targeted content. Used in conjunction with a blob +// store, a descriptor can be used to fetch, store and target any kind of +// blob. The struct also describes the wire protocol format. Fields should +// only be added but never changed. +type Descriptor struct { + // MediaType describe the type of the content. All text based formats are + // encoded as utf-8. + MediaType string `json:"mediaType,omitempty"` + + // Size in bytes of content. + Size int64 `json:"size,omitempty"` + + // Digest uniquely identifies the content. A byte stream can be verified + // against this digest. + Digest digest.Digest `json:"digest,omitempty"` + + // URLs contains the source URLs of this content. + URLs []string `json:"urls,omitempty"` + + // Annotations contains arbitrary metadata relating to the targeted content. + Annotations map[string]string `json:"annotations,omitempty"` + + // Platform describes the platform which the image in the manifest runs on. + // This should only be used when referring to a manifest. + Platform *v1.Platform `json:"platform,omitempty"` + + // NOTE: Before adding a field here, please ensure that all + // other options have been exhausted. Much of the type relationships + // depend on the simplicity of this type. +} + +// Descriptor returns the descriptor, to make it satisfy the Describable +// interface. Note that implementations of Describable are generally objects +// which can be described, not simply descriptors; this exception is in place +// to make it more convenient to pass actual descriptors to functions that +// expect Describable objects. +func (d Descriptor) Descriptor() Descriptor { + return d +} + +// BlobStatter makes blob descriptors available by digest. The service may +// provide a descriptor of a different digest if the provided digest is not +// canonical. +type BlobStatter interface { + // Stat provides metadata about a blob identified by the digest. If the + // blob is unknown to the describer, ErrBlobUnknown will be returned. + Stat(ctx context.Context, dgst digest.Digest) (Descriptor, error) +} + +// BlobDeleter enables deleting blobs from storage. +type BlobDeleter interface { + Delete(ctx context.Context, dgst digest.Digest) error +} + +// BlobEnumerator enables iterating over blobs from storage +type BlobEnumerator interface { + Enumerate(ctx context.Context, ingester func(dgst digest.Digest) error) error +} + +// BlobDescriptorService manages metadata about a blob by digest. Most +// implementations will not expose such an interface explicitly. Such mappings +// should be maintained by interacting with the BlobIngester. Hence, this is +// left off of BlobService and BlobStore. +type BlobDescriptorService interface { + BlobStatter + + // SetDescriptor assigns the descriptor to the digest. The provided digest and + // the digest in the descriptor must map to identical content but they may + // differ on their algorithm. The descriptor must have the canonical + // digest of the content and the digest algorithm must match the + // annotators canonical algorithm. + // + // Such a facility can be used to map blobs between digest domains, with + // the restriction that the algorithm of the descriptor must match the + // canonical algorithm (ie sha256) of the annotator. + SetDescriptor(ctx context.Context, dgst digest.Digest, desc Descriptor) error + + // Clear enables descriptors to be unlinked + Clear(ctx context.Context, dgst digest.Digest) error +} + +// BlobDescriptorServiceFactory creates middleware for BlobDescriptorService. +type BlobDescriptorServiceFactory interface { + BlobAccessController(svc BlobDescriptorService) BlobDescriptorService +} + +// ReadSeekCloser is the primary reader type for blob data, combining +// io.ReadSeeker with io.Closer. +type ReadSeekCloser interface { + io.ReadSeeker + io.Closer +} + +// BlobProvider describes operations for getting blob data. +type BlobProvider interface { + // Get returns the entire blob identified by digest along with the descriptor. + Get(ctx context.Context, dgst digest.Digest) ([]byte, error) + + // Open provides a ReadSeekCloser to the blob identified by the provided + // descriptor. If the blob is not known to the service, an error will be + // returned. + Open(ctx context.Context, dgst digest.Digest) (ReadSeekCloser, error) +} + +// BlobServer can serve blobs via http. +type BlobServer interface { + // ServeBlob attempts to serve the blob, identified by dgst, via http. The + // service may decide to redirect the client elsewhere or serve the data + // directly. + // + // This handler only issues successful responses, such as 2xx or 3xx, + // meaning it serves data or issues a redirect. If the blob is not + // available, an error will be returned and the caller may still issue a + // response. + // + // The implementation may serve the same blob from a different digest + // domain. The appropriate headers will be set for the blob, unless they + // have already been set by the caller. + ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) error +} + +// BlobIngester ingests blob data. +type BlobIngester interface { + // Put inserts the content p into the blob service, returning a descriptor + // or an error. + Put(ctx context.Context, mediaType string, p []byte) (Descriptor, error) + + // Create allocates a new blob writer to add a blob to this service. The + // returned handle can be written to and later resumed using an opaque + // identifier. With this approach, one can Close and Resume a BlobWriter + // multiple times until the BlobWriter is committed or cancelled. + Create(ctx context.Context, options ...BlobCreateOption) (BlobWriter, error) + + // Resume attempts to resume a write to a blob, identified by an id. + Resume(ctx context.Context, id string) (BlobWriter, error) +} + +// BlobCreateOption is a general extensible function argument for blob creation +// methods. A BlobIngester may choose to honor any or none of the given +// BlobCreateOptions, which can be specific to the implementation of the +// BlobIngester receiving them. +// TODO (brianbland): unify this with ManifestServiceOption in the future +type BlobCreateOption interface { + Apply(interface{}) error +} + +// CreateOptions is a collection of blob creation modifiers relevant to general +// blob storage intended to be configured by the BlobCreateOption.Apply method. +type CreateOptions struct { + Mount struct { + ShouldMount bool + From reference.Canonical + // Stat allows to pass precalculated descriptor to link and return. + // Blob access check will be skipped if set. + Stat *Descriptor + } +} + +// BlobWriter provides a handle for inserting data into a blob store. +// Instances should be obtained from BlobWriteService.Writer and +// BlobWriteService.Resume. If supported by the store, a writer can be +// recovered with the id. +type BlobWriter interface { + io.WriteCloser + io.ReaderFrom + + // Size returns the number of bytes written to this blob. + Size() int64 + + // ID returns the identifier for this writer. The ID can be used with the + // Blob service to later resume the write. + ID() string + + // StartedAt returns the time this blob write was started. + StartedAt() time.Time + + // Commit completes the blob writer process. The content is verified + // against the provided provisional descriptor, which may result in an + // error. Depending on the implementation, written data may be validated + // against the provisional descriptor fields. If MediaType is not present, + // the implementation may reject the commit or assign "application/octet- + // stream" to the blob. The returned descriptor may have a different + // digest depending on the blob store, referred to as the canonical + // descriptor. + Commit(ctx context.Context, provisional Descriptor) (canonical Descriptor, err error) + + // Cancel ends the blob write without storing any data and frees any + // associated resources. Any data written thus far will be lost. Cancel + // implementations should allow multiple calls even after a commit that + // result in a no-op. This allows use of Cancel in a defer statement, + // increasing the assurance that it is correctly called. + Cancel(ctx context.Context) error +} + +// BlobService combines the operations to access, read and write blobs. This +// can be used to describe remote blob services. +type BlobService interface { + BlobStatter + BlobProvider + BlobIngester +} + +// BlobStore represent the entire suite of blob related operations. Such an +// implementation can access, read, write, delete and serve blobs. +type BlobStore interface { + BlobService + BlobServer + BlobDeleter +} diff --git a/vendor/github.com/docker/distribution/configuration/configuration.go b/vendor/github.com/docker/distribution/configuration/configuration.go new file mode 100644 index 000000000..7076df85d --- /dev/null +++ b/vendor/github.com/docker/distribution/configuration/configuration.go @@ -0,0 +1,706 @@ +package configuration + +import ( + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "reflect" + "strings" + "time" +) + +// Configuration is a versioned registry configuration, intended to be provided by a yaml file, and +// optionally modified by environment variables. +// +// Note that yaml field names should never include _ characters, since this is the separator used +// in environment variable names. +type Configuration struct { + // Version is the version which defines the format of the rest of the configuration + Version Version `yaml:"version"` + + // Log supports setting various parameters related to the logging + // subsystem. + Log struct { + // AccessLog configures access logging. + AccessLog struct { + // Disabled disables access logging. + Disabled bool `yaml:"disabled,omitempty"` + } `yaml:"accesslog,omitempty"` + + // Level is the granularity at which registry operations are logged. + Level Loglevel `yaml:"level,omitempty"` + + // Formatter overrides the default formatter with another. Options + // include "text", "json" and "logstash". + Formatter string `yaml:"formatter,omitempty"` + + // Fields allows users to specify static string fields to include in + // the logger context. + Fields map[string]interface{} `yaml:"fields,omitempty"` + + // Hooks allows users to configure the log hooks, to enabling the + // sequent handling behavior, when defined levels of log message emit. + Hooks []LogHook `yaml:"hooks,omitempty"` + } + + // Loglevel is the level at which registry operations are logged. + // + // Deprecated: Use Log.Level instead. + Loglevel Loglevel `yaml:"loglevel,omitempty"` + + // Storage is the configuration for the registry's storage driver + Storage Storage `yaml:"storage"` + + // Auth allows configuration of various authorization methods that may be + // used to gate requests. + Auth Auth `yaml:"auth,omitempty"` + + // Middleware lists all middlewares to be used by the registry. + Middleware map[string][]Middleware `yaml:"middleware,omitempty"` + + // Reporting is the configuration for error reporting + Reporting Reporting `yaml:"reporting,omitempty"` + + // HTTP contains configuration parameters for the registry's http + // interface. + HTTP struct { + // Addr specifies the bind address for the registry instance. + Addr string `yaml:"addr,omitempty"` + + // Net specifies the net portion of the bind address. A default empty value means tcp. + Net string `yaml:"net,omitempty"` + + // Host specifies an externally-reachable address for the registry, as a fully + // qualified URL. + Host string `yaml:"host,omitempty"` + + Prefix string `yaml:"prefix,omitempty"` + + // Secret specifies the secret key which HMAC tokens are created with. + Secret string `yaml:"secret,omitempty"` + + // RelativeURLs specifies that relative URLs should be returned in + // Location headers + RelativeURLs bool `yaml:"relativeurls,omitempty"` + + // Amount of time to wait for connection to drain before shutting down when registry + // receives a stop signal + DrainTimeout time.Duration `yaml:"draintimeout,omitempty"` + + // TLS instructs the http server to listen with a TLS configuration. + // This only support simple tls configuration with a cert and key. + // Mostly, this is useful for testing situations or simple deployments + // that require tls. If more complex configurations are required, use + // a proxy or make a proposal to add support here. + TLS struct { + // Certificate specifies the path to an x509 certificate file to + // be used for TLS. + Certificate string `yaml:"certificate,omitempty"` + + // Key specifies the path to the x509 key file, which should + // contain the private portion for the file specified in + // Certificate. + Key string `yaml:"key,omitempty"` + + // Specifies the CA certs for client authentication + // A file may contain multiple CA certificates encoded as PEM + ClientCAs []string `yaml:"clientcas,omitempty"` + + // Specifies the lowest TLS version allowed + MinimumTLS string `yaml:"minimumtls,omitempty"` + + // Specifies a list of cipher suites allowed + CipherSuites []string `yaml:"ciphersuites,omitempty"` + + // LetsEncrypt is used to configuration setting up TLS through + // Let's Encrypt instead of manually specifying certificate and + // key. If a TLS certificate is specified, the Let's Encrypt + // section will not be used. + LetsEncrypt struct { + // CacheFile specifies cache file to use for lets encrypt + // certificates and keys. + CacheFile string `yaml:"cachefile,omitempty"` + + // Email is the email to use during Let's Encrypt registration + Email string `yaml:"email,omitempty"` + + // Hosts specifies the hosts which are allowed to obtain Let's + // Encrypt certificates. + Hosts []string `yaml:"hosts,omitempty"` + } `yaml:"letsencrypt,omitempty"` + } `yaml:"tls,omitempty"` + + // Headers is a set of headers to include in HTTP responses. A common + // use case for this would be security headers such as + // Strict-Transport-Security. The map keys are the header names, and + // the values are the associated header payloads. + Headers http.Header `yaml:"headers,omitempty"` + + // Debug configures the http debug interface, if specified. This can + // include services such as pprof, expvar and other data that should + // not be exposed externally. Left disabled by default. + Debug struct { + // Addr specifies the bind address for the debug server. + Addr string `yaml:"addr,omitempty"` + // Prometheus configures the Prometheus telemetry endpoint. + Prometheus struct { + Enabled bool `yaml:"enabled,omitempty"` + Path string `yaml:"path,omitempty"` + } `yaml:"prometheus,omitempty"` + } `yaml:"debug,omitempty"` + + // HTTP2 configuration options + HTTP2 struct { + // Specifies whether the registry should disallow clients attempting + // to connect via http2. If set to true, only http/1.1 is supported. + Disabled bool `yaml:"disabled,omitempty"` + } `yaml:"http2,omitempty"` + } `yaml:"http,omitempty"` + + // Notifications specifies configuration about various endpoint to which + // registry events are dispatched. + Notifications Notifications `yaml:"notifications,omitempty"` + + // Redis configures the redis pool available to the registry webapp. + Redis struct { + // Addr specifies the the redis instance available to the application. + Addr string `yaml:"addr,omitempty"` + + // Password string to use when making a connection. + Password string `yaml:"password,omitempty"` + + // DB specifies the database to connect to on the redis instance. + DB int `yaml:"db,omitempty"` + + DialTimeout time.Duration `yaml:"dialtimeout,omitempty"` // timeout for connect + ReadTimeout time.Duration `yaml:"readtimeout,omitempty"` // timeout for reads of data + WriteTimeout time.Duration `yaml:"writetimeout,omitempty"` // timeout for writes of data + + // Pool configures the behavior of the redis connection pool. + Pool struct { + // MaxIdle sets the maximum number of idle connections. + MaxIdle int `yaml:"maxidle,omitempty"` + + // MaxActive sets the maximum number of connections that should be + // opened before blocking a connection request. + MaxActive int `yaml:"maxactive,omitempty"` + + // IdleTimeout sets the amount time to wait before closing + // inactive connections. + IdleTimeout time.Duration `yaml:"idletimeout,omitempty"` + } `yaml:"pool,omitempty"` + } `yaml:"redis,omitempty"` + + Health Health `yaml:"health,omitempty"` + Catalog Catalog `yaml:"catalog,omitempty"` + + Proxy Proxy `yaml:"proxy,omitempty"` + + // Compatibility is used for configurations of working with older or deprecated features. + Compatibility struct { + // Schema1 configures how schema1 manifests will be handled + Schema1 struct { + // TrustKey is the signing key to use for adding the signature to + // schema1 manifests. + TrustKey string `yaml:"signingkeyfile,omitempty"` + // Enabled determines if schema1 manifests should be pullable + Enabled bool `yaml:"enabled,omitempty"` + } `yaml:"schema1,omitempty"` + } `yaml:"compatibility,omitempty"` + + // Validation configures validation options for the registry. + Validation struct { + // Enabled enables the other options in this section. This field is + // deprecated in favor of Disabled. + Enabled bool `yaml:"enabled,omitempty"` + // Disabled disables the other options in this section. + Disabled bool `yaml:"disabled,omitempty"` + // Manifests configures manifest validation. + Manifests struct { + // URLs configures validation for URLs in pushed manifests. + URLs struct { + // Allow specifies regular expressions (https://godoc.org/regexp/syntax) + // that URLs in pushed manifests must match. + Allow []string `yaml:"allow,omitempty"` + // Deny specifies regular expressions (https://godoc.org/regexp/syntax) + // that URLs in pushed manifests must not match. + Deny []string `yaml:"deny,omitempty"` + } `yaml:"urls,omitempty"` + } `yaml:"manifests,omitempty"` + } `yaml:"validation,omitempty"` + + // Policy configures registry policy options. + Policy struct { + // Repository configures policies for repositories + Repository struct { + // Classes is a list of repository classes which the + // registry allows content for. This class is matched + // against the configuration media type inside uploaded + // manifests. When non-empty, the registry will enforce + // the class in authorized resources. + Classes []string `yaml:"classes"` + } `yaml:"repository,omitempty"` + } `yaml:"policy,omitempty"` +} + +// Catalog is composed of MaxEntries. +// Catalog endpoint (/v2/_catalog) configuration, it provides the configuration +// options to control the maximum number of entries returned by the catalog endpoint. +type Catalog struct { + // Max number of entries returned by the catalog endpoint. Requesting n entries + // to the catalog endpoint will return at most MaxEntries entries. + // An empty or a negative value will set a default of 1000 maximum entries by default. + MaxEntries int `yaml:"maxentries,omitempty"` +} + +// LogHook is composed of hook Level and Type. +// After hooks configuration, it can execute the next handling automatically, +// when defined levels of log message emitted. +// Example: hook can sending an email notification when error log happens in app. +type LogHook struct { + // Disable lets user select to enable hook or not. + Disabled bool `yaml:"disabled,omitempty"` + + // Type allows user to select which type of hook handler they want. + Type string `yaml:"type,omitempty"` + + // Levels set which levels of log message will let hook executed. + Levels []string `yaml:"levels,omitempty"` + + // MailOptions allows user to configure email parameters. + MailOptions MailOptions `yaml:"options,omitempty"` +} + +// MailOptions provides the configuration sections to user, for specific handler. +type MailOptions struct { + SMTP struct { + // Addr defines smtp host address + Addr string `yaml:"addr,omitempty"` + + // Username defines user name to smtp host + Username string `yaml:"username,omitempty"` + + // Password defines password of login user + Password string `yaml:"password,omitempty"` + + // Insecure defines if smtp login skips the secure certification. + Insecure bool `yaml:"insecure,omitempty"` + } `yaml:"smtp,omitempty"` + + // From defines mail sending address + From string `yaml:"from,omitempty"` + + // To defines mail receiving address + To []string `yaml:"to,omitempty"` +} + +// FileChecker is a type of entry in the health section for checking files. +type FileChecker struct { + // Interval is the duration in between checks + Interval time.Duration `yaml:"interval,omitempty"` + // File is the path to check + File string `yaml:"file,omitempty"` + // Threshold is the number of times a check must fail to trigger an + // unhealthy state + Threshold int `yaml:"threshold,omitempty"` +} + +// HTTPChecker is a type of entry in the health section for checking HTTP URIs. +type HTTPChecker struct { + // Timeout is the duration to wait before timing out the HTTP request + Timeout time.Duration `yaml:"timeout,omitempty"` + // StatusCode is the expected status code + StatusCode int + // Interval is the duration in between checks + Interval time.Duration `yaml:"interval,omitempty"` + // URI is the HTTP URI to check + URI string `yaml:"uri,omitempty"` + // Headers lists static headers that should be added to all requests + Headers http.Header `yaml:"headers"` + // Threshold is the number of times a check must fail to trigger an + // unhealthy state + Threshold int `yaml:"threshold,omitempty"` +} + +// TCPChecker is a type of entry in the health section for checking TCP servers. +type TCPChecker struct { + // Timeout is the duration to wait before timing out the TCP connection + Timeout time.Duration `yaml:"timeout,omitempty"` + // Interval is the duration in between checks + Interval time.Duration `yaml:"interval,omitempty"` + // Addr is the TCP address to check + Addr string `yaml:"addr,omitempty"` + // Threshold is the number of times a check must fail to trigger an + // unhealthy state + Threshold int `yaml:"threshold,omitempty"` +} + +// Health provides the configuration section for health checks. +type Health struct { + // FileCheckers is a list of paths to check + FileCheckers []FileChecker `yaml:"file,omitempty"` + // HTTPCheckers is a list of URIs to check + HTTPCheckers []HTTPChecker `yaml:"http,omitempty"` + // TCPCheckers is a list of URIs to check + TCPCheckers []TCPChecker `yaml:"tcp,omitempty"` + // StorageDriver configures a health check on the configured storage + // driver + StorageDriver struct { + // Enabled turns on the health check for the storage driver + Enabled bool `yaml:"enabled,omitempty"` + // Interval is the duration in between checks + Interval time.Duration `yaml:"interval,omitempty"` + // Threshold is the number of times a check must fail to trigger an + // unhealthy state + Threshold int `yaml:"threshold,omitempty"` + } `yaml:"storagedriver,omitempty"` +} + +// v0_1Configuration is a Version 0.1 Configuration struct +// This is currently aliased to Configuration, as it is the current version +type v0_1Configuration Configuration + +// UnmarshalYAML implements the yaml.Unmarshaler interface +// Unmarshals a string of the form X.Y into a Version, validating that X and Y can represent unsigned integers +func (version *Version) UnmarshalYAML(unmarshal func(interface{}) error) error { + var versionString string + err := unmarshal(&versionString) + if err != nil { + return err + } + + newVersion := Version(versionString) + if _, err := newVersion.major(); err != nil { + return err + } + + if _, err := newVersion.minor(); err != nil { + return err + } + + *version = newVersion + return nil +} + +// CurrentVersion is the most recent Version that can be parsed +var CurrentVersion = MajorMinorVersion(0, 1) + +// Loglevel is the level at which operations are logged +// This can be error, warn, info, or debug +type Loglevel string + +// UnmarshalYAML implements the yaml.Umarshaler interface +// Unmarshals a string into a Loglevel, lowercasing the string and validating that it represents a +// valid loglevel +func (loglevel *Loglevel) UnmarshalYAML(unmarshal func(interface{}) error) error { + var loglevelString string + err := unmarshal(&loglevelString) + if err != nil { + return err + } + + loglevelString = strings.ToLower(loglevelString) + switch loglevelString { + case "error", "warn", "info", "debug": + default: + return fmt.Errorf("invalid loglevel %s Must be one of [error, warn, info, debug]", loglevelString) + } + + *loglevel = Loglevel(loglevelString) + return nil +} + +// Parameters defines a key-value parameters mapping +type Parameters map[string]interface{} + +// Storage defines the configuration for registry object storage +type Storage map[string]Parameters + +// Type returns the storage driver type, such as filesystem or s3 +func (storage Storage) Type() string { + var storageType []string + + // Return only key in this map + for k := range storage { + switch k { + case "maintenance": + // allow configuration of maintenance + case "cache": + // allow configuration of caching + case "delete": + // allow configuration of delete + case "redirect": + // allow configuration of redirect + default: + storageType = append(storageType, k) + } + } + if len(storageType) > 1 { + panic("multiple storage drivers specified in configuration or environment: " + strings.Join(storageType, ", ")) + } + if len(storageType) == 1 { + return storageType[0] + } + return "" +} + +// Parameters returns the Parameters map for a Storage configuration +func (storage Storage) Parameters() Parameters { + return storage[storage.Type()] +} + +// setParameter changes the parameter at the provided key to the new value +func (storage Storage) setParameter(key string, value interface{}) { + storage[storage.Type()][key] = value +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface +// Unmarshals a single item map into a Storage or a string into a Storage type with no parameters +func (storage *Storage) UnmarshalYAML(unmarshal func(interface{}) error) error { + var storageMap map[string]Parameters + err := unmarshal(&storageMap) + if err == nil { + if len(storageMap) > 1 { + types := make([]string, 0, len(storageMap)) + for k := range storageMap { + switch k { + case "maintenance": + // allow for configuration of maintenance + case "cache": + // allow configuration of caching + case "delete": + // allow configuration of delete + case "redirect": + // allow configuration of redirect + default: + types = append(types, k) + } + } + + if len(types) > 1 { + return fmt.Errorf("must provide exactly one storage type. Provided: %v", types) + } + } + *storage = storageMap + return nil + } + + var storageType string + err = unmarshal(&storageType) + if err == nil { + *storage = Storage{storageType: Parameters{}} + return nil + } + + return err +} + +// MarshalYAML implements the yaml.Marshaler interface +func (storage Storage) MarshalYAML() (interface{}, error) { + if storage.Parameters() == nil { + return storage.Type(), nil + } + return map[string]Parameters(storage), nil +} + +// Auth defines the configuration for registry authorization. +type Auth map[string]Parameters + +// Type returns the auth type, such as htpasswd or token +func (auth Auth) Type() string { + // Return only key in this map + for k := range auth { + return k + } + return "" +} + +// Parameters returns the Parameters map for an Auth configuration +func (auth Auth) Parameters() Parameters { + return auth[auth.Type()] +} + +// setParameter changes the parameter at the provided key to the new value +func (auth Auth) setParameter(key string, value interface{}) { + auth[auth.Type()][key] = value +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface +// Unmarshals a single item map into a Storage or a string into a Storage type with no parameters +func (auth *Auth) UnmarshalYAML(unmarshal func(interface{}) error) error { + var m map[string]Parameters + err := unmarshal(&m) + if err == nil { + if len(m) > 1 { + types := make([]string, 0, len(m)) + for k := range m { + types = append(types, k) + } + + // TODO(stevvooe): May want to change this slightly for + // authorization to allow multiple challenges. + return fmt.Errorf("must provide exactly one type. Provided: %v", types) + + } + *auth = m + return nil + } + + var authType string + err = unmarshal(&authType) + if err == nil { + *auth = Auth{authType: Parameters{}} + return nil + } + + return err +} + +// MarshalYAML implements the yaml.Marshaler interface +func (auth Auth) MarshalYAML() (interface{}, error) { + if auth.Parameters() == nil { + return auth.Type(), nil + } + return map[string]Parameters(auth), nil +} + +// Notifications configures multiple http endpoints. +type Notifications struct { + // EventConfig is the configuration for the event format that is sent to each Endpoint. + EventConfig Events `yaml:"events,omitempty"` + // Endpoints is a list of http configurations for endpoints that + // respond to webhook notifications. In the future, we may allow other + // kinds of endpoints, such as external queues. + Endpoints []Endpoint `yaml:"endpoints,omitempty"` +} + +// Endpoint describes the configuration of an http webhook notification +// endpoint. +type Endpoint struct { + Name string `yaml:"name"` // identifies the endpoint in the registry instance. + Disabled bool `yaml:"disabled"` // disables the endpoint + URL string `yaml:"url"` // post url for the endpoint. + Headers http.Header `yaml:"headers"` // static headers that should be added to all requests + Timeout time.Duration `yaml:"timeout"` // HTTP timeout + Threshold int `yaml:"threshold"` // circuit breaker threshold before backing off on failure + Backoff time.Duration `yaml:"backoff"` // backoff duration + IgnoredMediaTypes []string `yaml:"ignoredmediatypes"` // target media types to ignore + Ignore Ignore `yaml:"ignore"` // ignore event types +} + +// Events configures notification events. +type Events struct { + IncludeReferences bool `yaml:"includereferences"` // include reference data in manifest events +} + +// Ignore configures mediaTypes and actions of the event, that it won't be propagated +type Ignore struct { + MediaTypes []string `yaml:"mediatypes"` // target media types to ignore + Actions []string `yaml:"actions"` // ignore action types +} + +// Reporting defines error reporting methods. +type Reporting struct { + // Bugsnag configures error reporting for Bugsnag (bugsnag.com). + Bugsnag BugsnagReporting `yaml:"bugsnag,omitempty"` + // NewRelic configures error reporting for NewRelic (newrelic.com) + NewRelic NewRelicReporting `yaml:"newrelic,omitempty"` +} + +// BugsnagReporting configures error reporting for Bugsnag (bugsnag.com). +type BugsnagReporting struct { + // APIKey is the Bugsnag api key. + APIKey string `yaml:"apikey,omitempty"` + // ReleaseStage tracks where the registry is deployed. + // Examples: production, staging, development + ReleaseStage string `yaml:"releasestage,omitempty"` + // Endpoint is used for specifying an enterprise Bugsnag endpoint. + Endpoint string `yaml:"endpoint,omitempty"` +} + +// NewRelicReporting configures error reporting for NewRelic (newrelic.com) +type NewRelicReporting struct { + // LicenseKey is the NewRelic user license key + LicenseKey string `yaml:"licensekey,omitempty"` + // Name is the component name of the registry in NewRelic + Name string `yaml:"name,omitempty"` + // Verbose configures debug output to STDOUT + Verbose bool `yaml:"verbose,omitempty"` +} + +// Middleware configures named middlewares to be applied at injection points. +type Middleware struct { + // Name the middleware registers itself as + Name string `yaml:"name"` + // Flag to disable middleware easily + Disabled bool `yaml:"disabled,omitempty"` + // Map of parameters that will be passed to the middleware's initialization function + Options Parameters `yaml:"options"` +} + +// Proxy configures the registry as a pull through cache +type Proxy struct { + // RemoteURL is the URL of the remote registry + RemoteURL string `yaml:"remoteurl"` + + // Username of the hub user + Username string `yaml:"username"` + + // Password of the hub user + Password string `yaml:"password"` +} + +// Parse parses an input configuration yaml document into a Configuration struct +// This should generally be capable of handling old configuration format versions +// +// Environment variables may be used to override configuration parameters other than version, +// following the scheme below: +// Configuration.Abc may be replaced by the value of REGISTRY_ABC, +// Configuration.Abc.Xyz may be replaced by the value of REGISTRY_ABC_XYZ, and so forth +func Parse(rd io.Reader) (*Configuration, error) { + in, err := ioutil.ReadAll(rd) + if err != nil { + return nil, err + } + + p := NewParser("registry", []VersionedParseInfo{ + { + Version: MajorMinorVersion(0, 1), + ParseAs: reflect.TypeOf(v0_1Configuration{}), + ConversionFunc: func(c interface{}) (interface{}, error) { + if v0_1, ok := c.(*v0_1Configuration); ok { + if v0_1.Log.Level == Loglevel("") { + if v0_1.Loglevel != Loglevel("") { + v0_1.Log.Level = v0_1.Loglevel + } else { + v0_1.Log.Level = Loglevel("info") + } + } + if v0_1.Loglevel != Loglevel("") { + v0_1.Loglevel = Loglevel("") + } + + if v0_1.Catalog.MaxEntries <= 0 { + v0_1.Catalog.MaxEntries = 1000 + } + + if v0_1.Storage.Type() == "" { + return nil, errors.New("no storage configuration provided") + } + return (*Configuration)(v0_1), nil + } + return nil, fmt.Errorf("expected *v0_1Configuration, received %#v", c) + }, + }, + }) + + config := new(Configuration) + err = p.Parse(in, config) + if err != nil { + return nil, err + } + + return config, nil +} diff --git a/vendor/github.com/docker/distribution/configuration/parser.go b/vendor/github.com/docker/distribution/configuration/parser.go new file mode 100644 index 000000000..2bf59d219 --- /dev/null +++ b/vendor/github.com/docker/distribution/configuration/parser.go @@ -0,0 +1,283 @@ +package configuration + +import ( + "fmt" + "os" + "reflect" + "sort" + "strconv" + "strings" + + "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" +) + +// Version is a major/minor version pair of the form Major.Minor +// Major version upgrades indicate structure or type changes +// Minor version upgrades should be strictly additive +type Version string + +// MajorMinorVersion constructs a Version from its Major and Minor components +func MajorMinorVersion(major, minor uint) Version { + return Version(fmt.Sprintf("%d.%d", major, minor)) +} + +func (version Version) major() (uint, error) { + majorPart := strings.Split(string(version), ".")[0] + major, err := strconv.ParseUint(majorPart, 10, 0) + return uint(major), err +} + +// Major returns the major version portion of a Version +func (version Version) Major() uint { + major, _ := version.major() + return major +} + +func (version Version) minor() (uint, error) { + minorPart := strings.Split(string(version), ".")[1] + minor, err := strconv.ParseUint(minorPart, 10, 0) + return uint(minor), err +} + +// Minor returns the minor version portion of a Version +func (version Version) Minor() uint { + minor, _ := version.minor() + return minor +} + +// VersionedParseInfo defines how a specific version of a configuration should +// be parsed into the current version +type VersionedParseInfo struct { + // Version is the version which this parsing information relates to + Version Version + // ParseAs defines the type which a configuration file of this version + // should be parsed into + ParseAs reflect.Type + // ConversionFunc defines a method for converting the parsed configuration + // (of type ParseAs) into the current configuration version + // Note: this method signature is very unclear with the absence of generics + ConversionFunc func(interface{}) (interface{}, error) +} + +type envVar struct { + name string + value string +} + +type envVars []envVar + +func (a envVars) Len() int { return len(a) } +func (a envVars) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a envVars) Less(i, j int) bool { return a[i].name < a[j].name } + +// Parser can be used to parse a configuration file and environment of a defined +// version into a unified output structure +type Parser struct { + prefix string + mapping map[Version]VersionedParseInfo + env envVars +} + +// NewParser returns a *Parser with the given environment prefix which handles +// versioned configurations which match the given parseInfos +func NewParser(prefix string, parseInfos []VersionedParseInfo) *Parser { + p := Parser{prefix: prefix, mapping: make(map[Version]VersionedParseInfo)} + + for _, parseInfo := range parseInfos { + p.mapping[parseInfo.Version] = parseInfo + } + + for _, env := range os.Environ() { + envParts := strings.SplitN(env, "=", 2) + p.env = append(p.env, envVar{envParts[0], envParts[1]}) + } + + // We must sort the environment variables lexically by name so that + // more specific variables are applied before less specific ones + // (i.e. REGISTRY_STORAGE before + // REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY). This sucks, but it's a + // lot simpler and easier to get right than unmarshalling map entries + // into temporaries and merging with the existing entry. + sort.Sort(p.env) + + return &p +} + +// Parse reads in the given []byte and environment and writes the resulting +// configuration into the input v +// +// Environment variables may be used to override configuration parameters other +// than version, following the scheme below: +// v.Abc may be replaced by the value of PREFIX_ABC, +// v.Abc.Xyz may be replaced by the value of PREFIX_ABC_XYZ, and so forth +func (p *Parser) Parse(in []byte, v interface{}) error { + var versionedStruct struct { + Version Version + } + + if err := yaml.Unmarshal(in, &versionedStruct); err != nil { + return err + } + + parseInfo, ok := p.mapping[versionedStruct.Version] + if !ok { + return fmt.Errorf("unsupported version: %q", versionedStruct.Version) + } + + parseAs := reflect.New(parseInfo.ParseAs) + err := yaml.Unmarshal(in, parseAs.Interface()) + if err != nil { + return err + } + + for _, envVar := range p.env { + pathStr := envVar.name + if strings.HasPrefix(pathStr, strings.ToUpper(p.prefix)+"_") { + path := strings.Split(pathStr, "_") + + err = p.overwriteFields(parseAs, pathStr, path[1:], envVar.value) + if err != nil { + return err + } + } + } + + c, err := parseInfo.ConversionFunc(parseAs.Interface()) + if err != nil { + return err + } + reflect.ValueOf(v).Elem().Set(reflect.Indirect(reflect.ValueOf(c))) + return nil +} + +// overwriteFields replaces configuration values with alternate values specified +// through the environment. Precondition: an empty path slice must never be +// passed in. +func (p *Parser) overwriteFields(v reflect.Value, fullpath string, path []string, payload string) error { + for v.Kind() == reflect.Ptr { + if v.IsNil() { + panic("encountered nil pointer while handling environment variable " + fullpath) + } + v = reflect.Indirect(v) + } + switch v.Kind() { + case reflect.Struct: + return p.overwriteStruct(v, fullpath, path, payload) + case reflect.Map: + return p.overwriteMap(v, fullpath, path, payload) + case reflect.Interface: + if v.NumMethod() == 0 { + if !v.IsNil() { + return p.overwriteFields(v.Elem(), fullpath, path, payload) + } + // Interface was empty; create an implicit map + var template map[string]interface{} + wrappedV := reflect.MakeMap(reflect.TypeOf(template)) + v.Set(wrappedV) + return p.overwriteMap(wrappedV, fullpath, path, payload) + } + } + return nil +} + +func (p *Parser) overwriteStruct(v reflect.Value, fullpath string, path []string, payload string) error { + // Generate case-insensitive map of struct fields + byUpperCase := make(map[string]int) + for i := 0; i < v.NumField(); i++ { + sf := v.Type().Field(i) + upper := strings.ToUpper(sf.Name) + if _, present := byUpperCase[upper]; present { + panic(fmt.Sprintf("field name collision in configuration object: %s", sf.Name)) + } + byUpperCase[upper] = i + } + + fieldIndex, present := byUpperCase[path[0]] + if !present { + logrus.Warnf("Ignoring unrecognized environment variable %s", fullpath) + return nil + } + field := v.Field(fieldIndex) + sf := v.Type().Field(fieldIndex) + + if len(path) == 1 { + // Env var specifies this field directly + fieldVal := reflect.New(sf.Type) + err := yaml.Unmarshal([]byte(payload), fieldVal.Interface()) + if err != nil { + return err + } + field.Set(reflect.Indirect(fieldVal)) + return nil + } + + // If the field is nil, must create an object + switch sf.Type.Kind() { + case reflect.Map: + if field.IsNil() { + field.Set(reflect.MakeMap(sf.Type)) + } + case reflect.Ptr: + if field.IsNil() { + field.Set(reflect.New(sf.Type)) + } + } + + err := p.overwriteFields(field, fullpath, path[1:], payload) + if err != nil { + return err + } + + return nil +} + +func (p *Parser) overwriteMap(m reflect.Value, fullpath string, path []string, payload string) error { + if m.Type().Key().Kind() != reflect.String { + // non-string keys unsupported + logrus.Warnf("Ignoring environment variable %s involving map with non-string keys", fullpath) + return nil + } + + if len(path) > 1 { + // If a matching key exists, get its value and continue the + // overwriting process. + for _, k := range m.MapKeys() { + if strings.ToUpper(k.String()) == path[0] { + mapValue := m.MapIndex(k) + // If the existing value is nil, we want to + // recreate it instead of using this value. + if (mapValue.Kind() == reflect.Ptr || + mapValue.Kind() == reflect.Interface || + mapValue.Kind() == reflect.Map) && + mapValue.IsNil() { + break + } + return p.overwriteFields(mapValue, fullpath, path[1:], payload) + } + } + } + + // (Re)create this key + var mapValue reflect.Value + if m.Type().Elem().Kind() == reflect.Map { + mapValue = reflect.MakeMap(m.Type().Elem()) + } else { + mapValue = reflect.New(m.Type().Elem()) + } + if len(path) > 1 { + err := p.overwriteFields(mapValue, fullpath, path[1:], payload) + if err != nil { + return err + } + } else { + err := yaml.Unmarshal([]byte(payload), mapValue.Interface()) + if err != nil { + return err + } + } + + m.SetMapIndex(reflect.ValueOf(strings.ToLower(path[0])), reflect.Indirect(mapValue)) + + return nil +} diff --git a/vendor/github.com/docker/distribution/context/context.go b/vendor/github.com/docker/distribution/context/context.go new file mode 100644 index 000000000..ab6865467 --- /dev/null +++ b/vendor/github.com/docker/distribution/context/context.go @@ -0,0 +1,73 @@ +package context + +import ( + "context" + "sync" + + "github.com/docker/distribution/uuid" +) + +// instanceContext is a context that provides only an instance id. It is +// provided as the main background context. +type instanceContext struct { + context.Context + id string // id of context, logged as "instance.id" + once sync.Once // once protect generation of the id +} + +func (ic *instanceContext) Value(key interface{}) interface{} { + if key == "instance.id" { + ic.once.Do(func() { + // We want to lazy initialize the UUID such that we don't + // call a random generator from the package initialization + // code. For various reasons random could not be available + // https://github.com/docker/distribution/issues/782 + ic.id = uuid.Generate().String() + }) + return ic.id + } + + return ic.Context.Value(key) +} + +var background = &instanceContext{ + Context: context.Background(), +} + +// Background returns a non-nil, empty Context. The background context +// provides a single key, "instance.id" that is globally unique to the +// process. +func Background() context.Context { + return background +} + +// stringMapContext is a simple context implementation that checks a map for a +// key, falling back to a parent if not present. +type stringMapContext struct { + context.Context + m map[string]interface{} +} + +// WithValues returns a context that proxies lookups through a map. Only +// supports string keys. +func WithValues(ctx context.Context, m map[string]interface{}) context.Context { + mo := make(map[string]interface{}, len(m)) // make our own copy. + for k, v := range m { + mo[k] = v + } + + return stringMapContext{ + Context: ctx, + m: mo, + } +} + +func (smc stringMapContext) Value(key interface{}) interface{} { + if ks, ok := key.(string); ok { + if v, ok := smc.m[ks]; ok { + return v + } + } + + return smc.Context.Value(key) +} diff --git a/vendor/github.com/docker/distribution/context/doc.go b/vendor/github.com/docker/distribution/context/doc.go new file mode 100644 index 000000000..51376dd69 --- /dev/null +++ b/vendor/github.com/docker/distribution/context/doc.go @@ -0,0 +1,88 @@ +// Package context provides several utilities for working with +// Go's context in http requests. Primarily, the focus is on logging relevant +// request information but this package is not limited to that purpose. +// +// The easiest way to get started is to get the background context: +// +// ctx := context.Background() +// +// The returned context should be passed around your application and be the +// root of all other context instances. If the application has a version, this +// line should be called before anything else: +// +// ctx := context.WithVersion(context.Background(), version) +// +// The above will store the version in the context and will be available to +// the logger. +// +// # Logging +// +// The most useful aspect of this package is GetLogger. This function takes +// any context.Context interface and returns the current logger from the +// context. Canonical usage looks like this: +// +// GetLogger(ctx).Infof("something interesting happened") +// +// GetLogger also takes optional key arguments. The keys will be looked up in +// the context and reported with the logger. The following example would +// return a logger that prints the version with each log message: +// +// ctx := context.Context(context.Background(), "version", version) +// GetLogger(ctx, "version").Infof("this log message has a version field") +// +// The above would print out a log message like this: +// +// INFO[0000] this log message has a version field version=v2.0.0-alpha.2.m +// +// When used with WithLogger, we gain the ability to decorate the context with +// loggers that have information from disparate parts of the call stack. +// Following from the version example, we can build a new context with the +// configured logger such that we always print the version field: +// +// ctx = WithLogger(ctx, GetLogger(ctx, "version")) +// +// Since the logger has been pushed to the context, we can now get the version +// field for free with our log messages. Future calls to GetLogger on the new +// context will have the version field: +// +// GetLogger(ctx).Infof("this log message has a version field") +// +// This becomes more powerful when we start stacking loggers. Let's say we +// have the version logger from above but also want a request id. Using the +// context above, in our request scoped function, we place another logger in +// the context: +// +// ctx = context.WithValue(ctx, "http.request.id", "unique id") // called when building request context +// ctx = WithLogger(ctx, GetLogger(ctx, "http.request.id")) +// +// When GetLogger is called on the new context, "http.request.id" will be +// included as a logger field, along with the original "version" field: +// +// INFO[0000] this log message has a version field http.request.id=unique id version=v2.0.0-alpha.2.m +// +// Note that this only affects the new context, the previous context, with the +// version field, can be used independently. Put another way, the new logger, +// added to the request context, is unique to that context and can have +// request scoped variables. +// +// # HTTP Requests +// +// This package also contains several methods for working with http requests. +// The concepts are very similar to those described above. We simply place the +// request in the context using WithRequest. This makes the request variables +// available. GetRequestLogger can then be called to get request specific +// variables in a log line: +// +// ctx = WithRequest(ctx, req) +// GetRequestLogger(ctx).Infof("request variables") +// +// Like above, if we want to include the request data in all log messages in +// the context, we push the logger to a new context and use that one: +// +// ctx = WithLogger(ctx, GetRequestLogger(ctx)) +// +// The concept is fairly powerful and ensures that calls throughout the stack +// can be traced in log messages. Using the fields like "http.request.id", one +// can analyze call flow for a particular request with a simple grep of the +// logs. +package context diff --git a/vendor/github.com/docker/distribution/context/http.go b/vendor/github.com/docker/distribution/context/http.go new file mode 100644 index 000000000..48e354865 --- /dev/null +++ b/vendor/github.com/docker/distribution/context/http.go @@ -0,0 +1,333 @@ +package context + +import ( + "context" + "errors" + "net" + "net/http" + "strings" + "sync" + "time" + + "github.com/docker/distribution/uuid" + "github.com/gorilla/mux" + log "github.com/sirupsen/logrus" +) + +// Common errors used with this package. +var ( + ErrNoRequestContext = errors.New("no http request in context") + ErrNoResponseWriterContext = errors.New("no http response in context") +) + +func parseIP(ipStr string) net.IP { + ip := net.ParseIP(ipStr) + if ip == nil { + log.Warnf("invalid remote IP address: %q", ipStr) + } + return ip +} + +// RemoteAddr extracts the remote address of the request, taking into +// account proxy headers. +func RemoteAddr(r *http.Request) string { + if prior := r.Header.Get("X-Forwarded-For"); prior != "" { + proxies := strings.Split(prior, ",") + if len(proxies) > 0 { + remoteAddr := strings.Trim(proxies[0], " ") + if parseIP(remoteAddr) != nil { + return remoteAddr + } + } + } + // X-Real-Ip is less supported, but worth checking in the + // absence of X-Forwarded-For + if realIP := r.Header.Get("X-Real-Ip"); realIP != "" { + if parseIP(realIP) != nil { + return realIP + } + } + + return r.RemoteAddr +} + +// RemoteIP extracts the remote IP of the request, taking into +// account proxy headers. +func RemoteIP(r *http.Request) string { + addr := RemoteAddr(r) + + // Try parsing it as "IP:port" + if ip, _, err := net.SplitHostPort(addr); err == nil { + return ip + } + + return addr +} + +// WithRequest places the request on the context. The context of the request +// is assigned a unique id, available at "http.request.id". The request itself +// is available at "http.request". Other common attributes are available under +// the prefix "http.request.". If a request is already present on the context, +// this method will panic. +func WithRequest(ctx context.Context, r *http.Request) context.Context { + if ctx.Value("http.request") != nil { + // NOTE(stevvooe): This needs to be considered a programming error. It + // is unlikely that we'd want to have more than one request in + // context. + panic("only one request per context") + } + + return &httpRequestContext{ + Context: ctx, + startedAt: time.Now(), + id: uuid.Generate().String(), + r: r, + } +} + +// GetRequest returns the http request in the given context. Returns +// ErrNoRequestContext if the context does not have an http request associated +// with it. +func GetRequest(ctx context.Context) (*http.Request, error) { + if r, ok := ctx.Value("http.request").(*http.Request); r != nil && ok { + return r, nil + } + return nil, ErrNoRequestContext +} + +// GetRequestID attempts to resolve the current request id, if possible. An +// error is return if it is not available on the context. +func GetRequestID(ctx context.Context) string { + return GetStringValue(ctx, "http.request.id") +} + +// WithResponseWriter returns a new context and response writer that makes +// interesting response statistics available within the context. +func WithResponseWriter(ctx context.Context, w http.ResponseWriter) (context.Context, http.ResponseWriter) { + irw := instrumentedResponseWriter{ + ResponseWriter: w, + Context: ctx, + } + return &irw, &irw +} + +// GetResponseWriter returns the http.ResponseWriter from the provided +// context. If not present, ErrNoResponseWriterContext is returned. The +// returned instance provides instrumentation in the context. +func GetResponseWriter(ctx context.Context) (http.ResponseWriter, error) { + v := ctx.Value("http.response") + + rw, ok := v.(http.ResponseWriter) + if !ok || rw == nil { + return nil, ErrNoResponseWriterContext + } + + return rw, nil +} + +// getVarsFromRequest let's us change request vars implementation for testing +// and maybe future changes. +var getVarsFromRequest = mux.Vars + +// WithVars extracts gorilla/mux vars and makes them available on the returned +// context. Variables are available at keys with the prefix "vars.". For +// example, if looking for the variable "name", it can be accessed as +// "vars.name". Implementations that are accessing values need not know that +// the underlying context is implemented with gorilla/mux vars. +func WithVars(ctx context.Context, r *http.Request) context.Context { + return &muxVarsContext{ + Context: ctx, + vars: getVarsFromRequest(r), + } +} + +// GetRequestLogger returns a logger that contains fields from the request in +// the current context. If the request is not available in the context, no +// fields will display. Request loggers can safely be pushed onto the context. +func GetRequestLogger(ctx context.Context) Logger { + return GetLogger(ctx, + "http.request.id", + "http.request.method", + "http.request.host", + "http.request.uri", + "http.request.referer", + "http.request.useragent", + "http.request.remoteaddr", + "http.request.contenttype") +} + +// GetResponseLogger reads the current response stats and builds a logger. +// Because the values are read at call time, pushing a logger returned from +// this function on the context will lead to missing or invalid data. Only +// call this at the end of a request, after the response has been written. +func GetResponseLogger(ctx context.Context) Logger { + l := getLogrusLogger(ctx, + "http.response.written", + "http.response.status", + "http.response.contenttype") + + duration := Since(ctx, "http.request.startedat") + + if duration > 0 { + l = l.WithField("http.response.duration", duration.String()) + } + + return l +} + +// httpRequestContext makes information about a request available to context. +type httpRequestContext struct { + context.Context + + startedAt time.Time + id string + r *http.Request +} + +// Value returns a keyed element of the request for use in the context. To get +// the request itself, query "request". For other components, access them as +// "request.". For example, r.RequestURI +func (ctx *httpRequestContext) Value(key interface{}) interface{} { + if keyStr, ok := key.(string); ok { + if keyStr == "http.request" { + return ctx.r + } + + if !strings.HasPrefix(keyStr, "http.request.") { + goto fallback + } + + parts := strings.Split(keyStr, ".") + + if len(parts) != 3 { + goto fallback + } + + switch parts[2] { + case "uri": + return ctx.r.RequestURI + case "remoteaddr": + return RemoteAddr(ctx.r) + case "method": + return ctx.r.Method + case "host": + return ctx.r.Host + case "referer": + referer := ctx.r.Referer() + if referer != "" { + return referer + } + case "useragent": + return ctx.r.UserAgent() + case "id": + return ctx.id + case "startedat": + return ctx.startedAt + case "contenttype": + ct := ctx.r.Header.Get("Content-Type") + if ct != "" { + return ct + } + } + } + +fallback: + return ctx.Context.Value(key) +} + +type muxVarsContext struct { + context.Context + vars map[string]string +} + +func (ctx *muxVarsContext) Value(key interface{}) interface{} { + if keyStr, ok := key.(string); ok { + if keyStr == "vars" { + return ctx.vars + } + + if v, ok := ctx.vars[strings.TrimPrefix(keyStr, "vars.")]; ok { + return v + } + } + + return ctx.Context.Value(key) +} + +// instrumentedResponseWriter provides response writer information in a +// context. This variant is only used in the case where CloseNotifier is not +// implemented by the parent ResponseWriter. +type instrumentedResponseWriter struct { + http.ResponseWriter + context.Context + + mu sync.Mutex + status int + written int64 +} + +func (irw *instrumentedResponseWriter) Write(p []byte) (n int, err error) { + n, err = irw.ResponseWriter.Write(p) + + irw.mu.Lock() + irw.written += int64(n) + + // Guess the likely status if not set. + if irw.status == 0 { + irw.status = http.StatusOK + } + + irw.mu.Unlock() + + return +} + +func (irw *instrumentedResponseWriter) WriteHeader(status int) { + irw.ResponseWriter.WriteHeader(status) + + irw.mu.Lock() + irw.status = status + irw.mu.Unlock() +} + +func (irw *instrumentedResponseWriter) Flush() { + if flusher, ok := irw.ResponseWriter.(http.Flusher); ok { + flusher.Flush() + } +} + +func (irw *instrumentedResponseWriter) Value(key interface{}) interface{} { + if keyStr, ok := key.(string); ok { + if keyStr == "http.response" { + return irw + } + + if !strings.HasPrefix(keyStr, "http.response.") { + goto fallback + } + + parts := strings.Split(keyStr, ".") + + if len(parts) != 3 { + goto fallback + } + + irw.mu.Lock() + defer irw.mu.Unlock() + + switch parts[2] { + case "written": + return irw.written + case "status": + return irw.status + case "contenttype": + contentType := irw.Header().Get("Content-Type") + if contentType != "" { + return contentType + } + } + } + +fallback: + return irw.Context.Value(key) +} diff --git a/vendor/github.com/docker/distribution/context/logger.go b/vendor/github.com/docker/distribution/context/logger.go new file mode 100644 index 000000000..3e5b81bbf --- /dev/null +++ b/vendor/github.com/docker/distribution/context/logger.go @@ -0,0 +1,121 @@ +package context + +import ( + "context" + "fmt" + "runtime" + + "github.com/sirupsen/logrus" +) + +// Logger provides a leveled-logging interface. +type Logger interface { + // standard logger methods + Print(args ...interface{}) + Printf(format string, args ...interface{}) + Println(args ...interface{}) + + Fatal(args ...interface{}) + Fatalf(format string, args ...interface{}) + Fatalln(args ...interface{}) + + Panic(args ...interface{}) + Panicf(format string, args ...interface{}) + Panicln(args ...interface{}) + + // Leveled methods, from logrus + Debug(args ...interface{}) + Debugf(format string, args ...interface{}) + Debugln(args ...interface{}) + + Error(args ...interface{}) + Errorf(format string, args ...interface{}) + Errorln(args ...interface{}) + + Info(args ...interface{}) + Infof(format string, args ...interface{}) + Infoln(args ...interface{}) + + Warn(args ...interface{}) + Warnf(format string, args ...interface{}) + Warnln(args ...interface{}) + + WithError(err error) *logrus.Entry +} + +type loggerKey struct{} + +// WithLogger creates a new context with provided logger. +func WithLogger(ctx context.Context, logger Logger) context.Context { + return context.WithValue(ctx, loggerKey{}, logger) +} + +// GetLoggerWithField returns a logger instance with the specified field key +// and value without affecting the context. Extra specified keys will be +// resolved from the context. +func GetLoggerWithField(ctx context.Context, key, value interface{}, keys ...interface{}) Logger { + return getLogrusLogger(ctx, keys...).WithField(fmt.Sprint(key), value) +} + +// GetLoggerWithFields returns a logger instance with the specified fields +// without affecting the context. Extra specified keys will be resolved from +// the context. +func GetLoggerWithFields(ctx context.Context, fields map[interface{}]interface{}, keys ...interface{}) Logger { + // must convert from interface{} -> interface{} to string -> interface{} for logrus. + lfields := make(logrus.Fields, len(fields)) + for key, value := range fields { + lfields[fmt.Sprint(key)] = value + } + + return getLogrusLogger(ctx, keys...).WithFields(lfields) +} + +// GetLogger returns the logger from the current context, if present. If one +// or more keys are provided, they will be resolved on the context and +// included in the logger. While context.Value takes an interface, any key +// argument passed to GetLogger will be passed to fmt.Sprint when expanded as +// a logging key field. If context keys are integer constants, for example, +// its recommended that a String method is implemented. +func GetLogger(ctx context.Context, keys ...interface{}) Logger { + return getLogrusLogger(ctx, keys...) +} + +// GetLogrusLogger returns the logrus logger for the context. If one more keys +// are provided, they will be resolved on the context and included in the +// logger. Only use this function if specific logrus functionality is +// required. +func getLogrusLogger(ctx context.Context, keys ...interface{}) *logrus.Entry { + var logger *logrus.Entry + + // Get a logger, if it is present. + loggerInterface := ctx.Value(loggerKey{}) + if loggerInterface != nil { + if lgr, ok := loggerInterface.(*logrus.Entry); ok { + logger = lgr + } + } + + if logger == nil { + fields := logrus.Fields{} + + // Fill in the instance id, if we have it. + instanceID := ctx.Value("instance.id") + if instanceID != nil { + fields["instance.id"] = instanceID + } + + fields["go.version"] = runtime.Version() + // If no logger is found, just return the standard logger. + logger = logrus.StandardLogger().WithFields(fields) + } + + fields := logrus.Fields{} + for _, key := range keys { + v := ctx.Value(key) + if v != nil { + fields[fmt.Sprint(key)] = v + } + } + + return logger.WithFields(fields) +} diff --git a/vendor/github.com/docker/distribution/context/trace.go b/vendor/github.com/docker/distribution/context/trace.go new file mode 100644 index 000000000..092ef608c --- /dev/null +++ b/vendor/github.com/docker/distribution/context/trace.go @@ -0,0 +1,105 @@ +package context + +import ( + "context" + "runtime" + "time" + + "github.com/docker/distribution/uuid" +) + +// WithTrace allocates a traced timing span in a new context. This allows a +// caller to track the time between calling WithTrace and the returned done +// function. When the done function is called, a log message is emitted with a +// "trace.duration" field, corresponding to the elapsed time and a +// "trace.func" field, corresponding to the function that called WithTrace. +// +// The logging keys "trace.id" and "trace.parent.id" are provided to implement +// dapper-like tracing. This function should be complemented with a WithSpan +// method that could be used for tracing distributed RPC calls. +// +// The main benefit of this function is to post-process log messages or +// intercept them in a hook to provide timing data. Trace ids and parent ids +// can also be linked to provide call tracing, if so required. +// +// Here is an example of the usage: +// +// func timedOperation(ctx Context) { +// ctx, done := WithTrace(ctx) +// defer done("this will be the log message") +// // ... function body ... +// } +// +// If the function ran for roughly 1s, such a usage would emit a log message +// as follows: +// +// INFO[0001] this will be the log message trace.duration=1.004575763s trace.func=github.com/docker/distribution/context.traceOperation trace.id= ... +// +// Notice that the function name is automatically resolved, along with the +// package and a trace id is emitted that can be linked with parent ids. +func WithTrace(ctx context.Context) (context.Context, func(format string, a ...interface{})) { + if ctx == nil { + ctx = Background() + } + + pc, file, line, _ := runtime.Caller(1) + f := runtime.FuncForPC(pc) + ctx = &traced{ + Context: ctx, + id: uuid.Generate().String(), + start: time.Now(), + parent: GetStringValue(ctx, "trace.id"), + fnname: f.Name(), + file: file, + line: line, + } + + return ctx, func(format string, a ...interface{}) { + GetLogger(ctx, + "trace.duration", + "trace.id", + "trace.parent.id", + "trace.func", + "trace.file", + "trace.line"). + Debugf(format, a...) + } +} + +// traced represents a context that is traced for function call timing. It +// also provides fast lookup for the various attributes that are available on +// the trace. +type traced struct { + context.Context + id string + parent string + start time.Time + fnname string + file string + line int +} + +func (ts *traced) Value(key interface{}) interface{} { + switch key { + case "trace.start": + return ts.start + case "trace.duration": + return time.Since(ts.start) + case "trace.id": + return ts.id + case "trace.parent.id": + if ts.parent == "" { + return nil // must return nil to signal no parent. + } + + return ts.parent + case "trace.func": + return ts.fnname + case "trace.file": + return ts.file + case "trace.line": + return ts.line + } + + return ts.Context.Value(key) +} diff --git a/vendor/github.com/docker/distribution/context/util.go b/vendor/github.com/docker/distribution/context/util.go new file mode 100644 index 000000000..c462e7563 --- /dev/null +++ b/vendor/github.com/docker/distribution/context/util.go @@ -0,0 +1,25 @@ +package context + +import ( + "context" + "time" +) + +// Since looks up key, which should be a time.Time, and returns the duration +// since that time. If the key is not found, the value returned will be zero. +// This is helpful when inferring metrics related to context execution times. +func Since(ctx context.Context, key interface{}) time.Duration { + if startedAt, ok := ctx.Value(key).(time.Time); ok { + return time.Since(startedAt) + } + return 0 +} + +// GetStringValue returns a string value from the context. The empty string +// will be returned if not found. +func GetStringValue(ctx context.Context, key interface{}) (value string) { + if valuev, ok := ctx.Value(key).(string); ok { + value = valuev + } + return value +} diff --git a/vendor/github.com/docker/distribution/context/version.go b/vendor/github.com/docker/distribution/context/version.go new file mode 100644 index 000000000..97cf9d665 --- /dev/null +++ b/vendor/github.com/docker/distribution/context/version.go @@ -0,0 +1,22 @@ +package context + +import "context" + +type versionKey struct{} + +func (versionKey) String() string { return "version" } + +// WithVersion stores the application version in the context. The new context +// gets a logger to ensure log messages are marked with the application +// version. +func WithVersion(ctx context.Context, version string) context.Context { + ctx = context.WithValue(ctx, versionKey{}, version) + // push a new logger onto the stack + return WithLogger(ctx, GetLogger(ctx, versionKey{})) +} + +// GetVersion returns the application version from the context. An empty +// string may returned if the version was not set on the context. +func GetVersion(ctx context.Context) string { + return GetStringValue(ctx, versionKey{}) +} diff --git a/vendor/github.com/docker/distribution/doc.go b/vendor/github.com/docker/distribution/doc.go new file mode 100644 index 000000000..bdd8cb708 --- /dev/null +++ b/vendor/github.com/docker/distribution/doc.go @@ -0,0 +1,7 @@ +// Package distribution will define the interfaces for the components of +// docker distribution. The goal is to allow users to reliably package, ship +// and store content related to docker images. +// +// This is currently a work in progress. More details are available in the +// README.md. +package distribution diff --git a/vendor/github.com/docker/distribution/docker-bake.hcl b/vendor/github.com/docker/distribution/docker-bake.hcl new file mode 100644 index 000000000..91686e608 --- /dev/null +++ b/vendor/github.com/docker/distribution/docker-bake.hcl @@ -0,0 +1,56 @@ +group "default" { + targets = ["image-local"] +} + +// Special target: https://github.com/docker/metadata-action#bake-definition +target "docker-metadata-action" { + tags = ["registry:local"] +} + +target "binary" { + target = "binary" + output = ["./bin"] +} + +target "artifact" { + target = "artifact" + output = ["./bin"] +} + +target "artifact-all" { + inherits = ["artifact"] + platforms = [ + "linux/amd64", + "linux/arm/v6", + "linux/arm/v7", + "linux/arm64", + "linux/ppc64le", + "linux/s390x" + ] +} + +// Special target: https://github.com/docker/metadata-action#bake-definition +target "docker-metadata-action" { + tags = ["registry:local"] +} + +target "image" { + inherits = ["docker-metadata-action"] +} + +target "image-local" { + inherits = ["image"] + output = ["type=docker"] +} + +target "image-all" { + inherits = ["image"] + platforms = [ + "linux/amd64", + "linux/arm/v6", + "linux/arm/v7", + "linux/arm64", + "linux/ppc64le", + "linux/s390x" + ] +} diff --git a/vendor/github.com/docker/distribution/errors.go b/vendor/github.com/docker/distribution/errors.go new file mode 100644 index 000000000..8e0b788d6 --- /dev/null +++ b/vendor/github.com/docker/distribution/errors.go @@ -0,0 +1,119 @@ +package distribution + +import ( + "errors" + "fmt" + "strings" + + "github.com/opencontainers/go-digest" +) + +// ErrAccessDenied is returned when an access to a requested resource is +// denied. +var ErrAccessDenied = errors.New("access denied") + +// ErrManifestNotModified is returned when a conditional manifest GetByTag +// returns nil due to the client indicating it has the latest version +var ErrManifestNotModified = errors.New("manifest not modified") + +// ErrUnsupported is returned when an unimplemented or unsupported action is +// performed +var ErrUnsupported = errors.New("operation unsupported") + +// ErrSchemaV1Unsupported is returned when a client tries to upload a schema v1 +// manifest but the registry is configured to reject it +var ErrSchemaV1Unsupported = errors.New("manifest schema v1 unsupported") + +// ErrTagUnknown is returned if the given tag is not known by the tag service +type ErrTagUnknown struct { + Tag string +} + +func (err ErrTagUnknown) Error() string { + return fmt.Sprintf("unknown tag=%s", err.Tag) +} + +// ErrRepositoryUnknown is returned if the named repository is not known by +// the registry. +type ErrRepositoryUnknown struct { + Name string +} + +func (err ErrRepositoryUnknown) Error() string { + return fmt.Sprintf("unknown repository name=%s", err.Name) +} + +// ErrRepositoryNameInvalid should be used to denote an invalid repository +// name. Reason may set, indicating the cause of invalidity. +type ErrRepositoryNameInvalid struct { + Name string + Reason error +} + +func (err ErrRepositoryNameInvalid) Error() string { + return fmt.Sprintf("repository name %q invalid: %v", err.Name, err.Reason) +} + +// ErrManifestUnknown is returned if the manifest is not known by the +// registry. +type ErrManifestUnknown struct { + Name string + Tag string +} + +func (err ErrManifestUnknown) Error() string { + return fmt.Sprintf("unknown manifest name=%s tag=%s", err.Name, err.Tag) +} + +// ErrManifestUnknownRevision is returned when a manifest cannot be found by +// revision within a repository. +type ErrManifestUnknownRevision struct { + Name string + Revision digest.Digest +} + +func (err ErrManifestUnknownRevision) Error() string { + return fmt.Sprintf("unknown manifest name=%s revision=%s", err.Name, err.Revision) +} + +// ErrManifestUnverified is returned when the registry is unable to verify +// the manifest. +type ErrManifestUnverified struct{} + +func (ErrManifestUnverified) Error() string { + return "unverified manifest" +} + +// ErrManifestVerification provides a type to collect errors encountered +// during manifest verification. Currently, it accepts errors of all types, +// but it may be narrowed to those involving manifest verification. +type ErrManifestVerification []error + +func (errs ErrManifestVerification) Error() string { + var parts []string + for _, err := range errs { + parts = append(parts, err.Error()) + } + + return fmt.Sprintf("errors verifying manifest: %v", strings.Join(parts, ",")) +} + +// ErrManifestBlobUnknown returned when a referenced blob cannot be found. +type ErrManifestBlobUnknown struct { + Digest digest.Digest +} + +func (err ErrManifestBlobUnknown) Error() string { + return fmt.Sprintf("unknown blob %v on manifest", err.Digest) +} + +// ErrManifestNameInvalid should be used to denote an invalid manifest +// name. Reason may set, indicating the cause of invalidity. +type ErrManifestNameInvalid struct { + Name string + Reason error +} + +func (err ErrManifestNameInvalid) Error() string { + return fmt.Sprintf("manifest name %q invalid: %v", err.Name, err.Reason) +} diff --git a/vendor/github.com/docker/distribution/health/checks/checks.go b/vendor/github.com/docker/distribution/health/checks/checks.go new file mode 100644 index 000000000..7760f6105 --- /dev/null +++ b/vendor/github.com/docker/distribution/health/checks/checks.go @@ -0,0 +1,73 @@ +package checks + +import ( + "errors" + "fmt" + "net" + "net/http" + "os" + "path/filepath" + "strconv" + "time" + + "github.com/docker/distribution/health" +) + +// FileChecker checks the existence of a file and returns an error +// if the file exists. +func FileChecker(f string) health.Checker { + return health.CheckFunc(func() error { + absoluteFilePath, err := filepath.Abs(f) + if err != nil { + return fmt.Errorf("failed to get absolute path for %q: %v", f, err) + } + + _, err = os.Stat(absoluteFilePath) + if err == nil { + return errors.New("file exists") + } else if os.IsNotExist(err) { + return nil + } + + return err + }) +} + +// HTTPChecker does a HEAD request and verifies that the HTTP status code +// returned matches statusCode. +func HTTPChecker(r string, statusCode int, timeout time.Duration, headers http.Header) health.Checker { + return health.CheckFunc(func() error { + client := http.Client{ + Timeout: timeout, + } + req, err := http.NewRequest("HEAD", r, nil) + if err != nil { + return errors.New("error creating request: " + r) + } + for headerName, headerValues := range headers { + for _, headerValue := range headerValues { + req.Header.Add(headerName, headerValue) + } + } + response, err := client.Do(req) + if err != nil { + return errors.New("error while checking: " + r) + } + if response.StatusCode != statusCode { + return errors.New("downstream service returned unexpected status: " + strconv.Itoa(response.StatusCode)) + } + return nil + }) +} + +// TCPChecker attempts to open a TCP connection. +func TCPChecker(addr string, timeout time.Duration) health.Checker { + return health.CheckFunc(func() error { + conn, err := net.DialTimeout("tcp", addr, timeout) + if err != nil { + return errors.New("connection to " + addr + " failed") + } + conn.Close() + return nil + }) +} diff --git a/vendor/github.com/docker/distribution/health/doc.go b/vendor/github.com/docker/distribution/health/doc.go new file mode 100644 index 000000000..5e5ba2fc0 --- /dev/null +++ b/vendor/github.com/docker/distribution/health/doc.go @@ -0,0 +1,136 @@ +// Package health provides a generic health checking framework. +// The health package works expvar style. By importing the package the debug +// server is getting a "/debug/health" endpoint that returns the current +// status of the application. +// If there are no errors, "/debug/health" will return an HTTP 200 status, +// together with an empty JSON reply "{}". If there are any checks +// with errors, the JSON reply will include all the failed checks, and the +// response will be have an HTTP 503 status. +// +// A Check can either be run synchronously, or asynchronously. We recommend +// that most checks are registered as an asynchronous check, so a call to the +// "/debug/health" endpoint always returns immediately. This pattern is +// particularly useful for checks that verify upstream connectivity or +// database status, since they might take a long time to return/timeout. +// +// # Installing +// +// To install health, just import it in your application: +// +// import "github.com/docker/distribution/health" +// +// You can also (optionally) import "health/api" that will add two convenience +// endpoints: "/debug/health/down" and "/debug/health/up". These endpoints add +// "manual" checks that allow the service to quickly be brought in/out of +// rotation. +// +// import _ "github.com/docker/distribution/health/api" +// +// # curl localhost:5001/debug/health +// {} +// # curl -X POST localhost:5001/debug/health/down +// # curl localhost:5001/debug/health +// {"manual_http_status":"Manual Check"} +// +// After importing these packages to your main application, you can start +// registering checks. +// +// # Registering Checks +// +// The recommended way of registering checks is using a periodic Check. +// PeriodicChecks run on a certain schedule and asynchronously update the +// status of the check. This allows CheckStatus to return without blocking +// on an expensive check. +// +// A trivial example of a check that runs every 5 seconds and shuts down our +// server if the current minute is even, could be added as follows: +// +// func currentMinuteEvenCheck() error { +// m := time.Now().Minute() +// if m%2 == 0 { +// return errors.New("Current minute is even!") +// } +// return nil +// } +// +// health.RegisterPeriodicFunc("minute_even", currentMinuteEvenCheck, time.Second*5) +// +// Alternatively, you can also make use of "RegisterPeriodicThresholdFunc" to +// implement the exact same check, but add a threshold of failures after which +// the check will be unhealthy. This is particularly useful for flaky Checks, +// ensuring some stability of the service when handling them. +// +// health.RegisterPeriodicThresholdFunc("minute_even", currentMinuteEvenCheck, time.Second*5, 4) +// +// The lowest-level way to interact with the health package is calling +// "Register" directly. Register allows you to pass in an arbitrary string and +// something that implements "Checker" and runs your check. If your method +// returns an error with nil, it is considered a healthy check, otherwise it +// will make the health check endpoint "/debug/health" start returning a 503 +// and list the specific check that failed. +// +// Assuming you wish to register a method called "currentMinuteEvenCheck() +// error" you could do that by doing: +// +// health.Register("even_minute", health.CheckFunc(currentMinuteEvenCheck)) +// +// CheckFunc is a convenience type that implements Checker. +// +// Another way of registering a check could be by using an anonymous function +// and the convenience method RegisterFunc. An example that makes the status +// endpoint always return an error: +// +// health.RegisterFunc("my_check", func() error { +// return Errors.new("This is an error!") +// })) +// +// # Examples +// +// You could also use the health checker mechanism to ensure your application +// only comes up if certain conditions are met, or to allow the developer to +// take the service out of rotation immediately. An example that checks +// database connectivity and immediately takes the server out of rotation on +// err: +// +// updater = health.NewStatusUpdater() +// health.RegisterFunc("database_check", func() error { +// return updater.Check() +// })) +// +// conn, err := Connect(...) // database call here +// if err != nil { +// updater.Update(errors.New("Error connecting to the database: " + err.Error())) +// } +// +// You can also use the predefined Checkers that come included with the health +// package. First, import the checks: +// +// import "github.com/docker/distribution/health/checks +// +// After that you can make use of any of the provided checks. An example of +// using a `FileChecker` to take the application out of rotation if a certain +// file exists can be done as follows: +// +// health.Register("fileChecker", health.PeriodicChecker(checks.FileChecker("/tmp/disable"), time.Second*5)) +// +// After registering the check, it is trivial to take an application out of +// rotation from the console: +// +// # curl localhost:5001/debug/health +// {} +// # touch /tmp/disable +// # curl localhost:5001/debug/health +// {"fileChecker":"file exists"} +// +// FileChecker only accepts absolute or relative file path. It does not work +// properly with tilde(~). You should make sure that the application has +// proper permission(read and execute permission for directory along with +// the specified file path). Otherwise, the FileChecker will report error +// and file health check is not ok. +// +// You could also test the connectivity to a downstream service by using a +// "HTTPChecker", but ensure that you only mark the test unhealthy if there +// are a minimum of two failures in a row: +// +// health.Register("httpChecker", health.PeriodicThresholdChecker(checks.HTTPChecker("https://www.google.pt"), time.Second*5, 2)) +package health diff --git a/vendor/github.com/docker/distribution/health/health.go b/vendor/github.com/docker/distribution/health/health.go new file mode 100644 index 000000000..592bce576 --- /dev/null +++ b/vendor/github.com/docker/distribution/health/health.go @@ -0,0 +1,306 @@ +package health + +import ( + "encoding/json" + "fmt" + "net/http" + "sync" + "time" + + "github.com/docker/distribution/context" + "github.com/docker/distribution/registry/api/errcode" +) + +// A Registry is a collection of checks. Most applications will use the global +// registry defined in DefaultRegistry. However, unit tests may need to create +// separate registries to isolate themselves from other tests. +type Registry struct { + mu sync.RWMutex + registeredChecks map[string]Checker +} + +// NewRegistry creates a new registry. This isn't necessary for normal use of +// the package, but may be useful for unit tests so individual tests have their +// own set of checks. +func NewRegistry() *Registry { + return &Registry{ + registeredChecks: make(map[string]Checker), + } +} + +// DefaultRegistry is the default registry where checks are registered. It is +// the registry used by the HTTP handler. +var DefaultRegistry *Registry + +// Checker is the interface for a Health Checker +type Checker interface { + // Check returns nil if the service is okay. + Check() error +} + +// CheckFunc is a convenience type to create functions that implement +// the Checker interface +type CheckFunc func() error + +// Check Implements the Checker interface to allow for any func() error method +// to be passed as a Checker +func (cf CheckFunc) Check() error { + return cf() +} + +// Updater implements a health check that is explicitly set. +type Updater interface { + Checker + + // Update updates the current status of the health check. + Update(status error) +} + +// updater implements Checker and Updater, providing an asynchronous Update +// method. +// This allows us to have a Checker that returns the Check() call immediately +// not blocking on a potentially expensive check. +type updater struct { + mu sync.Mutex + status error +} + +// Check implements the Checker interface +func (u *updater) Check() error { + u.mu.Lock() + defer u.mu.Unlock() + + return u.status +} + +// Update implements the Updater interface, allowing asynchronous access to +// the status of a Checker. +func (u *updater) Update(status error) { + u.mu.Lock() + defer u.mu.Unlock() + + u.status = status +} + +// NewStatusUpdater returns a new updater +func NewStatusUpdater() Updater { + return &updater{} +} + +// thresholdUpdater implements Checker and Updater, providing an asynchronous Update +// method. +// This allows us to have a Checker that returns the Check() call immediately +// not blocking on a potentially expensive check. +type thresholdUpdater struct { + mu sync.Mutex + status error + threshold int + count int +} + +// Check implements the Checker interface +func (tu *thresholdUpdater) Check() error { + tu.mu.Lock() + defer tu.mu.Unlock() + + if tu.count >= tu.threshold { + return tu.status + } + + return nil +} + +// thresholdUpdater implements the Updater interface, allowing asynchronous +// access to the status of a Checker. +func (tu *thresholdUpdater) Update(status error) { + tu.mu.Lock() + defer tu.mu.Unlock() + + if status == nil { + tu.count = 0 + } else if tu.count < tu.threshold { + tu.count++ + } + + tu.status = status +} + +// NewThresholdStatusUpdater returns a new thresholdUpdater +func NewThresholdStatusUpdater(t int) Updater { + return &thresholdUpdater{threshold: t} +} + +// PeriodicChecker wraps an updater to provide a periodic checker +func PeriodicChecker(check Checker, period time.Duration) Checker { + u := NewStatusUpdater() + go func() { + t := time.NewTicker(period) + for { + <-t.C + u.Update(check.Check()) + } + }() + + return u +} + +// PeriodicThresholdChecker wraps an updater to provide a periodic checker that +// uses a threshold before it changes status +func PeriodicThresholdChecker(check Checker, period time.Duration, threshold int) Checker { + tu := NewThresholdStatusUpdater(threshold) + go func() { + t := time.NewTicker(period) + for { + <-t.C + tu.Update(check.Check()) + } + }() + + return tu +} + +// CheckStatus returns a map with all the current health check errors +func (registry *Registry) CheckStatus() map[string]string { // TODO(stevvooe) this needs a proper type + registry.mu.RLock() + defer registry.mu.RUnlock() + statusKeys := make(map[string]string) + for k, v := range registry.registeredChecks { + err := v.Check() + if err != nil { + statusKeys[k] = err.Error() + } + } + + return statusKeys +} + +// CheckStatus returns a map with all the current health check errors from the +// default registry. +func CheckStatus() map[string]string { + return DefaultRegistry.CheckStatus() +} + +// Register associates the checker with the provided name. +func (registry *Registry) Register(name string, check Checker) { + if registry == nil { + registry = DefaultRegistry + } + registry.mu.Lock() + defer registry.mu.Unlock() + _, ok := registry.registeredChecks[name] + if ok { + panic("Check already exists: " + name) + } + registry.registeredChecks[name] = check +} + +// Register associates the checker with the provided name in the default +// registry. +func Register(name string, check Checker) { + DefaultRegistry.Register(name, check) +} + +// RegisterFunc allows the convenience of registering a checker directly from +// an arbitrary func() error. +func (registry *Registry) RegisterFunc(name string, check func() error) { + registry.Register(name, CheckFunc(check)) +} + +// RegisterFunc allows the convenience of registering a checker in the default +// registry directly from an arbitrary func() error. +func RegisterFunc(name string, check func() error) { + DefaultRegistry.RegisterFunc(name, check) +} + +// RegisterPeriodicFunc allows the convenience of registering a PeriodicChecker +// from an arbitrary func() error. +func (registry *Registry) RegisterPeriodicFunc(name string, period time.Duration, check CheckFunc) { + registry.Register(name, PeriodicChecker(check, period)) +} + +// RegisterPeriodicFunc allows the convenience of registering a PeriodicChecker +// in the default registry from an arbitrary func() error. +func RegisterPeriodicFunc(name string, period time.Duration, check CheckFunc) { + DefaultRegistry.RegisterPeriodicFunc(name, period, check) +} + +// RegisterPeriodicThresholdFunc allows the convenience of registering a +// PeriodicChecker from an arbitrary func() error. +func (registry *Registry) RegisterPeriodicThresholdFunc(name string, period time.Duration, threshold int, check CheckFunc) { + registry.Register(name, PeriodicThresholdChecker(check, period, threshold)) +} + +// RegisterPeriodicThresholdFunc allows the convenience of registering a +// PeriodicChecker in the default registry from an arbitrary func() error. +func RegisterPeriodicThresholdFunc(name string, period time.Duration, threshold int, check CheckFunc) { + DefaultRegistry.RegisterPeriodicThresholdFunc(name, period, threshold, check) +} + +// StatusHandler returns a JSON blob with all the currently registered Health Checks +// and their corresponding status. +// Returns 503 if any Error status exists, 200 otherwise +func StatusHandler(w http.ResponseWriter, r *http.Request) { + if r.Method == "GET" { + checks := CheckStatus() + status := http.StatusOK + + // If there is an error, return 503 + if len(checks) != 0 { + status = http.StatusServiceUnavailable + } + + statusResponse(w, r, status, checks) + } else { + http.NotFound(w, r) + } +} + +// Handler returns a handler that will return 503 response code if the health +// checks have failed. If everything is okay with the health checks, the +// handler will pass through to the provided handler. Use this handler to +// disable a web application when the health checks fail. +func Handler(handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + checks := CheckStatus() + if len(checks) != 0 { + errcode.ServeJSON(w, errcode.ErrorCodeUnavailable. + WithDetail("health check failed: please see /debug/health")) + return + } + + handler.ServeHTTP(w, r) // pass through + }) +} + +// statusResponse completes the request with a response describing the health +// of the service. +func statusResponse(w http.ResponseWriter, r *http.Request, status int, checks map[string]string) { + p, err := json.Marshal(checks) + if err != nil { + context.GetLogger(context.Background()).Errorf("error serializing health status: %v", err) + p, err = json.Marshal(struct { + ServerError string `json:"server_error"` + }{ + ServerError: "Could not parse error message", + }) + status = http.StatusInternalServerError + + if err != nil { + context.GetLogger(context.Background()).Errorf("error serializing health status failure message: %v", err) + return + } + } + + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("Content-Length", fmt.Sprint(len(p))) + w.WriteHeader(status) + if _, err := w.Write(p); err != nil { + context.GetLogger(context.Background()).Errorf("error writing health status response body: %v", err) + } +} + +// Registers global /debug/health api endpoint, creates default registry +func init() { + DefaultRegistry = NewRegistry() + http.HandleFunc("/debug/health", StatusHandler) +} diff --git a/vendor/github.com/docker/distribution/manifest/doc.go b/vendor/github.com/docker/distribution/manifest/doc.go new file mode 100644 index 000000000..88367b0a0 --- /dev/null +++ b/vendor/github.com/docker/distribution/manifest/doc.go @@ -0,0 +1 @@ +package manifest diff --git a/vendor/github.com/docker/distribution/manifest/manifestlist/manifestlist.go b/vendor/github.com/docker/distribution/manifest/manifestlist/manifestlist.go new file mode 100644 index 000000000..bea2341c7 --- /dev/null +++ b/vendor/github.com/docker/distribution/manifest/manifestlist/manifestlist.go @@ -0,0 +1,239 @@ +package manifestlist + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/docker/distribution" + "github.com/docker/distribution/manifest" + "github.com/opencontainers/go-digest" + v1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +const ( + // MediaTypeManifestList specifies the mediaType for manifest lists. + MediaTypeManifestList = "application/vnd.docker.distribution.manifest.list.v2+json" +) + +// SchemaVersion provides a pre-initialized version structure for this +// packages version of the manifest. +var SchemaVersion = manifest.Versioned{ + SchemaVersion: 2, + MediaType: MediaTypeManifestList, +} + +// OCISchemaVersion provides a pre-initialized version structure for this +// packages OCIschema version of the manifest. +var OCISchemaVersion = manifest.Versioned{ + SchemaVersion: 2, + MediaType: v1.MediaTypeImageIndex, +} + +func init() { + manifestListFunc := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) { + m := new(DeserializedManifestList) + err := m.UnmarshalJSON(b) + if err != nil { + return nil, distribution.Descriptor{}, err + } + + if m.MediaType != MediaTypeManifestList { + err = fmt.Errorf("mediaType in manifest list should be '%s' not '%s'", + MediaTypeManifestList, m.MediaType) + + return nil, distribution.Descriptor{}, err + } + + dgst := digest.FromBytes(b) + return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: MediaTypeManifestList}, err + } + err := distribution.RegisterManifestSchema(MediaTypeManifestList, manifestListFunc) + if err != nil { + panic(fmt.Sprintf("Unable to register manifest: %s", err)) + } + + imageIndexFunc := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) { + if err := validateIndex(b); err != nil { + return nil, distribution.Descriptor{}, err + } + m := new(DeserializedManifestList) + err := m.UnmarshalJSON(b) + if err != nil { + return nil, distribution.Descriptor{}, err + } + + if m.MediaType != "" && m.MediaType != v1.MediaTypeImageIndex { + err = fmt.Errorf("if present, mediaType in image index should be '%s' not '%s'", + v1.MediaTypeImageIndex, m.MediaType) + + return nil, distribution.Descriptor{}, err + } + + dgst := digest.FromBytes(b) + return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: v1.MediaTypeImageIndex}, err + } + err = distribution.RegisterManifestSchema(v1.MediaTypeImageIndex, imageIndexFunc) + if err != nil { + panic(fmt.Sprintf("Unable to register OCI Image Index: %s", err)) + } +} + +// PlatformSpec specifies a platform where a particular image manifest is +// applicable. +type PlatformSpec struct { + // Architecture field specifies the CPU architecture, for example + // `amd64` or `ppc64`. + Architecture string `json:"architecture"` + + // OS specifies the operating system, for example `linux` or `windows`. + OS string `json:"os"` + + // OSVersion is an optional field specifying the operating system + // version, for example `10.0.10586`. + OSVersion string `json:"os.version,omitempty"` + + // OSFeatures is an optional field specifying an array of strings, + // each listing a required OS feature (for example on Windows `win32k`). + OSFeatures []string `json:"os.features,omitempty"` + + // Variant is an optional field specifying a variant of the CPU, for + // example `ppc64le` to specify a little-endian version of a PowerPC CPU. + Variant string `json:"variant,omitempty"` + + // Features is an optional field specifying an array of strings, each + // listing a required CPU feature (for example `sse4` or `aes`). + Features []string `json:"features,omitempty"` +} + +// A ManifestDescriptor references a platform-specific manifest. +type ManifestDescriptor struct { + distribution.Descriptor + + // Platform specifies which platform the manifest pointed to by the + // descriptor runs on. + Platform PlatformSpec `json:"platform"` +} + +// ManifestList references manifests for various platforms. +type ManifestList struct { + manifest.Versioned + + // Config references the image configuration as a blob. + Manifests []ManifestDescriptor `json:"manifests"` +} + +// References returns the distribution descriptors for the referenced image +// manifests. +func (m ManifestList) References() []distribution.Descriptor { + dependencies := make([]distribution.Descriptor, len(m.Manifests)) + for i := range m.Manifests { + dependencies[i] = m.Manifests[i].Descriptor + } + + return dependencies +} + +// DeserializedManifestList wraps ManifestList with a copy of the original +// JSON. +type DeserializedManifestList struct { + ManifestList + + // canonical is the canonical byte representation of the Manifest. + canonical []byte +} + +// FromDescriptors takes a slice of descriptors, and returns a +// DeserializedManifestList which contains the resulting manifest list +// and its JSON representation. +func FromDescriptors(descriptors []ManifestDescriptor) (*DeserializedManifestList, error) { + var mediaType string + if len(descriptors) > 0 && descriptors[0].Descriptor.MediaType == v1.MediaTypeImageManifest { + mediaType = v1.MediaTypeImageIndex + } else { + mediaType = MediaTypeManifestList + } + + return FromDescriptorsWithMediaType(descriptors, mediaType) +} + +// FromDescriptorsWithMediaType is for testing purposes, it's useful to be able to specify the media type explicitly +func FromDescriptorsWithMediaType(descriptors []ManifestDescriptor, mediaType string) (*DeserializedManifestList, error) { + m := ManifestList{ + Versioned: manifest.Versioned{ + SchemaVersion: 2, + MediaType: mediaType, + }, + } + + m.Manifests = make([]ManifestDescriptor, len(descriptors)) + copy(m.Manifests, descriptors) + + deserialized := DeserializedManifestList{ + ManifestList: m, + } + + var err error + deserialized.canonical, err = json.MarshalIndent(&m, "", " ") + return &deserialized, err +} + +// UnmarshalJSON populates a new ManifestList struct from JSON data. +func (m *DeserializedManifestList) UnmarshalJSON(b []byte) error { + m.canonical = make([]byte, len(b)) + // store manifest list in canonical + copy(m.canonical, b) + + // Unmarshal canonical JSON into ManifestList object + var manifestList ManifestList + if err := json.Unmarshal(m.canonical, &manifestList); err != nil { + return err + } + + m.ManifestList = manifestList + + return nil +} + +// MarshalJSON returns the contents of canonical. If canonical is empty, +// marshals the inner contents. +func (m *DeserializedManifestList) MarshalJSON() ([]byte, error) { + if len(m.canonical) > 0 { + return m.canonical, nil + } + + return nil, errors.New("JSON representation not initialized in DeserializedManifestList") +} + +// Payload returns the raw content of the manifest list. The contents can be +// used to calculate the content identifier. +func (m DeserializedManifestList) Payload() (string, []byte, error) { + var mediaType string + if m.MediaType == "" { + mediaType = v1.MediaTypeImageIndex + } else { + mediaType = m.MediaType + } + + return mediaType, m.canonical, nil +} + +// unknownDocument represents a manifest, manifest list, or index that has not +// yet been validated +type unknownDocument struct { + Config interface{} `json:"config,omitempty"` + Layers interface{} `json:"layers,omitempty"` +} + +// validateIndex returns an error if the byte slice is invalid JSON or if it +// contains fields that belong to a manifest +func validateIndex(b []byte) error { + var doc unknownDocument + if err := json.Unmarshal(b, &doc); err != nil { + return err + } + if doc.Config != nil || doc.Layers != nil { + return errors.New("index: expected index but found manifest") + } + return nil +} diff --git a/vendor/github.com/docker/distribution/manifest/ocischema/builder.go b/vendor/github.com/docker/distribution/manifest/ocischema/builder.go new file mode 100644 index 000000000..b89bf5b71 --- /dev/null +++ b/vendor/github.com/docker/distribution/manifest/ocischema/builder.go @@ -0,0 +1,107 @@ +package ocischema + +import ( + "context" + "errors" + + "github.com/docker/distribution" + "github.com/docker/distribution/manifest" + "github.com/opencontainers/go-digest" + v1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +// Builder is a type for constructing manifests. +type Builder struct { + // bs is a BlobService used to publish the configuration blob. + bs distribution.BlobService + + // configJSON references + configJSON []byte + + // layers is a list of layer descriptors that gets built by successive + // calls to AppendReference. + layers []distribution.Descriptor + + // Annotations contains arbitrary metadata relating to the targeted content. + annotations map[string]string + + // For testing purposes + mediaType string +} + +// NewManifestBuilder is used to build new manifests for the current schema +// version. It takes a BlobService so it can publish the configuration blob +// as part of the Build process, and annotations. +func NewManifestBuilder(bs distribution.BlobService, configJSON []byte, annotations map[string]string) distribution.ManifestBuilder { + mb := &Builder{ + bs: bs, + configJSON: make([]byte, len(configJSON)), + annotations: annotations, + mediaType: v1.MediaTypeImageManifest, + } + copy(mb.configJSON, configJSON) + + return mb +} + +// SetMediaType assigns the passed mediatype or error if the mediatype is not a +// valid media type for oci image manifests currently: "" or "application/vnd.oci.image.manifest.v1+json" +func (mb *Builder) SetMediaType(mediaType string) error { + if mediaType != "" && mediaType != v1.MediaTypeImageManifest { + return errors.New("invalid media type for OCI image manifest") + } + + mb.mediaType = mediaType + return nil +} + +// Build produces a final manifest from the given references. +func (mb *Builder) Build(ctx context.Context) (distribution.Manifest, error) { + m := Manifest{ + Versioned: manifest.Versioned{ + SchemaVersion: 2, + MediaType: mb.mediaType, + }, + Layers: make([]distribution.Descriptor, len(mb.layers)), + Annotations: mb.annotations, + } + copy(m.Layers, mb.layers) + + configDigest := digest.FromBytes(mb.configJSON) + + var err error + m.Config, err = mb.bs.Stat(ctx, configDigest) + switch err { + case nil: + // Override MediaType, since Put always replaces the specified media + // type with application/octet-stream in the descriptor it returns. + m.Config.MediaType = v1.MediaTypeImageConfig + return FromStruct(m) + case distribution.ErrBlobUnknown: + // nop + default: + return nil, err + } + + // Add config to the blob store + m.Config, err = mb.bs.Put(ctx, v1.MediaTypeImageConfig, mb.configJSON) + // Override MediaType, since Put always replaces the specified media + // type with application/octet-stream in the descriptor it returns. + m.Config.MediaType = v1.MediaTypeImageConfig + if err != nil { + return nil, err + } + + return FromStruct(m) +} + +// AppendReference adds a reference to the current ManifestBuilder. +func (mb *Builder) AppendReference(d distribution.Describable) error { + mb.layers = append(mb.layers, d.Descriptor()) + return nil +} + +// References returns the current references added to this builder. +func (mb *Builder) References() []distribution.Descriptor { + return mb.layers +} diff --git a/vendor/github.com/docker/distribution/manifest/ocischema/manifest.go b/vendor/github.com/docker/distribution/manifest/ocischema/manifest.go new file mode 100644 index 000000000..d51f8debb --- /dev/null +++ b/vendor/github.com/docker/distribution/manifest/ocischema/manifest.go @@ -0,0 +1,146 @@ +package ocischema + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/docker/distribution" + "github.com/docker/distribution/manifest" + "github.com/opencontainers/go-digest" + v1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +var ( + // SchemaVersion provides a pre-initialized version structure for this + // packages version of the manifest. + SchemaVersion = manifest.Versioned{ + SchemaVersion: 2, // historical value here.. does not pertain to OCI or docker version + MediaType: v1.MediaTypeImageManifest, + } +) + +func init() { + ocischemaFunc := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) { + if err := validateManifest(b); err != nil { + return nil, distribution.Descriptor{}, err + } + m := new(DeserializedManifest) + err := m.UnmarshalJSON(b) + if err != nil { + return nil, distribution.Descriptor{}, err + } + + dgst := digest.FromBytes(b) + return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: v1.MediaTypeImageManifest}, err + } + err := distribution.RegisterManifestSchema(v1.MediaTypeImageManifest, ocischemaFunc) + if err != nil { + panic(fmt.Sprintf("Unable to register manifest: %s", err)) + } +} + +// Manifest defines a ocischema manifest. +type Manifest struct { + manifest.Versioned + + // Config references the image configuration as a blob. + Config distribution.Descriptor `json:"config"` + + // Layers lists descriptors for the layers referenced by the + // configuration. + Layers []distribution.Descriptor `json:"layers"` + + // Annotations contains arbitrary metadata for the image manifest. + Annotations map[string]string `json:"annotations,omitempty"` +} + +// References returns the descriptors of this manifests references. +func (m Manifest) References() []distribution.Descriptor { + references := make([]distribution.Descriptor, 0, 1+len(m.Layers)) + references = append(references, m.Config) + references = append(references, m.Layers...) + return references +} + +// Target returns the target of this manifest. +func (m Manifest) Target() distribution.Descriptor { + return m.Config +} + +// DeserializedManifest wraps Manifest with a copy of the original JSON. +// It satisfies the distribution.Manifest interface. +type DeserializedManifest struct { + Manifest + + // canonical is the canonical byte representation of the Manifest. + canonical []byte +} + +// FromStruct takes a Manifest structure, marshals it to JSON, and returns a +// DeserializedManifest which contains the manifest and its JSON representation. +func FromStruct(m Manifest) (*DeserializedManifest, error) { + var deserialized DeserializedManifest + deserialized.Manifest = m + + var err error + deserialized.canonical, err = json.MarshalIndent(&m, "", " ") + return &deserialized, err +} + +// UnmarshalJSON populates a new Manifest struct from JSON data. +func (m *DeserializedManifest) UnmarshalJSON(b []byte) error { + m.canonical = make([]byte, len(b)) + // store manifest in canonical + copy(m.canonical, b) + + // Unmarshal canonical JSON into Manifest object + var manifest Manifest + if err := json.Unmarshal(m.canonical, &manifest); err != nil { + return err + } + + if manifest.MediaType != "" && manifest.MediaType != v1.MediaTypeImageManifest { + return fmt.Errorf("if present, mediaType in manifest should be '%s' not '%s'", + v1.MediaTypeImageManifest, manifest.MediaType) + } + + m.Manifest = manifest + + return nil +} + +// MarshalJSON returns the contents of canonical. If canonical is empty, +// marshals the inner contents. +func (m *DeserializedManifest) MarshalJSON() ([]byte, error) { + if len(m.canonical) > 0 { + return m.canonical, nil + } + + return nil, errors.New("JSON representation not initialized in DeserializedManifest") +} + +// Payload returns the raw content of the manifest. The contents can be used to +// calculate the content identifier. +func (m DeserializedManifest) Payload() (string, []byte, error) { + return v1.MediaTypeImageManifest, m.canonical, nil +} + +// unknownDocument represents a manifest, manifest list, or index that has not +// yet been validated +type unknownDocument struct { + Manifests interface{} `json:"manifests,omitempty"` +} + +// validateManifest returns an error if the byte slice is invalid JSON or if it +// contains fields that belong to a index +func validateManifest(b []byte) error { + var doc unknownDocument + if err := json.Unmarshal(b, &doc); err != nil { + return err + } + if doc.Manifests != nil { + return errors.New("ocimanifest: expected manifest but found index") + } + return nil +} diff --git a/vendor/github.com/docker/distribution/manifest/schema1/config_builder.go b/vendor/github.com/docker/distribution/manifest/schema1/config_builder.go new file mode 100644 index 000000000..3e60885fc --- /dev/null +++ b/vendor/github.com/docker/distribution/manifest/schema1/config_builder.go @@ -0,0 +1,287 @@ +package schema1 + +import ( + "context" + "crypto/sha512" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/distribution/reference" + "github.com/docker/distribution" + "github.com/docker/distribution/manifest" + "github.com/docker/libtrust" + "github.com/opencontainers/go-digest" +) + +type diffID digest.Digest + +// gzippedEmptyTar is a gzip-compressed version of an empty tar file +// (1024 NULL bytes) +var gzippedEmptyTar = []byte{ + 31, 139, 8, 0, 0, 9, 110, 136, 0, 255, 98, 24, 5, 163, 96, 20, 140, 88, + 0, 8, 0, 0, 255, 255, 46, 175, 181, 239, 0, 4, 0, 0, +} + +// digestSHA256GzippedEmptyTar is the canonical sha256 digest of +// gzippedEmptyTar +const digestSHA256GzippedEmptyTar = digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4") + +// configManifestBuilder is a type for constructing manifests from an image +// configuration and generic descriptors. +type configManifestBuilder struct { + // bs is a BlobService used to create empty layer tars in the + // blob store if necessary. + bs distribution.BlobService + // pk is the libtrust private key used to sign the final manifest. + pk libtrust.PrivateKey + // configJSON is configuration supplied when the ManifestBuilder was + // created. + configJSON []byte + // ref contains the name and optional tag provided to NewConfigManifestBuilder. + ref reference.Named + // descriptors is the set of descriptors referencing the layers. + descriptors []distribution.Descriptor + // emptyTarDigest is set to a valid digest if an empty tar has been + // put in the blob store; otherwise it is empty. + emptyTarDigest digest.Digest +} + +// NewConfigManifestBuilder is used to build new manifests for the current +// schema version from an image configuration and a set of descriptors. +// It takes a BlobService so that it can add an empty tar to the blob store +// if the resulting manifest needs empty layers. +func NewConfigManifestBuilder(bs distribution.BlobService, pk libtrust.PrivateKey, ref reference.Named, configJSON []byte) distribution.ManifestBuilder { + return &configManifestBuilder{ + bs: bs, + pk: pk, + configJSON: configJSON, + ref: ref, + } +} + +// Build produces a final manifest from the given references +func (mb *configManifestBuilder) Build(ctx context.Context) (m distribution.Manifest, err error) { + type imageRootFS struct { + Type string `json:"type"` + DiffIDs []diffID `json:"diff_ids,omitempty"` + BaseLayer string `json:"base_layer,omitempty"` + } + + type imageHistory struct { + Created time.Time `json:"created"` + Author string `json:"author,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + Comment string `json:"comment,omitempty"` + EmptyLayer bool `json:"empty_layer,omitempty"` + } + + type imageConfig struct { + RootFS *imageRootFS `json:"rootfs,omitempty"` + History []imageHistory `json:"history,omitempty"` + Architecture string `json:"architecture,omitempty"` + } + + var img imageConfig + + if err := json.Unmarshal(mb.configJSON, &img); err != nil { + return nil, err + } + + if len(img.History) == 0 { + return nil, errors.New("empty history when trying to create schema1 manifest") + } + + if len(img.RootFS.DiffIDs) != len(mb.descriptors) { + return nil, fmt.Errorf("number of descriptors and number of layers in rootfs must match: len(%v) != len(%v)", img.RootFS.DiffIDs, mb.descriptors) + } + + // Generate IDs for each layer + // For non-top-level layers, create fake V1Compatibility strings that + // fit the format and don't collide with anything else, but don't + // result in runnable images on their own. + type v1Compatibility struct { + ID string `json:"id"` + Parent string `json:"parent,omitempty"` + Comment string `json:"comment,omitempty"` + Created time.Time `json:"created"` + ContainerConfig struct { + Cmd []string + } `json:"container_config,omitempty"` + Author string `json:"author,omitempty"` + ThrowAway bool `json:"throwaway,omitempty"` + } + + fsLayerList := make([]FSLayer, len(img.History)) + history := make([]History, len(img.History)) + + parent := "" + layerCounter := 0 + for i, h := range img.History[:len(img.History)-1] { + var blobsum digest.Digest + if h.EmptyLayer { + if blobsum, err = mb.emptyTar(ctx); err != nil { + return nil, err + } + } else { + if len(img.RootFS.DiffIDs) <= layerCounter { + return nil, errors.New("too many non-empty layers in History section") + } + blobsum = mb.descriptors[layerCounter].Digest + layerCounter++ + } + + v1ID := digest.FromBytes([]byte(blobsum.Hex() + " " + parent)).Hex() + + if i == 0 && img.RootFS.BaseLayer != "" { + // windows-only baselayer setup + baseID := sha512.Sum384([]byte(img.RootFS.BaseLayer)) + parent = fmt.Sprintf("%x", baseID[:32]) + } + + v1Compatibility := v1Compatibility{ + ID: v1ID, + Parent: parent, + Comment: h.Comment, + Created: h.Created, + Author: h.Author, + } + v1Compatibility.ContainerConfig.Cmd = []string{img.History[i].CreatedBy} + if h.EmptyLayer { + v1Compatibility.ThrowAway = true + } + jsonBytes, err := json.Marshal(&v1Compatibility) + if err != nil { + return nil, err + } + + reversedIndex := len(img.History) - i - 1 + history[reversedIndex].V1Compatibility = string(jsonBytes) + fsLayerList[reversedIndex] = FSLayer{BlobSum: blobsum} + + parent = v1ID + } + + latestHistory := img.History[len(img.History)-1] + + var blobsum digest.Digest + if latestHistory.EmptyLayer { + if blobsum, err = mb.emptyTar(ctx); err != nil { + return nil, err + } + } else { + if len(img.RootFS.DiffIDs) <= layerCounter { + return nil, errors.New("too many non-empty layers in History section") + } + blobsum = mb.descriptors[layerCounter].Digest + } + + fsLayerList[0] = FSLayer{BlobSum: blobsum} + dgst := digest.FromBytes([]byte(blobsum.Hex() + " " + parent + " " + string(mb.configJSON))) + + // Top-level v1compatibility string should be a modified version of the + // image config. + transformedConfig, err := MakeV1ConfigFromConfig(mb.configJSON, dgst.Hex(), parent, latestHistory.EmptyLayer) + if err != nil { + return nil, err + } + + history[0].V1Compatibility = string(transformedConfig) + + tag := "" + if tagged, isTagged := mb.ref.(reference.Tagged); isTagged { + tag = tagged.Tag() + } + + mfst := Manifest{ + Versioned: manifest.Versioned{ + SchemaVersion: 1, + }, + Name: mb.ref.Name(), + Tag: tag, + Architecture: img.Architecture, + FSLayers: fsLayerList, + History: history, + } + + return Sign(&mfst, mb.pk) +} + +// emptyTar pushes a compressed empty tar to the blob store if one doesn't +// already exist, and returns its blobsum. +func (mb *configManifestBuilder) emptyTar(ctx context.Context) (digest.Digest, error) { + if mb.emptyTarDigest != "" { + // Already put an empty tar + return mb.emptyTarDigest, nil + } + + descriptor, err := mb.bs.Stat(ctx, digestSHA256GzippedEmptyTar) + switch err { + case nil: + mb.emptyTarDigest = descriptor.Digest + return descriptor.Digest, nil + case distribution.ErrBlobUnknown: + // nop + default: + return "", err + } + + // Add gzipped empty tar to the blob store + descriptor, err = mb.bs.Put(ctx, "", gzippedEmptyTar) + if err != nil { + return "", err + } + + mb.emptyTarDigest = descriptor.Digest + + return descriptor.Digest, nil +} + +// AppendReference adds a reference to the current ManifestBuilder +func (mb *configManifestBuilder) AppendReference(d distribution.Describable) error { + descriptor := d.Descriptor() + + if err := descriptor.Digest.Validate(); err != nil { + return err + } + + mb.descriptors = append(mb.descriptors, descriptor) + return nil +} + +// References returns the current references added to this builder +func (mb *configManifestBuilder) References() []distribution.Descriptor { + return mb.descriptors +} + +// MakeV1ConfigFromConfig creates an legacy V1 image config from image config JSON +func MakeV1ConfigFromConfig(configJSON []byte, v1ID, parentV1ID string, throwaway bool) ([]byte, error) { + // Top-level v1compatibility string should be a modified version of the + // image config. + var configAsMap map[string]*json.RawMessage + if err := json.Unmarshal(configJSON, &configAsMap); err != nil { + return nil, err + } + + // Delete fields that didn't exist in old manifest + delete(configAsMap, "rootfs") + delete(configAsMap, "history") + configAsMap["id"] = rawJSON(v1ID) + if parentV1ID != "" { + configAsMap["parent"] = rawJSON(parentV1ID) + } + if throwaway { + configAsMap["throwaway"] = rawJSON(true) + } + + return json.Marshal(configAsMap) +} + +func rawJSON(value interface{}) *json.RawMessage { + jsonval, err := json.Marshal(value) + if err != nil { + return nil + } + return (*json.RawMessage)(&jsonval) +} diff --git a/vendor/github.com/docker/distribution/manifest/schema1/manifest.go b/vendor/github.com/docker/distribution/manifest/schema1/manifest.go new file mode 100644 index 000000000..9fef4dc7e --- /dev/null +++ b/vendor/github.com/docker/distribution/manifest/schema1/manifest.go @@ -0,0 +1,184 @@ +package schema1 + +import ( + "encoding/json" + "fmt" + + "github.com/docker/distribution" + "github.com/docker/distribution/manifest" + "github.com/docker/libtrust" + "github.com/opencontainers/go-digest" +) + +const ( + // MediaTypeManifest specifies the mediaType for the current version. Note + // that for schema version 1, the the media is optionally "application/json". + MediaTypeManifest = "application/vnd.docker.distribution.manifest.v1+json" + // MediaTypeSignedManifest specifies the mediatype for current SignedManifest version + MediaTypeSignedManifest = "application/vnd.docker.distribution.manifest.v1+prettyjws" + // MediaTypeManifestLayer specifies the media type for manifest layers + MediaTypeManifestLayer = "application/vnd.docker.container.image.rootfs.diff+x-gtar" +) + +var ( + // SchemaVersion provides a pre-initialized version structure for this + // packages version of the manifest. + SchemaVersion = manifest.Versioned{ + SchemaVersion: 1, + } +) + +func init() { + schema1Func := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) { + sm := new(SignedManifest) + err := sm.UnmarshalJSON(b) + if err != nil { + return nil, distribution.Descriptor{}, err + } + + desc := distribution.Descriptor{ + Digest: digest.FromBytes(sm.Canonical), + Size: int64(len(sm.Canonical)), + MediaType: MediaTypeSignedManifest, + } + return sm, desc, err + } + err := distribution.RegisterManifestSchema(MediaTypeSignedManifest, schema1Func) + if err != nil { + panic(fmt.Sprintf("Unable to register manifest: %s", err)) + } + err = distribution.RegisterManifestSchema("", schema1Func) + if err != nil { + panic(fmt.Sprintf("Unable to register manifest: %s", err)) + } + err = distribution.RegisterManifestSchema("application/json", schema1Func) + if err != nil { + panic(fmt.Sprintf("Unable to register manifest: %s", err)) + } +} + +// FSLayer is a container struct for BlobSums defined in an image manifest +type FSLayer struct { + // BlobSum is the tarsum of the referenced filesystem image layer + BlobSum digest.Digest `json:"blobSum"` +} + +// History stores unstructured v1 compatibility information +type History struct { + // V1Compatibility is the raw v1 compatibility information + V1Compatibility string `json:"v1Compatibility"` +} + +// Manifest provides the base accessible fields for working with V2 image +// format in the registry. +type Manifest struct { + manifest.Versioned + + // Name is the name of the image's repository + Name string `json:"name"` + + // Tag is the tag of the image specified by this manifest + Tag string `json:"tag"` + + // Architecture is the host architecture on which this image is intended to + // run + Architecture string `json:"architecture"` + + // FSLayers is a list of filesystem layer blobSums contained in this image + FSLayers []FSLayer `json:"fsLayers"` + + // History is a list of unstructured historical data for v1 compatibility + History []History `json:"history"` +} + +// SignedManifest provides an envelope for a signed image manifest, including +// the format sensitive raw bytes. +type SignedManifest struct { + Manifest + + // Canonical is the canonical byte representation of the ImageManifest, + // without any attached signatures. The manifest byte + // representation cannot change or it will have to be re-signed. + Canonical []byte `json:"-"` + + // all contains the byte representation of the Manifest including signatures + // and is returned by Payload() + all []byte +} + +// UnmarshalJSON populates a new SignedManifest struct from JSON data. +func (sm *SignedManifest) UnmarshalJSON(b []byte) error { + sm.all = make([]byte, len(b)) + // store manifest and signatures in all + copy(sm.all, b) + + jsig, err := libtrust.ParsePrettySignature(b, "signatures") + if err != nil { + return err + } + + // Resolve the payload in the manifest. + bytes, err := jsig.Payload() + if err != nil { + return err + } + + // sm.Canonical stores the canonical manifest JSON + sm.Canonical = make([]byte, len(bytes)) + copy(sm.Canonical, bytes) + + // Unmarshal canonical JSON into Manifest object + var manifest Manifest + if err := json.Unmarshal(sm.Canonical, &manifest); err != nil { + return err + } + + sm.Manifest = manifest + + return nil +} + +// References returns the descriptors of this manifests references +func (sm SignedManifest) References() []distribution.Descriptor { + dependencies := make([]distribution.Descriptor, len(sm.FSLayers)) + for i, fsLayer := range sm.FSLayers { + dependencies[i] = distribution.Descriptor{ + MediaType: "application/vnd.docker.container.image.rootfs.diff+x-gtar", + Digest: fsLayer.BlobSum, + } + } + + return dependencies + +} + +// MarshalJSON returns the contents of raw. If Raw is nil, marshals the inner +// contents. Applications requiring a marshaled signed manifest should simply +// use Raw directly, since the the content produced by json.Marshal will be +// compacted and will fail signature checks. +func (sm *SignedManifest) MarshalJSON() ([]byte, error) { + if len(sm.all) > 0 { + return sm.all, nil + } + + // If the raw data is not available, just dump the inner content. + return json.Marshal(&sm.Manifest) +} + +// Payload returns the signed content of the signed manifest. +func (sm SignedManifest) Payload() (string, []byte, error) { + return MediaTypeSignedManifest, sm.all, nil +} + +// Signatures returns the signatures as provided by +// (*libtrust.JSONSignature).Signatures. The byte slices are opaque jws +// signatures. +func (sm *SignedManifest) Signatures() ([][]byte, error) { + jsig, err := libtrust.ParsePrettySignature(sm.all, "signatures") + if err != nil { + return nil, err + } + + // Resolve the payload in the manifest. + return jsig.Signatures() +} diff --git a/vendor/github.com/docker/distribution/manifest/schema1/reference_builder.go b/vendor/github.com/docker/distribution/manifest/schema1/reference_builder.go new file mode 100644 index 000000000..692a1d323 --- /dev/null +++ b/vendor/github.com/docker/distribution/manifest/schema1/reference_builder.go @@ -0,0 +1,98 @@ +package schema1 + +import ( + "context" + "errors" + "fmt" + + "github.com/distribution/reference" + "github.com/docker/distribution" + "github.com/docker/distribution/manifest" + "github.com/docker/libtrust" + "github.com/opencontainers/go-digest" +) + +// referenceManifestBuilder is a type for constructing manifests from schema1 +// dependencies. +type referenceManifestBuilder struct { + Manifest + pk libtrust.PrivateKey +} + +// NewReferenceManifestBuilder is used to build new manifests for the current +// schema version using schema1 dependencies. +func NewReferenceManifestBuilder(pk libtrust.PrivateKey, ref reference.Named, architecture string) distribution.ManifestBuilder { + tag := "" + if tagged, isTagged := ref.(reference.Tagged); isTagged { + tag = tagged.Tag() + } + + return &referenceManifestBuilder{ + Manifest: Manifest{ + Versioned: manifest.Versioned{ + SchemaVersion: 1, + }, + Name: ref.Name(), + Tag: tag, + Architecture: architecture, + }, + pk: pk, + } +} + +func (mb *referenceManifestBuilder) Build(ctx context.Context) (distribution.Manifest, error) { + m := mb.Manifest + if len(m.FSLayers) == 0 { + return nil, errors.New("cannot build manifest with zero layers or history") + } + + m.FSLayers = make([]FSLayer, len(mb.Manifest.FSLayers)) + m.History = make([]History, len(mb.Manifest.History)) + copy(m.FSLayers, mb.Manifest.FSLayers) + copy(m.History, mb.Manifest.History) + + return Sign(&m, mb.pk) +} + +// AppendReference adds a reference to the current ManifestBuilder +func (mb *referenceManifestBuilder) AppendReference(d distribution.Describable) error { + r, ok := d.(Reference) + if !ok { + return fmt.Errorf("unable to add non-reference type to v1 builder") + } + + // Entries need to be prepended + mb.Manifest.FSLayers = append([]FSLayer{{BlobSum: r.Digest}}, mb.Manifest.FSLayers...) + mb.Manifest.History = append([]History{r.History}, mb.Manifest.History...) + return nil + +} + +// References returns the current references added to this builder +func (mb *referenceManifestBuilder) References() []distribution.Descriptor { + refs := make([]distribution.Descriptor, len(mb.Manifest.FSLayers)) + for i := range mb.Manifest.FSLayers { + layerDigest := mb.Manifest.FSLayers[i].BlobSum + history := mb.Manifest.History[i] + ref := Reference{layerDigest, 0, history} + refs[i] = ref.Descriptor() + } + return refs +} + +// Reference describes a manifest v2, schema version 1 dependency. +// An FSLayer associated with a history entry. +type Reference struct { + Digest digest.Digest + Size int64 // if we know it, set it for the descriptor. + History History +} + +// Descriptor describes a reference +func (r Reference) Descriptor() distribution.Descriptor { + return distribution.Descriptor{ + MediaType: MediaTypeManifestLayer, + Digest: r.Digest, + Size: r.Size, + } +} diff --git a/vendor/github.com/docker/distribution/manifest/schema1/sign.go b/vendor/github.com/docker/distribution/manifest/schema1/sign.go new file mode 100644 index 000000000..c862dd812 --- /dev/null +++ b/vendor/github.com/docker/distribution/manifest/schema1/sign.go @@ -0,0 +1,68 @@ +package schema1 + +import ( + "crypto/x509" + "encoding/json" + + "github.com/docker/libtrust" +) + +// Sign signs the manifest with the provided private key, returning a +// SignedManifest. This typically won't be used within the registry, except +// for testing. +func Sign(m *Manifest, pk libtrust.PrivateKey) (*SignedManifest, error) { + p, err := json.MarshalIndent(m, "", " ") + if err != nil { + return nil, err + } + + js, err := libtrust.NewJSONSignature(p) + if err != nil { + return nil, err + } + + if err := js.Sign(pk); err != nil { + return nil, err + } + + pretty, err := js.PrettySignature("signatures") + if err != nil { + return nil, err + } + + return &SignedManifest{ + Manifest: *m, + all: pretty, + Canonical: p, + }, nil +} + +// SignWithChain signs the manifest with the given private key and x509 chain. +// The public key of the first element in the chain must be the public key +// corresponding with the sign key. +func SignWithChain(m *Manifest, key libtrust.PrivateKey, chain []*x509.Certificate) (*SignedManifest, error) { + p, err := json.MarshalIndent(m, "", " ") + if err != nil { + return nil, err + } + + js, err := libtrust.NewJSONSignature(p) + if err != nil { + return nil, err + } + + if err := js.SignWithChain(key, chain); err != nil { + return nil, err + } + + pretty, err := js.PrettySignature("signatures") + if err != nil { + return nil, err + } + + return &SignedManifest{ + Manifest: *m, + all: pretty, + Canonical: p, + }, nil +} diff --git a/vendor/github.com/docker/distribution/manifest/schema1/verify.go b/vendor/github.com/docker/distribution/manifest/schema1/verify.go new file mode 100644 index 000000000..ef59065cd --- /dev/null +++ b/vendor/github.com/docker/distribution/manifest/schema1/verify.go @@ -0,0 +1,32 @@ +package schema1 + +import ( + "crypto/x509" + + "github.com/docker/libtrust" + "github.com/sirupsen/logrus" +) + +// Verify verifies the signature of the signed manifest returning the public +// keys used during signing. +func Verify(sm *SignedManifest) ([]libtrust.PublicKey, error) { + js, err := libtrust.ParsePrettySignature(sm.all, "signatures") + if err != nil { + logrus.WithField("err", err).Debugf("(*SignedManifest).Verify") + return nil, err + } + + return js.Verify() +} + +// VerifyChains verifies the signature of the signed manifest against the +// certificate pool returning the list of verified chains. Signatures without +// an x509 chain are not checked. +func VerifyChains(sm *SignedManifest, ca *x509.CertPool) ([][]*x509.Certificate, error) { + js, err := libtrust.ParsePrettySignature(sm.all, "signatures") + if err != nil { + return nil, err + } + + return js.VerifyChains(ca) +} diff --git a/vendor/github.com/docker/distribution/manifest/schema2/builder.go b/vendor/github.com/docker/distribution/manifest/schema2/builder.go new file mode 100644 index 000000000..3facaae62 --- /dev/null +++ b/vendor/github.com/docker/distribution/manifest/schema2/builder.go @@ -0,0 +1,85 @@ +package schema2 + +import ( + "context" + + "github.com/docker/distribution" + "github.com/opencontainers/go-digest" +) + +// builder is a type for constructing manifests. +type builder struct { + // bs is a BlobService used to publish the configuration blob. + bs distribution.BlobService + + // configMediaType is media type used to describe configuration + configMediaType string + + // configJSON references + configJSON []byte + + // dependencies is a list of descriptors that gets built by successive + // calls to AppendReference. In case of image configuration these are layers. + dependencies []distribution.Descriptor +} + +// NewManifestBuilder is used to build new manifests for the current schema +// version. It takes a BlobService so it can publish the configuration blob +// as part of the Build process. +func NewManifestBuilder(bs distribution.BlobService, configMediaType string, configJSON []byte) distribution.ManifestBuilder { + mb := &builder{ + bs: bs, + configMediaType: configMediaType, + configJSON: make([]byte, len(configJSON)), + } + copy(mb.configJSON, configJSON) + + return mb +} + +// Build produces a final manifest from the given references. +func (mb *builder) Build(ctx context.Context) (distribution.Manifest, error) { + m := Manifest{ + Versioned: SchemaVersion, + Layers: make([]distribution.Descriptor, len(mb.dependencies)), + } + copy(m.Layers, mb.dependencies) + + configDigest := digest.FromBytes(mb.configJSON) + + var err error + m.Config, err = mb.bs.Stat(ctx, configDigest) + switch err { + case nil: + // Override MediaType, since Put always replaces the specified media + // type with application/octet-stream in the descriptor it returns. + m.Config.MediaType = mb.configMediaType + return FromStruct(m) + case distribution.ErrBlobUnknown: + // nop + default: + return nil, err + } + + // Add config to the blob store + m.Config, err = mb.bs.Put(ctx, mb.configMediaType, mb.configJSON) + // Override MediaType, since Put always replaces the specified media + // type with application/octet-stream in the descriptor it returns. + m.Config.MediaType = mb.configMediaType + if err != nil { + return nil, err + } + + return FromStruct(m) +} + +// AppendReference adds a reference to the current ManifestBuilder. +func (mb *builder) AppendReference(d distribution.Describable) error { + mb.dependencies = append(mb.dependencies, d.Descriptor()) + return nil +} + +// References returns the current references added to this builder. +func (mb *builder) References() []distribution.Descriptor { + return mb.dependencies +} diff --git a/vendor/github.com/docker/distribution/manifest/schema2/manifest.go b/vendor/github.com/docker/distribution/manifest/schema2/manifest.go new file mode 100644 index 000000000..41f480292 --- /dev/null +++ b/vendor/github.com/docker/distribution/manifest/schema2/manifest.go @@ -0,0 +1,144 @@ +package schema2 + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/docker/distribution" + "github.com/docker/distribution/manifest" + "github.com/opencontainers/go-digest" +) + +const ( + // MediaTypeManifest specifies the mediaType for the current version. + MediaTypeManifest = "application/vnd.docker.distribution.manifest.v2+json" + + // MediaTypeImageConfig specifies the mediaType for the image configuration. + MediaTypeImageConfig = "application/vnd.docker.container.image.v1+json" + + // MediaTypePluginConfig specifies the mediaType for plugin configuration. + MediaTypePluginConfig = "application/vnd.docker.plugin.v1+json" + + // MediaTypeLayer is the mediaType used for layers referenced by the + // manifest. + MediaTypeLayer = "application/vnd.docker.image.rootfs.diff.tar.gzip" + + // MediaTypeForeignLayer is the mediaType used for layers that must be + // downloaded from foreign URLs. + MediaTypeForeignLayer = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip" + + // MediaTypeUncompressedLayer is the mediaType used for layers which + // are not compressed. + MediaTypeUncompressedLayer = "application/vnd.docker.image.rootfs.diff.tar" +) + +var ( + // SchemaVersion provides a pre-initialized version structure for this + // packages version of the manifest. + SchemaVersion = manifest.Versioned{ + SchemaVersion: 2, + MediaType: MediaTypeManifest, + } +) + +func init() { + schema2Func := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) { + m := new(DeserializedManifest) + err := m.UnmarshalJSON(b) + if err != nil { + return nil, distribution.Descriptor{}, err + } + + dgst := digest.FromBytes(b) + return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: MediaTypeManifest}, err + } + err := distribution.RegisterManifestSchema(MediaTypeManifest, schema2Func) + if err != nil { + panic(fmt.Sprintf("Unable to register manifest: %s", err)) + } +} + +// Manifest defines a schema2 manifest. +type Manifest struct { + manifest.Versioned + + // Config references the image configuration as a blob. + Config distribution.Descriptor `json:"config"` + + // Layers lists descriptors for the layers referenced by the + // configuration. + Layers []distribution.Descriptor `json:"layers"` +} + +// References returns the descriptors of this manifests references. +func (m Manifest) References() []distribution.Descriptor { + references := make([]distribution.Descriptor, 0, 1+len(m.Layers)) + references = append(references, m.Config) + references = append(references, m.Layers...) + return references +} + +// Target returns the target of this manifest. +func (m Manifest) Target() distribution.Descriptor { + return m.Config +} + +// DeserializedManifest wraps Manifest with a copy of the original JSON. +// It satisfies the distribution.Manifest interface. +type DeserializedManifest struct { + Manifest + + // canonical is the canonical byte representation of the Manifest. + canonical []byte +} + +// FromStruct takes a Manifest structure, marshals it to JSON, and returns a +// DeserializedManifest which contains the manifest and its JSON representation. +func FromStruct(m Manifest) (*DeserializedManifest, error) { + var deserialized DeserializedManifest + deserialized.Manifest = m + + var err error + deserialized.canonical, err = json.MarshalIndent(&m, "", " ") + return &deserialized, err +} + +// UnmarshalJSON populates a new Manifest struct from JSON data. +func (m *DeserializedManifest) UnmarshalJSON(b []byte) error { + m.canonical = make([]byte, len(b)) + // store manifest in canonical + copy(m.canonical, b) + + // Unmarshal canonical JSON into Manifest object + var manifest Manifest + if err := json.Unmarshal(m.canonical, &manifest); err != nil { + return err + } + + if manifest.MediaType != MediaTypeManifest { + return fmt.Errorf("mediaType in manifest should be '%s' not '%s'", + MediaTypeManifest, manifest.MediaType) + + } + + m.Manifest = manifest + + return nil +} + +// MarshalJSON returns the contents of canonical. If canonical is empty, +// marshals the inner contents. +func (m *DeserializedManifest) MarshalJSON() ([]byte, error) { + if len(m.canonical) > 0 { + return m.canonical, nil + } + + return nil, errors.New("JSON representation not initialized in DeserializedManifest") +} + +// Payload returns the raw content of the manifest. The contents can be used to +// calculate the content identifier. +func (m DeserializedManifest) Payload() (string, []byte, error) { + return m.MediaType, m.canonical, nil +} diff --git a/vendor/github.com/docker/distribution/manifest/versioned.go b/vendor/github.com/docker/distribution/manifest/versioned.go new file mode 100644 index 000000000..caa6b14e8 --- /dev/null +++ b/vendor/github.com/docker/distribution/manifest/versioned.go @@ -0,0 +1,12 @@ +package manifest + +// Versioned provides a struct with the manifest schemaVersion and mediaType. +// Incoming content with unknown schema version can be decoded against this +// struct to check the version. +type Versioned struct { + // SchemaVersion is the image manifest schema that this image follows + SchemaVersion int `json:"schemaVersion"` + + // MediaType is the media type of this schema. + MediaType string `json:"mediaType,omitempty"` +} diff --git a/vendor/github.com/docker/distribution/manifests.go b/vendor/github.com/docker/distribution/manifests.go new file mode 100644 index 000000000..8f84a220a --- /dev/null +++ b/vendor/github.com/docker/distribution/manifests.go @@ -0,0 +1,125 @@ +package distribution + +import ( + "context" + "fmt" + "mime" + + "github.com/opencontainers/go-digest" +) + +// Manifest represents a registry object specifying a set of +// references and an optional target +type Manifest interface { + // References returns a list of objects which make up this manifest. + // A reference is anything which can be represented by a + // distribution.Descriptor. These can consist of layers, resources or other + // manifests. + // + // While no particular order is required, implementations should return + // them from highest to lowest priority. For example, one might want to + // return the base layer before the top layer. + References() []Descriptor + + // Payload provides the serialized format of the manifest, in addition to + // the media type. + Payload() (mediaType string, payload []byte, err error) +} + +// ManifestBuilder creates a manifest allowing one to include dependencies. +// Instances can be obtained from a version-specific manifest package. Manifest +// specific data is passed into the function which creates the builder. +type ManifestBuilder interface { + // Build creates the manifest from his builder. + Build(ctx context.Context) (Manifest, error) + + // References returns a list of objects which have been added to this + // builder. The dependencies are returned in the order they were added, + // which should be from base to head. + References() []Descriptor + + // AppendReference includes the given object in the manifest after any + // existing dependencies. If the add fails, such as when adding an + // unsupported dependency, an error may be returned. + // + // The destination of the reference is dependent on the manifest type and + // the dependency type. + AppendReference(dependency Describable) error +} + +// ManifestService describes operations on image manifests. +type ManifestService interface { + // Exists returns true if the manifest exists. + Exists(ctx context.Context, dgst digest.Digest) (bool, error) + + // Get retrieves the manifest specified by the given digest + Get(ctx context.Context, dgst digest.Digest, options ...ManifestServiceOption) (Manifest, error) + + // Put creates or updates the given manifest returning the manifest digest + Put(ctx context.Context, manifest Manifest, options ...ManifestServiceOption) (digest.Digest, error) + + // Delete removes the manifest specified by the given digest. Deleting + // a manifest that doesn't exist will return ErrManifestNotFound + Delete(ctx context.Context, dgst digest.Digest) error +} + +// ManifestEnumerator enables iterating over manifests +type ManifestEnumerator interface { + // Enumerate calls ingester for each manifest. + Enumerate(ctx context.Context, ingester func(digest.Digest) error) error +} + +// Describable is an interface for descriptors +type Describable interface { + Descriptor() Descriptor +} + +// ManifestMediaTypes returns the supported media types for manifests. +func ManifestMediaTypes() (mediaTypes []string) { + for t := range mappings { + if t != "" { + mediaTypes = append(mediaTypes, t) + } + } + return +} + +// UnmarshalFunc implements manifest unmarshalling a given MediaType +type UnmarshalFunc func([]byte) (Manifest, Descriptor, error) + +var mappings = make(map[string]UnmarshalFunc) + +// UnmarshalManifest looks up manifest unmarshal functions based on +// MediaType +func UnmarshalManifest(ctHeader string, p []byte) (Manifest, Descriptor, error) { + // Need to look up by the actual media type, not the raw contents of + // the header. Strip semicolons and anything following them. + var mediaType string + if ctHeader != "" { + var err error + mediaType, _, err = mime.ParseMediaType(ctHeader) + if err != nil { + return nil, Descriptor{}, err + } + } + + unmarshalFunc, ok := mappings[mediaType] + if !ok { + unmarshalFunc, ok = mappings[""] + if !ok { + return nil, Descriptor{}, fmt.Errorf("unsupported manifest media type and no default available: %s", mediaType) + } + } + + return unmarshalFunc(p) +} + +// RegisterManifestSchema registers an UnmarshalFunc for a given schema type. This +// should be called from specific +func RegisterManifestSchema(mediaType string, u UnmarshalFunc) error { + if _, ok := mappings[mediaType]; ok { + return fmt.Errorf("manifest media type registration would overwrite existing: %s", mediaType) + } + mappings[mediaType] = u + return nil +} diff --git a/vendor/github.com/docker/distribution/metrics/prometheus.go b/vendor/github.com/docker/distribution/metrics/prometheus.go new file mode 100644 index 000000000..b5a532144 --- /dev/null +++ b/vendor/github.com/docker/distribution/metrics/prometheus.go @@ -0,0 +1,13 @@ +package metrics + +import "github.com/docker/go-metrics" + +const ( + // NamespacePrefix is the namespace of prometheus metrics + NamespacePrefix = "registry" +) + +var ( + // StorageNamespace is the prometheus namespace of blob/cache related operations + StorageNamespace = metrics.NewNamespace(NamespacePrefix, "storage", nil) +) diff --git a/vendor/github.com/docker/distribution/notifications/bridge.go b/vendor/github.com/docker/distribution/notifications/bridge.go new file mode 100644 index 000000000..d68811c65 --- /dev/null +++ b/vendor/github.com/docker/distribution/notifications/bridge.go @@ -0,0 +1,225 @@ +package notifications + +import ( + "net/http" + "time" + + "github.com/distribution/reference" + "github.com/docker/distribution" + "github.com/docker/distribution/context" + "github.com/docker/distribution/uuid" + "github.com/opencontainers/go-digest" +) + +type bridge struct { + ub URLBuilder + includeReferences bool + actor ActorRecord + source SourceRecord + request RequestRecord + sink Sink +} + +var _ Listener = &bridge{} + +// URLBuilder defines a subset of url builder to be used by the event listener. +type URLBuilder interface { + BuildManifestURL(name reference.Named) (string, error) + BuildBlobURL(ref reference.Canonical) (string, error) +} + +// NewBridge returns a notification listener that writes records to sink, +// using the actor and source. Any urls populated in the events created by +// this bridge will be created using the URLBuilder. +// TODO(stevvooe): Update this to simply take a context.Context object. +func NewBridge(ub URLBuilder, source SourceRecord, actor ActorRecord, request RequestRecord, sink Sink, includeReferences bool) Listener { + return &bridge{ + ub: ub, + includeReferences: includeReferences, + actor: actor, + source: source, + request: request, + sink: sink, + } +} + +// NewRequestRecord builds a RequestRecord for use in NewBridge from an +// http.Request, associating it with a request id. +func NewRequestRecord(id string, r *http.Request) RequestRecord { + return RequestRecord{ + ID: id, + Addr: context.RemoteAddr(r), + Host: r.Host, + Method: r.Method, + UserAgent: r.UserAgent(), + } +} + +func (b *bridge) ManifestPushed(repo reference.Named, sm distribution.Manifest, options ...distribution.ManifestServiceOption) error { + manifestEvent, err := b.createManifestEvent(EventActionPush, repo, sm) + if err != nil { + return err + } + + for _, option := range options { + if opt, ok := option.(distribution.WithTagOption); ok { + manifestEvent.Target.Tag = opt.Tag + break + } + } + return b.sink.Write(*manifestEvent) +} + +func (b *bridge) ManifestPulled(repo reference.Named, sm distribution.Manifest, options ...distribution.ManifestServiceOption) error { + manifestEvent, err := b.createManifestEvent(EventActionPull, repo, sm) + if err != nil { + return err + } + + for _, option := range options { + if opt, ok := option.(distribution.WithTagOption); ok { + manifestEvent.Target.Tag = opt.Tag + break + } + } + return b.sink.Write(*manifestEvent) +} + +func (b *bridge) ManifestDeleted(repo reference.Named, dgst digest.Digest) error { + return b.createManifestDeleteEventAndWrite(EventActionDelete, repo, dgst) +} + +func (b *bridge) BlobPushed(repo reference.Named, desc distribution.Descriptor) error { + return b.createBlobEventAndWrite(EventActionPush, repo, desc) +} + +func (b *bridge) BlobPulled(repo reference.Named, desc distribution.Descriptor) error { + return b.createBlobEventAndWrite(EventActionPull, repo, desc) +} + +func (b *bridge) BlobMounted(repo reference.Named, desc distribution.Descriptor, fromRepo reference.Named) error { + event, err := b.createBlobEvent(EventActionMount, repo, desc) + if err != nil { + return err + } + event.Target.FromRepository = fromRepo.Name() + return b.sink.Write(*event) +} + +func (b *bridge) BlobDeleted(repo reference.Named, dgst digest.Digest) error { + return b.createBlobDeleteEventAndWrite(EventActionDelete, repo, dgst) +} + +func (b *bridge) TagDeleted(repo reference.Named, tag string) error { + event := b.createEvent(EventActionDelete) + event.Target.Repository = repo.Name() + event.Target.Tag = tag + + return b.sink.Write(*event) +} + +func (b *bridge) RepoDeleted(repo reference.Named) error { + event := b.createEvent(EventActionDelete) + event.Target.Repository = repo.Name() + + return b.sink.Write(*event) +} + +func (b *bridge) createManifestDeleteEventAndWrite(action string, repo reference.Named, dgst digest.Digest) error { + event := b.createEvent(action) + event.Target.Repository = repo.Name() + event.Target.Digest = dgst + + return b.sink.Write(*event) +} + +func (b *bridge) createManifestEvent(action string, repo reference.Named, sm distribution.Manifest) (*Event, error) { + event := b.createEvent(action) + event.Target.Repository = repo.Name() + + mt, p, err := sm.Payload() + if err != nil { + return nil, err + } + + // Ensure we have the canonical manifest descriptor here + manifest, desc, err := distribution.UnmarshalManifest(mt, p) + if err != nil { + return nil, err + } + + event.Target.MediaType = mt + event.Target.Length = desc.Size + event.Target.Size = desc.Size + event.Target.Digest = desc.Digest + if b.includeReferences { + event.Target.References = append(event.Target.References, manifest.References()...) + } + + ref, err := reference.WithDigest(repo, event.Target.Digest) + if err != nil { + return nil, err + } + + event.Target.URL, err = b.ub.BuildManifestURL(ref) + if err != nil { + return nil, err + } + + return event, nil +} + +func (b *bridge) createBlobDeleteEventAndWrite(action string, repo reference.Named, dgst digest.Digest) error { + event := b.createEvent(action) + event.Target.Digest = dgst + event.Target.Repository = repo.Name() + + return b.sink.Write(*event) +} + +func (b *bridge) createBlobEventAndWrite(action string, repo reference.Named, desc distribution.Descriptor) error { + event, err := b.createBlobEvent(action, repo, desc) + if err != nil { + return err + } + + return b.sink.Write(*event) +} + +func (b *bridge) createBlobEvent(action string, repo reference.Named, desc distribution.Descriptor) (*Event, error) { + event := b.createEvent(action) + event.Target.Descriptor = desc + event.Target.Length = desc.Size + event.Target.Repository = repo.Name() + + ref, err := reference.WithDigest(repo, desc.Digest) + if err != nil { + return nil, err + } + + event.Target.URL, err = b.ub.BuildBlobURL(ref) + if err != nil { + return nil, err + } + + return event, nil +} + +// createEvent creates an event with actor and source populated. +func (b *bridge) createEvent(action string) *Event { + event := createEvent(action) + event.Source = b.source + event.Actor = b.actor + event.Request = b.request + + return event +} + +// createEvent returns a new event, timestamped, with the specified action. +func createEvent(action string) *Event { + return &Event{ + ID: uuid.Generate().String(), + Timestamp: time.Now(), + Action: action, + } +} diff --git a/vendor/github.com/docker/distribution/notifications/endpoint.go b/vendor/github.com/docker/distribution/notifications/endpoint.go new file mode 100644 index 000000000..a8a52d0c9 --- /dev/null +++ b/vendor/github.com/docker/distribution/notifications/endpoint.go @@ -0,0 +1,97 @@ +package notifications + +import ( + "net/http" + "time" + + "github.com/docker/distribution/configuration" +) + +// EndpointConfig covers the optional configuration parameters for an active +// endpoint. +type EndpointConfig struct { + Headers http.Header + Timeout time.Duration + Threshold int + Backoff time.Duration + IgnoredMediaTypes []string + Transport *http.Transport `json:"-"` + Ignore configuration.Ignore +} + +// defaults set any zero-valued fields to a reasonable default. +func (ec *EndpointConfig) defaults() { + if ec.Timeout <= 0 { + ec.Timeout = time.Second + } + + if ec.Threshold <= 0 { + ec.Threshold = 10 + } + + if ec.Backoff <= 0 { + ec.Backoff = time.Second + } + + if ec.Transport == nil { + ec.Transport = http.DefaultTransport.(*http.Transport) + } +} + +// Endpoint is a reliable, queued, thread-safe sink that notify external http +// services when events are written. Writes are non-blocking and always +// succeed for callers but events may be queued internally. +type Endpoint struct { + Sink + url string + name string + + EndpointConfig + + metrics *safeMetrics +} + +// NewEndpoint returns a running endpoint, ready to receive events. +func NewEndpoint(name, url string, config EndpointConfig) *Endpoint { + var endpoint Endpoint + endpoint.name = name + endpoint.url = url + endpoint.EndpointConfig = config + endpoint.defaults() + endpoint.metrics = newSafeMetrics() + + // Configures the inmemory queue, retry, http pipeline. + endpoint.Sink = newHTTPSink( + endpoint.url, endpoint.Timeout, endpoint.Headers, + endpoint.Transport, endpoint.metrics.httpStatusListener()) + endpoint.Sink = newRetryingSink(endpoint.Sink, endpoint.Threshold, endpoint.Backoff) + endpoint.Sink = newEventQueue(endpoint.Sink, endpoint.metrics.eventQueueListener()) + mediaTypes := append(config.Ignore.MediaTypes, config.IgnoredMediaTypes...) + endpoint.Sink = newIgnoredSink(endpoint.Sink, mediaTypes, config.Ignore.Actions) + + register(&endpoint) + return &endpoint +} + +// Name returns the name of the endpoint, generally used for debugging. +func (e *Endpoint) Name() string { + return e.name +} + +// URL returns the url of the endpoint. +func (e *Endpoint) URL() string { + return e.url +} + +// ReadMetrics populates em with metrics from the endpoint. +func (e *Endpoint) ReadMetrics(em *EndpointMetrics) { + e.metrics.Lock() + defer e.metrics.Unlock() + + *em = e.metrics.EndpointMetrics + // Map still need to copied in a threadsafe manner. + em.Statuses = make(map[string]int) + for k, v := range e.metrics.Statuses { + em.Statuses[k] = v + } +} diff --git a/vendor/github.com/docker/distribution/notifications/event.go b/vendor/github.com/docker/distribution/notifications/event.go new file mode 100644 index 000000000..54940a46f --- /dev/null +++ b/vendor/github.com/docker/distribution/notifications/event.go @@ -0,0 +1,163 @@ +package notifications + +import ( + "fmt" + "time" + + "github.com/docker/distribution" +) + +// EventAction constants used in action field of Event. +const ( + EventActionPull = "pull" + EventActionPush = "push" + EventActionMount = "mount" + EventActionDelete = "delete" +) + +const ( + // EventsMediaType is the mediatype for the json event envelope. If the + // Event, ActorRecord, SourceRecord or Envelope structs change, the version + // number should be incremented. + EventsMediaType = "application/vnd.docker.distribution.events.v1+json" + // LayerMediaType is the media type for image rootfs diffs (aka "layers") + // used by Docker. We don't expect this to change for quite a while. + layerMediaType = "application/vnd.docker.container.image.rootfs.diff+x-gtar" +) + +// Envelope defines the fields of a json event envelope message that can hold +// one or more events. +type Envelope struct { + // Events make up the contents of the envelope. Events present in a single + // envelope are not necessarily related. + Events []Event `json:"events,omitempty"` +} + +// TODO(stevvooe): The event type should be separate from the json format. It +// should be defined as an interface. Leaving as is for now since we don't +// need that at this time. If we make this change, the struct below would be +// called "EventRecord". + +// Event provides the fields required to describe a registry event. +type Event struct { + // ID provides a unique identifier for the event. + ID string `json:"id,omitempty"` + + // Timestamp is the time at which the event occurred. + Timestamp time.Time `json:"timestamp,omitempty"` + + // Action indicates what action encompasses the provided event. + Action string `json:"action,omitempty"` + + // Target uniquely describes the target of the event. + Target struct { + // TODO(stevvooe): Use http.DetectContentType for layers, maybe. + + distribution.Descriptor + + // Length in bytes of content. Same as Size field in Descriptor. + // Provided for backwards compatibility. + Length int64 `json:"length,omitempty"` + + // Repository identifies the named repository. + Repository string `json:"repository,omitempty"` + + // FromRepository identifies the named repository which a blob was mounted + // from if appropriate. + FromRepository string `json:"fromRepository,omitempty"` + + // URL provides a direct link to the content. + URL string `json:"url,omitempty"` + + // Tag provides the tag + Tag string `json:"tag,omitempty"` + + // References provides the references descriptors. + References []distribution.Descriptor `json:"references,omitempty"` + } `json:"target,omitempty"` + + // Request covers the request that generated the event. + Request RequestRecord `json:"request,omitempty"` + + // Actor specifies the agent that initiated the event. For most + // situations, this could be from the authorization context of the request. + Actor ActorRecord `json:"actor,omitempty"` + + // Source identifies the registry node that generated the event. Put + // differently, while the actor "initiates" the event, the source + // "generates" it. + Source SourceRecord `json:"source,omitempty"` +} + +// ActorRecord specifies the agent that initiated the event. For most +// situations, this could be from the authorization context of the request. +// Data in this record can refer to both the initiating client and the +// generating request. +type ActorRecord struct { + // Name corresponds to the subject or username associated with the + // request context that generated the event. + Name string `json:"name,omitempty"` + + // TODO(stevvooe): Look into setting a session cookie to get this + // without docker daemon. + // SessionID + + // TODO(stevvooe): Push the "Docker-Command" header to replace cookie and + // get the actual command. + // Command +} + +// RequestRecord covers the request that generated the event. +type RequestRecord struct { + // ID uniquely identifies the request that initiated the event. + ID string `json:"id"` + + // Addr contains the ip or hostname and possibly port of the client + // connection that initiated the event. This is the RemoteAddr from + // the standard http request. + Addr string `json:"addr,omitempty"` + + // Host is the externally accessible host name of the registry instance, + // as specified by the http host header on incoming requests. + Host string `json:"host,omitempty"` + + // Method has the request method that generated the event. + Method string `json:"method"` + + // UserAgent contains the user agent header of the request. + UserAgent string `json:"useragent"` +} + +// SourceRecord identifies the registry node that generated the event. Put +// differently, while the actor "initiates" the event, the source "generates" +// it. +type SourceRecord struct { + // Addr contains the ip or hostname and the port of the registry node + // that generated the event. Generally, this will be resolved by + // os.Hostname() along with the running port. + Addr string `json:"addr,omitempty"` + + // InstanceID identifies a running instance of an application. Changes + // after each restart. + InstanceID string `json:"instanceID,omitempty"` +} + +var ( + // ErrSinkClosed is returned if a write is issued to a sink that has been + // closed. If encountered, the error should be considered terminal and + // retries will not be successful. + ErrSinkClosed = fmt.Errorf("sink: closed") +) + +// Sink accepts and sends events. +type Sink interface { + // Write writes one or more events to the sink. If no error is returned, + // the caller will assume that all events have been committed and will not + // try to send them again. If an error is received, the caller may retry + // sending the event. The caller should cede the slice of memory to the + // sink and not modify it after calling this method. + Write(events ...Event) error + + // Close the sink, possibly waiting for pending events to flush. + Close() error +} diff --git a/vendor/github.com/docker/distribution/notifications/http.go b/vendor/github.com/docker/distribution/notifications/http.go new file mode 100644 index 000000000..46f47af2b --- /dev/null +++ b/vendor/github.com/docker/distribution/notifications/http.go @@ -0,0 +1,149 @@ +package notifications + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "sync" + "time" +) + +// httpSink implements a single-flight, http notification endpoint. This is +// very lightweight in that it only makes an attempt at an http request. +// Reliability should be provided by the caller. +type httpSink struct { + url string + + mu sync.Mutex + closed bool + client *http.Client + listeners []httpStatusListener + + // TODO(stevvooe): Allow one to configure the media type accepted by this + // sink and choose the serialization based on that. +} + +// newHTTPSink returns an unreliable, single-flight http sink. Wrap in other +// sinks for increased reliability. +func newHTTPSink(u string, timeout time.Duration, headers http.Header, transport *http.Transport, listeners ...httpStatusListener) *httpSink { + if transport == nil { + transport = http.DefaultTransport.(*http.Transport) + } + return &httpSink{ + url: u, + listeners: listeners, + client: &http.Client{ + Transport: &headerRoundTripper{ + Transport: transport, + headers: headers, + }, + Timeout: timeout, + }, + } +} + +// httpStatusListener is called on various outcomes of sending notifications. +type httpStatusListener interface { + success(status int, events ...Event) + failure(status int, events ...Event) + err(err error, events ...Event) +} + +// Accept makes an attempt to notify the endpoint, returning an error if it +// fails. It is the caller's responsibility to retry on error. The events are +// accepted or rejected as a group. +func (hs *httpSink) Write(events ...Event) error { + hs.mu.Lock() + defer hs.mu.Unlock() + defer hs.client.Transport.(*headerRoundTripper).CloseIdleConnections() + + if hs.closed { + return ErrSinkClosed + } + + envelope := Envelope{ + Events: events, + } + + // TODO(stevvooe): It is not ideal to keep re-encoding the request body on + // retry but we are going to do it to keep the code simple. It is likely + // we could change the event struct to manage its own buffer. + + p, err := json.MarshalIndent(envelope, "", " ") + if err != nil { + for _, listener := range hs.listeners { + listener.err(err, events...) + } + return fmt.Errorf("%v: error marshaling event envelope: %v", hs, err) + } + + body := bytes.NewReader(p) + resp, err := hs.client.Post(hs.url, EventsMediaType, body) + if err != nil { + for _, listener := range hs.listeners { + listener.err(err, events...) + } + + return fmt.Errorf("%v: error posting: %v", hs, err) + } + defer resp.Body.Close() + + // The notifier will treat any 2xx or 3xx response as accepted by the + // endpoint. + switch { + case resp.StatusCode >= 200 && resp.StatusCode < 400: + for _, listener := range hs.listeners { + listener.success(resp.StatusCode, events...) + } + + // TODO(stevvooe): This is a little accepting: we may want to support + // unsupported media type responses with retries using the correct + // media type. There may also be cases that will never work. + + return nil + default: + for _, listener := range hs.listeners { + listener.failure(resp.StatusCode, events...) + } + return fmt.Errorf("%v: response status %v unaccepted", hs, resp.Status) + } +} + +// Close the endpoint +func (hs *httpSink) Close() error { + hs.mu.Lock() + defer hs.mu.Unlock() + + if hs.closed { + return fmt.Errorf("httpsink: already closed") + } + + hs.closed = true + return nil +} + +func (hs *httpSink) String() string { + return fmt.Sprintf("httpSink{%s}", hs.url) +} + +type headerRoundTripper struct { + *http.Transport // must be transport to support CancelRequest + headers http.Header +} + +func (hrt *headerRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + var nreq = *req + nreq.Header = make(http.Header) + + merge := func(headers http.Header) { + for k, v := range headers { + nreq.Header[k] = append(nreq.Header[k], v...) + } + } + + merge(req.Header) + merge(hrt.headers) + + return hrt.Transport.RoundTrip(&nreq) +} diff --git a/vendor/github.com/docker/distribution/notifications/listener.go b/vendor/github.com/docker/distribution/notifications/listener.go new file mode 100644 index 000000000..51d833d4d --- /dev/null +++ b/vendor/github.com/docker/distribution/notifications/listener.go @@ -0,0 +1,263 @@ +package notifications + +import ( + "context" + "net/http" + + "github.com/docker/distribution" + + "github.com/distribution/reference" + dcontext "github.com/docker/distribution/context" + "github.com/opencontainers/go-digest" +) + +// ManifestListener describes a set of methods for listening to events related to manifests. +type ManifestListener interface { + ManifestPushed(repo reference.Named, sm distribution.Manifest, options ...distribution.ManifestServiceOption) error + ManifestPulled(repo reference.Named, sm distribution.Manifest, options ...distribution.ManifestServiceOption) error + ManifestDeleted(repo reference.Named, dgst digest.Digest) error +} + +// BlobListener describes a listener that can respond to layer related events. +type BlobListener interface { + BlobPushed(repo reference.Named, desc distribution.Descriptor) error + BlobPulled(repo reference.Named, desc distribution.Descriptor) error + BlobMounted(repo reference.Named, desc distribution.Descriptor, fromRepo reference.Named) error + BlobDeleted(repo reference.Named, desc digest.Digest) error +} + +// RepoListener provides repository methods that respond to repository lifecycle +type RepoListener interface { + TagDeleted(repo reference.Named, tag string) error + RepoDeleted(repo reference.Named) error +} + +// Listener combines all repository events into a single interface. +type Listener interface { + ManifestListener + BlobListener + RepoListener +} + +type repositoryListener struct { + distribution.Repository + listener Listener +} + +type removerListener struct { + distribution.RepositoryRemover + listener Listener +} + +// Listen dispatches events on the repository to the listener. +func Listen(repo distribution.Repository, remover distribution.RepositoryRemover, listener Listener) (distribution.Repository, distribution.RepositoryRemover) { + return &repositoryListener{ + Repository: repo, + listener: listener, + }, &removerListener{ + RepositoryRemover: remover, + listener: listener, + } +} + +func (nl *removerListener) Remove(ctx context.Context, name reference.Named) error { + err := nl.RepositoryRemover.Remove(ctx, name) + if err != nil { + return err + } + return nl.listener.RepoDeleted(name) +} + +func (rl *repositoryListener) Manifests(ctx context.Context, options ...distribution.ManifestServiceOption) (distribution.ManifestService, error) { + manifests, err := rl.Repository.Manifests(ctx, options...) + if err != nil { + return nil, err + } + return &manifestServiceListener{ + ManifestService: manifests, + parent: rl, + }, nil +} + +func (rl *repositoryListener) Blobs(ctx context.Context) distribution.BlobStore { + return &blobServiceListener{ + BlobStore: rl.Repository.Blobs(ctx), + parent: rl, + } +} + +type manifestServiceListener struct { + distribution.ManifestService + parent *repositoryListener +} + +func (msl *manifestServiceListener) Delete(ctx context.Context, dgst digest.Digest) error { + err := msl.ManifestService.Delete(ctx, dgst) + if err == nil { + if err := msl.parent.listener.ManifestDeleted(msl.parent.Repository.Named(), dgst); err != nil { + dcontext.GetLogger(ctx).Errorf("error dispatching manifest delete to listener: %v", err) + } + } + + return err +} + +func (msl *manifestServiceListener) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) { + sm, err := msl.ManifestService.Get(ctx, dgst, options...) + if err == nil { + if err := msl.parent.listener.ManifestPulled(msl.parent.Repository.Named(), sm, options...); err != nil { + dcontext.GetLogger(ctx).Errorf("error dispatching manifest pull to listener: %v", err) + } + } + + return sm, err +} + +func (msl *manifestServiceListener) Put(ctx context.Context, sm distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) { + dgst, err := msl.ManifestService.Put(ctx, sm, options...) + + if err == nil { + if err := msl.parent.listener.ManifestPushed(msl.parent.Repository.Named(), sm, options...); err != nil { + dcontext.GetLogger(ctx).Errorf("error dispatching manifest push to listener: %v", err) + } + } + + return dgst, err +} + +type blobServiceListener struct { + distribution.BlobStore + parent *repositoryListener +} + +var _ distribution.BlobStore = &blobServiceListener{} + +func (bsl *blobServiceListener) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) { + p, err := bsl.BlobStore.Get(ctx, dgst) + if err == nil { + if desc, err := bsl.Stat(ctx, dgst); err != nil { + dcontext.GetLogger(ctx).Errorf("error resolving descriptor in ServeBlob listener: %v", err) + } else { + if err := bsl.parent.listener.BlobPulled(bsl.parent.Repository.Named(), desc); err != nil { + dcontext.GetLogger(ctx).Errorf("error dispatching layer pull to listener: %v", err) + } + } + } + + return p, err +} + +func (bsl *blobServiceListener) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) { + rc, err := bsl.BlobStore.Open(ctx, dgst) + if err == nil { + if desc, err := bsl.Stat(ctx, dgst); err != nil { + dcontext.GetLogger(ctx).Errorf("error resolving descriptor in ServeBlob listener: %v", err) + } else { + if err := bsl.parent.listener.BlobPulled(bsl.parent.Repository.Named(), desc); err != nil { + dcontext.GetLogger(ctx).Errorf("error dispatching layer pull to listener: %v", err) + } + } + } + + return rc, err +} + +func (bsl *blobServiceListener) ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) error { + err := bsl.BlobStore.ServeBlob(ctx, w, r, dgst) + if err == nil { + if desc, err := bsl.Stat(ctx, dgst); err != nil { + dcontext.GetLogger(ctx).Errorf("error resolving descriptor in ServeBlob listener: %v", err) + } else { + if err := bsl.parent.listener.BlobPulled(bsl.parent.Repository.Named(), desc); err != nil { + dcontext.GetLogger(ctx).Errorf("error dispatching layer pull to listener: %v", err) + } + } + } + + return err +} + +func (bsl *blobServiceListener) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) { + desc, err := bsl.BlobStore.Put(ctx, mediaType, p) + if err == nil { + if err := bsl.parent.listener.BlobPushed(bsl.parent.Repository.Named(), desc); err != nil { + dcontext.GetLogger(ctx).Errorf("error dispatching layer push to listener: %v", err) + } + } + + return desc, err +} + +func (bsl *blobServiceListener) Create(ctx context.Context, options ...distribution.BlobCreateOption) (distribution.BlobWriter, error) { + wr, err := bsl.BlobStore.Create(ctx, options...) + switch err := err.(type) { + case distribution.ErrBlobMounted: + if err := bsl.parent.listener.BlobMounted(bsl.parent.Repository.Named(), err.Descriptor, err.From); err != nil { + dcontext.GetLogger(ctx).Errorf("error dispatching blob mount to listener: %v", err) + } + return nil, err + } + return bsl.decorateWriter(wr), err +} + +func (bsl *blobServiceListener) Delete(ctx context.Context, dgst digest.Digest) error { + err := bsl.BlobStore.Delete(ctx, dgst) + if err == nil { + if err := bsl.parent.listener.BlobDeleted(bsl.parent.Repository.Named(), dgst); err != nil { + dcontext.GetLogger(ctx).Errorf("error dispatching layer delete to listener: %v", err) + } + } + + return err +} + +func (bsl *blobServiceListener) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) { + wr, err := bsl.BlobStore.Resume(ctx, id) + return bsl.decorateWriter(wr), err +} + +func (bsl *blobServiceListener) decorateWriter(wr distribution.BlobWriter) distribution.BlobWriter { + return &blobWriterListener{ + BlobWriter: wr, + parent: bsl, + } +} + +type blobWriterListener struct { + distribution.BlobWriter + parent *blobServiceListener +} + +func (bwl *blobWriterListener) Commit(ctx context.Context, desc distribution.Descriptor) (distribution.Descriptor, error) { + committed, err := bwl.BlobWriter.Commit(ctx, desc) + if err == nil { + if err := bwl.parent.parent.listener.BlobPushed(bwl.parent.parent.Repository.Named(), committed); err != nil { + dcontext.GetLogger(ctx).Errorf("error dispatching blob push to listener: %v", err) + } + } + + return committed, err +} + +type tagServiceListener struct { + distribution.TagService + parent *repositoryListener +} + +func (rl *repositoryListener) Tags(ctx context.Context) distribution.TagService { + return &tagServiceListener{ + TagService: rl.Repository.Tags(ctx), + parent: rl, + } +} + +func (tagSL *tagServiceListener) Untag(ctx context.Context, tag string) error { + if err := tagSL.TagService.Untag(ctx, tag); err != nil { + return err + } + if err := tagSL.parent.listener.TagDeleted(tagSL.parent.Repository.Named(), tag); err != nil { + dcontext.GetLogger(ctx).Errorf("error dispatching tag deleted to listener: %v", err) + return err + } + return nil +} diff --git a/vendor/github.com/docker/distribution/notifications/metrics.go b/vendor/github.com/docker/distribution/notifications/metrics.go new file mode 100644 index 000000000..a20af1687 --- /dev/null +++ b/vendor/github.com/docker/distribution/notifications/metrics.go @@ -0,0 +1,152 @@ +package notifications + +import ( + "expvar" + "fmt" + "net/http" + "sync" +) + +// EndpointMetrics track various actions taken by the endpoint, typically by +// number of events. The goal of this to export it via expvar but we may find +// some other future solution to be better. +type EndpointMetrics struct { + Pending int // events pending in queue + Events int // total events incoming + Successes int // total events written successfully + Failures int // total events failed + Errors int // total events errored + Statuses map[string]int // status code histogram, per call event +} + +// safeMetrics guards the metrics implementation with a lock and provides a +// safe update function. +type safeMetrics struct { + EndpointMetrics + sync.Mutex // protects statuses map +} + +// newSafeMetrics returns safeMetrics with map allocated. +func newSafeMetrics() *safeMetrics { + var sm safeMetrics + sm.Statuses = make(map[string]int) + return &sm +} + +// httpStatusListener returns the listener for the http sink that updates the +// relevant counters. +func (sm *safeMetrics) httpStatusListener() httpStatusListener { + return &endpointMetricsHTTPStatusListener{ + safeMetrics: sm, + } +} + +// eventQueueListener returns a listener that maintains queue related counters. +func (sm *safeMetrics) eventQueueListener() eventQueueListener { + return &endpointMetricsEventQueueListener{ + safeMetrics: sm, + } +} + +// endpointMetricsHTTPStatusListener increments counters related to http sinks +// for the relevant events. +type endpointMetricsHTTPStatusListener struct { + *safeMetrics +} + +var _ httpStatusListener = &endpointMetricsHTTPStatusListener{} + +func (emsl *endpointMetricsHTTPStatusListener) success(status int, events ...Event) { + emsl.safeMetrics.Lock() + defer emsl.safeMetrics.Unlock() + emsl.Statuses[fmt.Sprintf("%d %s", status, http.StatusText(status))] += len(events) + emsl.Successes += len(events) +} + +func (emsl *endpointMetricsHTTPStatusListener) failure(status int, events ...Event) { + emsl.safeMetrics.Lock() + defer emsl.safeMetrics.Unlock() + emsl.Statuses[fmt.Sprintf("%d %s", status, http.StatusText(status))] += len(events) + emsl.Failures += len(events) +} + +func (emsl *endpointMetricsHTTPStatusListener) err(err error, events ...Event) { + emsl.safeMetrics.Lock() + defer emsl.safeMetrics.Unlock() + emsl.Errors += len(events) +} + +// endpointMetricsEventQueueListener maintains the incoming events counter and +// the queues pending count. +type endpointMetricsEventQueueListener struct { + *safeMetrics +} + +func (eqc *endpointMetricsEventQueueListener) ingress(events ...Event) { + eqc.Lock() + defer eqc.Unlock() + eqc.Events += len(events) + eqc.Pending += len(events) +} + +func (eqc *endpointMetricsEventQueueListener) egress(events ...Event) { + eqc.Lock() + defer eqc.Unlock() + eqc.Pending -= len(events) +} + +// endpoints is global registry of endpoints used to report metrics to expvar +var endpoints struct { + registered []*Endpoint + mu sync.Mutex +} + +// register places the endpoint into expvar so that stats are tracked. +func register(e *Endpoint) { + endpoints.mu.Lock() + defer endpoints.mu.Unlock() + + endpoints.registered = append(endpoints.registered, e) +} + +func init() { + // NOTE(stevvooe): Setup registry metrics structure to report to expvar. + // Ideally, we do more metrics through logging but we need some nice + // realtime metrics for queue state for now. + + registry := expvar.Get("registry") + + if registry == nil { + registry = expvar.NewMap("registry") + } + + var notifications expvar.Map + notifications.Init() + notifications.Set("endpoints", expvar.Func(func() interface{} { + endpoints.mu.Lock() + defer endpoints.mu.Unlock() + + var names []interface{} + for _, v := range endpoints.registered { + var epjson struct { + Name string `json:"name"` + URL string `json:"url"` + EndpointConfig + + Metrics EndpointMetrics + } + + epjson.Name = v.Name() + epjson.URL = v.URL() + epjson.EndpointConfig = v.EndpointConfig + + v.ReadMetrics(&epjson.Metrics) + + names = append(names, epjson) + } + + return names + })) + + registry.(*expvar.Map).Set("notifications", ¬ifications) +} diff --git a/vendor/github.com/docker/distribution/notifications/sinks.go b/vendor/github.com/docker/distribution/notifications/sinks.go new file mode 100644 index 000000000..816f75d7b --- /dev/null +++ b/vendor/github.com/docker/distribution/notifications/sinks.go @@ -0,0 +1,387 @@ +package notifications + +import ( + "container/list" + "fmt" + "sync" + "time" + + "github.com/sirupsen/logrus" +) + +// NOTE(stevvooe): This file contains definitions for several utility sinks. +// Typically, the broadcaster is the only sink that should be required +// externally, but others are suitable for export if the need arises. Albeit, +// the tight integration with endpoint metrics should be removed. + +// Broadcaster sends events to multiple, reliable Sinks. The goal of this +// component is to dispatch events to configured endpoints. Reliability can be +// provided by wrapping incoming sinks. +type Broadcaster struct { + sinks []Sink + events chan []Event + closed chan chan struct{} +} + +// NewBroadcaster ... +// Add appends one or more sinks to the list of sinks. The broadcaster +// behavior will be affected by the properties of the sink. Generally, the +// sink should accept all messages and deal with reliability on its own. Use +// of EventQueue and RetryingSink should be used here. +func NewBroadcaster(sinks ...Sink) *Broadcaster { + b := Broadcaster{ + sinks: sinks, + events: make(chan []Event), + closed: make(chan chan struct{}), + } + + // Start the broadcaster + go b.run() + + return &b +} + +// Write accepts a block of events to be dispatched to all sinks. This method +// will never fail and should never block (hopefully!). The caller cedes the +// slice memory to the broadcaster and should not modify it after calling +// write. +func (b *Broadcaster) Write(events ...Event) error { + select { + case b.events <- events: + case <-b.closed: + return ErrSinkClosed + } + return nil +} + +// Close the broadcaster, ensuring that all messages are flushed to the +// underlying sink before returning. +func (b *Broadcaster) Close() error { + logrus.Infof("broadcaster: closing") + select { + case <-b.closed: + // already closed + return fmt.Errorf("broadcaster: already closed") + default: + // do a little chan handoff dance to synchronize closing + closed := make(chan struct{}) + b.closed <- closed + close(b.closed) + <-closed + return nil + } +} + +// run is the main broadcast loop, started when the broadcaster is created. +// Under normal conditions, it waits for events on the event channel. After +// Close is called, this goroutine will exit. +func (b *Broadcaster) run() { + for { + select { + case block := <-b.events: + for _, sink := range b.sinks { + if err := sink.Write(block...); err != nil { + logrus.Errorf("broadcaster: error writing events to %v, these events will be lost: %v", sink, err) + } + } + case closing := <-b.closed: + + // close all the underlying sinks + for _, sink := range b.sinks { + if err := sink.Close(); err != nil { + logrus.Errorf("broadcaster: error closing sink %v: %v", sink, err) + } + } + closing <- struct{}{} + + logrus.Debugf("broadcaster: closed") + return + } + } +} + +// eventQueue accepts all messages into a queue for asynchronous consumption +// by a sink. It is unbounded and thread safe but the sink must be reliable or +// events will be dropped. +type eventQueue struct { + sink Sink + events *list.List + listeners []eventQueueListener + cond *sync.Cond + mu sync.Mutex + closed bool +} + +// eventQueueListener is called when various events happen on the queue. +type eventQueueListener interface { + ingress(events ...Event) + egress(events ...Event) +} + +// newEventQueue returns a queue to the provided sink. If the updater is non- +// nil, it will be called to update pending metrics on ingress and egress. +func newEventQueue(sink Sink, listeners ...eventQueueListener) *eventQueue { + eq := eventQueue{ + sink: sink, + events: list.New(), + listeners: listeners, + } + + eq.cond = sync.NewCond(&eq.mu) + go eq.run() + return &eq +} + +// Write accepts the events into the queue, only failing if the queue has +// beend closed. +func (eq *eventQueue) Write(events ...Event) error { + eq.mu.Lock() + defer eq.mu.Unlock() + + if eq.closed { + return ErrSinkClosed + } + + for _, listener := range eq.listeners { + listener.ingress(events...) + } + eq.events.PushBack(events) + eq.cond.Signal() // signal waiters + + return nil +} + +// Close shuts down the event queue, flushing +func (eq *eventQueue) Close() error { + eq.mu.Lock() + defer eq.mu.Unlock() + + if eq.closed { + return fmt.Errorf("eventqueue: already closed") + } + + // set closed flag + eq.closed = true + eq.cond.Signal() // signal flushes queue + eq.cond.Wait() // wait for signal from last flush + + return eq.sink.Close() +} + +// run is the main goroutine to flush events to the target sink. +func (eq *eventQueue) run() { + for { + block := eq.next() + + if block == nil { + return // nil block means event queue is closed. + } + + if err := eq.sink.Write(block...); err != nil { + logrus.Warnf("eventqueue: error writing events to %v, these events will be lost: %v", eq.sink, err) + } + + for _, listener := range eq.listeners { + listener.egress(block...) + } + } +} + +// next encompasses the critical section of the run loop. When the queue is +// empty, it will block on the condition. If new data arrives, it will wake +// and return a block. When closed, a nil slice will be returned. +func (eq *eventQueue) next() []Event { + eq.mu.Lock() + defer eq.mu.Unlock() + + for eq.events.Len() < 1 { + if eq.closed { + eq.cond.Broadcast() + return nil + } + + eq.cond.Wait() + } + + front := eq.events.Front() + block := front.Value.([]Event) + eq.events.Remove(front) + + return block +} + +// ignoredSink discards events with ignored target media types and actions. +// passes the rest along. +type ignoredSink struct { + Sink + ignoreMediaTypes map[string]bool + ignoreActions map[string]bool +} + +func newIgnoredSink(sink Sink, ignored []string, ignoreActions []string) Sink { + if len(ignored) == 0 { + return sink + } + + ignoredMap := make(map[string]bool) + for _, mediaType := range ignored { + ignoredMap[mediaType] = true + } + + ignoredActionsMap := make(map[string]bool) + for _, action := range ignoreActions { + ignoredActionsMap[action] = true + } + + return &ignoredSink{ + Sink: sink, + ignoreMediaTypes: ignoredMap, + ignoreActions: ignoredActionsMap, + } +} + +// Write discards events with ignored target media types and passes the rest +// along. +func (imts *ignoredSink) Write(events ...Event) error { + var kept []Event + for _, e := range events { + if !imts.ignoreMediaTypes[e.Target.MediaType] { + kept = append(kept, e) + } + } + if len(kept) == 0 { + return nil + } + + var results []Event + for _, e := range kept { + if !imts.ignoreActions[e.Action] { + results = append(results, e) + } + } + if len(results) == 0 { + return nil + } + return imts.Sink.Write(results...) +} + +// retryingSink retries the write until success or an ErrSinkClosed is +// returned. Underlying sink must have p > 0 of succeeding or the sink will +// block. Internally, it is a circuit breaker retries to manage reset. +// Concurrent calls to a retrying sink are serialized through the sink, +// meaning that if one is in-flight, another will not proceed. +type retryingSink struct { + mu sync.Mutex + sink Sink + closed bool + + // circuit breaker heuristics + failures struct { + threshold int + recent int + last time.Time + backoff time.Duration // time after which we retry after failure. + } +} + +// TODO(stevvooe): We are using circuit break here, which actually doesn't +// make a whole lot of sense for this use case, since we always retry. Move +// this to use bounded exponential backoff. + +// newRetryingSink returns a sink that will retry writes to a sink, backing +// off on failure. Parameters threshold and backoff adjust the behavior of the +// circuit breaker. +func newRetryingSink(sink Sink, threshold int, backoff time.Duration) *retryingSink { + rs := &retryingSink{ + sink: sink, + } + rs.failures.threshold = threshold + rs.failures.backoff = backoff + + return rs +} + +// Write attempts to flush the events to the downstream sink until it succeeds +// or the sink is closed. +func (rs *retryingSink) Write(events ...Event) error { + rs.mu.Lock() + defer rs.mu.Unlock() + +retry: + + if rs.closed { + return ErrSinkClosed + } + + if !rs.proceed() { + logrus.Warnf("%v encountered too many errors, backing off", rs.sink) + rs.wait(rs.failures.backoff) + goto retry + } + + if err := rs.write(events...); err != nil { + if err == ErrSinkClosed { + // terminal! + return err + } + + logrus.Errorf("retryingsink: error writing events: %v, retrying", err) + goto retry + } + + return nil +} + +// Close closes the sink and the underlying sink. +func (rs *retryingSink) Close() error { + rs.mu.Lock() + defer rs.mu.Unlock() + + if rs.closed { + return fmt.Errorf("retryingsink: already closed") + } + + rs.closed = true + return rs.sink.Close() +} + +// write provides a helper that dispatches failure and success properly. Used +// by write as the single-flight write call. +func (rs *retryingSink) write(events ...Event) error { + if err := rs.sink.Write(events...); err != nil { + rs.failure() + return err + } + + rs.reset() + return nil +} + +// wait backoff time against the sink, unlocking so others can proceed. Should +// only be called by methods that currently have the mutex. +func (rs *retryingSink) wait(backoff time.Duration) { + rs.mu.Unlock() + defer rs.mu.Lock() + + // backoff here + time.Sleep(backoff) +} + +// reset marks a successful call. +func (rs *retryingSink) reset() { + rs.failures.recent = 0 + rs.failures.last = time.Time{} +} + +// failure records a failure. +func (rs *retryingSink) failure() { + rs.failures.recent++ + rs.failures.last = time.Now().UTC() +} + +// proceed returns true if the call should proceed based on circuit breaker +// heuristics. +func (rs *retryingSink) proceed() bool { + return rs.failures.recent < rs.failures.threshold || + time.Now().UTC().After(rs.failures.last.Add(rs.failures.backoff)) +} diff --git a/vendor/github.com/docker/distribution/registry.go b/vendor/github.com/docker/distribution/registry.go new file mode 100644 index 000000000..d0deee65d --- /dev/null +++ b/vendor/github.com/docker/distribution/registry.go @@ -0,0 +1,118 @@ +package distribution + +import ( + "context" + + "github.com/distribution/reference" +) + +// Scope defines the set of items that match a namespace. +type Scope interface { + // Contains returns true if the name belongs to the namespace. + Contains(name string) bool +} + +type fullScope struct{} + +func (f fullScope) Contains(string) bool { + return true +} + +// GlobalScope represents the full namespace scope which contains +// all other scopes. +var GlobalScope = Scope(fullScope{}) + +// Namespace represents a collection of repositories, addressable by name. +// Generally, a namespace is backed by a set of one or more services, +// providing facilities such as registry access, trust, and indexing. +type Namespace interface { + // Scope describes the names that can be used with this Namespace. The + // global namespace will have a scope that matches all names. The scope + // effectively provides an identity for the namespace. + Scope() Scope + + // Repository should return a reference to the named repository. The + // registry may or may not have the repository but should always return a + // reference. + Repository(ctx context.Context, name reference.Named) (Repository, error) + + // Repositories fills 'repos' with a lexicographically sorted catalog of repositories + // up to the size of 'repos' and returns the value 'n' for the number of entries + // which were filled. 'last' contains an offset in the catalog, and 'err' will be + // set to io.EOF if there are no more entries to obtain. + Repositories(ctx context.Context, repos []string, last string) (n int, err error) + + // Blobs returns a blob enumerator to access all blobs + Blobs() BlobEnumerator + + // BlobStatter returns a BlobStatter to control + BlobStatter() BlobStatter +} + +// RepositoryEnumerator describes an operation to enumerate repositories +type RepositoryEnumerator interface { + Enumerate(ctx context.Context, ingester func(string) error) error +} + +// RepositoryRemover removes given repository +type RepositoryRemover interface { + Remove(ctx context.Context, name reference.Named) error +} + +// ManifestServiceOption is a function argument for Manifest Service methods +type ManifestServiceOption interface { + Apply(ManifestService) error +} + +// WithTag allows a tag to be passed into Put +func WithTag(tag string) ManifestServiceOption { + return WithTagOption{tag} +} + +// WithTagOption holds a tag +type WithTagOption struct{ Tag string } + +// Apply conforms to the ManifestServiceOption interface +func (o WithTagOption) Apply(m ManifestService) error { + // no implementation + return nil +} + +// WithManifestMediaTypes lists the media types the client wishes +// the server to provide. +func WithManifestMediaTypes(mediaTypes []string) ManifestServiceOption { + return WithManifestMediaTypesOption{mediaTypes} +} + +// WithManifestMediaTypesOption holds a list of accepted media types +type WithManifestMediaTypesOption struct{ MediaTypes []string } + +// Apply conforms to the ManifestServiceOption interface +func (o WithManifestMediaTypesOption) Apply(m ManifestService) error { + // no implementation + return nil +} + +// Repository is a named collection of manifests and layers. +type Repository interface { + // Named returns the name of the repository. + Named() reference.Named + + // Manifests returns a reference to this repository's manifest service. + // with the supplied options applied. + Manifests(ctx context.Context, options ...ManifestServiceOption) (ManifestService, error) + + // Blobs returns a reference to this repository's blob service. + Blobs(ctx context.Context) BlobStore + + // TODO(stevvooe): The above BlobStore return can probably be relaxed to + // be a BlobService for use with clients. This will allow such + // implementations to avoid implementing ServeBlob. + + // Tags returns a reference to this repositories tag service + Tags(ctx context.Context) TagService +} + +// TODO(stevvooe): Must add close methods to all these. May want to change the +// way instances are created to better reflect internal dependency +// relationships. diff --git a/vendor/github.com/docker/distribution/registry/api/errcode/errors.go b/vendor/github.com/docker/distribution/registry/api/errcode/errors.go new file mode 100644 index 000000000..4c35b879a --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/api/errcode/errors.go @@ -0,0 +1,267 @@ +package errcode + +import ( + "encoding/json" + "fmt" + "strings" +) + +// ErrorCoder is the base interface for ErrorCode and Error allowing +// users of each to just call ErrorCode to get the real ID of each +type ErrorCoder interface { + ErrorCode() ErrorCode +} + +// ErrorCode represents the error type. The errors are serialized via strings +// and the integer format may change and should *never* be exported. +type ErrorCode int + +var _ error = ErrorCode(0) + +// ErrorCode just returns itself +func (ec ErrorCode) ErrorCode() ErrorCode { + return ec +} + +// Error returns the ID/Value +func (ec ErrorCode) Error() string { + // NOTE(stevvooe): Cannot use message here since it may have unpopulated args. + return strings.ToLower(strings.Replace(ec.String(), "_", " ", -1)) +} + +// Descriptor returns the descriptor for the error code. +func (ec ErrorCode) Descriptor() ErrorDescriptor { + d, ok := errorCodeToDescriptors[ec] + + if !ok { + return ErrorCodeUnknown.Descriptor() + } + + return d +} + +// String returns the canonical identifier for this error code. +func (ec ErrorCode) String() string { + return ec.Descriptor().Value +} + +// Message returned the human-readable error message for this error code. +func (ec ErrorCode) Message() string { + return ec.Descriptor().Message +} + +// MarshalText encodes the receiver into UTF-8-encoded text and returns the +// result. +func (ec ErrorCode) MarshalText() (text []byte, err error) { + return []byte(ec.String()), nil +} + +// UnmarshalText decodes the form generated by MarshalText. +func (ec *ErrorCode) UnmarshalText(text []byte) error { + desc, ok := idToDescriptors[string(text)] + + if !ok { + desc = ErrorCodeUnknown.Descriptor() + } + + *ec = desc.Code + + return nil +} + +// WithMessage creates a new Error struct based on the passed-in info and +// overrides the Message property. +func (ec ErrorCode) WithMessage(message string) Error { + return Error{ + Code: ec, + Message: message, + } +} + +// WithDetail creates a new Error struct based on the passed-in info and +// set the Detail property appropriately +func (ec ErrorCode) WithDetail(detail interface{}) Error { + return Error{ + Code: ec, + Message: ec.Message(), + }.WithDetail(detail) +} + +// WithArgs creates a new Error struct and sets the Args slice +func (ec ErrorCode) WithArgs(args ...interface{}) Error { + return Error{ + Code: ec, + Message: ec.Message(), + }.WithArgs(args...) +} + +// Error provides a wrapper around ErrorCode with extra Details provided. +type Error struct { + Code ErrorCode `json:"code"` + Message string `json:"message"` + Detail interface{} `json:"detail,omitempty"` + + // TODO(duglin): See if we need an "args" property so we can do the + // variable substitution right before showing the message to the user +} + +var _ error = Error{} + +// ErrorCode returns the ID/Value of this Error +func (e Error) ErrorCode() ErrorCode { + return e.Code +} + +// Error returns a human readable representation of the error. +func (e Error) Error() string { + return fmt.Sprintf("%s: %s", e.Code.Error(), e.Message) +} + +// WithDetail will return a new Error, based on the current one, but with +// some Detail info added +func (e Error) WithDetail(detail interface{}) Error { + return Error{ + Code: e.Code, + Message: e.Message, + Detail: detail, + } +} + +// WithArgs uses the passed-in list of interface{} as the substitution +// variables in the Error's Message string, but returns a new Error +func (e Error) WithArgs(args ...interface{}) Error { + return Error{ + Code: e.Code, + Message: fmt.Sprintf(e.Code.Message(), args...), + Detail: e.Detail, + } +} + +// ErrorDescriptor provides relevant information about a given error code. +type ErrorDescriptor struct { + // Code is the error code that this descriptor describes. + Code ErrorCode + + // Value provides a unique, string key, often captilized with + // underscores, to identify the error code. This value is used as the + // keyed value when serializing api errors. + Value string + + // Message is a short, human readable decription of the error condition + // included in API responses. + Message string + + // Description provides a complete account of the errors purpose, suitable + // for use in documentation. + Description string + + // HTTPStatusCode provides the http status code that is associated with + // this error condition. + HTTPStatusCode int +} + +// ParseErrorCode returns the value by the string error code. +// `ErrorCodeUnknown` will be returned if the error is not known. +func ParseErrorCode(value string) ErrorCode { + ed, ok := idToDescriptors[value] + if ok { + return ed.Code + } + + return ErrorCodeUnknown +} + +// Errors provides the envelope for multiple errors and a few sugar methods +// for use within the application. +type Errors []error + +var _ error = Errors{} + +func (errs Errors) Error() string { + switch len(errs) { + case 0: + return "" + case 1: + return errs[0].Error() + default: + msg := "errors:\n" + for _, err := range errs { + msg += err.Error() + "\n" + } + return msg + } +} + +// Len returns the current number of errors. +func (errs Errors) Len() int { + return len(errs) +} + +// MarshalJSON converts slice of error, ErrorCode or Error into a +// slice of Error - then serializes +func (errs Errors) MarshalJSON() ([]byte, error) { + var tmpErrs struct { + Errors []Error `json:"errors,omitempty"` + } + + for _, daErr := range errs { + var err Error + + switch daErr := daErr.(type) { + case ErrorCode: + err = daErr.WithDetail(nil) + case Error: + err = daErr + default: + err = ErrorCodeUnknown.WithDetail(daErr) + + } + + // If the Error struct was setup and they forgot to set the + // Message field (meaning its "") then grab it from the ErrCode + msg := err.Message + if msg == "" { + msg = err.Code.Message() + } + + tmpErrs.Errors = append(tmpErrs.Errors, Error{ + Code: err.Code, + Message: msg, + Detail: err.Detail, + }) + } + + return json.Marshal(tmpErrs) +} + +// UnmarshalJSON deserializes []Error and then converts it into slice of +// Error or ErrorCode +func (errs *Errors) UnmarshalJSON(data []byte) error { + var tmpErrs struct { + Errors []Error + } + + if err := json.Unmarshal(data, &tmpErrs); err != nil { + return err + } + + var newErrs Errors + for _, daErr := range tmpErrs.Errors { + // If Message is empty or exactly matches the Code's message string + // then just use the Code, no need for a full Error struct + if daErr.Detail == nil && (daErr.Message == "" || daErr.Message == daErr.Code.Message()) { + // Error's w/o details get converted to ErrorCode + newErrs = append(newErrs, daErr.Code) + } else { + // Error's w/ details are untouched + newErrs = append(newErrs, Error{ + Code: daErr.Code, + Message: daErr.Message, + Detail: daErr.Detail, + }) + } + } + + *errs = newErrs + return nil +} diff --git a/vendor/github.com/docker/distribution/registry/api/errcode/handler.go b/vendor/github.com/docker/distribution/registry/api/errcode/handler.go new file mode 100644 index 000000000..d77e70473 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/api/errcode/handler.go @@ -0,0 +1,40 @@ +package errcode + +import ( + "encoding/json" + "net/http" +) + +// ServeJSON attempts to serve the errcode in a JSON envelope. It marshals err +// and sets the content-type header to 'application/json'. It will handle +// ErrorCoder and Errors, and if necessary will create an envelope. +func ServeJSON(w http.ResponseWriter, err error) error { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + var sc int + + switch errs := err.(type) { + case Errors: + if len(errs) < 1 { + break + } + + if err, ok := errs[0].(ErrorCoder); ok { + sc = err.ErrorCode().Descriptor().HTTPStatusCode + } + case ErrorCoder: + sc = errs.ErrorCode().Descriptor().HTTPStatusCode + err = Errors{err} // create an envelope. + default: + // We just have an unhandled error type, so just place in an envelope + // and move along. + err = Errors{err} + } + + if sc == 0 { + sc = http.StatusInternalServerError + } + + w.WriteHeader(sc) + + return json.NewEncoder(w).Encode(err) +} diff --git a/vendor/github.com/docker/distribution/registry/api/errcode/register.go b/vendor/github.com/docker/distribution/registry/api/errcode/register.go new file mode 100644 index 000000000..d1e8826c6 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/api/errcode/register.go @@ -0,0 +1,138 @@ +package errcode + +import ( + "fmt" + "net/http" + "sort" + "sync" +) + +var ( + errorCodeToDescriptors = map[ErrorCode]ErrorDescriptor{} + idToDescriptors = map[string]ErrorDescriptor{} + groupToDescriptors = map[string][]ErrorDescriptor{} +) + +var ( + // ErrorCodeUnknown is a generic error that can be used as a last + // resort if there is no situation-specific error message that can be used + ErrorCodeUnknown = Register("errcode", ErrorDescriptor{ + Value: "UNKNOWN", + Message: "unknown error", + Description: `Generic error returned when the error does not have an + API classification.`, + HTTPStatusCode: http.StatusInternalServerError, + }) + + // ErrorCodeUnsupported is returned when an operation is not supported. + ErrorCodeUnsupported = Register("errcode", ErrorDescriptor{ + Value: "UNSUPPORTED", + Message: "The operation is unsupported.", + Description: `The operation was unsupported due to a missing + implementation or invalid set of parameters.`, + HTTPStatusCode: http.StatusMethodNotAllowed, + }) + + // ErrorCodeUnauthorized is returned if a request requires + // authentication. + ErrorCodeUnauthorized = Register("errcode", ErrorDescriptor{ + Value: "UNAUTHORIZED", + Message: "authentication required", + Description: `The access controller was unable to authenticate + the client. Often this will be accompanied by a + Www-Authenticate HTTP response header indicating how to + authenticate.`, + HTTPStatusCode: http.StatusUnauthorized, + }) + + // ErrorCodeDenied is returned if a client does not have sufficient + // permission to perform an action. + ErrorCodeDenied = Register("errcode", ErrorDescriptor{ + Value: "DENIED", + Message: "requested access to the resource is denied", + Description: `The access controller denied access for the + operation on a resource.`, + HTTPStatusCode: http.StatusForbidden, + }) + + // ErrorCodeUnavailable provides a common error to report unavailability + // of a service or endpoint. + ErrorCodeUnavailable = Register("errcode", ErrorDescriptor{ + Value: "UNAVAILABLE", + Message: "service unavailable", + Description: "Returned when a service is not available", + HTTPStatusCode: http.StatusServiceUnavailable, + }) + + // ErrorCodeTooManyRequests is returned if a client attempts too many + // times to contact a service endpoint. + ErrorCodeTooManyRequests = Register("errcode", ErrorDescriptor{ + Value: "TOOMANYREQUESTS", + Message: "too many requests", + Description: `Returned when a client attempts to contact a + service too many times`, + HTTPStatusCode: http.StatusTooManyRequests, + }) +) + +var nextCode = 1000 +var registerLock sync.Mutex + +// Register will make the passed-in error known to the environment and +// return a new ErrorCode +func Register(group string, descriptor ErrorDescriptor) ErrorCode { + registerLock.Lock() + defer registerLock.Unlock() + + descriptor.Code = ErrorCode(nextCode) + + if _, ok := idToDescriptors[descriptor.Value]; ok { + panic(fmt.Sprintf("ErrorValue %q is already registered", descriptor.Value)) + } + if _, ok := errorCodeToDescriptors[descriptor.Code]; ok { + panic(fmt.Sprintf("ErrorCode %v is already registered", descriptor.Code)) + } + + groupToDescriptors[group] = append(groupToDescriptors[group], descriptor) + errorCodeToDescriptors[descriptor.Code] = descriptor + idToDescriptors[descriptor.Value] = descriptor + + nextCode++ + return descriptor.Code +} + +type byValue []ErrorDescriptor + +func (a byValue) Len() int { return len(a) } +func (a byValue) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a byValue) Less(i, j int) bool { return a[i].Value < a[j].Value } + +// GetGroupNames returns the list of Error group names that are registered +func GetGroupNames() []string { + keys := []string{} + + for k := range groupToDescriptors { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// GetErrorCodeGroup returns the named group of error descriptors +func GetErrorCodeGroup(name string) []ErrorDescriptor { + desc := groupToDescriptors[name] + sort.Sort(byValue(desc)) + return desc +} + +// GetErrorAllDescriptors returns a slice of all ErrorDescriptors that are +// registered, irrespective of what group they're in +func GetErrorAllDescriptors() []ErrorDescriptor { + result := []ErrorDescriptor{} + + for _, group := range GetGroupNames() { + result = append(result, GetErrorCodeGroup(group)...) + } + sort.Sort(byValue(result)) + return result +} diff --git a/vendor/github.com/docker/distribution/registry/api/v2/descriptors.go b/vendor/github.com/docker/distribution/registry/api/v2/descriptors.go new file mode 100644 index 000000000..7fceefbc6 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/api/v2/descriptors.go @@ -0,0 +1,1613 @@ +package v2 + +import ( + "net/http" + "regexp" + + "github.com/distribution/reference" + "github.com/docker/distribution/registry/api/errcode" + "github.com/opencontainers/go-digest" +) + +var ( + nameParameterDescriptor = ParameterDescriptor{ + Name: "name", + Type: "string", + Format: reference.NameRegexp.String(), + Required: true, + Description: `Name of the target repository.`, + } + + referenceParameterDescriptor = ParameterDescriptor{ + Name: "reference", + Type: "string", + Format: reference.TagRegexp.String(), + Required: true, + Description: `Tag or digest of the target manifest.`, + } + + uuidParameterDescriptor = ParameterDescriptor{ + Name: "uuid", + Type: "opaque", + Required: true, + Description: "A uuid identifying the upload. This field can accept characters that match `[a-zA-Z0-9-_.=]+`.", + } + + digestPathParameter = ParameterDescriptor{ + Name: "digest", + Type: "path", + Required: true, + Format: digest.DigestRegexp.String(), + Description: `Digest of desired blob.`, + } + + hostHeader = ParameterDescriptor{ + Name: "Host", + Type: "string", + Description: "Standard HTTP Host Header. Should be set to the registry host.", + Format: "", + Examples: []string{"registry-1.docker.io"}, + } + + authHeader = ParameterDescriptor{ + Name: "Authorization", + Type: "string", + Description: "An RFC7235 compliant authorization header.", + Format: " ", + Examples: []string{"Bearer dGhpcyBpcyBhIGZha2UgYmVhcmVyIHRva2VuIQ=="}, + } + + authChallengeHeader = ParameterDescriptor{ + Name: "WWW-Authenticate", + Type: "string", + Description: "An RFC7235 compliant authentication challenge header.", + Format: ` realm="", ..."`, + Examples: []string{ + `Bearer realm="https://auth.docker.com/", service="registry.docker.com", scopes="repository:library/ubuntu:pull"`, + }, + } + + contentLengthZeroHeader = ParameterDescriptor{ + Name: "Content-Length", + Description: "The `Content-Length` header must be zero and the body must be empty.", + Type: "integer", + Format: "0", + } + + dockerUploadUUIDHeader = ParameterDescriptor{ + Name: "Docker-Upload-UUID", + Description: "Identifies the docker upload uuid for the current request.", + Type: "uuid", + Format: "", + } + + digestHeader = ParameterDescriptor{ + Name: "Docker-Content-Digest", + Description: "Digest of the targeted content for the request.", + Type: "digest", + Format: "", + } + + linkHeader = ParameterDescriptor{ + Name: "Link", + Type: "link", + Description: "RFC5988 compliant rel='next' with URL to next result set, if available", + Format: `<?n=&last=>; rel="next"`, + } + + paginationParameters = []ParameterDescriptor{ + { + Name: "n", + Type: "integer", + Description: "Limit the number of entries in each response. It not present, all entries will be returned.", + Format: "", + Required: false, + }, + { + Name: "last", + Type: "string", + Description: "Result set will include values lexically after last.", + Format: "", + Required: false, + }, + } + + unauthorizedResponseDescriptor = ResponseDescriptor{ + Name: "Authentication Required", + StatusCode: http.StatusUnauthorized, + Description: "The client is not authenticated.", + Headers: []ParameterDescriptor{ + authChallengeHeader, + { + Name: "Content-Length", + Type: "integer", + Description: "Length of the JSON response body.", + Format: "", + }, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + ErrorCodes: []errcode.ErrorCode{ + errcode.ErrorCodeUnauthorized, + }, + } + + invalidPaginationResponseDescriptor = ResponseDescriptor{ + Name: "Invalid pagination number", + Description: "The received parameter n was invalid in some way, as described by the error code. The client should resolve the issue and retry the request.", + StatusCode: http.StatusBadRequest, + Body: BodyDescriptor{ + ContentType: "application/json", + Format: errorsBody, + }, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodePaginationNumberInvalid, + }, + } + + repositoryNotFoundResponseDescriptor = ResponseDescriptor{ + Name: "No Such Repository Error", + StatusCode: http.StatusNotFound, + Description: "The repository is not known to the registry.", + Headers: []ParameterDescriptor{ + { + Name: "Content-Length", + Type: "integer", + Description: "Length of the JSON response body.", + Format: "", + }, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeNameUnknown, + }, + } + + deniedResponseDescriptor = ResponseDescriptor{ + Name: "Access Denied", + StatusCode: http.StatusForbidden, + Description: "The client does not have required access to the repository.", + Headers: []ParameterDescriptor{ + { + Name: "Content-Length", + Type: "integer", + Description: "Length of the JSON response body.", + Format: "", + }, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + ErrorCodes: []errcode.ErrorCode{ + errcode.ErrorCodeDenied, + }, + } + + tooManyRequestsDescriptor = ResponseDescriptor{ + Name: "Too Many Requests", + StatusCode: http.StatusTooManyRequests, + Description: "The client made too many requests within a time interval.", + Headers: []ParameterDescriptor{ + { + Name: "Content-Length", + Type: "integer", + Description: "Length of the JSON response body.", + Format: "", + }, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + ErrorCodes: []errcode.ErrorCode{ + errcode.ErrorCodeTooManyRequests, + }, + } +) + +const ( + manifestBody = `{ + "name": , + "tag": , + "fsLayers": [ + { + "blobSum": "" + }, + ... + ] + ], + "history": , + "signature": +}` + + errorsBody = `{ + "errors:" [ + { + "code": , + "message": "", + "detail": ... + }, + ... + ] +}` +) + +// APIDescriptor exports descriptions of the layout of the v2 registry API. +var APIDescriptor = struct { + // RouteDescriptors provides a list of the routes available in the API. + RouteDescriptors []RouteDescriptor +}{ + RouteDescriptors: routeDescriptors, +} + +// RouteDescriptor describes a route specified by name. +type RouteDescriptor struct { + // Name is the name of the route, as specified in RouteNameXXX exports. + // These names a should be considered a unique reference for a route. If + // the route is registered with gorilla, this is the name that will be + // used. + Name string + + // Path is a gorilla/mux-compatible regexp that can be used to match the + // route. For any incoming method and path, only one route descriptor + // should match. + Path string + + // Entity should be a short, human-readalbe description of the object + // targeted by the endpoint. + Entity string + + // Description should provide an accurate overview of the functionality + // provided by the route. + Description string + + // Methods should describe the various HTTP methods that may be used on + // this route, including request and response formats. + Methods []MethodDescriptor +} + +// MethodDescriptor provides a description of the requests that may be +// conducted with the target method. +type MethodDescriptor struct { + + // Method is an HTTP method, such as GET, PUT or POST. + Method string + + // Description should provide an overview of the functionality provided by + // the covered method, suitable for use in documentation. Use of markdown + // here is encouraged. + Description string + + // Requests is a slice of request descriptors enumerating how this + // endpoint may be used. + Requests []RequestDescriptor +} + +// RequestDescriptor covers a particular set of headers and parameters that +// can be carried out with the parent method. Its most helpful to have one +// RequestDescriptor per API use case. +type RequestDescriptor struct { + // Name provides a short identifier for the request, usable as a title or + // to provide quick context for the particular request. + Name string + + // Description should cover the requests purpose, covering any details for + // this particular use case. + Description string + + // Headers describes headers that must be used with the HTTP request. + Headers []ParameterDescriptor + + // PathParameters enumerate the parameterized path components for the + // given request, as defined in the route's regular expression. + PathParameters []ParameterDescriptor + + // QueryParameters provides a list of query parameters for the given + // request. + QueryParameters []ParameterDescriptor + + // Body describes the format of the request body. + Body BodyDescriptor + + // Successes enumerates the possible responses that are considered to be + // the result of a successful request. + Successes []ResponseDescriptor + + // Failures covers the possible failures from this particular request. + Failures []ResponseDescriptor +} + +// ResponseDescriptor describes the components of an API response. +type ResponseDescriptor struct { + // Name provides a short identifier for the response, usable as a title or + // to provide quick context for the particular response. + Name string + + // Description should provide a brief overview of the role of the + // response. + Description string + + // StatusCode specifies the status received by this particular response. + StatusCode int + + // Headers covers any headers that may be returned from the response. + Headers []ParameterDescriptor + + // Fields describes any fields that may be present in the response. + Fields []ParameterDescriptor + + // ErrorCodes enumerates the error codes that may be returned along with + // the response. + ErrorCodes []errcode.ErrorCode + + // Body describes the body of the response, if any. + Body BodyDescriptor +} + +// BodyDescriptor describes a request body and its expected content type. For +// the most part, it should be example json or some placeholder for body +// data in documentation. +type BodyDescriptor struct { + ContentType string + Format string +} + +// ParameterDescriptor describes the format of a request parameter, which may +// be a header, path parameter or query parameter. +type ParameterDescriptor struct { + // Name is the name of the parameter, either of the path component or + // query parameter. + Name string + + // Type specifies the type of the parameter, such as string, integer, etc. + Type string + + // Description provides a human-readable description of the parameter. + Description string + + // Required means the field is required when set. + Required bool + + // Format is a specifying the string format accepted by this parameter. + Format string + + // Regexp is a compiled regular expression that can be used to validate + // the contents of the parameter. + Regexp *regexp.Regexp + + // Examples provides multiple examples for the values that might be valid + // for this parameter. + Examples []string +} + +var routeDescriptors = []RouteDescriptor{ + { + Name: RouteNameBase, + Path: "/v2/", + Entity: "Base", + Description: `Base V2 API route. Typically, this can be used for lightweight version checks and to validate registry authentication.`, + Methods: []MethodDescriptor{ + { + Method: "GET", + Description: "Check that the endpoint implements Docker Registry API V2.", + Requests: []RequestDescriptor{ + { + Headers: []ParameterDescriptor{ + hostHeader, + authHeader, + }, + Successes: []ResponseDescriptor{ + { + Description: "The API implements V2 protocol and is accessible.", + StatusCode: http.StatusOK, + }, + }, + Failures: []ResponseDescriptor{ + { + Description: "The registry does not implement the V2 API.", + StatusCode: http.StatusNotFound, + }, + unauthorizedResponseDescriptor, + tooManyRequestsDescriptor, + }, + }, + }, + }, + }, + }, + { + Name: RouteNameTags, + Path: "/v2/{name:" + reference.NameRegexp.String() + "}/tags/list", + Entity: "Tags", + Description: "Retrieve information about tags.", + Methods: []MethodDescriptor{ + { + Method: "GET", + Description: "Fetch the tags under the repository identified by `name`.", + Requests: []RequestDescriptor{ + { + Name: "Tags", + Description: "Return all tags for the repository", + Headers: []ParameterDescriptor{ + hostHeader, + authHeader, + }, + PathParameters: []ParameterDescriptor{ + nameParameterDescriptor, + }, + Successes: []ResponseDescriptor{ + { + StatusCode: http.StatusOK, + Description: "A list of tags for the named repository.", + Headers: []ParameterDescriptor{ + { + Name: "Content-Length", + Type: "integer", + Description: "Length of the JSON response body.", + Format: "", + }, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: `{ + "name": , + "tags": [ + , + ... + ] +}`, + }, + }, + }, + Failures: []ResponseDescriptor{ + unauthorizedResponseDescriptor, + repositoryNotFoundResponseDescriptor, + deniedResponseDescriptor, + tooManyRequestsDescriptor, + }, + }, + { + Name: "Tags Paginated", + Description: "Return a portion of the tags for the specified repository.", + PathParameters: []ParameterDescriptor{nameParameterDescriptor}, + QueryParameters: paginationParameters, + Successes: []ResponseDescriptor{ + { + StatusCode: http.StatusOK, + Description: "A list of tags for the named repository.", + Headers: []ParameterDescriptor{ + { + Name: "Content-Length", + Type: "integer", + Description: "Length of the JSON response body.", + Format: "", + }, + linkHeader, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: `{ + "name": , + "tags": [ + , + ... + ], +}`, + }, + }, + }, + Failures: []ResponseDescriptor{ + invalidPaginationResponseDescriptor, + unauthorizedResponseDescriptor, + repositoryNotFoundResponseDescriptor, + deniedResponseDescriptor, + tooManyRequestsDescriptor, + }, + }, + }, + }, + }, + }, + { + Name: RouteNameManifest, + Path: "/v2/{name:" + reference.NameRegexp.String() + "}/manifests/{reference:" + reference.TagRegexp.String() + "|" + digest.DigestRegexp.String() + "}", + Entity: "Manifest", + Description: "Create, update, delete and retrieve manifests.", + Methods: []MethodDescriptor{ + { + Method: "GET", + Description: "Fetch the manifest identified by `name` and `reference` where `reference` can be a tag or digest. A `HEAD` request can also be issued to this endpoint to obtain resource information without receiving all data.", + Requests: []RequestDescriptor{ + { + Headers: []ParameterDescriptor{ + hostHeader, + authHeader, + }, + PathParameters: []ParameterDescriptor{ + nameParameterDescriptor, + referenceParameterDescriptor, + }, + Successes: []ResponseDescriptor{ + { + Description: "The manifest identified by `name` and `reference`. The contents can be used to identify and resolve resources required to run the specified image.", + StatusCode: http.StatusOK, + Headers: []ParameterDescriptor{ + digestHeader, + }, + Body: BodyDescriptor{ + ContentType: "", + Format: manifestBody, + }, + }, + }, + Failures: []ResponseDescriptor{ + { + Description: "The name or reference was invalid.", + StatusCode: http.StatusBadRequest, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeNameInvalid, + ErrorCodeTagInvalid, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + }, + unauthorizedResponseDescriptor, + repositoryNotFoundResponseDescriptor, + deniedResponseDescriptor, + tooManyRequestsDescriptor, + }, + }, + }, + }, + { + Method: "PUT", + Description: "Put the manifest identified by `name` and `reference` where `reference` can be a tag or digest.", + Requests: []RequestDescriptor{ + { + Headers: []ParameterDescriptor{ + hostHeader, + authHeader, + }, + PathParameters: []ParameterDescriptor{ + nameParameterDescriptor, + referenceParameterDescriptor, + }, + Body: BodyDescriptor{ + ContentType: "", + Format: manifestBody, + }, + Successes: []ResponseDescriptor{ + { + Description: "The manifest has been accepted by the registry and is stored under the specified `name` and `tag`.", + StatusCode: http.StatusCreated, + Headers: []ParameterDescriptor{ + { + Name: "Location", + Type: "url", + Description: "The canonical location url of the uploaded manifest.", + Format: "", + }, + contentLengthZeroHeader, + digestHeader, + }, + }, + }, + Failures: []ResponseDescriptor{ + { + Name: "Invalid Manifest", + Description: "The received manifest was invalid in some way, as described by the error codes. The client should resolve the issue and retry the request.", + StatusCode: http.StatusBadRequest, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeNameInvalid, + ErrorCodeTagInvalid, + ErrorCodeManifestInvalid, + ErrorCodeManifestUnverified, + ErrorCodeBlobUnknown, + }, + }, + unauthorizedResponseDescriptor, + repositoryNotFoundResponseDescriptor, + deniedResponseDescriptor, + tooManyRequestsDescriptor, + { + Name: "Missing Layer(s)", + Description: "One or more layers may be missing during a manifest upload. If so, the missing layers will be enumerated in the error response.", + StatusCode: http.StatusBadRequest, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeBlobUnknown, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: `{ + "errors:" [{ + "code": "BLOB_UNKNOWN", + "message": "blob unknown to registry", + "detail": { + "digest": "" + } + }, + ... + ] +}`, + }, + }, + { + Name: "Not allowed", + Description: "Manifest put is not allowed because the registry is configured as a pull-through cache or for some other reason", + StatusCode: http.StatusMethodNotAllowed, + ErrorCodes: []errcode.ErrorCode{ + errcode.ErrorCodeUnsupported, + }, + }, + }, + }, + }, + }, + { + Method: "DELETE", + Description: "Delete the manifest identified by `name` and `reference`. Note that a manifest can _only_ be deleted by `digest`.", + Requests: []RequestDescriptor{ + { + Headers: []ParameterDescriptor{ + hostHeader, + authHeader, + }, + PathParameters: []ParameterDescriptor{ + nameParameterDescriptor, + referenceParameterDescriptor, + }, + Successes: []ResponseDescriptor{ + { + StatusCode: http.StatusAccepted, + }, + }, + Failures: []ResponseDescriptor{ + { + Name: "Invalid Name or Reference", + Description: "The specified `name` or `reference` were invalid and the delete was unable to proceed.", + StatusCode: http.StatusBadRequest, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeNameInvalid, + ErrorCodeTagInvalid, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + }, + unauthorizedResponseDescriptor, + repositoryNotFoundResponseDescriptor, + deniedResponseDescriptor, + tooManyRequestsDescriptor, + { + Name: "Unknown Manifest", + Description: "The specified `name` or `reference` are unknown to the registry and the delete was unable to proceed. Clients can assume the manifest was already deleted if this response is returned.", + StatusCode: http.StatusNotFound, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeNameUnknown, + ErrorCodeManifestUnknown, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + }, + { + Name: "Not allowed", + Description: "Manifest delete is not allowed because the registry is configured as a pull-through cache or `delete` has been disabled.", + StatusCode: http.StatusMethodNotAllowed, + ErrorCodes: []errcode.ErrorCode{ + errcode.ErrorCodeUnsupported, + }, + }, + }, + }, + }, + }, + }, + }, + + { + Name: RouteNameBlob, + Path: "/v2/{name:" + reference.NameRegexp.String() + "}/blobs/{digest:" + digest.DigestRegexp.String() + "}", + Entity: "Blob", + Description: "Operations on blobs identified by `name` and `digest`. Used to fetch or delete layers by digest.", + Methods: []MethodDescriptor{ + { + Method: "GET", + Description: "Retrieve the blob from the registry identified by `digest`. A `HEAD` request can also be issued to this endpoint to obtain resource information without receiving all data.", + Requests: []RequestDescriptor{ + { + Name: "Fetch Blob", + Headers: []ParameterDescriptor{ + hostHeader, + authHeader, + }, + PathParameters: []ParameterDescriptor{ + nameParameterDescriptor, + digestPathParameter, + }, + Successes: []ResponseDescriptor{ + { + Description: "The blob identified by `digest` is available. The blob content will be present in the body of the request.", + StatusCode: http.StatusOK, + Headers: []ParameterDescriptor{ + { + Name: "Content-Length", + Type: "integer", + Description: "The length of the requested blob content.", + Format: "", + }, + digestHeader, + }, + Body: BodyDescriptor{ + ContentType: "application/octet-stream", + Format: "", + }, + }, + { + Description: "The blob identified by `digest` is available at the provided location.", + StatusCode: http.StatusTemporaryRedirect, + Headers: []ParameterDescriptor{ + { + Name: "Location", + Type: "url", + Description: "The location where the layer should be accessible.", + Format: "", + }, + digestHeader, + }, + }, + }, + Failures: []ResponseDescriptor{ + { + Description: "There was a problem with the request that needs to be addressed by the client, such as an invalid `name` or `tag`.", + StatusCode: http.StatusBadRequest, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeNameInvalid, + ErrorCodeDigestInvalid, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + }, + { + Description: "The blob, identified by `name` and `digest`, is unknown to the registry.", + StatusCode: http.StatusNotFound, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeNameUnknown, + ErrorCodeBlobUnknown, + }, + }, + unauthorizedResponseDescriptor, + repositoryNotFoundResponseDescriptor, + deniedResponseDescriptor, + tooManyRequestsDescriptor, + }, + }, + { + Name: "Fetch Blob Part", + Description: "This endpoint may also support RFC7233 compliant range requests. Support can be detected by issuing a HEAD request. If the header `Accept-Range: bytes` is returned, range requests can be used to fetch partial content.", + Headers: []ParameterDescriptor{ + hostHeader, + authHeader, + { + Name: "Range", + Type: "string", + Description: "HTTP Range header specifying blob chunk.", + Format: "bytes=-", + }, + }, + PathParameters: []ParameterDescriptor{ + nameParameterDescriptor, + digestPathParameter, + }, + Successes: []ResponseDescriptor{ + { + Description: "The blob identified by `digest` is available. The specified chunk of blob content will be present in the body of the request.", + StatusCode: http.StatusPartialContent, + Headers: []ParameterDescriptor{ + { + Name: "Content-Length", + Type: "integer", + Description: "The length of the requested blob chunk.", + Format: "", + }, + { + Name: "Content-Range", + Type: "byte range", + Description: "Content range of blob chunk.", + Format: "bytes -/", + }, + }, + Body: BodyDescriptor{ + ContentType: "application/octet-stream", + Format: "", + }, + }, + }, + Failures: []ResponseDescriptor{ + { + Description: "There was a problem with the request that needs to be addressed by the client, such as an invalid `name` or `tag`.", + StatusCode: http.StatusBadRequest, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeNameInvalid, + ErrorCodeDigestInvalid, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + }, + { + StatusCode: http.StatusNotFound, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeNameUnknown, + ErrorCodeBlobUnknown, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + }, + { + Description: "The range specification cannot be satisfied for the requested content. This can happen when the range is not formatted correctly or if the range is outside of the valid size of the content.", + StatusCode: http.StatusRequestedRangeNotSatisfiable, + }, + unauthorizedResponseDescriptor, + repositoryNotFoundResponseDescriptor, + deniedResponseDescriptor, + tooManyRequestsDescriptor, + }, + }, + }, + }, + { + Method: "DELETE", + Description: "Delete the blob identified by `name` and `digest`", + Requests: []RequestDescriptor{ + { + Headers: []ParameterDescriptor{ + hostHeader, + authHeader, + }, + PathParameters: []ParameterDescriptor{ + nameParameterDescriptor, + digestPathParameter, + }, + Successes: []ResponseDescriptor{ + { + StatusCode: http.StatusAccepted, + Headers: []ParameterDescriptor{ + { + Name: "Content-Length", + Type: "integer", + Description: "0", + Format: "0", + }, + digestHeader, + }, + }, + }, + Failures: []ResponseDescriptor{ + { + Name: "Invalid Name or Digest", + StatusCode: http.StatusBadRequest, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeDigestInvalid, + ErrorCodeNameInvalid, + }, + }, + { + Description: "The blob, identified by `name` and `digest`, is unknown to the registry.", + StatusCode: http.StatusNotFound, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeNameUnknown, + ErrorCodeBlobUnknown, + }, + }, + { + Description: "Blob delete is not allowed because the registry is configured as a pull-through cache or `delete` has been disabled", + StatusCode: http.StatusMethodNotAllowed, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + ErrorCodes: []errcode.ErrorCode{ + errcode.ErrorCodeUnsupported, + }, + }, + unauthorizedResponseDescriptor, + repositoryNotFoundResponseDescriptor, + deniedResponseDescriptor, + tooManyRequestsDescriptor, + }, + }, + }, + }, + + // TODO(stevvooe): We may want to add a PUT request here to + // kickoff an upload of a blob, integrated with the blob upload + // API. + }, + }, + + { + Name: RouteNameBlobUpload, + Path: "/v2/{name:" + reference.NameRegexp.String() + "}/blobs/uploads/", + Entity: "Initiate Blob Upload", + Description: "Initiate a blob upload. This endpoint can be used to create resumable uploads or monolithic uploads.", + Methods: []MethodDescriptor{ + { + Method: "POST", + Description: "Initiate a resumable blob upload. If successful, an upload location will be provided to complete the upload. Optionally, if the `digest` parameter is present, the request body will be used to complete the upload in a single request.", + Requests: []RequestDescriptor{ + { + Name: "Initiate Monolithic Blob Upload", + Description: "Upload a blob identified by the `digest` parameter in single request. This upload will not be resumable unless a recoverable error is returned.", + Headers: []ParameterDescriptor{ + hostHeader, + authHeader, + { + Name: "Content-Length", + Type: "integer", + Format: "", + }, + }, + PathParameters: []ParameterDescriptor{ + nameParameterDescriptor, + }, + QueryParameters: []ParameterDescriptor{ + { + Name: "digest", + Type: "query", + Format: "", + Regexp: digest.DigestRegexp, + Description: `Digest of uploaded blob. If present, the upload will be completed, in a single request, with contents of the request body as the resulting blob.`, + }, + }, + Body: BodyDescriptor{ + ContentType: "application/octect-stream", + Format: "", + }, + Successes: []ResponseDescriptor{ + { + Description: "The blob has been created in the registry and is available at the provided location.", + StatusCode: http.StatusCreated, + Headers: []ParameterDescriptor{ + { + Name: "Location", + Type: "url", + Format: "", + }, + contentLengthZeroHeader, + dockerUploadUUIDHeader, + }, + }, + }, + Failures: []ResponseDescriptor{ + { + Name: "Invalid Name or Digest", + StatusCode: http.StatusBadRequest, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeDigestInvalid, + ErrorCodeNameInvalid, + }, + }, + { + Name: "Not allowed", + Description: "Blob upload is not allowed because the registry is configured as a pull-through cache or for some other reason", + StatusCode: http.StatusMethodNotAllowed, + ErrorCodes: []errcode.ErrorCode{ + errcode.ErrorCodeUnsupported, + }, + }, + unauthorizedResponseDescriptor, + repositoryNotFoundResponseDescriptor, + deniedResponseDescriptor, + tooManyRequestsDescriptor, + }, + }, + { + Name: "Initiate Resumable Blob Upload", + Description: "Initiate a resumable blob upload with an empty request body.", + Headers: []ParameterDescriptor{ + hostHeader, + authHeader, + contentLengthZeroHeader, + }, + PathParameters: []ParameterDescriptor{ + nameParameterDescriptor, + }, + Successes: []ResponseDescriptor{ + { + Description: "The upload has been created. The `Location` header must be used to complete the upload. The response should be identical to a `GET` request on the contents of the returned `Location` header.", + StatusCode: http.StatusAccepted, + Headers: []ParameterDescriptor{ + contentLengthZeroHeader, + { + Name: "Location", + Type: "url", + Format: "/v2//blobs/uploads/", + Description: "The location of the created upload. Clients should use the contents verbatim to complete the upload, adding parameters where required.", + }, + { + Name: "Range", + Format: "0-0", + Description: "Range header indicating the progress of the upload. When starting an upload, it will return an empty range, since no content has been received.", + }, + dockerUploadUUIDHeader, + }, + }, + }, + Failures: []ResponseDescriptor{ + { + Name: "Invalid Name or Digest", + StatusCode: http.StatusBadRequest, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeDigestInvalid, + ErrorCodeNameInvalid, + }, + }, + unauthorizedResponseDescriptor, + repositoryNotFoundResponseDescriptor, + deniedResponseDescriptor, + tooManyRequestsDescriptor, + }, + }, + { + Name: "Mount Blob", + Description: "Mount a blob identified by the `mount` parameter from another repository.", + Headers: []ParameterDescriptor{ + hostHeader, + authHeader, + contentLengthZeroHeader, + }, + PathParameters: []ParameterDescriptor{ + nameParameterDescriptor, + }, + QueryParameters: []ParameterDescriptor{ + { + Name: "mount", + Type: "query", + Format: "", + Regexp: digest.DigestRegexp, + Description: `Digest of blob to mount from the source repository.`, + }, + { + Name: "from", + Type: "query", + Format: "", + Regexp: reference.NameRegexp, + Description: `Name of the source repository.`, + }, + }, + Successes: []ResponseDescriptor{ + { + Description: "The blob has been mounted in the repository and is available at the provided location.", + StatusCode: http.StatusCreated, + Headers: []ParameterDescriptor{ + { + Name: "Location", + Type: "url", + Format: "", + }, + contentLengthZeroHeader, + dockerUploadUUIDHeader, + }, + }, + }, + Failures: []ResponseDescriptor{ + { + Name: "Invalid Name or Digest", + StatusCode: http.StatusBadRequest, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeDigestInvalid, + ErrorCodeNameInvalid, + }, + }, + { + Name: "Not allowed", + Description: "Blob mount is not allowed because the registry is configured as a pull-through cache or for some other reason", + StatusCode: http.StatusMethodNotAllowed, + ErrorCodes: []errcode.ErrorCode{ + errcode.ErrorCodeUnsupported, + }, + }, + unauthorizedResponseDescriptor, + repositoryNotFoundResponseDescriptor, + deniedResponseDescriptor, + tooManyRequestsDescriptor, + }, + }, + }, + }, + }, + }, + + { + Name: RouteNameBlobUploadChunk, + Path: "/v2/{name:" + reference.NameRegexp.String() + "}/blobs/uploads/{uuid:[a-zA-Z0-9-_.=]+}", + Entity: "Blob Upload", + Description: "Interact with blob uploads. Clients should never assemble URLs for this endpoint and should only take it through the `Location` header on related API requests. The `Location` header and its parameters should be preserved by clients, using the latest value returned via upload related API calls.", + Methods: []MethodDescriptor{ + { + Method: "GET", + Description: "Retrieve status of upload identified by `uuid`. The primary purpose of this endpoint is to resolve the current status of a resumable upload.", + Requests: []RequestDescriptor{ + { + Description: "Retrieve the progress of the current upload, as reported by the `Range` header.", + Headers: []ParameterDescriptor{ + hostHeader, + authHeader, + }, + PathParameters: []ParameterDescriptor{ + nameParameterDescriptor, + uuidParameterDescriptor, + }, + Successes: []ResponseDescriptor{ + { + Name: "Upload Progress", + Description: "The upload is known and in progress. The last received offset is available in the `Range` header.", + StatusCode: http.StatusNoContent, + Headers: []ParameterDescriptor{ + { + Name: "Range", + Type: "header", + Format: "0-", + Description: "Range indicating the current progress of the upload.", + }, + contentLengthZeroHeader, + dockerUploadUUIDHeader, + }, + }, + }, + Failures: []ResponseDescriptor{ + { + Description: "There was an error processing the upload and it must be restarted.", + StatusCode: http.StatusBadRequest, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeDigestInvalid, + ErrorCodeNameInvalid, + ErrorCodeBlobUploadInvalid, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + }, + { + Description: "The upload is unknown to the registry. The upload must be restarted.", + StatusCode: http.StatusNotFound, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeBlobUploadUnknown, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + }, + unauthorizedResponseDescriptor, + repositoryNotFoundResponseDescriptor, + deniedResponseDescriptor, + tooManyRequestsDescriptor, + }, + }, + }, + }, + { + Method: "PATCH", + Description: "Upload a chunk of data for the specified upload.", + Requests: []RequestDescriptor{ + { + Name: "Stream upload", + Description: "Upload a stream of data to upload without completing the upload.", + PathParameters: []ParameterDescriptor{ + nameParameterDescriptor, + uuidParameterDescriptor, + }, + Headers: []ParameterDescriptor{ + hostHeader, + authHeader, + }, + Body: BodyDescriptor{ + ContentType: "application/octet-stream", + Format: "", + }, + Successes: []ResponseDescriptor{ + { + Name: "Data Accepted", + Description: "The stream of data has been accepted and the current progress is available in the range header. The updated upload location is available in the `Location` header.", + StatusCode: http.StatusNoContent, + Headers: []ParameterDescriptor{ + { + Name: "Location", + Type: "url", + Format: "/v2//blobs/uploads/", + Description: "The location of the upload. Clients should assume this changes after each request. Clients should use the contents verbatim to complete the upload, adding parameters where required.", + }, + { + Name: "Range", + Type: "header", + Format: "0-", + Description: "Range indicating the current progress of the upload.", + }, + contentLengthZeroHeader, + dockerUploadUUIDHeader, + }, + }, + }, + Failures: []ResponseDescriptor{ + { + Description: "There was an error processing the upload and it must be restarted.", + StatusCode: http.StatusBadRequest, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeDigestInvalid, + ErrorCodeNameInvalid, + ErrorCodeBlobUploadInvalid, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + }, + { + Description: "The upload is unknown to the registry. The upload must be restarted.", + StatusCode: http.StatusNotFound, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeBlobUploadUnknown, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + }, + unauthorizedResponseDescriptor, + repositoryNotFoundResponseDescriptor, + deniedResponseDescriptor, + tooManyRequestsDescriptor, + }, + }, + { + Name: "Chunked upload", + Description: "Upload a chunk of data to specified upload without completing the upload. The data will be uploaded to the specified Content Range.", + PathParameters: []ParameterDescriptor{ + nameParameterDescriptor, + uuidParameterDescriptor, + }, + Headers: []ParameterDescriptor{ + hostHeader, + authHeader, + { + Name: "Content-Range", + Type: "header", + Format: "-", + Required: true, + Description: "Range of bytes identifying the desired block of content represented by the body. Start must the end offset retrieved via status check plus one. Note that this is a non-standard use of the `Content-Range` header.", + }, + { + Name: "Content-Length", + Type: "integer", + Format: "", + Description: "Length of the chunk being uploaded, corresponding the length of the request body.", + }, + }, + Body: BodyDescriptor{ + ContentType: "application/octet-stream", + Format: "", + }, + Successes: []ResponseDescriptor{ + { + Name: "Chunk Accepted", + Description: "The chunk of data has been accepted and the current progress is available in the range header. The updated upload location is available in the `Location` header.", + StatusCode: http.StatusNoContent, + Headers: []ParameterDescriptor{ + { + Name: "Location", + Type: "url", + Format: "/v2//blobs/uploads/", + Description: "The location of the upload. Clients should assume this changes after each request. Clients should use the contents verbatim to complete the upload, adding parameters where required.", + }, + { + Name: "Range", + Type: "header", + Format: "0-", + Description: "Range indicating the current progress of the upload.", + }, + contentLengthZeroHeader, + dockerUploadUUIDHeader, + }, + }, + }, + Failures: []ResponseDescriptor{ + { + Description: "There was an error processing the upload and it must be restarted.", + StatusCode: http.StatusBadRequest, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeDigestInvalid, + ErrorCodeNameInvalid, + ErrorCodeBlobUploadInvalid, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + }, + { + Description: "The upload is unknown to the registry. The upload must be restarted.", + StatusCode: http.StatusNotFound, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeBlobUploadUnknown, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + }, + { + Description: "The `Content-Range` specification cannot be accepted, either because it does not overlap with the current progress or it is invalid.", + StatusCode: http.StatusRequestedRangeNotSatisfiable, + }, + unauthorizedResponseDescriptor, + repositoryNotFoundResponseDescriptor, + deniedResponseDescriptor, + tooManyRequestsDescriptor, + }, + }, + }, + }, + { + Method: "PUT", + Description: "Complete the upload specified by `uuid`, optionally appending the body as the final chunk.", + Requests: []RequestDescriptor{ + { + Description: "Complete the upload, providing all the data in the body, if necessary. A request without a body will just complete the upload with previously uploaded content.", + Headers: []ParameterDescriptor{ + hostHeader, + authHeader, + { + Name: "Content-Length", + Type: "integer", + Format: "", + Description: "Length of the data being uploaded, corresponding to the length of the request body. May be zero if no data is provided.", + }, + }, + PathParameters: []ParameterDescriptor{ + nameParameterDescriptor, + uuidParameterDescriptor, + }, + QueryParameters: []ParameterDescriptor{ + { + Name: "digest", + Type: "string", + Format: "", + Regexp: digest.DigestRegexp, + Required: true, + Description: `Digest of uploaded blob.`, + }, + }, + Body: BodyDescriptor{ + ContentType: "application/octet-stream", + Format: "", + }, + Successes: []ResponseDescriptor{ + { + Name: "Upload Complete", + Description: "The upload has been completed and accepted by the registry. The canonical location will be available in the `Location` header.", + StatusCode: http.StatusNoContent, + Headers: []ParameterDescriptor{ + { + Name: "Location", + Type: "url", + Format: "", + Description: "The canonical location of the blob for retrieval", + }, + { + Name: "Content-Range", + Type: "header", + Format: "-", + Description: "Range of bytes identifying the desired block of content represented by the body. Start must match the end of offset retrieved via status check. Note that this is a non-standard use of the `Content-Range` header.", + }, + contentLengthZeroHeader, + digestHeader, + }, + }, + }, + Failures: []ResponseDescriptor{ + { + Description: "There was an error processing the upload and it must be restarted.", + StatusCode: http.StatusBadRequest, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeDigestInvalid, + ErrorCodeNameInvalid, + ErrorCodeBlobUploadInvalid, + errcode.ErrorCodeUnsupported, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + }, + { + Description: "The upload is unknown to the registry. The upload must be restarted.", + StatusCode: http.StatusNotFound, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeBlobUploadUnknown, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + }, + unauthorizedResponseDescriptor, + repositoryNotFoundResponseDescriptor, + deniedResponseDescriptor, + tooManyRequestsDescriptor, + }, + }, + }, + }, + { + Method: "DELETE", + Description: "Cancel outstanding upload processes, releasing associated resources. If this is not called, the unfinished uploads will eventually timeout.", + Requests: []RequestDescriptor{ + { + Description: "Cancel the upload specified by `uuid`.", + PathParameters: []ParameterDescriptor{ + nameParameterDescriptor, + uuidParameterDescriptor, + }, + Headers: []ParameterDescriptor{ + hostHeader, + authHeader, + contentLengthZeroHeader, + }, + Successes: []ResponseDescriptor{ + { + Name: "Upload Deleted", + Description: "The upload has been successfully deleted.", + StatusCode: http.StatusNoContent, + Headers: []ParameterDescriptor{ + contentLengthZeroHeader, + }, + }, + }, + Failures: []ResponseDescriptor{ + { + Description: "An error was encountered processing the delete. The client may ignore this error.", + StatusCode: http.StatusBadRequest, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeNameInvalid, + ErrorCodeBlobUploadInvalid, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + }, + { + Description: "The upload is unknown to the registry. The client may ignore this error and assume the upload has been deleted.", + StatusCode: http.StatusNotFound, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeBlobUploadUnknown, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + }, + unauthorizedResponseDescriptor, + repositoryNotFoundResponseDescriptor, + deniedResponseDescriptor, + tooManyRequestsDescriptor, + }, + }, + }, + }, + }, + }, + { + Name: RouteNameCatalog, + Path: "/v2/_catalog", + Entity: "Catalog", + Description: "List a set of available repositories in the local registry cluster. Does not provide any indication of what may be available upstream. Applications can only determine if a repository is available but not if it is not available.", + Methods: []MethodDescriptor{ + { + Method: "GET", + Description: "Retrieve a sorted, json list of repositories available in the registry.", + Requests: []RequestDescriptor{ + { + Name: "Catalog Fetch", + Description: "Request an unabridged list of repositories available. The implementation may impose a maximum limit and return a partial set with pagination links.", + Successes: []ResponseDescriptor{ + { + Description: "Returns the unabridged list of repositories as a json response.", + StatusCode: http.StatusOK, + Headers: []ParameterDescriptor{ + { + Name: "Content-Length", + Type: "integer", + Description: "Length of the JSON response body.", + Format: "", + }, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: `{ + "repositories": [ + , + ... + ] +}`, + }, + }, + }, + }, + { + Name: "Catalog Fetch Paginated", + Description: "Return the specified portion of repositories.", + QueryParameters: paginationParameters, + Successes: []ResponseDescriptor{ + { + StatusCode: http.StatusOK, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: `{ + "repositories": [ + , + ... + ] + "next": "?last=&n=" +}`, + }, + Headers: []ParameterDescriptor{ + { + Name: "Content-Length", + Type: "integer", + Description: "Length of the JSON response body.", + Format: "", + }, + linkHeader, + }, + }, + }, + Failures: []ResponseDescriptor{ + invalidPaginationResponseDescriptor, + }, + }, + }, + }, + }, + }, +} + +var routeDescriptorsMap map[string]RouteDescriptor + +func init() { + routeDescriptorsMap = make(map[string]RouteDescriptor, len(routeDescriptors)) + + for _, descriptor := range routeDescriptors { + routeDescriptorsMap[descriptor.Name] = descriptor + } +} diff --git a/vendor/github.com/docker/distribution/registry/api/v2/doc.go b/vendor/github.com/docker/distribution/registry/api/v2/doc.go new file mode 100644 index 000000000..cde011959 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/api/v2/doc.go @@ -0,0 +1,9 @@ +// Package v2 describes routes, urls and the error codes used in the Docker +// Registry JSON HTTP API V2. In addition to declarations, descriptors are +// provided for routes and error codes that can be used for implementation and +// automatically generating documentation. +// +// Definitions here are considered to be locked down for the V2 registry api. +// Any changes must be considered carefully and should not proceed without a +// change proposal in docker core. +package v2 diff --git a/vendor/github.com/docker/distribution/registry/api/v2/errors.go b/vendor/github.com/docker/distribution/registry/api/v2/errors.go new file mode 100644 index 000000000..87e9f3c14 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/api/v2/errors.go @@ -0,0 +1,145 @@ +package v2 + +import ( + "net/http" + + "github.com/docker/distribution/registry/api/errcode" +) + +const errGroup = "registry.api.v2" + +var ( + // ErrorCodeDigestInvalid is returned when uploading a blob if the + // provided digest does not match the blob contents. + ErrorCodeDigestInvalid = errcode.Register(errGroup, errcode.ErrorDescriptor{ + Value: "DIGEST_INVALID", + Message: "provided digest did not match uploaded content", + Description: `When a blob is uploaded, the registry will check that + the content matches the digest provided by the client. The error may + include a detail structure with the key "digest", including the + invalid digest string. This error may also be returned when a manifest + includes an invalid layer digest.`, + HTTPStatusCode: http.StatusBadRequest, + }) + + // ErrorCodeSizeInvalid is returned when uploading a blob if the provided + ErrorCodeSizeInvalid = errcode.Register(errGroup, errcode.ErrorDescriptor{ + Value: "SIZE_INVALID", + Message: "provided length did not match content length", + Description: `When a layer is uploaded, the provided size will be + checked against the uploaded content. If they do not match, this error + will be returned.`, + HTTPStatusCode: http.StatusBadRequest, + }) + + // ErrorCodeNameInvalid is returned when the name in the manifest does not + // match the provided name. + ErrorCodeNameInvalid = errcode.Register(errGroup, errcode.ErrorDescriptor{ + Value: "NAME_INVALID", + Message: "invalid repository name", + Description: `Invalid repository name encountered either during + manifest validation or any API operation.`, + HTTPStatusCode: http.StatusBadRequest, + }) + + // ErrorCodeTagInvalid is returned when the tag in the manifest does not + // match the provided tag. + ErrorCodeTagInvalid = errcode.Register(errGroup, errcode.ErrorDescriptor{ + Value: "TAG_INVALID", + Message: "manifest tag did not match URI", + Description: `During a manifest upload, if the tag in the manifest + does not match the uri tag, this error will be returned.`, + HTTPStatusCode: http.StatusBadRequest, + }) + + // ErrorCodeNameUnknown when the repository name is not known. + ErrorCodeNameUnknown = errcode.Register(errGroup, errcode.ErrorDescriptor{ + Value: "NAME_UNKNOWN", + Message: "repository name not known to registry", + Description: `This is returned if the name used during an operation is + unknown to the registry.`, + HTTPStatusCode: http.StatusNotFound, + }) + + // ErrorCodeManifestUnknown returned when image manifest is unknown. + ErrorCodeManifestUnknown = errcode.Register(errGroup, errcode.ErrorDescriptor{ + Value: "MANIFEST_UNKNOWN", + Message: "manifest unknown", + Description: `This error is returned when the manifest, identified by + name and tag is unknown to the repository.`, + HTTPStatusCode: http.StatusNotFound, + }) + + // ErrorCodeManifestInvalid returned when an image manifest is invalid, + // typically during a PUT operation. This error encompasses all errors + // encountered during manifest validation that aren't signature errors. + ErrorCodeManifestInvalid = errcode.Register(errGroup, errcode.ErrorDescriptor{ + Value: "MANIFEST_INVALID", + Message: "manifest invalid", + Description: `During upload, manifests undergo several checks ensuring + validity. If those checks fail, this error may be returned, unless a + more specific error is included. The detail will contain information + the failed validation.`, + HTTPStatusCode: http.StatusBadRequest, + }) + + // ErrorCodeManifestUnverified is returned when the manifest fails + // signature verification. + ErrorCodeManifestUnverified = errcode.Register(errGroup, errcode.ErrorDescriptor{ + Value: "MANIFEST_UNVERIFIED", + Message: "manifest failed signature verification", + Description: `During manifest upload, if the manifest fails signature + verification, this error will be returned.`, + HTTPStatusCode: http.StatusBadRequest, + }) + + // ErrorCodeManifestBlobUnknown is returned when a manifest blob is + // unknown to the registry. + ErrorCodeManifestBlobUnknown = errcode.Register(errGroup, errcode.ErrorDescriptor{ + Value: "MANIFEST_BLOB_UNKNOWN", + Message: "blob unknown to registry", + Description: `This error may be returned when a manifest blob is + unknown to the registry.`, + HTTPStatusCode: http.StatusBadRequest, + }) + + // ErrorCodeBlobUnknown is returned when a blob is unknown to the + // registry. This can happen when the manifest references a nonexistent + // layer or the result is not found by a blob fetch. + ErrorCodeBlobUnknown = errcode.Register(errGroup, errcode.ErrorDescriptor{ + Value: "BLOB_UNKNOWN", + Message: "blob unknown to registry", + Description: `This error may be returned when a blob is unknown to the + registry in a specified repository. This can be returned with a + standard get or if a manifest references an unknown layer during + upload.`, + HTTPStatusCode: http.StatusNotFound, + }) + + // ErrorCodeBlobUploadUnknown is returned when an upload is unknown. + ErrorCodeBlobUploadUnknown = errcode.Register(errGroup, errcode.ErrorDescriptor{ + Value: "BLOB_UPLOAD_UNKNOWN", + Message: "blob upload unknown to registry", + Description: `If a blob upload has been cancelled or was never + started, this error code may be returned.`, + HTTPStatusCode: http.StatusNotFound, + }) + + // ErrorCodeBlobUploadInvalid is returned when an upload is invalid. + ErrorCodeBlobUploadInvalid = errcode.Register(errGroup, errcode.ErrorDescriptor{ + Value: "BLOB_UPLOAD_INVALID", + Message: "blob upload invalid", + Description: `The blob upload encountered an error and can no + longer proceed.`, + HTTPStatusCode: http.StatusNotFound, + }) + + ErrorCodePaginationNumberInvalid = errcode.Register(errGroup, errcode.ErrorDescriptor{ + Value: "PAGINATION_NUMBER_INVALID", + Message: "invalid number of results requested", + Description: `Returned when the "n" parameter (number of results + to return) is not an integer, "n" is negative or "n" is bigger than + the maximum allowed.`, + HTTPStatusCode: http.StatusBadRequest, + }) +) diff --git a/vendor/github.com/docker/distribution/registry/api/v2/headerparser.go b/vendor/github.com/docker/distribution/registry/api/v2/headerparser.go new file mode 100644 index 000000000..9bc41a3a6 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/api/v2/headerparser.go @@ -0,0 +1,161 @@ +package v2 + +import ( + "fmt" + "regexp" + "strings" + "unicode" +) + +var ( + // according to rfc7230 + reToken = regexp.MustCompile(`^[^"(),/:;<=>?@[\]{}[:space:][:cntrl:]]+`) + reQuotedValue = regexp.MustCompile(`^[^\\"]+`) + reEscapedCharacter = regexp.MustCompile(`^[[:blank:][:graph:]]`) +) + +// parseForwardedHeader is a benevolent parser of Forwarded header defined in rfc7239. The header contains +// a comma-separated list of forwarding key-value pairs. Each list element is set by single proxy. The +// function parses only the first element of the list, which is set by the very first proxy. It returns a map +// of corresponding key-value pairs and an unparsed slice of the input string. +// +// Examples of Forwarded header values: +// +// 1. Forwarded: For=192.0.2.43; Proto=https,For="[2001:db8:cafe::17]",For=unknown +// 2. Forwarded: for="192.0.2.43:443"; host="registry.example.org", for="10.10.05.40:80" +// +// The first will be parsed into {"for": "192.0.2.43", "proto": "https"} while the second into +// {"for": "192.0.2.43:443", "host": "registry.example.org"}. +func parseForwardedHeader(forwarded string) (map[string]string, string, error) { + // Following are states of forwarded header parser. Any state could transition to a failure. + const ( + // terminating state; can transition to Parameter + stateElement = iota + // terminating state; can transition to KeyValueDelimiter + stateParameter + // can transition to Value + stateKeyValueDelimiter + // can transition to one of { QuotedValue, PairEnd } + stateValue + // can transition to one of { EscapedCharacter, PairEnd } + stateQuotedValue + // can transition to one of { QuotedValue } + stateEscapedCharacter + // terminating state; can transition to one of { Parameter, Element } + statePairEnd + ) + + var ( + parameter string + value string + parse = forwarded[:] + res = map[string]string{} + state = stateElement + ) + +Loop: + for { + // skip spaces unless in quoted value + if state != stateQuotedValue && state != stateEscapedCharacter { + parse = strings.TrimLeftFunc(parse, unicode.IsSpace) + } + + if len(parse) == 0 { + if state != stateElement && state != statePairEnd && state != stateParameter { + return nil, parse, fmt.Errorf("unexpected end of input") + } + // terminating + break + } + + switch state { + // terminate at list element delimiter + case stateElement: + if parse[0] == ',' { + parse = parse[1:] + break Loop + } + state = stateParameter + + // parse parameter (the key of key-value pair) + case stateParameter: + match := reToken.FindString(parse) + if len(match) == 0 { + return nil, parse, fmt.Errorf("failed to parse token at position %d", len(forwarded)-len(parse)) + } + parameter = strings.ToLower(match) + parse = parse[len(match):] + state = stateKeyValueDelimiter + + // parse '=' + case stateKeyValueDelimiter: + if parse[0] != '=' { + return nil, parse, fmt.Errorf("expected '=', not '%c' at position %d", parse[0], len(forwarded)-len(parse)) + } + parse = parse[1:] + state = stateValue + + // parse value or quoted value + case stateValue: + if parse[0] == '"' { + parse = parse[1:] + state = stateQuotedValue + } else { + value = reToken.FindString(parse) + if len(value) == 0 { + return nil, parse, fmt.Errorf("failed to parse value at position %d", len(forwarded)-len(parse)) + } + if _, exists := res[parameter]; exists { + return nil, parse, fmt.Errorf("duplicate parameter %q at position %d", parameter, len(forwarded)-len(parse)) + } + res[parameter] = value + parse = parse[len(value):] + value = "" + state = statePairEnd + } + + // parse a part of quoted value until the first backslash + case stateQuotedValue: + match := reQuotedValue.FindString(parse) + value += match + parse = parse[len(match):] + switch { + case len(parse) == 0: + return nil, parse, fmt.Errorf("unterminated quoted string") + case parse[0] == '"': + res[parameter] = value + value = "" + parse = parse[1:] + state = statePairEnd + case parse[0] == '\\': + parse = parse[1:] + state = stateEscapedCharacter + } + + // parse escaped character in a quoted string, ignore the backslash + // transition back to QuotedValue state + case stateEscapedCharacter: + c := reEscapedCharacter.FindString(parse) + if len(c) == 0 { + return nil, parse, fmt.Errorf("invalid escape sequence at position %d", len(forwarded)-len(parse)-1) + } + value += c + parse = parse[1:] + state = stateQuotedValue + + // expect either a new key-value pair, new list or end of input + case statePairEnd: + switch parse[0] { + case ';': + parse = parse[1:] + state = stateParameter + case ',': + state = stateElement + default: + return nil, parse, fmt.Errorf("expected ',' or ';', not %c at position %d", parse[0], len(forwarded)-len(parse)) + } + } + } + + return res, parse, nil +} diff --git a/vendor/github.com/docker/distribution/registry/api/v2/routes.go b/vendor/github.com/docker/distribution/registry/api/v2/routes.go new file mode 100644 index 000000000..9612ac2e5 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/api/v2/routes.go @@ -0,0 +1,40 @@ +package v2 + +import "github.com/gorilla/mux" + +// The following are definitions of the name under which all V2 routes are +// registered. These symbols can be used to look up a route based on the name. +const ( + RouteNameBase = "base" + RouteNameManifest = "manifest" + RouteNameTags = "tags" + RouteNameBlob = "blob" + RouteNameBlobUpload = "blob-upload" + RouteNameBlobUploadChunk = "blob-upload-chunk" + RouteNameCatalog = "catalog" +) + +// Router builds a gorilla router with named routes for the various API +// methods. This can be used directly by both server implementations and +// clients. +func Router() *mux.Router { + return RouterWithPrefix("") +} + +// RouterWithPrefix builds a gorilla router with a configured prefix +// on all routes. +func RouterWithPrefix(prefix string) *mux.Router { + rootRouter := mux.NewRouter() + router := rootRouter + if prefix != "" { + router = router.PathPrefix(prefix).Subrouter() + } + + router.StrictSlash(true) + + for _, descriptor := range routeDescriptors { + router.Path(descriptor.Path).Name(descriptor.Name) + } + + return rootRouter +} diff --git a/vendor/github.com/docker/distribution/registry/api/v2/urls.go b/vendor/github.com/docker/distribution/registry/api/v2/urls.go new file mode 100644 index 000000000..ab6406335 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/api/v2/urls.go @@ -0,0 +1,254 @@ +package v2 + +import ( + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/distribution/reference" + "github.com/gorilla/mux" +) + +// URLBuilder creates registry API urls from a single base endpoint. It can be +// used to create urls for use in a registry client or server. +// +// All urls will be created from the given base, including the api version. +// For example, if a root of "/foo/" is provided, urls generated will be fall +// under "/foo/v2/...". Most application will only provide a schema, host and +// port, such as "https://localhost:5000/". +type URLBuilder struct { + root *url.URL // url root (ie http://localhost/) + router *mux.Router + relative bool +} + +// NewURLBuilder creates a URLBuilder with provided root url object. +func NewURLBuilder(root *url.URL, relative bool) *URLBuilder { + return &URLBuilder{ + root: root, + router: Router(), + relative: relative, + } +} + +// NewURLBuilderFromString workes identically to NewURLBuilder except it takes +// a string argument for the root, returning an error if it is not a valid +// url. +func NewURLBuilderFromString(root string, relative bool) (*URLBuilder, error) { + u, err := url.Parse(root) + if err != nil { + return nil, err + } + + return NewURLBuilder(u, relative), nil +} + +// NewURLBuilderFromRequest uses information from an *http.Request to +// construct the root url. +func NewURLBuilderFromRequest(r *http.Request, relative bool) *URLBuilder { + var ( + scheme = "http" + host = r.Host + ) + + if r.TLS != nil { + scheme = "https" + } else if len(r.URL.Scheme) > 0 { + scheme = r.URL.Scheme + } + + // Handle fowarded headers + // Prefer "Forwarded" header as defined by rfc7239 if given + // see https://tools.ietf.org/html/rfc7239 + if forwarded := r.Header.Get("Forwarded"); len(forwarded) > 0 { + forwardedHeader, _, err := parseForwardedHeader(forwarded) + if err == nil { + if fproto := forwardedHeader["proto"]; len(fproto) > 0 { + scheme = fproto + } + if fhost := forwardedHeader["host"]; len(fhost) > 0 { + host = fhost + } + } + } else { + if forwardedProto := r.Header.Get("X-Forwarded-Proto"); len(forwardedProto) > 0 { + scheme = forwardedProto + } + if forwardedHost := r.Header.Get("X-Forwarded-Host"); len(forwardedHost) > 0 { + // According to the Apache mod_proxy docs, X-Forwarded-Host can be a + // comma-separated list of hosts, to which each proxy appends the + // requested host. We want to grab the first from this comma-separated + // list. + hosts := strings.SplitN(forwardedHost, ",", 2) + host = strings.TrimSpace(hosts[0]) + } + } + + basePath := routeDescriptorsMap[RouteNameBase].Path + + requestPath := r.URL.Path + index := strings.Index(requestPath, basePath) + + u := &url.URL{ + Scheme: scheme, + Host: host, + } + + if index > 0 { + // N.B. index+1 is important because we want to include the trailing / + u.Path = requestPath[0 : index+1] + } + + return NewURLBuilder(u, relative) +} + +// BuildBaseURL constructs a base url for the API, typically just "/v2/". +func (ub *URLBuilder) BuildBaseURL() (string, error) { + route := ub.cloneRoute(RouteNameBase) + + baseURL, err := route.URL() + if err != nil { + return "", err + } + + return baseURL.String(), nil +} + +// BuildCatalogURL constructs a url get a catalog of repositories +func (ub *URLBuilder) BuildCatalogURL(values ...url.Values) (string, error) { + route := ub.cloneRoute(RouteNameCatalog) + + catalogURL, err := route.URL() + if err != nil { + return "", err + } + + return appendValuesURL(catalogURL, values...).String(), nil +} + +// BuildTagsURL constructs a url to list the tags in the named repository. +func (ub *URLBuilder) BuildTagsURL(name reference.Named) (string, error) { + route := ub.cloneRoute(RouteNameTags) + + tagsURL, err := route.URL("name", name.Name()) + if err != nil { + return "", err + } + + return tagsURL.String(), nil +} + +// BuildManifestURL constructs a url for the manifest identified by name and +// reference. The argument reference may be either a tag or digest. +func (ub *URLBuilder) BuildManifestURL(ref reference.Named) (string, error) { + route := ub.cloneRoute(RouteNameManifest) + + tagOrDigest := "" + switch v := ref.(type) { + case reference.Tagged: + tagOrDigest = v.Tag() + case reference.Digested: + tagOrDigest = v.Digest().String() + default: + return "", fmt.Errorf("reference must have a tag or digest") + } + + manifestURL, err := route.URL("name", ref.Name(), "reference", tagOrDigest) + if err != nil { + return "", err + } + + return manifestURL.String(), nil +} + +// BuildBlobURL constructs the url for the blob identified by name and dgst. +func (ub *URLBuilder) BuildBlobURL(ref reference.Canonical) (string, error) { + route := ub.cloneRoute(RouteNameBlob) + + layerURL, err := route.URL("name", ref.Name(), "digest", ref.Digest().String()) + if err != nil { + return "", err + } + + return layerURL.String(), nil +} + +// BuildBlobUploadURL constructs a url to begin a blob upload in the +// repository identified by name. +func (ub *URLBuilder) BuildBlobUploadURL(name reference.Named, values ...url.Values) (string, error) { + route := ub.cloneRoute(RouteNameBlobUpload) + + uploadURL, err := route.URL("name", name.Name()) + if err != nil { + return "", err + } + + return appendValuesURL(uploadURL, values...).String(), nil +} + +// BuildBlobUploadChunkURL constructs a url for the upload identified by uuid, +// including any url values. This should generally not be used by clients, as +// this url is provided by server implementations during the blob upload +// process. +func (ub *URLBuilder) BuildBlobUploadChunkURL(name reference.Named, uuid string, values ...url.Values) (string, error) { + route := ub.cloneRoute(RouteNameBlobUploadChunk) + + uploadURL, err := route.URL("name", name.Name(), "uuid", uuid) + if err != nil { + return "", err + } + + return appendValuesURL(uploadURL, values...).String(), nil +} + +// clondedRoute returns a clone of the named route from the router. Routes +// must be cloned to avoid modifying them during url generation. +func (ub *URLBuilder) cloneRoute(name string) clonedRoute { + route := new(mux.Route) + root := new(url.URL) + + *route = *ub.router.GetRoute(name) // clone the route + *root = *ub.root + + return clonedRoute{Route: route, root: root, relative: ub.relative} +} + +type clonedRoute struct { + *mux.Route + root *url.URL + relative bool +} + +func (cr clonedRoute) URL(pairs ...string) (*url.URL, error) { + routeURL, err := cr.Route.URL(pairs...) + if err != nil { + return nil, err + } + + if cr.relative { + return routeURL, nil + } + + if routeURL.Scheme == "" && routeURL.User == nil && routeURL.Host == "" { + routeURL.Path = routeURL.Path[1:] + } + + url := cr.root.ResolveReference(routeURL) + url.Scheme = cr.root.Scheme + return url, nil +} + +// appendValuesURL appends the parameters to the url. +func appendValuesURL(u *url.URL, values ...url.Values) *url.URL { + merged := u.Query() + + for _, v := range values { + for k, vv := range v { + merged[k] = append(merged[k], vv...) + } + } + + u.RawQuery = merged.Encode() + return u +} diff --git a/vendor/github.com/docker/distribution/registry/auth/auth.go b/vendor/github.com/docker/distribution/registry/auth/auth.go new file mode 100644 index 000000000..9cb036f1f --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/auth/auth.go @@ -0,0 +1,200 @@ +// Package auth defines a standard interface for request access controllers. +// +// An access controller has a simple interface with a single `Authorized` +// method which checks that a given request is authorized to perform one or +// more actions on one or more resources. This method should return a non-nil +// error if the request is not authorized. +// +// An implementation registers its access controller by name with a constructor +// which accepts an options map for configuring the access controller. +// +// options := map[string]interface{}{"sillySecret": "whysosilly?"} +// accessController, _ := auth.GetAccessController("silly", options) +// +// This `accessController` can then be used in a request handler like so: +// +// func updateOrder(w http.ResponseWriter, r *http.Request) { +// orderNumber := r.FormValue("orderNumber") +// resource := auth.Resource{Type: "customerOrder", Name: orderNumber} +// access := auth.Access{Resource: resource, Action: "update"} +// +// if ctx, err := accessController.Authorized(ctx, access); err != nil { +// if challenge, ok := err.(auth.Challenge) { +// // Let the challenge write the response. +// challenge.SetHeaders(r, w) +// w.WriteHeader(http.StatusUnauthorized) +// return +// } else { +// // Some other error. +// } +// } +// } +package auth + +import ( + "context" + "errors" + "fmt" + "net/http" +) + +const ( + // UserKey is used to get the user object from + // a user context + UserKey = "auth.user" + + // UserNameKey is used to get the user name from + // a user context + UserNameKey = "auth.user.name" +) + +var ( + // ErrInvalidCredential is returned when the auth token does not authenticate correctly. + ErrInvalidCredential = errors.New("invalid authorization credential") + + // ErrAuthenticationFailure returned when authentication fails. + ErrAuthenticationFailure = errors.New("authentication failure") +) + +// UserInfo carries information about +// an autenticated/authorized client. +type UserInfo struct { + Name string +} + +// Resource describes a resource by type and name. +type Resource struct { + Type string + Class string + Name string +} + +// Access describes a specific action that is +// requested or allowed for a given resource. +type Access struct { + Resource + Action string +} + +// Challenge is a special error type which is used for HTTP 401 Unauthorized +// responses and is able to write the response with WWW-Authenticate challenge +// header values based on the error. +type Challenge interface { + error + + // SetHeaders prepares the request to conduct a challenge response by + // adding the an HTTP challenge header on the response message. Callers + // are expected to set the appropriate HTTP status code (e.g. 401) + // themselves. + SetHeaders(r *http.Request, w http.ResponseWriter) +} + +// AccessController controls access to registry resources based on a request +// and required access levels for a request. Implementations can support both +// complete denial and http authorization challenges. +type AccessController interface { + // Authorized returns a non-nil error if the context is granted access and + // returns a new authorized context. If one or more Access structs are + // provided, the requested access will be compared with what is available + // to the context. The given context will contain a "http.request" key with + // a `*http.Request` value. If the error is non-nil, access should always + // be denied. The error may be of type Challenge, in which case the caller + // may have the Challenge handle the request or choose what action to take + // based on the Challenge header or response status. The returned context + // object should have a "auth.user" value set to a UserInfo struct. + Authorized(ctx context.Context, access ...Access) (context.Context, error) +} + +// CredentialAuthenticator is an object which is able to authenticate credentials +type CredentialAuthenticator interface { + AuthenticateUser(username, password string) error +} + +// WithUser returns a context with the authorized user info. +func WithUser(ctx context.Context, user UserInfo) context.Context { + return userInfoContext{ + Context: ctx, + user: user, + } +} + +type userInfoContext struct { + context.Context + user UserInfo +} + +func (uic userInfoContext) Value(key interface{}) interface{} { + switch key { + case UserKey: + return uic.user + case UserNameKey: + return uic.user.Name + } + + return uic.Context.Value(key) +} + +// WithResources returns a context with the authorized resources. +func WithResources(ctx context.Context, resources []Resource) context.Context { + return resourceContext{ + Context: ctx, + resources: resources, + } +} + +type resourceContext struct { + context.Context + resources []Resource +} + +type resourceKey struct{} + +func (rc resourceContext) Value(key interface{}) interface{} { + if key == (resourceKey{}) { + return rc.resources + } + + return rc.Context.Value(key) +} + +// AuthorizedResources returns the list of resources which have +// been authorized for this request. +func AuthorizedResources(ctx context.Context) []Resource { + if resources, ok := ctx.Value(resourceKey{}).([]Resource); ok { + return resources + } + + return nil +} + +// InitFunc is the type of an AccessController factory function and is used +// to register the constructor for different AccesController backends. +type InitFunc func(options map[string]interface{}) (AccessController, error) + +var accessControllers map[string]InitFunc + +func init() { + accessControllers = make(map[string]InitFunc) +} + +// Register is used to register an InitFunc for +// an AccessController backend with the given name. +func Register(name string, initFunc InitFunc) error { + if _, exists := accessControllers[name]; exists { + return fmt.Errorf("name already registered: %s", name) + } + + accessControllers[name] = initFunc + + return nil +} + +// GetAccessController constructs an AccessController +// with the given options using the named backend. +func GetAccessController(name string, options map[string]interface{}) (AccessController, error) { + if initFunc, exists := accessControllers[name]; exists { + return initFunc(options) + } + + return nil, fmt.Errorf("no access controller registered with name: %s", name) +} diff --git a/vendor/github.com/docker/distribution/registry/client/auth/api_version.go b/vendor/github.com/docker/distribution/registry/client/auth/api_version.go new file mode 100644 index 000000000..7d8f1d957 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/client/auth/api_version.go @@ -0,0 +1,58 @@ +package auth + +import ( + "net/http" + "strings" +) + +// APIVersion represents a version of an API including its +// type and version number. +type APIVersion struct { + // Type refers to the name of a specific API specification + // such as "registry" + Type string + + // Version is the version of the API specification implemented, + // This may omit the revision number and only include + // the major and minor version, such as "2.0" + Version string +} + +// String returns the string formatted API Version +func (v APIVersion) String() string { + return v.Type + "/" + v.Version +} + +// APIVersions gets the API versions out of an HTTP response using the provided +// version header as the key for the HTTP header. +func APIVersions(resp *http.Response, versionHeader string) []APIVersion { + versions := []APIVersion{} + if versionHeader != "" { + for _, supportedVersions := range resp.Header[http.CanonicalHeaderKey(versionHeader)] { + for _, version := range strings.Fields(supportedVersions) { + versions = append(versions, ParseAPIVersion(version)) + } + } + } + return versions +} + +// ParseAPIVersion parses an API version string into an APIVersion +// Format (Expected, not enforced): +// API version string = '/' +// API type = [a-z][a-z0-9]* +// API version = [0-9]+(\.[0-9]+)? +// TODO(dmcgowan): Enforce format, add error condition, remove unknown type +func ParseAPIVersion(versionStr string) APIVersion { + idx := strings.IndexRune(versionStr, '/') + if idx == -1 { + return APIVersion{ + Type: "unknown", + Version: versionStr, + } + } + return APIVersion{ + Type: strings.ToLower(versionStr[:idx]), + Version: versionStr[idx+1:], + } +} diff --git a/vendor/github.com/docker/distribution/registry/client/auth/challenge/addr.go b/vendor/github.com/docker/distribution/registry/client/auth/challenge/addr.go new file mode 100644 index 000000000..2c3ebe165 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/client/auth/challenge/addr.go @@ -0,0 +1,27 @@ +package challenge + +import ( + "net/url" + "strings" +) + +// FROM: https://golang.org/src/net/http/http.go +// Given a string of the form "host", "host:port", or "[ipv6::address]:port", +// return true if the string includes a port. +func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") } + +// FROM: http://golang.org/src/net/http/transport.go +var portMap = map[string]string{ + "http": "80", + "https": "443", +} + +// canonicalAddr returns url.Host but always with a ":port" suffix +// FROM: http://golang.org/src/net/http/transport.go +func canonicalAddr(url *url.URL) string { + addr := url.Host + if !hasPort(addr) { + return addr + ":" + portMap[url.Scheme] + } + return addr +} diff --git a/vendor/github.com/docker/distribution/registry/client/auth/challenge/authchallenge.go b/vendor/github.com/docker/distribution/registry/client/auth/challenge/authchallenge.go new file mode 100644 index 000000000..fe238210c --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/client/auth/challenge/authchallenge.go @@ -0,0 +1,237 @@ +package challenge + +import ( + "fmt" + "net/http" + "net/url" + "strings" + "sync" +) + +// Challenge carries information from a WWW-Authenticate response header. +// See RFC 2617. +type Challenge struct { + // Scheme is the auth-scheme according to RFC 2617 + Scheme string + + // Parameters are the auth-params according to RFC 2617 + Parameters map[string]string +} + +// Manager manages the challenges for endpoints. +// The challenges are pulled out of HTTP responses. Only +// responses which expect challenges should be added to +// the manager, since a non-unauthorized request will be +// viewed as not requiring challenges. +type Manager interface { + // GetChallenges returns the challenges for the given + // endpoint URL. + GetChallenges(endpoint url.URL) ([]Challenge, error) + + // AddResponse adds the response to the challenge + // manager. The challenges will be parsed out of + // the WWW-Authenicate headers and added to the + // URL which was produced the response. If the + // response was authorized, any challenges for the + // endpoint will be cleared. + AddResponse(resp *http.Response) error +} + +// NewSimpleManager returns an instance of +// Manger which only maps endpoints to challenges +// based on the responses which have been added the +// manager. The simple manager will make no attempt to +// perform requests on the endpoints or cache the responses +// to a backend. +func NewSimpleManager() Manager { + return &simpleManager{ + Challenges: make(map[string][]Challenge), + } +} + +type simpleManager struct { + sync.RWMutex + Challenges map[string][]Challenge +} + +func normalizeURL(endpoint *url.URL) { + endpoint.Host = strings.ToLower(endpoint.Host) + endpoint.Host = canonicalAddr(endpoint) +} + +func (m *simpleManager) GetChallenges(endpoint url.URL) ([]Challenge, error) { + normalizeURL(&endpoint) + + m.RLock() + defer m.RUnlock() + challenges := m.Challenges[endpoint.String()] + return challenges, nil +} + +func (m *simpleManager) AddResponse(resp *http.Response) error { + challenges := ResponseChallenges(resp) + if resp.Request == nil { + return fmt.Errorf("missing request reference") + } + urlCopy := url.URL{ + Path: resp.Request.URL.Path, + Host: resp.Request.URL.Host, + Scheme: resp.Request.URL.Scheme, + } + normalizeURL(&urlCopy) + + m.Lock() + defer m.Unlock() + m.Challenges[urlCopy.String()] = challenges + return nil +} + +// Octet types from RFC 2616. +type octetType byte + +var octetTypes [256]octetType + +const ( + isToken octetType = 1 << iota + isSpace +) + +func init() { + // OCTET = + // CHAR = + // CTL = + // CR = + // LF = + // SP = + // HT = + // <"> = + // CRLF = CR LF + // LWS = [CRLF] 1*( SP | HT ) + // TEXT = + // separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <"> + // | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT + // token = 1* + // qdtext = > + + for c := 0; c < 256; c++ { + var t octetType + isCtl := c <= 31 || c == 127 + isChar := 0 <= c && c <= 127 + isSeparator := strings.ContainsRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) + if strings.ContainsRune(" \t\r\n", rune(c)) { + t |= isSpace + } + if isChar && !isCtl && !isSeparator { + t |= isToken + } + octetTypes[c] = t + } +} + +// ResponseChallenges returns a list of authorization challenges +// for the given http Response. Challenges are only checked if +// the response status code was a 401. +func ResponseChallenges(resp *http.Response) []Challenge { + if resp.StatusCode == http.StatusUnauthorized { + // Parse the WWW-Authenticate Header and store the challenges + // on this endpoint object. + return parseAuthHeader(resp.Header) + } + + return nil +} + +func parseAuthHeader(header http.Header) []Challenge { + challenges := []Challenge{} + for _, h := range header[http.CanonicalHeaderKey("WWW-Authenticate")] { + v, p := parseValueAndParams(h) + if v != "" { + challenges = append(challenges, Challenge{Scheme: v, Parameters: p}) + } + } + return challenges +} + +func parseValueAndParams(header string) (value string, params map[string]string) { + params = make(map[string]string) + value, s := expectToken(header) + if value == "" { + return + } + value = strings.ToLower(value) + s = "," + skipSpace(s) + for strings.HasPrefix(s, ",") { + var pkey string + pkey, s = expectToken(skipSpace(s[1:])) + if pkey == "" { + return + } + if !strings.HasPrefix(s, "=") { + return + } + var pvalue string + pvalue, s = expectTokenOrQuoted(s[1:]) + if pvalue == "" { + return + } + pkey = strings.ToLower(pkey) + params[pkey] = pvalue + s = skipSpace(s) + } + return +} + +func skipSpace(s string) (rest string) { + i := 0 + for ; i < len(s); i++ { + if octetTypes[s[i]]&isSpace == 0 { + break + } + } + return s[i:] +} + +func expectToken(s string) (token, rest string) { + i := 0 + for ; i < len(s); i++ { + if octetTypes[s[i]]&isToken == 0 { + break + } + } + return s[:i], s[i:] +} + +func expectTokenOrQuoted(s string) (value string, rest string) { + if !strings.HasPrefix(s, "\"") { + return expectToken(s) + } + s = s[1:] + for i := 0; i < len(s); i++ { + switch s[i] { + case '"': + return s[:i], s[i+1:] + case '\\': + p := make([]byte, len(s)-1) + j := copy(p, s[:i]) + escape := true + for i = i + 1; i < len(s); i++ { + b := s[i] + switch { + case escape: + escape = false + p[j] = b + j++ + case b == '\\': + escape = true + case b == '"': + return string(p[:j]), s[i+1:] + default: + p[j] = b + j++ + } + } + return "", "" + } + } + return "", "" +} diff --git a/vendor/github.com/docker/distribution/registry/client/auth/session.go b/vendor/github.com/docker/distribution/registry/client/auth/session.go new file mode 100644 index 000000000..aad8a0e6f --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/client/auth/session.go @@ -0,0 +1,530 @@ +package auth + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "github.com/docker/distribution/registry/client" + "github.com/docker/distribution/registry/client/auth/challenge" + "github.com/docker/distribution/registry/client/transport" +) + +var ( + // ErrNoBasicAuthCredentials is returned if a request can't be authorized with + // basic auth due to lack of credentials. + ErrNoBasicAuthCredentials = errors.New("no basic auth credentials") + + // ErrNoToken is returned if a request is successful but the body does not + // contain an authorization token. + ErrNoToken = errors.New("authorization server did not include a token in the response") +) + +const defaultClientID = "registry-client" + +// AuthenticationHandler is an interface for authorizing a request from +// params from a "WWW-Authenicate" header for a single scheme. +type AuthenticationHandler interface { + // Scheme returns the scheme as expected from the "WWW-Authenicate" header. + Scheme() string + + // AuthorizeRequest adds the authorization header to a request (if needed) + // using the parameters from "WWW-Authenticate" method. The parameters + // values depend on the scheme. + AuthorizeRequest(req *http.Request, params map[string]string) error +} + +// CredentialStore is an interface for getting credentials for +// a given URL +type CredentialStore interface { + // Basic returns basic auth for the given URL + Basic(*url.URL) (string, string) + + // RefreshToken returns a refresh token for the + // given URL and service + RefreshToken(*url.URL, string) string + + // SetRefreshToken sets the refresh token if none + // is provided for the given url and service + SetRefreshToken(realm *url.URL, service, token string) +} + +// NewAuthorizer creates an authorizer which can handle multiple authentication +// schemes. The handlers are tried in order, the higher priority authentication +// methods should be first. The challengeMap holds a list of challenges for +// a given root API endpoint (for example "https://registry-1.docker.io/v2/"). +func NewAuthorizer(manager challenge.Manager, handlers ...AuthenticationHandler) transport.RequestModifier { + return &endpointAuthorizer{ + challenges: manager, + handlers: handlers, + } +} + +type endpointAuthorizer struct { + challenges challenge.Manager + handlers []AuthenticationHandler +} + +func (ea *endpointAuthorizer) ModifyRequest(req *http.Request) error { + pingPath := req.URL.Path + if v2Root := strings.Index(req.URL.Path, "/v2/"); v2Root != -1 { + pingPath = pingPath[:v2Root+4] + } else if v1Root := strings.Index(req.URL.Path, "/v1/"); v1Root != -1 { + pingPath = pingPath[:v1Root] + "/v2/" + } else { + return nil + } + + ping := url.URL{ + Host: req.URL.Host, + Scheme: req.URL.Scheme, + Path: pingPath, + } + + challenges, err := ea.challenges.GetChallenges(ping) + if err != nil { + return err + } + + if len(challenges) > 0 { + for _, handler := range ea.handlers { + for _, c := range challenges { + if c.Scheme != handler.Scheme() { + continue + } + if err := handler.AuthorizeRequest(req, c.Parameters); err != nil { + return err + } + } + } + } + + return nil +} + +// This is the minimum duration a token can last (in seconds). +// A token must not live less than 60 seconds because older versions +// of the Docker client didn't read their expiration from the token +// response and assumed 60 seconds. So to remain compatible with +// those implementations, a token must live at least this long. +const minimumTokenLifetimeSeconds = 60 + +// Private interface for time used by this package to enable tests to provide their own implementation. +type clock interface { + Now() time.Time +} + +type tokenHandler struct { + creds CredentialStore + transport http.RoundTripper + clock clock + + offlineAccess bool + forceOAuth bool + clientID string + scopes []Scope + + tokenLock sync.Mutex + tokenCache string + tokenExpiration time.Time + + logger Logger +} + +// Scope is a type which is serializable to a string +// using the allow scope grammar. +type Scope interface { + String() string +} + +// RepositoryScope represents a token scope for access +// to a repository. +type RepositoryScope struct { + Repository string + Class string + Actions []string +} + +// String returns the string representation of the repository +// using the scope grammar +func (rs RepositoryScope) String() string { + repoType := "repository" + // Keep existing format for image class to maintain backwards compatibility + // with authorization servers which do not support the expanded grammar. + if rs.Class != "" && rs.Class != "image" { + repoType = fmt.Sprintf("%s(%s)", repoType, rs.Class) + } + return fmt.Sprintf("%s:%s:%s", repoType, rs.Repository, strings.Join(rs.Actions, ",")) +} + +// RegistryScope represents a token scope for access +// to resources in the registry. +type RegistryScope struct { + Name string + Actions []string +} + +// String returns the string representation of the user +// using the scope grammar +func (rs RegistryScope) String() string { + return fmt.Sprintf("registry:%s:%s", rs.Name, strings.Join(rs.Actions, ",")) +} + +// Logger defines the injectable logging interface, used on TokenHandlers. +type Logger interface { + Debugf(format string, args ...interface{}) +} + +func logDebugf(logger Logger, format string, args ...interface{}) { + if logger == nil { + return + } + logger.Debugf(format, args...) +} + +// TokenHandlerOptions is used to configure a new token handler +type TokenHandlerOptions struct { + Transport http.RoundTripper + Credentials CredentialStore + + OfflineAccess bool + ForceOAuth bool + ClientID string + Scopes []Scope + Logger Logger +} + +// An implementation of clock for providing real time data. +type realClock struct{} + +// Now implements clock +func (realClock) Now() time.Time { return time.Now() } + +// NewTokenHandler creates a new AuthenicationHandler which supports +// fetching tokens from a remote token server. +func NewTokenHandler(transport http.RoundTripper, creds CredentialStore, scope string, actions ...string) AuthenticationHandler { + // Create options... + return NewTokenHandlerWithOptions(TokenHandlerOptions{ + Transport: transport, + Credentials: creds, + Scopes: []Scope{ + RepositoryScope{ + Repository: scope, + Actions: actions, + }, + }, + }) +} + +// NewTokenHandlerWithOptions creates a new token handler using the provided +// options structure. +func NewTokenHandlerWithOptions(options TokenHandlerOptions) AuthenticationHandler { + handler := &tokenHandler{ + transport: options.Transport, + creds: options.Credentials, + offlineAccess: options.OfflineAccess, + forceOAuth: options.ForceOAuth, + clientID: options.ClientID, + scopes: options.Scopes, + clock: realClock{}, + logger: options.Logger, + } + + return handler +} + +func (th *tokenHandler) client() *http.Client { + return &http.Client{ + Transport: th.transport, + Timeout: 15 * time.Second, + } +} + +func (th *tokenHandler) Scheme() string { + return "bearer" +} + +func (th *tokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error { + var additionalScopes []string + if fromParam := req.URL.Query().Get("from"); fromParam != "" { + additionalScopes = append(additionalScopes, RepositoryScope{ + Repository: fromParam, + Actions: []string{"pull"}, + }.String()) + } + + token, err := th.getToken(params, additionalScopes...) + if err != nil { + return err + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + + return nil +} + +func (th *tokenHandler) getToken(params map[string]string, additionalScopes ...string) (string, error) { + th.tokenLock.Lock() + defer th.tokenLock.Unlock() + scopes := make([]string, 0, len(th.scopes)+len(additionalScopes)) + for _, scope := range th.scopes { + scopes = append(scopes, scope.String()) + } + var addedScopes bool + for _, scope := range additionalScopes { + if hasScope(scopes, scope) { + continue + } + scopes = append(scopes, scope) + addedScopes = true + } + + now := th.clock.Now() + if now.After(th.tokenExpiration) || addedScopes { + token, expiration, err := th.fetchToken(params, scopes) + if err != nil { + return "", err + } + + // do not update cache for added scope tokens + if !addedScopes { + th.tokenCache = token + th.tokenExpiration = expiration + } + + return token, nil + } + + return th.tokenCache, nil +} + +func hasScope(scopes []string, scope string) bool { + for _, s := range scopes { + if s == scope { + return true + } + } + return false +} + +type postTokenResponse struct { + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + ExpiresIn int `json:"expires_in"` + IssuedAt time.Time `json:"issued_at"` + Scope string `json:"scope"` +} + +func (th *tokenHandler) fetchTokenWithOAuth(realm *url.URL, refreshToken, service string, scopes []string) (token string, expiration time.Time, err error) { + form := url.Values{} + form.Set("scope", strings.Join(scopes, " ")) + form.Set("service", service) + + clientID := th.clientID + if clientID == "" { + // Use default client, this is a required field + clientID = defaultClientID + } + form.Set("client_id", clientID) + + if refreshToken != "" { + form.Set("grant_type", "refresh_token") + form.Set("refresh_token", refreshToken) + } else if th.creds != nil { + form.Set("grant_type", "password") + username, password := th.creds.Basic(realm) + form.Set("username", username) + form.Set("password", password) + + // attempt to get a refresh token + form.Set("access_type", "offline") + } else { + // refuse to do oauth without a grant type + return "", time.Time{}, fmt.Errorf("no supported grant type") + } + + resp, err := th.client().PostForm(realm.String(), form) + if err != nil { + return "", time.Time{}, err + } + defer resp.Body.Close() + + if !client.SuccessStatus(resp.StatusCode) { + err := client.HandleErrorResponse(resp) + return "", time.Time{}, err + } + + decoder := json.NewDecoder(resp.Body) + + var tr postTokenResponse + if err = decoder.Decode(&tr); err != nil { + return "", time.Time{}, fmt.Errorf("unable to decode token response: %s", err) + } + + if tr.RefreshToken != "" && tr.RefreshToken != refreshToken { + th.creds.SetRefreshToken(realm, service, tr.RefreshToken) + } + + if tr.ExpiresIn < minimumTokenLifetimeSeconds { + // The default/minimum lifetime. + tr.ExpiresIn = minimumTokenLifetimeSeconds + logDebugf(th.logger, "Increasing token expiration to: %d seconds", tr.ExpiresIn) + } + + if tr.IssuedAt.IsZero() { + // issued_at is optional in the token response. + tr.IssuedAt = th.clock.Now().UTC() + } + + return tr.AccessToken, tr.IssuedAt.Add(time.Duration(tr.ExpiresIn) * time.Second), nil +} + +type getTokenResponse struct { + Token string `json:"token"` + AccessToken string `json:"access_token"` + ExpiresIn int `json:"expires_in"` + IssuedAt time.Time `json:"issued_at"` + RefreshToken string `json:"refresh_token"` +} + +func (th *tokenHandler) fetchTokenWithBasicAuth(realm *url.URL, service string, scopes []string) (token string, expiration time.Time, err error) { + + req, err := http.NewRequest("GET", realm.String(), nil) + if err != nil { + return "", time.Time{}, err + } + + reqParams := req.URL.Query() + + if service != "" { + reqParams.Add("service", service) + } + + for _, scope := range scopes { + reqParams.Add("scope", scope) + } + + if th.offlineAccess { + reqParams.Add("offline_token", "true") + clientID := th.clientID + if clientID == "" { + clientID = defaultClientID + } + reqParams.Add("client_id", clientID) + } + + if th.creds != nil { + username, password := th.creds.Basic(realm) + if username != "" && password != "" { + reqParams.Add("account", username) + req.SetBasicAuth(username, password) + } + } + + req.URL.RawQuery = reqParams.Encode() + + resp, err := th.client().Do(req) + if err != nil { + return "", time.Time{}, err + } + defer resp.Body.Close() + + if !client.SuccessStatus(resp.StatusCode) { + err := client.HandleErrorResponse(resp) + return "", time.Time{}, err + } + + decoder := json.NewDecoder(resp.Body) + + var tr getTokenResponse + if err = decoder.Decode(&tr); err != nil { + return "", time.Time{}, fmt.Errorf("unable to decode token response: %s", err) + } + + if tr.RefreshToken != "" && th.creds != nil { + th.creds.SetRefreshToken(realm, service, tr.RefreshToken) + } + + // `access_token` is equivalent to `token` and if both are specified + // the choice is undefined. Canonicalize `access_token` by sticking + // things in `token`. + if tr.AccessToken != "" { + tr.Token = tr.AccessToken + } + + if tr.Token == "" { + return "", time.Time{}, ErrNoToken + } + + if tr.ExpiresIn < minimumTokenLifetimeSeconds { + // The default/minimum lifetime. + tr.ExpiresIn = minimumTokenLifetimeSeconds + logDebugf(th.logger, "Increasing token expiration to: %d seconds", tr.ExpiresIn) + } + + if tr.IssuedAt.IsZero() { + // issued_at is optional in the token response. + tr.IssuedAt = th.clock.Now().UTC() + } + + return tr.Token, tr.IssuedAt.Add(time.Duration(tr.ExpiresIn) * time.Second), nil +} + +func (th *tokenHandler) fetchToken(params map[string]string, scopes []string) (token string, expiration time.Time, err error) { + realm, ok := params["realm"] + if !ok { + return "", time.Time{}, errors.New("no realm specified for token auth challenge") + } + + // TODO(dmcgowan): Handle empty scheme and relative realm + realmURL, err := url.Parse(realm) + if err != nil { + return "", time.Time{}, fmt.Errorf("invalid token auth challenge realm: %s", err) + } + + service := params["service"] + + var refreshToken string + + if th.creds != nil { + refreshToken = th.creds.RefreshToken(realmURL, service) + } + + if refreshToken != "" || th.forceOAuth { + return th.fetchTokenWithOAuth(realmURL, refreshToken, service, scopes) + } + + return th.fetchTokenWithBasicAuth(realmURL, service, scopes) +} + +type basicHandler struct { + creds CredentialStore +} + +// NewBasicHandler creaters a new authentiation handler which adds +// basic authentication credentials to a request. +func NewBasicHandler(creds CredentialStore) AuthenticationHandler { + return &basicHandler{ + creds: creds, + } +} + +func (*basicHandler) Scheme() string { + return "basic" +} + +func (bh *basicHandler) AuthorizeRequest(req *http.Request, params map[string]string) error { + if bh.creds != nil { + username, password := bh.creds.Basic(req.URL) + if username != "" && password != "" { + req.SetBasicAuth(username, password) + return nil + } + } + return ErrNoBasicAuthCredentials +} diff --git a/vendor/github.com/docker/distribution/registry/client/blob_writer.go b/vendor/github.com/docker/distribution/registry/client/blob_writer.go new file mode 100644 index 000000000..dac030c73 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/client/blob_writer.go @@ -0,0 +1,164 @@ +package client + +import ( + "bytes" + "context" + "fmt" + "io" + "io/ioutil" + "net/http" + "time" + + "github.com/docker/distribution" +) + +type httpBlobUpload struct { + statter distribution.BlobStatter + client *http.Client + + uuid string + startedAt time.Time + + location string // always the last value of the location header. + offset int64 + closed bool +} + +func (hbu *httpBlobUpload) Reader() (io.ReadCloser, error) { + panic("Not implemented") +} + +func (hbu *httpBlobUpload) handleErrorResponse(resp *http.Response) error { + if resp.StatusCode == http.StatusNotFound { + return distribution.ErrBlobUploadUnknown + } + return HandleErrorResponse(resp) +} + +func (hbu *httpBlobUpload) ReadFrom(r io.Reader) (n int64, err error) { + req, err := http.NewRequest("PATCH", hbu.location, ioutil.NopCloser(r)) + if err != nil { + return 0, err + } + defer req.Body.Close() + + req.Header.Set("Content-Type", "application/octet-stream") + + resp, err := hbu.client.Do(req) + if err != nil { + return 0, err + } + + if !SuccessStatus(resp.StatusCode) { + return 0, hbu.handleErrorResponse(resp) + } + + hbu.uuid = resp.Header.Get("Docker-Upload-UUID") + hbu.location, err = sanitizeLocation(resp.Header.Get("Location"), hbu.location) + if err != nil { + return 0, err + } + rng := resp.Header.Get("Range") + var start, end int64 + if n, err := fmt.Sscanf(rng, "%d-%d", &start, &end); err != nil { + return 0, err + } else if n != 2 || end < start { + return 0, fmt.Errorf("bad range format: %s", rng) + } + + return (end - start + 1), nil + +} + +func (hbu *httpBlobUpload) Write(p []byte) (n int, err error) { + req, err := http.NewRequest("PATCH", hbu.location, bytes.NewReader(p)) + if err != nil { + return 0, err + } + req.Header.Set("Content-Range", fmt.Sprintf("%d-%d", hbu.offset, hbu.offset+int64(len(p)-1))) + req.Header.Set("Content-Length", fmt.Sprintf("%d", len(p))) + req.Header.Set("Content-Type", "application/octet-stream") + + resp, err := hbu.client.Do(req) + if err != nil { + return 0, err + } + + if !SuccessStatus(resp.StatusCode) { + return 0, hbu.handleErrorResponse(resp) + } + + hbu.uuid = resp.Header.Get("Docker-Upload-UUID") + hbu.location, err = sanitizeLocation(resp.Header.Get("Location"), hbu.location) + if err != nil { + return 0, err + } + rng := resp.Header.Get("Range") + var start, end int + if n, err := fmt.Sscanf(rng, "%d-%d", &start, &end); err != nil { + return 0, err + } else if n != 2 || end < start { + return 0, fmt.Errorf("bad range format: %s", rng) + } + + return (end - start + 1), nil + +} + +func (hbu *httpBlobUpload) Size() int64 { + return hbu.offset +} + +func (hbu *httpBlobUpload) ID() string { + return hbu.uuid +} + +func (hbu *httpBlobUpload) StartedAt() time.Time { + return hbu.startedAt +} + +func (hbu *httpBlobUpload) Commit(ctx context.Context, desc distribution.Descriptor) (distribution.Descriptor, error) { + // TODO(dmcgowan): Check if already finished, if so just fetch + req, err := http.NewRequest("PUT", hbu.location, nil) + if err != nil { + return distribution.Descriptor{}, err + } + + values := req.URL.Query() + values.Set("digest", desc.Digest.String()) + req.URL.RawQuery = values.Encode() + + resp, err := hbu.client.Do(req) + if err != nil { + return distribution.Descriptor{}, err + } + defer resp.Body.Close() + + if !SuccessStatus(resp.StatusCode) { + return distribution.Descriptor{}, hbu.handleErrorResponse(resp) + } + + return hbu.statter.Stat(ctx, desc.Digest) +} + +func (hbu *httpBlobUpload) Cancel(ctx context.Context) error { + req, err := http.NewRequest("DELETE", hbu.location, nil) + if err != nil { + return err + } + resp, err := hbu.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound || SuccessStatus(resp.StatusCode) { + return nil + } + return hbu.handleErrorResponse(resp) +} + +func (hbu *httpBlobUpload) Close() error { + hbu.closed = true + return nil +} diff --git a/vendor/github.com/docker/distribution/registry/client/errors.go b/vendor/github.com/docker/distribution/registry/client/errors.go new file mode 100644 index 000000000..ce9902034 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/client/errors.go @@ -0,0 +1,160 @@ +package client + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "mime" + "net/http" + + "github.com/docker/distribution/registry/api/errcode" + "github.com/docker/distribution/registry/client/auth/challenge" +) + +// ErrNoErrorsInBody is returned when an HTTP response body parses to an empty +// errcode.Errors slice. +var ErrNoErrorsInBody = errors.New("no error details found in HTTP response body") + +// UnexpectedHTTPStatusError is returned when an unexpected HTTP status is +// returned when making a registry api call. +type UnexpectedHTTPStatusError struct { + Status string +} + +func (e *UnexpectedHTTPStatusError) Error() string { + return fmt.Sprintf("received unexpected HTTP status: %s", e.Status) +} + +// UnexpectedHTTPResponseError is returned when an expected HTTP status code +// is returned, but the content was unexpected and failed to be parsed. +type UnexpectedHTTPResponseError struct { + ParseErr error + StatusCode int + Response []byte +} + +func (e *UnexpectedHTTPResponseError) Error() string { + return fmt.Sprintf("error parsing HTTP %d response body: %s: %q", e.StatusCode, e.ParseErr.Error(), string(e.Response)) +} + +func parseHTTPErrorResponse(resp *http.Response) error { + var errors errcode.Errors + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + statusCode := resp.StatusCode + ctHeader := resp.Header.Get("Content-Type") + + if ctHeader == "" { + return makeError(statusCode, string(body)) + } + + contentType, _, err := mime.ParseMediaType(ctHeader) + if err != nil { + return fmt.Errorf("failed parsing content-type: %w", err) + } + + if contentType != "application/json" && contentType != "application/vnd.api+json" { + return makeError(statusCode, string(body)) + } + + // For backward compatibility, handle irregularly formatted + // messages that contain a "details" field. + var detailsErr struct { + Details string `json:"details"` + } + err = json.Unmarshal(body, &detailsErr) + if err == nil && detailsErr.Details != "" { + return makeError(statusCode, detailsErr.Details) + } + + if err := json.Unmarshal(body, &errors); err != nil { + return &UnexpectedHTTPResponseError{ + ParseErr: err, + StatusCode: statusCode, + Response: body, + } + } + + if len(errors) == 0 { + // If there was no error specified in the body, return + // UnexpectedHTTPResponseError. + return &UnexpectedHTTPResponseError{ + ParseErr: ErrNoErrorsInBody, + StatusCode: statusCode, + Response: body, + } + } + + return errors +} + +func makeError(statusCode int, details string) error { + switch statusCode { + case http.StatusUnauthorized: + return errcode.ErrorCodeUnauthorized.WithMessage(details) + case http.StatusForbidden: + return errcode.ErrorCodeDenied.WithMessage(details) + case http.StatusTooManyRequests: + return errcode.ErrorCodeTooManyRequests.WithMessage(details) + default: + return errcode.ErrorCodeUnknown.WithMessage(details) + } +} + +func makeErrorList(err error) []error { + if errL, ok := err.(errcode.Errors); ok { + return []error(errL) + } + return []error{err} +} + +func mergeErrors(err1, err2 error) error { + return errcode.Errors(append(makeErrorList(err1), makeErrorList(err2)...)) +} + +// HandleErrorResponse returns error parsed from HTTP response for an +// unsuccessful HTTP response code (in the range 400 - 499 inclusive). An +// UnexpectedHTTPStatusError returned for response code outside of expected +// range. +func HandleErrorResponse(resp *http.Response) error { + if resp.StatusCode >= 400 && resp.StatusCode < 500 { + // Check for OAuth errors within the `WWW-Authenticate` header first + // See https://tools.ietf.org/html/rfc6750#section-3 + for _, c := range challenge.ResponseChallenges(resp) { + if c.Scheme == "bearer" { + var err errcode.Error + // codes defined at https://tools.ietf.org/html/rfc6750#section-3.1 + switch c.Parameters["error"] { + case "invalid_token": + err.Code = errcode.ErrorCodeUnauthorized + case "insufficient_scope": + err.Code = errcode.ErrorCodeDenied + default: + continue + } + if description := c.Parameters["error_description"]; description != "" { + err.Message = description + } else { + err.Message = err.Code.Message() + } + return mergeErrors(err, parseHTTPErrorResponse(resp)) + } + } + err := parseHTTPErrorResponse(resp) + if uErr, ok := err.(*UnexpectedHTTPResponseError); ok && resp.StatusCode == 401 { + return errcode.ErrorCodeUnauthorized.WithDetail(uErr.Response) + } + return err + } + return &UnexpectedHTTPStatusError{Status: resp.Status} +} + +// SuccessStatus returns true if the argument is a successful HTTP response +// code (in the range 200 - 399 inclusive). +func SuccessStatus(status int) bool { + return status >= 200 && status <= 399 +} diff --git a/vendor/github.com/docker/distribution/registry/client/repository.go b/vendor/github.com/docker/distribution/registry/client/repository.go new file mode 100644 index 000000000..fd42a1e66 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/client/repository.go @@ -0,0 +1,870 @@ +package client + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/distribution/reference" + "github.com/docker/distribution" + v2 "github.com/docker/distribution/registry/api/v2" + "github.com/docker/distribution/registry/client/transport" + "github.com/docker/distribution/registry/storage/cache" + "github.com/docker/distribution/registry/storage/cache/memory" + "github.com/opencontainers/go-digest" +) + +// Registry provides an interface for calling Repositories, which returns a catalog of repositories. +type Registry interface { + Repositories(ctx context.Context, repos []string, last string) (n int, err error) +} + +// checkHTTPRedirect is a callback that can manipulate redirected HTTP +// requests. It is used to preserve Accept and Range headers. +func checkHTTPRedirect(req *http.Request, via []*http.Request) error { + if len(via) >= 10 { + return errors.New("stopped after 10 redirects") + } + + if len(via) > 0 { + for headerName, headerVals := range via[0].Header { + if headerName != "Accept" && headerName != "Range" { + continue + } + for _, val := range headerVals { + // Don't add to redirected request if redirected + // request already has a header with the same + // name and value. + hasValue := false + for _, existingVal := range req.Header[headerName] { + if existingVal == val { + hasValue = true + break + } + } + if !hasValue { + req.Header.Add(headerName, val) + } + } + } + } + + return nil +} + +// NewRegistry creates a registry namespace which can be used to get a listing of repositories +func NewRegistry(baseURL string, transport http.RoundTripper) (Registry, error) { + ub, err := v2.NewURLBuilderFromString(baseURL, false) + if err != nil { + return nil, err + } + + client := &http.Client{ + Transport: transport, + Timeout: 1 * time.Minute, + CheckRedirect: checkHTTPRedirect, + } + + return ®istry{ + client: client, + ub: ub, + }, nil +} + +type registry struct { + client *http.Client + ub *v2.URLBuilder +} + +// Repositories returns a lexigraphically sorted catalog given a base URL. The 'entries' slice will be filled up to the size +// of the slice, starting at the value provided in 'last'. The number of entries will be returned along with io.EOF if there +// are no more entries +func (r *registry) Repositories(ctx context.Context, entries []string, last string) (int, error) { + var numFilled int + var returnErr error + + values := buildCatalogValues(len(entries), last) + u, err := r.ub.BuildCatalogURL(values) + if err != nil { + return 0, err + } + + resp, err := r.client.Get(u) + if err != nil { + return 0, err + } + defer resp.Body.Close() + + if SuccessStatus(resp.StatusCode) { + var ctlg struct { + Repositories []string `json:"repositories"` + } + decoder := json.NewDecoder(resp.Body) + + if err := decoder.Decode(&ctlg); err != nil { + return 0, err + } + + copy(entries, ctlg.Repositories) + numFilled = len(ctlg.Repositories) + + link := resp.Header.Get("Link") + if link == "" { + returnErr = io.EOF + } + } else { + return 0, HandleErrorResponse(resp) + } + + return numFilled, returnErr +} + +// NewRepository creates a new Repository for the given repository name and base URL. +func NewRepository(name reference.Named, baseURL string, transport http.RoundTripper) (distribution.Repository, error) { + ub, err := v2.NewURLBuilderFromString(baseURL, false) + if err != nil { + return nil, err + } + + client := &http.Client{ + Transport: transport, + CheckRedirect: checkHTTPRedirect, + // TODO(dmcgowan): create cookie jar + } + + return &repository{ + client: client, + ub: ub, + name: name, + }, nil +} + +type repository struct { + client *http.Client + ub *v2.URLBuilder + name reference.Named +} + +func (r *repository) Named() reference.Named { + return r.name +} + +func (r *repository) Blobs(ctx context.Context) distribution.BlobStore { + statter := &blobStatter{ + name: r.name, + ub: r.ub, + client: r.client, + } + return &blobs{ + name: r.name, + ub: r.ub, + client: r.client, + statter: cache.NewCachedBlobStatter(memory.NewInMemoryBlobDescriptorCacheProvider(), statter), + } +} + +func (r *repository) Manifests(ctx context.Context, options ...distribution.ManifestServiceOption) (distribution.ManifestService, error) { + // todo(richardscothern): options should be sent over the wire + return &manifests{ + name: r.name, + ub: r.ub, + client: r.client, + etags: make(map[string]string), + }, nil +} + +func (r *repository) Tags(ctx context.Context) distribution.TagService { + return &tags{ + client: r.client, + ub: r.ub, + name: r.Named(), + } +} + +// tags implements remote tagging operations. +type tags struct { + client *http.Client + ub *v2.URLBuilder + name reference.Named +} + +// All returns all tags +func (t *tags) All(ctx context.Context) ([]string, error) { + var tags []string + + listURLStr, err := t.ub.BuildTagsURL(t.name) + if err != nil { + return tags, err + } + + listURL, err := url.Parse(listURLStr) + if err != nil { + return tags, err + } + + for { + resp, err := t.client.Get(listURL.String()) + if err != nil { + return tags, err + } + defer resp.Body.Close() + + if SuccessStatus(resp.StatusCode) { + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return tags, err + } + + tagsResponse := struct { + Tags []string `json:"tags"` + }{} + if err := json.Unmarshal(b, &tagsResponse); err != nil { + return tags, err + } + tags = append(tags, tagsResponse.Tags...) + if link := resp.Header.Get("Link"); link != "" { + linkURLStr := strings.Trim(strings.Split(link, ";")[0], "<>") + linkURL, err := url.Parse(linkURLStr) + if err != nil { + return tags, err + } + + listURL = listURL.ResolveReference(linkURL) + } else { + return tags, nil + } + } else { + return tags, HandleErrorResponse(resp) + } + } +} + +func descriptorFromResponse(response *http.Response) (distribution.Descriptor, error) { + desc := distribution.Descriptor{} + headers := response.Header + + ctHeader := headers.Get("Content-Type") + if ctHeader == "" { + return distribution.Descriptor{}, errors.New("missing or empty Content-Type header") + } + desc.MediaType = ctHeader + + digestHeader := headers.Get("Docker-Content-Digest") + if digestHeader == "" { + bytes, err := ioutil.ReadAll(response.Body) + if err != nil { + return distribution.Descriptor{}, err + } + _, desc, err := distribution.UnmarshalManifest(ctHeader, bytes) + if err != nil { + return distribution.Descriptor{}, err + } + return desc, nil + } + + dgst, err := digest.Parse(digestHeader) + if err != nil { + return distribution.Descriptor{}, err + } + desc.Digest = dgst + + lengthHeader := headers.Get("Content-Length") + if lengthHeader == "" { + return distribution.Descriptor{}, errors.New("missing or empty Content-Length header") + } + length, err := strconv.ParseInt(lengthHeader, 10, 64) + if err != nil { + return distribution.Descriptor{}, err + } + desc.Size = length + + return desc, nil + +} + +// Get issues a HEAD request for a Manifest against its named endpoint in order +// to construct a descriptor for the tag. If the registry doesn't support HEADing +// a manifest, fallback to GET. +func (t *tags) Get(ctx context.Context, tag string) (distribution.Descriptor, error) { + ref, err := reference.WithTag(t.name, tag) + if err != nil { + return distribution.Descriptor{}, err + } + u, err := t.ub.BuildManifestURL(ref) + if err != nil { + return distribution.Descriptor{}, err + } + + newRequest := func(method string) (*http.Response, error) { + req, err := http.NewRequest(method, u, nil) + if err != nil { + return nil, err + } + + for _, t := range distribution.ManifestMediaTypes() { + req.Header.Add("Accept", t) + } + resp, err := t.client.Do(req) + return resp, err + } + + resp, err := newRequest("HEAD") + if err != nil { + return distribution.Descriptor{}, err + } + defer resp.Body.Close() + + switch { + case resp.StatusCode >= 200 && resp.StatusCode < 400 && len(resp.Header.Get("Docker-Content-Digest")) > 0: + // if the response is a success AND a Docker-Content-Digest can be retrieved from the headers + return descriptorFromResponse(resp) + default: + // if the response is an error - there will be no body to decode. + // Issue a GET request: + // - for data from a server that does not handle HEAD + // - to get error details in case of a failure + resp, err = newRequest("GET") + if err != nil { + return distribution.Descriptor{}, err + } + defer resp.Body.Close() + + if resp.StatusCode >= 200 && resp.StatusCode < 400 { + return descriptorFromResponse(resp) + } + return distribution.Descriptor{}, HandleErrorResponse(resp) + } +} + +func (t *tags) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) { + panic("not implemented") +} + +func (t *tags) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error { + panic("not implemented") +} + +func (t *tags) Untag(ctx context.Context, tag string) error { + panic("not implemented") +} + +type manifests struct { + name reference.Named + ub *v2.URLBuilder + client *http.Client + etags map[string]string +} + +func (ms *manifests) Exists(ctx context.Context, dgst digest.Digest) (bool, error) { + ref, err := reference.WithDigest(ms.name, dgst) + if err != nil { + return false, err + } + u, err := ms.ub.BuildManifestURL(ref) + if err != nil { + return false, err + } + + resp, err := ms.client.Head(u) + if err != nil { + return false, err + } + + if SuccessStatus(resp.StatusCode) { + return true, nil + } else if resp.StatusCode == http.StatusNotFound { + return false, nil + } + return false, HandleErrorResponse(resp) +} + +// AddEtagToTag allows a client to supply an eTag to Get which will be +// used for a conditional HTTP request. If the eTag matches, a nil manifest +// and ErrManifestNotModified error will be returned. etag is automatically +// quoted when added to this map. +func AddEtagToTag(tag, etag string) distribution.ManifestServiceOption { + return etagOption{tag, etag} +} + +type etagOption struct{ tag, etag string } + +func (o etagOption) Apply(ms distribution.ManifestService) error { + if ms, ok := ms.(*manifests); ok { + ms.etags[o.tag] = fmt.Sprintf(`"%s"`, o.etag) + return nil + } + return fmt.Errorf("etag options is a client-only option") +} + +// ReturnContentDigest allows a client to set a the content digest on +// a successful request from the 'Docker-Content-Digest' header. This +// returned digest is represents the digest which the registry uses +// to refer to the content and can be used to delete the content. +func ReturnContentDigest(dgst *digest.Digest) distribution.ManifestServiceOption { + return contentDigestOption{dgst} +} + +type contentDigestOption struct{ digest *digest.Digest } + +func (o contentDigestOption) Apply(ms distribution.ManifestService) error { + return nil +} + +func (ms *manifests) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) { + var ( + digestOrTag string + ref reference.Named + err error + contentDgst *digest.Digest + mediaTypes []string + ) + + for _, option := range options { + switch opt := option.(type) { + case distribution.WithTagOption: + digestOrTag = opt.Tag + ref, err = reference.WithTag(ms.name, opt.Tag) + if err != nil { + return nil, err + } + case contentDigestOption: + contentDgst = opt.digest + case distribution.WithManifestMediaTypesOption: + mediaTypes = opt.MediaTypes + default: + err := option.Apply(ms) + if err != nil { + return nil, err + } + } + } + + if digestOrTag == "" { + digestOrTag = dgst.String() + ref, err = reference.WithDigest(ms.name, dgst) + if err != nil { + return nil, err + } + } + + if len(mediaTypes) == 0 { + mediaTypes = distribution.ManifestMediaTypes() + } + + u, err := ms.ub.BuildManifestURL(ref) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return nil, err + } + + for _, t := range mediaTypes { + req.Header.Add("Accept", t) + } + + if _, ok := ms.etags[digestOrTag]; ok { + req.Header.Set("If-None-Match", ms.etags[digestOrTag]) + } + + resp, err := ms.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotModified { + return nil, distribution.ErrManifestNotModified + } else if SuccessStatus(resp.StatusCode) { + if contentDgst != nil { + dgst, err := digest.Parse(resp.Header.Get("Docker-Content-Digest")) + if err == nil { + *contentDgst = dgst + } + } + mt := resp.Header.Get("Content-Type") + body, err := ioutil.ReadAll(resp.Body) + + if err != nil { + return nil, err + } + m, _, err := distribution.UnmarshalManifest(mt, body) + if err != nil { + return nil, err + } + return m, nil + } + return nil, HandleErrorResponse(resp) +} + +// Put puts a manifest. A tag can be specified using an options parameter which uses some shared state to hold the +// tag name in order to build the correct upload URL. +func (ms *manifests) Put(ctx context.Context, m distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) { + ref := ms.name + var tagged bool + + for _, option := range options { + if opt, ok := option.(distribution.WithTagOption); ok { + var err error + ref, err = reference.WithTag(ref, opt.Tag) + if err != nil { + return "", err + } + tagged = true + } else { + err := option.Apply(ms) + if err != nil { + return "", err + } + } + } + mediaType, p, err := m.Payload() + if err != nil { + return "", err + } + + if !tagged { + // generate a canonical digest and Put by digest + _, d, err := distribution.UnmarshalManifest(mediaType, p) + if err != nil { + return "", err + } + ref, err = reference.WithDigest(ref, d.Digest) + if err != nil { + return "", err + } + } + + manifestURL, err := ms.ub.BuildManifestURL(ref) + if err != nil { + return "", err + } + + putRequest, err := http.NewRequest("PUT", manifestURL, bytes.NewReader(p)) + if err != nil { + return "", err + } + + putRequest.Header.Set("Content-Type", mediaType) + + resp, err := ms.client.Do(putRequest) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if SuccessStatus(resp.StatusCode) { + dgstHeader := resp.Header.Get("Docker-Content-Digest") + dgst, err := digest.Parse(dgstHeader) + if err != nil { + return "", err + } + + return dgst, nil + } + + return "", HandleErrorResponse(resp) +} + +func (ms *manifests) Delete(ctx context.Context, dgst digest.Digest) error { + ref, err := reference.WithDigest(ms.name, dgst) + if err != nil { + return err + } + u, err := ms.ub.BuildManifestURL(ref) + if err != nil { + return err + } + req, err := http.NewRequest("DELETE", u, nil) + if err != nil { + return err + } + + resp, err := ms.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if SuccessStatus(resp.StatusCode) { + return nil + } + return HandleErrorResponse(resp) +} + +// todo(richardscothern): Restore interface and implementation with merge of #1050 +/*func (ms *manifests) Enumerate(ctx context.Context, manifests []distribution.Manifest, last distribution.Manifest) (n int, err error) { + panic("not supported") +}*/ + +type blobs struct { + name reference.Named + ub *v2.URLBuilder + client *http.Client + + statter distribution.BlobDescriptorService + distribution.BlobDeleter +} + +func sanitizeLocation(location, base string) (string, error) { + baseURL, err := url.Parse(base) + if err != nil { + return "", err + } + + locationURL, err := url.Parse(location) + if err != nil { + return "", err + } + + return baseURL.ResolveReference(locationURL).String(), nil +} + +func (bs *blobs) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { + return bs.statter.Stat(ctx, dgst) + +} + +func (bs *blobs) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) { + reader, err := bs.Open(ctx, dgst) + if err != nil { + return nil, err + } + defer reader.Close() + + return ioutil.ReadAll(reader) +} + +func (bs *blobs) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) { + ref, err := reference.WithDigest(bs.name, dgst) + if err != nil { + return nil, err + } + blobURL, err := bs.ub.BuildBlobURL(ref) + if err != nil { + return nil, err + } + + return transport.NewHTTPReadSeeker(bs.client, blobURL, + func(resp *http.Response) error { + if resp.StatusCode == http.StatusNotFound { + return distribution.ErrBlobUnknown + } + return HandleErrorResponse(resp) + }), nil +} + +func (bs *blobs) ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) error { + panic("not implemented") +} + +func (bs *blobs) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) { + writer, err := bs.Create(ctx) + if err != nil { + return distribution.Descriptor{}, err + } + dgstr := digest.Canonical.Digester() + n, err := io.Copy(writer, io.TeeReader(bytes.NewReader(p), dgstr.Hash())) + if err != nil { + return distribution.Descriptor{}, err + } + if n < int64(len(p)) { + return distribution.Descriptor{}, fmt.Errorf("short copy: wrote %d of %d", n, len(p)) + } + + desc := distribution.Descriptor{ + MediaType: mediaType, + Size: int64(len(p)), + Digest: dgstr.Digest(), + } + + return writer.Commit(ctx, desc) +} + +type optionFunc func(interface{}) error + +func (f optionFunc) Apply(v interface{}) error { + return f(v) +} + +// WithMountFrom returns a BlobCreateOption which designates that the blob should be +// mounted from the given canonical reference. +func WithMountFrom(ref reference.Canonical) distribution.BlobCreateOption { + return optionFunc(func(v interface{}) error { + opts, ok := v.(*distribution.CreateOptions) + if !ok { + return fmt.Errorf("unexpected options type: %T", v) + } + + opts.Mount.ShouldMount = true + opts.Mount.From = ref + + return nil + }) +} + +func (bs *blobs) Create(ctx context.Context, options ...distribution.BlobCreateOption) (distribution.BlobWriter, error) { + var opts distribution.CreateOptions + + for _, option := range options { + err := option.Apply(&opts) + if err != nil { + return nil, err + } + } + + var values []url.Values + + if opts.Mount.ShouldMount { + values = append(values, url.Values{"from": {opts.Mount.From.Name()}, "mount": {opts.Mount.From.Digest().String()}}) + } + + u, err := bs.ub.BuildBlobUploadURL(bs.name, values...) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", u, nil) + if err != nil { + return nil, err + } + + resp, err := bs.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusCreated: + desc, err := bs.statter.Stat(ctx, opts.Mount.From.Digest()) + if err != nil { + return nil, err + } + return nil, distribution.ErrBlobMounted{From: opts.Mount.From, Descriptor: desc} + case http.StatusAccepted: + // TODO(dmcgowan): Check for invalid UUID + uuid := resp.Header.Get("Docker-Upload-UUID") + location, err := sanitizeLocation(resp.Header.Get("Location"), u) + if err != nil { + return nil, err + } + + return &httpBlobUpload{ + statter: bs.statter, + client: bs.client, + uuid: uuid, + startedAt: time.Now(), + location: location, + }, nil + default: + return nil, HandleErrorResponse(resp) + } +} + +func (bs *blobs) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) { + panic("not implemented") +} + +func (bs *blobs) Delete(ctx context.Context, dgst digest.Digest) error { + return bs.statter.Clear(ctx, dgst) +} + +type blobStatter struct { + name reference.Named + ub *v2.URLBuilder + client *http.Client +} + +func (bs *blobStatter) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { + ref, err := reference.WithDigest(bs.name, dgst) + if err != nil { + return distribution.Descriptor{}, err + } + u, err := bs.ub.BuildBlobURL(ref) + if err != nil { + return distribution.Descriptor{}, err + } + + resp, err := bs.client.Head(u) + if err != nil { + return distribution.Descriptor{}, err + } + defer resp.Body.Close() + + if SuccessStatus(resp.StatusCode) { + lengthHeader := resp.Header.Get("Content-Length") + if lengthHeader == "" { + return distribution.Descriptor{}, fmt.Errorf("missing content-length header for request: %s", u) + } + + length, err := strconv.ParseInt(lengthHeader, 10, 64) + if err != nil { + return distribution.Descriptor{}, fmt.Errorf("error parsing content-length: %v", err) + } + + return distribution.Descriptor{ + MediaType: resp.Header.Get("Content-Type"), + Size: length, + Digest: dgst, + }, nil + } else if resp.StatusCode == http.StatusNotFound { + return distribution.Descriptor{}, distribution.ErrBlobUnknown + } + return distribution.Descriptor{}, HandleErrorResponse(resp) +} + +func buildCatalogValues(maxEntries int, last string) url.Values { + values := url.Values{} + + if maxEntries > 0 { + values.Add("n", strconv.Itoa(maxEntries)) + } + + if last != "" { + values.Add("last", last) + } + + return values +} + +func (bs *blobStatter) Clear(ctx context.Context, dgst digest.Digest) error { + ref, err := reference.WithDigest(bs.name, dgst) + if err != nil { + return err + } + blobURL, err := bs.ub.BuildBlobURL(ref) + if err != nil { + return err + } + + req, err := http.NewRequest("DELETE", blobURL, nil) + if err != nil { + return err + } + + resp, err := bs.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if SuccessStatus(resp.StatusCode) { + return nil + } + return HandleErrorResponse(resp) +} + +func (bs *blobStatter) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error { + return nil +} diff --git a/vendor/github.com/docker/distribution/registry/client/transport/http_reader.go b/vendor/github.com/docker/distribution/registry/client/transport/http_reader.go new file mode 100644 index 000000000..9120dbed6 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/client/transport/http_reader.go @@ -0,0 +1,249 @@ +package transport + +import ( + "errors" + "fmt" + "io" + "net/http" + "regexp" + "strconv" +) + +var ( + contentRangeRegexp = regexp.MustCompile(`bytes ([0-9]+)-([0-9]+)/([0-9]+|\\*)`) + + // ErrWrongCodeForByteRange is returned if the client sends a request + // with a Range header but the server returns a 2xx or 3xx code other + // than 206 Partial Content. + ErrWrongCodeForByteRange = errors.New("expected HTTP 206 from byte range request") +) + +// ReadSeekCloser combines io.ReadSeeker with io.Closer. +type ReadSeekCloser interface { + io.ReadSeeker + io.Closer +} + +// NewHTTPReadSeeker handles reading from an HTTP endpoint using a GET +// request. When seeking and starting a read from a non-zero offset +// the a "Range" header will be added which sets the offset. +// TODO(dmcgowan): Move this into a separate utility package +func NewHTTPReadSeeker(client *http.Client, url string, errorHandler func(*http.Response) error) ReadSeekCloser { + return &httpReadSeeker{ + client: client, + url: url, + errorHandler: errorHandler, + } +} + +type httpReadSeeker struct { + client *http.Client + url string + + // errorHandler creates an error from an unsuccessful HTTP response. + // This allows the error to be created with the HTTP response body + // without leaking the body through a returned error. + errorHandler func(*http.Response) error + + size int64 + + // rc is the remote read closer. + rc io.ReadCloser + // readerOffset tracks the offset as of the last read. + readerOffset int64 + // seekOffset allows Seek to override the offset. Seek changes + // seekOffset instead of changing readOffset directly so that + // connection resets can be delayed and possibly avoided if the + // seek is undone (i.e. seeking to the end and then back to the + // beginning). + seekOffset int64 + err error +} + +func (hrs *httpReadSeeker) Read(p []byte) (n int, err error) { + if hrs.err != nil { + return 0, hrs.err + } + + // If we sought to a different position, we need to reset the + // connection. This logic is here instead of Seek so that if + // a seek is undone before the next read, the connection doesn't + // need to be closed and reopened. A common example of this is + // seeking to the end to determine the length, and then seeking + // back to the original position. + if hrs.readerOffset != hrs.seekOffset { + hrs.reset() + } + + hrs.readerOffset = hrs.seekOffset + + rd, err := hrs.reader() + if err != nil { + return 0, err + } + + n, err = rd.Read(p) + hrs.seekOffset += int64(n) + hrs.readerOffset += int64(n) + + return n, err +} + +func (hrs *httpReadSeeker) Seek(offset int64, whence int) (int64, error) { + if hrs.err != nil { + return 0, hrs.err + } + + lastReaderOffset := hrs.readerOffset + + if whence == io.SeekStart && hrs.rc == nil { + // If no request has been made yet, and we are seeking to an + // absolute position, set the read offset as well to avoid an + // unnecessary request. + hrs.readerOffset = offset + } + + _, err := hrs.reader() + if err != nil { + hrs.readerOffset = lastReaderOffset + return 0, err + } + + newOffset := hrs.seekOffset + + switch whence { + case io.SeekCurrent: + newOffset += offset + case io.SeekEnd: + if hrs.size < 0 { + return 0, errors.New("content length not known") + } + newOffset = hrs.size + offset + case io.SeekStart: + newOffset = offset + } + + if newOffset < 0 { + err = errors.New("cannot seek to negative position") + } else { + hrs.seekOffset = newOffset + } + + return hrs.seekOffset, err +} + +func (hrs *httpReadSeeker) Close() error { + if hrs.err != nil { + return hrs.err + } + + // close and release reader chain + if hrs.rc != nil { + hrs.rc.Close() + } + + hrs.rc = nil + + hrs.err = errors.New("httpLayer: closed") + + return nil +} + +func (hrs *httpReadSeeker) reset() { + if hrs.err != nil { + return + } + if hrs.rc != nil { + hrs.rc.Close() + hrs.rc = nil + } +} + +func (hrs *httpReadSeeker) reader() (io.Reader, error) { + if hrs.err != nil { + return nil, hrs.err + } + + if hrs.rc != nil { + return hrs.rc, nil + } + + req, err := http.NewRequest("GET", hrs.url, nil) + if err != nil { + return nil, err + } + + if hrs.readerOffset > 0 { + // If we are at different offset, issue a range request from there. + req.Header.Add("Range", fmt.Sprintf("bytes=%d-", hrs.readerOffset)) + // TODO: get context in here + // context.GetLogger(hrs.context).Infof("Range: %s", req.Header.Get("Range")) + } + + resp, err := hrs.client.Do(req) + if err != nil { + return nil, err + } + + // Normally would use client.SuccessStatus, but that would be a cyclic + // import + if resp.StatusCode >= 200 && resp.StatusCode <= 399 { + if hrs.readerOffset > 0 { + if resp.StatusCode != http.StatusPartialContent { + return nil, ErrWrongCodeForByteRange + } + + contentRange := resp.Header.Get("Content-Range") + if contentRange == "" { + return nil, errors.New("no Content-Range header found in HTTP 206 response") + } + + submatches := contentRangeRegexp.FindStringSubmatch(contentRange) + if len(submatches) < 4 { + return nil, fmt.Errorf("could not parse Content-Range header: %s", contentRange) + } + + startByte, err := strconv.ParseUint(submatches[1], 10, 64) + if err != nil { + return nil, fmt.Errorf("could not parse start of range in Content-Range header: %s", contentRange) + } + + if startByte != uint64(hrs.readerOffset) { + return nil, fmt.Errorf("received Content-Range starting at offset %d instead of requested %d", startByte, hrs.readerOffset) + } + + endByte, err := strconv.ParseUint(submatches[2], 10, 64) + if err != nil { + return nil, fmt.Errorf("could not parse end of range in Content-Range header: %s", contentRange) + } + + if submatches[3] == "*" { + hrs.size = -1 + } else { + size, err := strconv.ParseUint(submatches[3], 10, 64) + if err != nil { + return nil, fmt.Errorf("could not parse total size in Content-Range header: %s", contentRange) + } + + if endByte+1 != size { + return nil, fmt.Errorf("range in Content-Range stops before the end of the content: %s", contentRange) + } + + hrs.size = int64(size) + } + } else if resp.StatusCode == http.StatusOK { + hrs.size = resp.ContentLength + } else { + hrs.size = -1 + } + hrs.rc = resp.Body + } else { + defer resp.Body.Close() + if hrs.errorHandler != nil { + return nil, hrs.errorHandler(resp) + } + return nil, fmt.Errorf("unexpected status resolving reader: %v", resp.Status) + } + + return hrs.rc, nil +} diff --git a/vendor/github.com/docker/distribution/registry/client/transport/transport.go b/vendor/github.com/docker/distribution/registry/client/transport/transport.go new file mode 100644 index 000000000..30e45fab0 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/client/transport/transport.go @@ -0,0 +1,147 @@ +package transport + +import ( + "io" + "net/http" + "sync" +) + +// RequestModifier represents an object which will do an inplace +// modification of an HTTP request. +type RequestModifier interface { + ModifyRequest(*http.Request) error +} + +type headerModifier http.Header + +// NewHeaderRequestModifier returns a new RequestModifier which will +// add the given headers to a request. +func NewHeaderRequestModifier(header http.Header) RequestModifier { + return headerModifier(header) +} + +func (h headerModifier) ModifyRequest(req *http.Request) error { + for k, s := range http.Header(h) { + req.Header[k] = append(req.Header[k], s...) + } + + return nil +} + +// NewTransport creates a new transport which will apply modifiers to +// the request on a RoundTrip call. +func NewTransport(base http.RoundTripper, modifiers ...RequestModifier) http.RoundTripper { + return &transport{ + Modifiers: modifiers, + Base: base, + } +} + +// transport is an http.RoundTripper that makes HTTP requests after +// copying and modifying the request +type transport struct { + Modifiers []RequestModifier + Base http.RoundTripper + + mu sync.Mutex // guards modReq + modReq map[*http.Request]*http.Request // original -> modified +} + +// RoundTrip authorizes and authenticates the request with an +// access token. If no token exists or token is expired, +// tries to refresh/fetch a new token. +func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) { + req2 := cloneRequest(req) + for _, modifier := range t.Modifiers { + if err := modifier.ModifyRequest(req2); err != nil { + return nil, err + } + } + + t.setModReq(req, req2) + res, err := t.base().RoundTrip(req2) + if err != nil { + t.setModReq(req, nil) + return nil, err + } + res.Body = &onEOFReader{ + rc: res.Body, + fn: func() { t.setModReq(req, nil) }, + } + return res, nil +} + +// CancelRequest cancels an in-flight request by closing its connection. +func (t *transport) CancelRequest(req *http.Request) { + type canceler interface { + CancelRequest(*http.Request) + } + if cr, ok := t.base().(canceler); ok { + t.mu.Lock() + modReq := t.modReq[req] + delete(t.modReq, req) + t.mu.Unlock() + cr.CancelRequest(modReq) + } +} + +func (t *transport) base() http.RoundTripper { + if t.Base != nil { + return t.Base + } + return http.DefaultTransport +} + +func (t *transport) setModReq(orig, mod *http.Request) { + t.mu.Lock() + defer t.mu.Unlock() + if t.modReq == nil { + t.modReq = make(map[*http.Request]*http.Request) + } + if mod == nil { + delete(t.modReq, orig) + } else { + t.modReq[orig] = mod + } +} + +// cloneRequest returns a clone of the provided *http.Request. +// The clone is a shallow copy of the struct and its Header map. +func cloneRequest(r *http.Request) *http.Request { + // shallow copy of the struct + r2 := new(http.Request) + *r2 = *r + // deep copy of the Header + r2.Header = make(http.Header, len(r.Header)) + for k, s := range r.Header { + r2.Header[k] = append([]string(nil), s...) + } + + return r2 +} + +type onEOFReader struct { + rc io.ReadCloser + fn func() +} + +func (r *onEOFReader) Read(p []byte) (n int, err error) { + n, err = r.rc.Read(p) + if err == io.EOF { + r.runFunc() + } + return +} + +func (r *onEOFReader) Close() error { + err := r.rc.Close() + r.runFunc() + return err +} + +func (r *onEOFReader) runFunc() { + if fn := r.fn; fn != nil { + fn() + r.fn = nil + } +} diff --git a/vendor/github.com/docker/distribution/registry/handlers/app.go b/vendor/github.com/docker/distribution/registry/handlers/app.go new file mode 100644 index 000000000..bf56cea22 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/handlers/app.go @@ -0,0 +1,1078 @@ +package handlers + +import ( + "context" + "crypto/rand" + "expvar" + "fmt" + "math" + "math/big" + "net" + "net/http" + "net/url" + "os" + "regexp" + "runtime" + "strings" + "time" + + "github.com/distribution/reference" + "github.com/docker/distribution" + "github.com/docker/distribution/configuration" + dcontext "github.com/docker/distribution/context" + "github.com/docker/distribution/health" + "github.com/docker/distribution/health/checks" + prometheus "github.com/docker/distribution/metrics" + "github.com/docker/distribution/notifications" + "github.com/docker/distribution/registry/api/errcode" + v2 "github.com/docker/distribution/registry/api/v2" + "github.com/docker/distribution/registry/auth" + registrymiddleware "github.com/docker/distribution/registry/middleware/registry" + repositorymiddleware "github.com/docker/distribution/registry/middleware/repository" + "github.com/docker/distribution/registry/proxy" + "github.com/docker/distribution/registry/storage" + memorycache "github.com/docker/distribution/registry/storage/cache/memory" + rediscache "github.com/docker/distribution/registry/storage/cache/redis" + storagedriver "github.com/docker/distribution/registry/storage/driver" + "github.com/docker/distribution/registry/storage/driver/factory" + storagemiddleware "github.com/docker/distribution/registry/storage/driver/middleware" + "github.com/docker/distribution/version" + "github.com/docker/go-metrics" + "github.com/docker/libtrust" + "github.com/garyburd/redigo/redis" + "github.com/gorilla/mux" + "github.com/sirupsen/logrus" +) + +// randomSecretSize is the number of random bytes to generate if no secret +// was specified. +const randomSecretSize = 32 + +// defaultCheckInterval is the default time in between health checks +const defaultCheckInterval = 10 * time.Second + +// App is a global registry application object. Shared resources can be placed +// on this object that will be accessible from all requests. Any writable +// fields should be protected. +type App struct { + context.Context + + Config *configuration.Configuration + + router *mux.Router // main application router, configured with dispatchers + driver storagedriver.StorageDriver // driver maintains the app global storage driver instance. + registry distribution.Namespace // registry is the primary registry backend for the app instance. + repoRemover distribution.RepositoryRemover // repoRemover provides ability to delete repos + accessController auth.AccessController // main access controller for application + + // httpHost is a parsed representation of the http.host parameter from + // the configuration. Only the Scheme and Host fields are used. + httpHost url.URL + + // events contains notification related configuration. + events struct { + sink notifications.Sink + source notifications.SourceRecord + } + + redis *redis.Pool + + // trustKey is a deprecated key used to sign manifests converted to + // schema1 for backward compatibility. It should not be used for any + // other purposes. + trustKey libtrust.PrivateKey + + // isCache is true if this registry is configured as a pull through cache + isCache bool + + // readOnly is true if the registry is in a read-only maintenance mode + readOnly bool +} + +// NewApp takes a configuration and returns a configured app, ready to serve +// requests. The app only implements ServeHTTP and can be wrapped in other +// handlers accordingly. +func NewApp(ctx context.Context, config *configuration.Configuration) *App { + app := &App{ + Config: config, + Context: ctx, + router: v2.RouterWithPrefix(config.HTTP.Prefix), + isCache: config.Proxy.RemoteURL != "", + } + + // Register the handler dispatchers. + app.register(v2.RouteNameBase, func(ctx *Context, r *http.Request) http.Handler { + return http.HandlerFunc(apiBase) + }) + app.register(v2.RouteNameManifest, manifestDispatcher) + app.register(v2.RouteNameCatalog, catalogDispatcher) + app.register(v2.RouteNameTags, tagsDispatcher) + app.register(v2.RouteNameBlob, blobDispatcher) + app.register(v2.RouteNameBlobUpload, blobUploadDispatcher) + app.register(v2.RouteNameBlobUploadChunk, blobUploadDispatcher) + + // override the storage driver's UA string for registry outbound HTTP requests + storageParams := config.Storage.Parameters() + if storageParams == nil { + storageParams = make(configuration.Parameters) + } + storageParams["useragent"] = fmt.Sprintf("docker-distribution/%s %s", version.Version, runtime.Version()) + + var err error + app.driver, err = factory.Create(config.Storage.Type(), storageParams) + if err != nil { + // TODO(stevvooe): Move the creation of a service into a protected + // method, where this is created lazily. Its status can be queried via + // a health check. + panic(err) + } + + purgeConfig := uploadPurgeDefaultConfig() + if mc, ok := config.Storage["maintenance"]; ok { + if v, ok := mc["uploadpurging"]; ok { + purgeConfig, ok = v.(map[interface{}]interface{}) + if !ok { + panic("uploadpurging config key must contain additional keys") + } + } + if v, ok := mc["readonly"]; ok { + readOnly, ok := v.(map[interface{}]interface{}) + if !ok { + panic("readonly config key must contain additional keys") + } + if readOnlyEnabled, ok := readOnly["enabled"]; ok { + app.readOnly, ok = readOnlyEnabled.(bool) + if !ok { + panic("readonly's enabled config key must have a boolean value") + } + } + } + } + + startUploadPurger(app, app.driver, dcontext.GetLogger(app), purgeConfig) + + app.driver, err = applyStorageMiddleware(app.driver, config.Middleware["storage"]) + if err != nil { + panic(err) + } + + app.configureSecret(config) + app.configureEvents(config) + app.configureRedis(config) + app.configureLogHook(config) + + options := registrymiddleware.GetRegistryOptions() + if config.Compatibility.Schema1.TrustKey != "" { + app.trustKey, err = libtrust.LoadKeyFile(config.Compatibility.Schema1.TrustKey) + if err != nil { + panic(fmt.Sprintf(`could not load schema1 "signingkey" parameter: %v`, err)) + } + } else { + // Generate an ephemeral key to be used for signing converted manifests + // for clients that don't support schema2. + app.trustKey, err = libtrust.GenerateECP256PrivateKey() + if err != nil { + panic(err) + } + } + + options = append(options, storage.Schema1SigningKey(app.trustKey)) + + if config.Compatibility.Schema1.Enabled { + options = append(options, storage.EnableSchema1) + } + + if config.HTTP.Host != "" { + u, err := url.Parse(config.HTTP.Host) + if err != nil { + panic(fmt.Sprintf(`could not parse http "host" parameter: %v`, err)) + } + app.httpHost = *u + } + + if app.isCache { + options = append(options, storage.DisableDigestResumption) + } + + // configure deletion + if d, ok := config.Storage["delete"]; ok { + e, ok := d["enabled"] + if ok { + if deleteEnabled, ok := e.(bool); ok && deleteEnabled { + options = append(options, storage.EnableDelete) + } + } + } + + // configure redirects + var redirectDisabled bool + if redirectConfig, ok := config.Storage["redirect"]; ok { + v := redirectConfig["disable"] + switch v := v.(type) { + case bool: + redirectDisabled = v + default: + panic(fmt.Sprintf("invalid type for redirect config: %#v", redirectConfig)) + } + } + if redirectDisabled { + dcontext.GetLogger(app).Infof("backend redirection disabled") + } else { + options = append(options, storage.EnableRedirect) + } + + if !config.Validation.Enabled { + config.Validation.Enabled = !config.Validation.Disabled + } + + // configure validation + if config.Validation.Enabled { + if len(config.Validation.Manifests.URLs.Allow) == 0 && len(config.Validation.Manifests.URLs.Deny) == 0 { + // If Allow and Deny are empty, allow nothing. + options = append(options, storage.ManifestURLsAllowRegexp(regexp.MustCompile("^$"))) + } else { + if len(config.Validation.Manifests.URLs.Allow) > 0 { + for i, s := range config.Validation.Manifests.URLs.Allow { + // Validate via compilation. + if _, err := regexp.Compile(s); err != nil { + panic(fmt.Sprintf("validation.manifests.urls.allow: %s", err)) + } + // Wrap with non-capturing group. + config.Validation.Manifests.URLs.Allow[i] = fmt.Sprintf("(?:%s)", s) + } + re := regexp.MustCompile(strings.Join(config.Validation.Manifests.URLs.Allow, "|")) + options = append(options, storage.ManifestURLsAllowRegexp(re)) + } + if len(config.Validation.Manifests.URLs.Deny) > 0 { + for i, s := range config.Validation.Manifests.URLs.Deny { + // Validate via compilation. + if _, err := regexp.Compile(s); err != nil { + panic(fmt.Sprintf("validation.manifests.urls.deny: %s", err)) + } + // Wrap with non-capturing group. + config.Validation.Manifests.URLs.Deny[i] = fmt.Sprintf("(?:%s)", s) + } + re := regexp.MustCompile(strings.Join(config.Validation.Manifests.URLs.Deny, "|")) + options = append(options, storage.ManifestURLsDenyRegexp(re)) + } + } + } + + // configure storage caches + if cc, ok := config.Storage["cache"]; ok { + v, ok := cc["blobdescriptor"] + if !ok { + // Backwards compatible: "layerinfo" == "blobdescriptor" + v = cc["layerinfo"] + } + + switch v { + case "redis": + if app.redis == nil { + panic("redis configuration required to use for layerinfo cache") + } + cacheProvider := rediscache.NewRedisBlobDescriptorCacheProvider(app.redis) + localOptions := append(options, storage.BlobDescriptorCacheProvider(cacheProvider)) + app.registry, err = storage.NewRegistry(app, app.driver, localOptions...) + if err != nil { + panic("could not create registry: " + err.Error()) + } + dcontext.GetLogger(app).Infof("using redis blob descriptor cache") + case "inmemory": + cacheProvider := memorycache.NewInMemoryBlobDescriptorCacheProvider() + localOptions := append(options, storage.BlobDescriptorCacheProvider(cacheProvider)) + app.registry, err = storage.NewRegistry(app, app.driver, localOptions...) + if err != nil { + panic("could not create registry: " + err.Error()) + } + dcontext.GetLogger(app).Infof("using inmemory blob descriptor cache") + default: + if v != "" { + dcontext.GetLogger(app).Warnf("unknown cache type %q, caching disabled", config.Storage["cache"]) + } + } + } + + if app.registry == nil { + // configure the registry if no cache section is available. + app.registry, err = storage.NewRegistry(app.Context, app.driver, options...) + if err != nil { + panic("could not create registry: " + err.Error()) + } + } + + app.registry, err = applyRegistryMiddleware(app, app.registry, config.Middleware["registry"]) + if err != nil { + panic(err) + } + + authType := config.Auth.Type() + + if authType != "" && !strings.EqualFold(authType, "none") { + accessController, err := auth.GetAccessController(config.Auth.Type(), config.Auth.Parameters()) + if err != nil { + panic(fmt.Sprintf("unable to configure authorization (%s): %v", authType, err)) + } + app.accessController = accessController + dcontext.GetLogger(app).Debugf("configured %q access controller", authType) + } + + // configure as a pull through cache + if config.Proxy.RemoteURL != "" { + app.registry, err = proxy.NewRegistryPullThroughCache(ctx, app.registry, app.driver, config.Proxy) + if err != nil { + panic(err.Error()) + } + app.isCache = true + dcontext.GetLogger(app).Info("Registry configured as a proxy cache to ", config.Proxy.RemoteURL) + } + var ok bool + app.repoRemover, ok = app.registry.(distribution.RepositoryRemover) + if !ok { + dcontext.GetLogger(app).Warnf("Registry does not implement RempositoryRemover. Will not be able to delete repos and tags") + } + + return app +} + +// RegisterHealthChecks is an awful hack to defer health check registration +// control to callers. This should only ever be called once per registry +// process, typically in a main function. The correct way would be register +// health checks outside of app, since multiple apps may exist in the same +// process. Because the configuration and app are tightly coupled, +// implementing this properly will require a refactor. This method may panic +// if called twice in the same process. +func (app *App) RegisterHealthChecks(healthRegistries ...*health.Registry) { + if len(healthRegistries) > 1 { + panic("RegisterHealthChecks called with more than one registry") + } + healthRegistry := health.DefaultRegistry + if len(healthRegistries) == 1 { + healthRegistry = healthRegistries[0] + } + + if app.Config.Health.StorageDriver.Enabled { + interval := app.Config.Health.StorageDriver.Interval + if interval == 0 { + interval = defaultCheckInterval + } + + storageDriverCheck := func() error { + _, err := app.driver.Stat(app, "/") // "/" should always exist + if _, ok := err.(storagedriver.PathNotFoundError); ok { + err = nil // pass this through, backend is responding, but this path doesn't exist. + } + return err + } + + if app.Config.Health.StorageDriver.Threshold != 0 { + healthRegistry.RegisterPeriodicThresholdFunc("storagedriver_"+app.Config.Storage.Type(), interval, app.Config.Health.StorageDriver.Threshold, storageDriverCheck) + } else { + healthRegistry.RegisterPeriodicFunc("storagedriver_"+app.Config.Storage.Type(), interval, storageDriverCheck) + } + } + + for _, fileChecker := range app.Config.Health.FileCheckers { + interval := fileChecker.Interval + if interval == 0 { + interval = defaultCheckInterval + } + dcontext.GetLogger(app).Infof("configuring file health check path=%s, interval=%d", fileChecker.File, interval/time.Second) + healthRegistry.Register(fileChecker.File, health.PeriodicChecker(checks.FileChecker(fileChecker.File), interval)) + } + + for _, httpChecker := range app.Config.Health.HTTPCheckers { + interval := httpChecker.Interval + if interval == 0 { + interval = defaultCheckInterval + } + + statusCode := httpChecker.StatusCode + if statusCode == 0 { + statusCode = 200 + } + + checker := checks.HTTPChecker(httpChecker.URI, statusCode, httpChecker.Timeout, httpChecker.Headers) + + if httpChecker.Threshold != 0 { + dcontext.GetLogger(app).Infof("configuring HTTP health check uri=%s, interval=%d, threshold=%d", httpChecker.URI, interval/time.Second, httpChecker.Threshold) + healthRegistry.Register(httpChecker.URI, health.PeriodicThresholdChecker(checker, interval, httpChecker.Threshold)) + } else { + dcontext.GetLogger(app).Infof("configuring HTTP health check uri=%s, interval=%d", httpChecker.URI, interval/time.Second) + healthRegistry.Register(httpChecker.URI, health.PeriodicChecker(checker, interval)) + } + } + + for _, tcpChecker := range app.Config.Health.TCPCheckers { + interval := tcpChecker.Interval + if interval == 0 { + interval = defaultCheckInterval + } + + checker := checks.TCPChecker(tcpChecker.Addr, tcpChecker.Timeout) + + if tcpChecker.Threshold != 0 { + dcontext.GetLogger(app).Infof("configuring TCP health check addr=%s, interval=%d, threshold=%d", tcpChecker.Addr, interval/time.Second, tcpChecker.Threshold) + healthRegistry.Register(tcpChecker.Addr, health.PeriodicThresholdChecker(checker, interval, tcpChecker.Threshold)) + } else { + dcontext.GetLogger(app).Infof("configuring TCP health check addr=%s, interval=%d", tcpChecker.Addr, interval/time.Second) + healthRegistry.Register(tcpChecker.Addr, health.PeriodicChecker(checker, interval)) + } + } +} + +// register a handler with the application, by route name. The handler will be +// passed through the application filters and context will be constructed at +// request time. +func (app *App) register(routeName string, dispatch dispatchFunc) { + handler := app.dispatcher(dispatch) + + // Chain the handler with prometheus instrumented handler + if app.Config.HTTP.Debug.Prometheus.Enabled { + namespace := metrics.NewNamespace(prometheus.NamespacePrefix, "http", nil) + httpMetrics := namespace.NewDefaultHttpMetrics(strings.Replace(routeName, "-", "_", -1)) + metrics.Register(namespace) + handler = metrics.InstrumentHandler(httpMetrics, handler) + } + + // TODO(stevvooe): This odd dispatcher/route registration is by-product of + // some limitations in the gorilla/mux router. We are using it to keep + // routing consistent between the client and server, but we may want to + // replace it with manual routing and structure-based dispatch for better + // control over the request execution. + + app.router.GetRoute(routeName).Handler(handler) +} + +// configureEvents prepares the event sink for action. +func (app *App) configureEvents(configuration *configuration.Configuration) { + // Configure all of the endpoint sinks. + var sinks []notifications.Sink + for _, endpoint := range configuration.Notifications.Endpoints { + if endpoint.Disabled { + dcontext.GetLogger(app).Infof("endpoint %s disabled, skipping", endpoint.Name) + continue + } + + dcontext.GetLogger(app).Infof("configuring endpoint %v (%v), timeout=%s, headers=%v", endpoint.Name, endpoint.URL, endpoint.Timeout, endpoint.Headers) + endpoint := notifications.NewEndpoint(endpoint.Name, endpoint.URL, notifications.EndpointConfig{ + Timeout: endpoint.Timeout, + Threshold: endpoint.Threshold, + Backoff: endpoint.Backoff, + Headers: endpoint.Headers, + IgnoredMediaTypes: endpoint.IgnoredMediaTypes, + Ignore: endpoint.Ignore, + }) + + sinks = append(sinks, endpoint) + } + + // NOTE(stevvooe): Moving to a new queuing implementation is as easy as + // replacing broadcaster with a rabbitmq implementation. It's recommended + // that the registry instances also act as the workers to keep deployment + // simple. + app.events.sink = notifications.NewBroadcaster(sinks...) + + // Populate registry event source + hostname, err := os.Hostname() + if err != nil { + hostname = configuration.HTTP.Addr + } else { + // try to pick the port off the config + _, port, err := net.SplitHostPort(configuration.HTTP.Addr) + if err == nil { + hostname = net.JoinHostPort(hostname, port) + } + } + + app.events.source = notifications.SourceRecord{ + Addr: hostname, + InstanceID: dcontext.GetStringValue(app, "instance.id"), + } +} + +type redisStartAtKey struct{} + +func (app *App) configureRedis(configuration *configuration.Configuration) { + if configuration.Redis.Addr == "" { + dcontext.GetLogger(app).Infof("redis not configured") + return + } + + pool := &redis.Pool{ + Dial: func() (redis.Conn, error) { + // TODO(stevvooe): Yet another use case for contextual timing. + ctx := context.WithValue(app, redisStartAtKey{}, time.Now()) + + done := func(err error) { + logger := dcontext.GetLoggerWithField(ctx, "redis.connect.duration", + dcontext.Since(ctx, redisStartAtKey{})) + if err != nil { + logger.Errorf("redis: error connecting: %v", err) + } else { + logger.Infof("redis: connect %v", configuration.Redis.Addr) + } + } + + conn, err := redis.DialTimeout("tcp", + configuration.Redis.Addr, + configuration.Redis.DialTimeout, + configuration.Redis.ReadTimeout, + configuration.Redis.WriteTimeout) + if err != nil { + dcontext.GetLogger(app).Errorf("error connecting to redis instance %s: %v", + configuration.Redis.Addr, err) + done(err) + return nil, err + } + + // authorize the connection + if configuration.Redis.Password != "" { + if _, err = conn.Do("AUTH", configuration.Redis.Password); err != nil { + defer conn.Close() + done(err) + return nil, err + } + } + + // select the database to use + if configuration.Redis.DB != 0 { + if _, err = conn.Do("SELECT", configuration.Redis.DB); err != nil { + defer conn.Close() + done(err) + return nil, err + } + } + + done(nil) + return conn, nil + }, + MaxIdle: configuration.Redis.Pool.MaxIdle, + MaxActive: configuration.Redis.Pool.MaxActive, + IdleTimeout: configuration.Redis.Pool.IdleTimeout, + TestOnBorrow: func(c redis.Conn, t time.Time) error { + // TODO(stevvooe): We can probably do something more interesting + // here with the health package. + _, err := c.Do("PING") + return err + }, + Wait: false, // if a connection is not available, proceed without cache. + } + + app.redis = pool + + // setup expvar + registry := expvar.Get("registry") + if registry == nil { + registry = expvar.NewMap("registry") + } + + registry.(*expvar.Map).Set("redis", expvar.Func(func() interface{} { + return map[string]interface{}{ + "Config": configuration.Redis, + "Active": app.redis.ActiveCount(), + } + })) +} + +// configureLogHook prepares logging hook parameters. +func (app *App) configureLogHook(configuration *configuration.Configuration) { + entry, ok := dcontext.GetLogger(app).(*logrus.Entry) + if !ok { + // somehow, we are not using logrus + return + } + + logger := entry.Logger + + for _, configHook := range configuration.Log.Hooks { + if !configHook.Disabled { + switch configHook.Type { + case "mail": + hook := &logHook{} + hook.LevelsParam = configHook.Levels + hook.Mail = &mailer{ + Addr: configHook.MailOptions.SMTP.Addr, + Username: configHook.MailOptions.SMTP.Username, + Password: configHook.MailOptions.SMTP.Password, + Insecure: configHook.MailOptions.SMTP.Insecure, + From: configHook.MailOptions.From, + To: configHook.MailOptions.To, + } + logger.Hooks.Add(hook) + default: + } + } + } +} + +// configureSecret creates a random secret if a secret wasn't included in the +// configuration. +func (app *App) configureSecret(configuration *configuration.Configuration) { + if configuration.HTTP.Secret == "" { + var secretBytes [randomSecretSize]byte + if _, err := rand.Read(secretBytes[:]); err != nil { + panic(fmt.Sprintf("could not generate random bytes for HTTP secret: %v", err)) + } + configuration.HTTP.Secret = string(secretBytes[:]) + dcontext.GetLogger(app).Warn("No HTTP secret provided - generated random secret. This may cause problems with uploads if multiple registries are behind a load-balancer. To provide a shared secret, fill in http.secret in the configuration file or set the REGISTRY_HTTP_SECRET environment variable.") + } +} + +func (app *App) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // Prepare the context with our own little decorations. + ctx := r.Context() + ctx = dcontext.WithRequest(ctx, r) + ctx, w = dcontext.WithResponseWriter(ctx, w) + ctx = dcontext.WithLogger(ctx, dcontext.GetRequestLogger(ctx)) + r = r.WithContext(ctx) + + defer func() { + status, ok := ctx.Value("http.response.status").(int) + if ok && status >= 200 && status <= 399 { + dcontext.GetResponseLogger(r.Context()).Infof("response completed") + } + }() + + // Set a header with the Docker Distribution API Version for all responses. + w.Header().Add("Docker-Distribution-API-Version", "registry/2.0") + app.router.ServeHTTP(w, r) +} + +// dispatchFunc takes a context and request and returns a constructed handler +// for the route. The dispatcher will use this to dynamically create request +// specific handlers for each endpoint without creating a new router for each +// request. +type dispatchFunc func(ctx *Context, r *http.Request) http.Handler + +// TODO(stevvooe): dispatchers should probably have some validation error +// chain with proper error reporting. + +// dispatcher returns a handler that constructs a request specific context and +// handler, using the dispatch factory function. +func (app *App) dispatcher(dispatch dispatchFunc) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + for headerName, headerValues := range app.Config.HTTP.Headers { + for _, value := range headerValues { + w.Header().Add(headerName, value) + } + } + + context := app.context(w, r) + + if err := app.authorized(w, r, context); err != nil { + dcontext.GetLogger(context).Warnf("error authorizing context: %v", err) + return + } + + // Add username to request logging + context.Context = dcontext.WithLogger(context.Context, dcontext.GetLogger(context.Context, auth.UserNameKey)) + + // sync up context on the request. + r = r.WithContext(context) + + if app.nameRequired(r) { + nameRef, err := reference.WithName(getName(context)) + if err != nil { + dcontext.GetLogger(context).Errorf("error parsing reference from context: %v", err) + context.Errors = append(context.Errors, distribution.ErrRepositoryNameInvalid{ + Name: getName(context), + Reason: err, + }) + if err := errcode.ServeJSON(w, context.Errors); err != nil { + dcontext.GetLogger(context).Errorf("error serving error json: %v (from %v)", err, context.Errors) + } + return + } + repository, err := app.registry.Repository(context, nameRef) + + if err != nil { + dcontext.GetLogger(context).Errorf("error resolving repository: %v", err) + + switch err := err.(type) { + case distribution.ErrRepositoryUnknown: + context.Errors = append(context.Errors, v2.ErrorCodeNameUnknown.WithDetail(err)) + case distribution.ErrRepositoryNameInvalid: + context.Errors = append(context.Errors, v2.ErrorCodeNameInvalid.WithDetail(err)) + case errcode.Error: + context.Errors = append(context.Errors, err) + } + + if err := errcode.ServeJSON(w, context.Errors); err != nil { + dcontext.GetLogger(context).Errorf("error serving error json: %v (from %v)", err, context.Errors) + } + return + } + + // assign and decorate the authorized repository with an event bridge. + context.Repository, context.RepositoryRemover = notifications.Listen( + repository, + context.App.repoRemover, + app.eventBridge(context, r)) + + context.Repository, err = applyRepoMiddleware(app, context.Repository, app.Config.Middleware["repository"]) + if err != nil { + dcontext.GetLogger(context).Errorf("error initializing repository middleware: %v", err) + context.Errors = append(context.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + + if err := errcode.ServeJSON(w, context.Errors); err != nil { + dcontext.GetLogger(context).Errorf("error serving error json: %v (from %v)", err, context.Errors) + } + return + } + } + + dispatch(context, r).ServeHTTP(w, r) + // Automated error response handling here. Handlers may return their + // own errors if they need different behavior (such as range errors + // for layer upload). + if context.Errors.Len() > 0 { + if err := errcode.ServeJSON(w, context.Errors); err != nil { + dcontext.GetLogger(context).Errorf("error serving error json: %v (from %v)", err, context.Errors) + } + + app.logError(context, context.Errors) + } + }) +} + +type errCodeKey struct{} + +func (errCodeKey) String() string { return "err.code" } + +type errMessageKey struct{} + +func (errMessageKey) String() string { return "err.message" } + +type errDetailKey struct{} + +func (errDetailKey) String() string { return "err.detail" } + +func (app *App) logError(ctx context.Context, errors errcode.Errors) { + for _, e1 := range errors { + var c context.Context + + switch e := e1.(type) { + case errcode.Error: + c = context.WithValue(ctx, errCodeKey{}, e.Code) + c = context.WithValue(c, errMessageKey{}, e.Message) + c = context.WithValue(c, errDetailKey{}, e.Detail) + case errcode.ErrorCode: + c = context.WithValue(ctx, errCodeKey{}, e) + c = context.WithValue(c, errMessageKey{}, e.Message()) + default: + // just normal go 'error' + c = context.WithValue(ctx, errCodeKey{}, errcode.ErrorCodeUnknown) + c = context.WithValue(c, errMessageKey{}, e.Error()) + } + + c = dcontext.WithLogger(c, dcontext.GetLogger(c, + errCodeKey{}, + errMessageKey{}, + errDetailKey{})) + dcontext.GetResponseLogger(c).Errorf("response completed with error") + } +} + +// context constructs the context object for the application. This only be +// called once per request. +func (app *App) context(w http.ResponseWriter, r *http.Request) *Context { + ctx := r.Context() + ctx = dcontext.WithVars(ctx, r) + ctx = dcontext.WithLogger(ctx, dcontext.GetLogger(ctx, + "vars.name", + "vars.reference", + "vars.digest", + "vars.uuid")) + + context := &Context{ + App: app, + Context: ctx, + } + + if app.httpHost.Scheme != "" && app.httpHost.Host != "" { + // A "host" item in the configuration takes precedence over + // X-Forwarded-Proto and X-Forwarded-Host headers, and the + // hostname in the request. + context.urlBuilder = v2.NewURLBuilder(&app.httpHost, false) + } else { + context.urlBuilder = v2.NewURLBuilderFromRequest(r, app.Config.HTTP.RelativeURLs) + } + + return context +} + +// authorized checks if the request can proceed with access to the requested +// repository. If it succeeds, the context may access the requested +// repository. An error will be returned if access is not available. +func (app *App) authorized(w http.ResponseWriter, r *http.Request, context *Context) error { + dcontext.GetLogger(context).Debug("authorizing request") + repo := getName(context) + + if app.accessController == nil { + return nil // access controller is not enabled. + } + + var accessRecords []auth.Access + + if repo != "" { + accessRecords = appendAccessRecords(accessRecords, r.Method, repo) + if fromRepo := r.FormValue("from"); fromRepo != "" { + // mounting a blob from one repository to another requires pull (GET) + // access to the source repository. + accessRecords = appendAccessRecords(accessRecords, "GET", fromRepo) + } + } else { + // Only allow the name not to be set on the base route. + if app.nameRequired(r) { + // For this to be properly secured, repo must always be set for a + // resource that may make a modification. The only condition under + // which name is not set and we still allow access is when the + // base route is accessed. This section prevents us from making + // that mistake elsewhere in the code, allowing any operation to + // proceed. + if err := errcode.ServeJSON(w, errcode.ErrorCodeUnauthorized); err != nil { + dcontext.GetLogger(context).Errorf("error serving error json: %v (from %v)", err, context.Errors) + } + return fmt.Errorf("forbidden: no repository name") + } + accessRecords = appendCatalogAccessRecord(accessRecords, r) + } + + ctx, err := app.accessController.Authorized(context.Context, accessRecords...) + if err != nil { + switch err := err.(type) { + case auth.Challenge: + // Add the appropriate WWW-Auth header + err.SetHeaders(r, w) + + if err := errcode.ServeJSON(w, errcode.ErrorCodeUnauthorized.WithDetail(accessRecords)); err != nil { + dcontext.GetLogger(context).Errorf("error serving error json: %v (from %v)", err, context.Errors) + } + default: + // This condition is a potential security problem either in + // the configuration or whatever is backing the access + // controller. Just return a bad request with no information + // to avoid exposure. The request should not proceed. + dcontext.GetLogger(context).Errorf("error checking authorization: %v", err) + w.WriteHeader(http.StatusBadRequest) + } + + return err + } + + dcontext.GetLogger(ctx).Info("authorized request") + // TODO(stevvooe): This pattern needs to be cleaned up a bit. One context + // should be replaced by another, rather than replacing the context on a + // mutable object. + context.Context = ctx + return nil +} + +// eventBridge returns a bridge for the current request, configured with the +// correct actor and source. +func (app *App) eventBridge(ctx *Context, r *http.Request) notifications.Listener { + actor := notifications.ActorRecord{ + Name: getUserName(ctx, r), + } + request := notifications.NewRequestRecord(dcontext.GetRequestID(ctx), r) + + return notifications.NewBridge(ctx.urlBuilder, app.events.source, actor, request, app.events.sink, app.Config.Notifications.EventConfig.IncludeReferences) +} + +// nameRequired returns true if the route requires a name. +func (app *App) nameRequired(r *http.Request) bool { + route := mux.CurrentRoute(r) + if route == nil { + return true + } + routeName := route.GetName() + return routeName != v2.RouteNameBase && routeName != v2.RouteNameCatalog +} + +// apiBase implements a simple yes-man for doing overall checks against the +// api. This can support auth roundtrips to support docker login. +func apiBase(w http.ResponseWriter, r *http.Request) { + const emptyJSON = "{}" + // Provide a simple /v2/ 200 OK response with empty json response. + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("Content-Length", fmt.Sprint(len(emptyJSON))) + + fmt.Fprint(w, emptyJSON) +} + +// appendAccessRecords checks the method and adds the appropriate Access records to the records list. +func appendAccessRecords(records []auth.Access, method string, repo string) []auth.Access { + resource := auth.Resource{ + Type: "repository", + Name: repo, + } + + switch method { + case "GET", "HEAD": + records = append(records, + auth.Access{ + Resource: resource, + Action: "pull", + }) + case "POST", "PUT", "PATCH": + records = append(records, + auth.Access{ + Resource: resource, + Action: "pull", + }, + auth.Access{ + Resource: resource, + Action: "push", + }) + case "DELETE": + records = append(records, + auth.Access{ + Resource: resource, + Action: "delete", + }) + } + return records +} + +// Add the access record for the catalog if it's our current route +func appendCatalogAccessRecord(accessRecords []auth.Access, r *http.Request) []auth.Access { + route := mux.CurrentRoute(r) + routeName := route.GetName() + + if routeName == v2.RouteNameCatalog { + resource := auth.Resource{ + Type: "registry", + Name: "catalog", + } + + accessRecords = append(accessRecords, + auth.Access{ + Resource: resource, + Action: "*", + }) + } + return accessRecords +} + +// applyRegistryMiddleware wraps a registry instance with the configured middlewares +func applyRegistryMiddleware(ctx context.Context, registry distribution.Namespace, middlewares []configuration.Middleware) (distribution.Namespace, error) { + for _, mw := range middlewares { + rmw, err := registrymiddleware.Get(ctx, mw.Name, mw.Options, registry) + if err != nil { + return nil, fmt.Errorf("unable to configure registry middleware (%s): %s", mw.Name, err) + } + registry = rmw + } + return registry, nil + +} + +// applyRepoMiddleware wraps a repository with the configured middlewares +func applyRepoMiddleware(ctx context.Context, repository distribution.Repository, middlewares []configuration.Middleware) (distribution.Repository, error) { + for _, mw := range middlewares { + rmw, err := repositorymiddleware.Get(ctx, mw.Name, mw.Options, repository) + if err != nil { + return nil, err + } + repository = rmw + } + return repository, nil +} + +// applyStorageMiddleware wraps a storage driver with the configured middlewares +func applyStorageMiddleware(driver storagedriver.StorageDriver, middlewares []configuration.Middleware) (storagedriver.StorageDriver, error) { + for _, mw := range middlewares { + smw, err := storagemiddleware.Get(mw.Name, mw.Options, driver) + if err != nil { + return nil, fmt.Errorf("unable to configure storage middleware (%s): %v", mw.Name, err) + } + driver = smw + } + return driver, nil +} + +// uploadPurgeDefaultConfig provides a default configuration for upload +// purging to be used in the absence of configuration in the +// configuration file +func uploadPurgeDefaultConfig() map[interface{}]interface{} { + config := map[interface{}]interface{}{} + config["enabled"] = true + config["age"] = "168h" + config["interval"] = "24h" + config["dryrun"] = false + return config +} + +func badPurgeUploadConfig(reason string) { + panic(fmt.Sprintf("Unable to parse upload purge configuration: %s", reason)) +} + +// startUploadPurger schedules a goroutine which will periodically +// check upload directories for old files and delete them +func startUploadPurger(ctx context.Context, storageDriver storagedriver.StorageDriver, log dcontext.Logger, config map[interface{}]interface{}) { + if config["enabled"] == false { + return + } + + var purgeAgeDuration time.Duration + var err error + purgeAge, ok := config["age"] + if ok { + ageStr, ok := purgeAge.(string) + if !ok { + badPurgeUploadConfig("age is not a string") + } + purgeAgeDuration, err = time.ParseDuration(ageStr) + if err != nil { + badPurgeUploadConfig(fmt.Sprintf("Cannot parse duration: %s", err.Error())) + } + } else { + badPurgeUploadConfig("age missing") + } + + var intervalDuration time.Duration + interval, ok := config["interval"] + if ok { + intervalStr, ok := interval.(string) + if !ok { + badPurgeUploadConfig("interval is not a string") + } + + intervalDuration, err = time.ParseDuration(intervalStr) + if err != nil { + badPurgeUploadConfig(fmt.Sprintf("Cannot parse interval: %s", err.Error())) + } + } else { + badPurgeUploadConfig("interval missing") + } + + var dryRunBool bool + dryRun, ok := config["dryrun"] + if ok { + dryRunBool, ok = dryRun.(bool) + if !ok { + badPurgeUploadConfig("cannot parse dryrun") + } + } else { + badPurgeUploadConfig("dryrun missing") + } + + go func() { + randInt, err := rand.Int(rand.Reader, new(big.Int).SetInt64(math.MaxInt64)) + if err != nil { + log.Infof("Failed to generate random jitter: %v", err) + // sleep 30min for failure case + randInt = big.NewInt(30) + } + jitter := time.Duration(randInt.Int64()%60) * time.Minute + log.Infof("Starting upload purge in %s", jitter) + time.Sleep(jitter) + + for { + storage.PurgeUploads(ctx, storageDriver, time.Now().Add(-purgeAgeDuration), !dryRunBool) + log.Infof("Starting upload purge in %s", intervalDuration) + time.Sleep(intervalDuration) + } + }() +} diff --git a/vendor/github.com/docker/distribution/registry/handlers/basicauth.go b/vendor/github.com/docker/distribution/registry/handlers/basicauth.go new file mode 100644 index 000000000..a41965dcf --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/handlers/basicauth.go @@ -0,0 +1,12 @@ +//go:build go1.4 +// +build go1.4 + +package handlers + +import ( + "net/http" +) + +func basicAuth(r *http.Request) (username, password string, ok bool) { + return r.BasicAuth() +} diff --git a/vendor/github.com/docker/distribution/registry/handlers/basicauth_prego14.go b/vendor/github.com/docker/distribution/registry/handlers/basicauth_prego14.go new file mode 100644 index 000000000..01290adfe --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/handlers/basicauth_prego14.go @@ -0,0 +1,42 @@ +//go:build !go1.4 +// +build !go1.4 + +package handlers + +import ( + "encoding/base64" + "net/http" + "strings" +) + +// NOTE(stevvooe): This is basic auth support from go1.4 present to ensure we +// can compile on go1.3 and earlier. + +// BasicAuth returns the username and password provided in the request's +// Authorization header, if the request uses HTTP Basic Authentication. +// See RFC 2617, Section 2. +func basicAuth(r *http.Request) (username, password string, ok bool) { + auth := r.Header.Get("Authorization") + if auth == "" { + return + } + return parseBasicAuth(auth) +} + +// parseBasicAuth parses an HTTP Basic Authentication string. +// "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==" returns ("Aladdin", "open sesame", true). +func parseBasicAuth(auth string) (username, password string, ok bool) { + if !strings.HasPrefix(auth, "Basic ") { + return + } + c, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(auth, "Basic ")) + if err != nil { + return + } + cs := string(c) + s := strings.IndexByte(cs, ':') + if s < 0 { + return + } + return cs[:s], cs[s+1:], true +} diff --git a/vendor/github.com/docker/distribution/registry/handlers/blob.go b/vendor/github.com/docker/distribution/registry/handlers/blob.go new file mode 100644 index 000000000..515fdad0a --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/handlers/blob.go @@ -0,0 +1,99 @@ +package handlers + +import ( + "net/http" + + "github.com/docker/distribution" + "github.com/docker/distribution/context" + "github.com/docker/distribution/registry/api/errcode" + v2 "github.com/docker/distribution/registry/api/v2" + "github.com/gorilla/handlers" + "github.com/opencontainers/go-digest" +) + +// blobDispatcher uses the request context to build a blobHandler. +func blobDispatcher(ctx *Context, r *http.Request) http.Handler { + dgst, err := getDigest(ctx) + if err != nil { + + if err == errDigestNotAvailable { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx.Errors = append(ctx.Errors, v2.ErrorCodeDigestInvalid.WithDetail(err)) + }) + } + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx.Errors = append(ctx.Errors, v2.ErrorCodeDigestInvalid.WithDetail(err)) + }) + } + + blobHandler := &blobHandler{ + Context: ctx, + Digest: dgst, + } + + mhandler := handlers.MethodHandler{ + "GET": http.HandlerFunc(blobHandler.GetBlob), + "HEAD": http.HandlerFunc(blobHandler.GetBlob), + } + + if !ctx.readOnly { + mhandler["DELETE"] = http.HandlerFunc(blobHandler.DeleteBlob) + } + + return mhandler +} + +// blobHandler serves http blob requests. +type blobHandler struct { + *Context + + Digest digest.Digest +} + +// GetBlob fetches the binary data from backend storage returns it in the +// response. +func (bh *blobHandler) GetBlob(w http.ResponseWriter, r *http.Request) { + context.GetLogger(bh).Debug("GetBlob") + blobs := bh.Repository.Blobs(bh) + desc, err := blobs.Stat(bh, bh.Digest) + if err != nil { + if err == distribution.ErrBlobUnknown { + bh.Errors = append(bh.Errors, v2.ErrorCodeBlobUnknown.WithDetail(bh.Digest)) + } else { + bh.Errors = append(bh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + } + return + } + + if err := blobs.ServeBlob(bh, w, r, desc.Digest); err != nil { + context.GetLogger(bh).Debugf("unexpected error getting blob HTTP handler: %v", err) + bh.Errors = append(bh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + return + } +} + +// DeleteBlob deletes a layer blob +func (bh *blobHandler) DeleteBlob(w http.ResponseWriter, r *http.Request) { + context.GetLogger(bh).Debug("DeleteBlob") + + blobs := bh.Repository.Blobs(bh) + err := blobs.Delete(bh, bh.Digest) + if err != nil { + switch err { + case distribution.ErrUnsupported: + bh.Errors = append(bh.Errors, errcode.ErrorCodeUnsupported) + return + case distribution.ErrBlobUnknown: + bh.Errors = append(bh.Errors, v2.ErrorCodeBlobUnknown) + return + default: + bh.Errors = append(bh.Errors, err) + context.GetLogger(bh).Errorf("Unknown error deleting blob: %s", err.Error()) + return + } + } + + w.Header().Set("Content-Length", "0") + w.WriteHeader(http.StatusAccepted) +} diff --git a/vendor/github.com/docker/distribution/registry/handlers/blobupload.go b/vendor/github.com/docker/distribution/registry/handlers/blobupload.go new file mode 100644 index 000000000..611cce3c8 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/handlers/blobupload.go @@ -0,0 +1,368 @@ +package handlers + +import ( + "fmt" + "net/http" + "net/url" + + "github.com/distribution/reference" + "github.com/docker/distribution" + dcontext "github.com/docker/distribution/context" + "github.com/docker/distribution/registry/api/errcode" + v2 "github.com/docker/distribution/registry/api/v2" + "github.com/docker/distribution/registry/storage" + "github.com/gorilla/handlers" + "github.com/opencontainers/go-digest" +) + +// blobUploadDispatcher constructs and returns the blob upload handler for the +// given request context. +func blobUploadDispatcher(ctx *Context, r *http.Request) http.Handler { + buh := &blobUploadHandler{ + Context: ctx, + UUID: getUploadUUID(ctx), + } + + handler := handlers.MethodHandler{ + "GET": http.HandlerFunc(buh.GetUploadStatus), + "HEAD": http.HandlerFunc(buh.GetUploadStatus), + } + + if !ctx.readOnly { + handler["POST"] = http.HandlerFunc(buh.StartBlobUpload) + handler["PATCH"] = http.HandlerFunc(buh.PatchBlobData) + handler["PUT"] = http.HandlerFunc(buh.PutBlobUploadComplete) + handler["DELETE"] = http.HandlerFunc(buh.CancelBlobUpload) + } + + if buh.UUID != "" { + state, err := hmacKey(ctx.Config.HTTP.Secret).unpackUploadState(r.FormValue("_state")) + if err != nil { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + dcontext.GetLogger(ctx).Infof("error resolving upload: %v", err) + buh.Errors = append(buh.Errors, v2.ErrorCodeBlobUploadInvalid.WithDetail(err)) + }) + } + buh.State = state + + if state.Name != ctx.Repository.Named().Name() { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + dcontext.GetLogger(ctx).Infof("mismatched repository name in upload state: %q != %q", state.Name, buh.Repository.Named().Name()) + buh.Errors = append(buh.Errors, v2.ErrorCodeBlobUploadInvalid.WithDetail(err)) + }) + } + + if state.UUID != buh.UUID { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + dcontext.GetLogger(ctx).Infof("mismatched uuid in upload state: %q != %q", state.UUID, buh.UUID) + buh.Errors = append(buh.Errors, v2.ErrorCodeBlobUploadInvalid.WithDetail(err)) + }) + } + + blobs := ctx.Repository.Blobs(buh) + upload, err := blobs.Resume(buh, buh.UUID) + if err != nil { + dcontext.GetLogger(ctx).Errorf("error resolving upload: %v", err) + if err == distribution.ErrBlobUploadUnknown { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + buh.Errors = append(buh.Errors, v2.ErrorCodeBlobUploadUnknown.WithDetail(err)) + }) + } + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + buh.Errors = append(buh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + }) + } + buh.Upload = upload + + if size := upload.Size(); size != buh.State.Offset { + defer upload.Close() + dcontext.GetLogger(ctx).Errorf("upload resumed at wrong offest: %d != %d", size, buh.State.Offset) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + buh.Errors = append(buh.Errors, v2.ErrorCodeBlobUploadInvalid.WithDetail(err)) + upload.Cancel(buh) + }) + } + return closeResources(handler, buh.Upload) + } + + return handler +} + +// blobUploadHandler handles the http blob upload process. +type blobUploadHandler struct { + *Context + + // UUID identifies the upload instance for the current request. Using UUID + // to key blob writers since this implementation uses UUIDs. + UUID string + + Upload distribution.BlobWriter + + State blobUploadState +} + +// StartBlobUpload begins the blob upload process and allocates a server-side +// blob writer session, optionally mounting the blob from a separate repository. +func (buh *blobUploadHandler) StartBlobUpload(w http.ResponseWriter, r *http.Request) { + var options []distribution.BlobCreateOption + + fromRepo := r.FormValue("from") + mountDigest := r.FormValue("mount") + + if mountDigest != "" && fromRepo != "" { + opt, err := buh.createBlobMountOption(fromRepo, mountDigest) + if opt != nil && err == nil { + options = append(options, opt) + } + } + + blobs := buh.Repository.Blobs(buh) + upload, err := blobs.Create(buh, options...) + + if err != nil { + if ebm, ok := err.(distribution.ErrBlobMounted); ok { + if err := buh.writeBlobCreatedHeaders(w, ebm.Descriptor); err != nil { + buh.Errors = append(buh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + } + } else if err == distribution.ErrUnsupported { + buh.Errors = append(buh.Errors, errcode.ErrorCodeUnsupported) + } else { + buh.Errors = append(buh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + } + return + } + + buh.Upload = upload + + if err := buh.blobUploadResponse(w, r, true); err != nil { + buh.Errors = append(buh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + return + } + + w.Header().Set("Docker-Upload-UUID", buh.Upload.ID()) + w.WriteHeader(http.StatusAccepted) +} + +// GetUploadStatus returns the status of a given upload, identified by id. +func (buh *blobUploadHandler) GetUploadStatus(w http.ResponseWriter, r *http.Request) { + if buh.Upload == nil { + buh.Errors = append(buh.Errors, v2.ErrorCodeBlobUploadUnknown) + return + } + + // TODO(dmcgowan): Set last argument to false in blobUploadResponse when + // resumable upload is supported. This will enable returning a non-zero + // range for clients to begin uploading at an offset. + if err := buh.blobUploadResponse(w, r, true); err != nil { + buh.Errors = append(buh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + return + } + + w.Header().Set("Docker-Upload-UUID", buh.UUID) + w.WriteHeader(http.StatusNoContent) +} + +// PatchBlobData writes data to an upload. +func (buh *blobUploadHandler) PatchBlobData(w http.ResponseWriter, r *http.Request) { + if buh.Upload == nil { + buh.Errors = append(buh.Errors, v2.ErrorCodeBlobUploadUnknown) + return + } + + ct := r.Header.Get("Content-Type") + if ct != "" && ct != "application/octet-stream" { + buh.Errors = append(buh.Errors, errcode.ErrorCodeUnknown.WithDetail(fmt.Errorf("bad Content-Type"))) + // TODO(dmcgowan): encode error + return + } + + // TODO(dmcgowan): support Content-Range header to seek and write range + + if err := copyFullPayload(buh, w, r, buh.Upload, -1, "blob PATCH"); err != nil { + buh.Errors = append(buh.Errors, errcode.ErrorCodeUnknown.WithDetail(err.Error())) + return + } + + if err := buh.blobUploadResponse(w, r, false); err != nil { + buh.Errors = append(buh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + return + } + + w.WriteHeader(http.StatusAccepted) +} + +// PutBlobUploadComplete takes the final request of a blob upload. The +// request may include all the blob data or no blob data. Any data +// provided is received and verified. If successful, the blob is linked +// into the blob store and 201 Created is returned with the canonical +// url of the blob. +func (buh *blobUploadHandler) PutBlobUploadComplete(w http.ResponseWriter, r *http.Request) { + if buh.Upload == nil { + buh.Errors = append(buh.Errors, v2.ErrorCodeBlobUploadUnknown) + return + } + + dgstStr := r.FormValue("digest") // TODO(stevvooe): Support multiple digest parameters! + + if dgstStr == "" { + // no digest? return error, but allow retry. + buh.Errors = append(buh.Errors, v2.ErrorCodeDigestInvalid.WithDetail("digest missing")) + return + } + + dgst, err := digest.Parse(dgstStr) + if err != nil { + // no digest? return error, but allow retry. + buh.Errors = append(buh.Errors, v2.ErrorCodeDigestInvalid.WithDetail("digest parsing failed")) + return + } + + if err := copyFullPayload(buh, w, r, buh.Upload, -1, "blob PUT"); err != nil { + buh.Errors = append(buh.Errors, errcode.ErrorCodeUnknown.WithDetail(err.Error())) + return + } + + desc, err := buh.Upload.Commit(buh, distribution.Descriptor{ + Digest: dgst, + + // TODO(stevvooe): This isn't wildly important yet, but we should + // really set the mediatype. For now, we can let the backend take care + // of this. + }) + + if err != nil { + switch err := err.(type) { + case distribution.ErrBlobInvalidDigest: + buh.Errors = append(buh.Errors, v2.ErrorCodeDigestInvalid.WithDetail(err)) + case errcode.Error: + buh.Errors = append(buh.Errors, err) + default: + switch err { + case distribution.ErrAccessDenied: + buh.Errors = append(buh.Errors, errcode.ErrorCodeDenied) + case distribution.ErrUnsupported: + buh.Errors = append(buh.Errors, errcode.ErrorCodeUnsupported) + case distribution.ErrBlobInvalidLength, distribution.ErrBlobDigestUnsupported: + buh.Errors = append(buh.Errors, v2.ErrorCodeBlobUploadInvalid.WithDetail(err)) + default: + dcontext.GetLogger(buh).Errorf("unknown error completing upload: %v", err) + buh.Errors = append(buh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + } + + } + + // Clean up the backend blob data if there was an error. + if err := buh.Upload.Cancel(buh); err != nil { + // If the cleanup fails, all we can do is observe and report. + dcontext.GetLogger(buh).Errorf("error canceling upload after error: %v", err) + } + + return + } + if err := buh.writeBlobCreatedHeaders(w, desc); err != nil { + buh.Errors = append(buh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + return + } +} + +// CancelBlobUpload cancels an in-progress upload of a blob. +func (buh *blobUploadHandler) CancelBlobUpload(w http.ResponseWriter, r *http.Request) { + if buh.Upload == nil { + buh.Errors = append(buh.Errors, v2.ErrorCodeBlobUploadUnknown) + return + } + + w.Header().Set("Docker-Upload-UUID", buh.UUID) + if err := buh.Upload.Cancel(buh); err != nil { + dcontext.GetLogger(buh).Errorf("error encountered canceling upload: %v", err) + buh.Errors = append(buh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + } + + w.WriteHeader(http.StatusNoContent) +} + +// blobUploadResponse provides a standard request for uploading blobs and +// chunk responses. This sets the correct headers but the response status is +// left to the caller. The fresh argument is used to ensure that new blob +// uploads always start at a 0 offset. This allows disabling resumable push by +// always returning a 0 offset on check status. +func (buh *blobUploadHandler) blobUploadResponse(w http.ResponseWriter, r *http.Request, fresh bool) error { + // TODO(stevvooe): Need a better way to manage the upload state automatically. + buh.State.Name = buh.Repository.Named().Name() + buh.State.UUID = buh.Upload.ID() + buh.Upload.Close() + buh.State.Offset = buh.Upload.Size() + buh.State.StartedAt = buh.Upload.StartedAt() + + token, err := hmacKey(buh.Config.HTTP.Secret).packUploadState(buh.State) + if err != nil { + dcontext.GetLogger(buh).Infof("error building upload state token: %s", err) + return err + } + + uploadURL, err := buh.urlBuilder.BuildBlobUploadChunkURL( + buh.Repository.Named(), buh.Upload.ID(), + url.Values{ + "_state": []string{token}, + }) + if err != nil { + dcontext.GetLogger(buh).Infof("error building upload url: %s", err) + return err + } + + endRange := buh.Upload.Size() + if endRange > 0 { + endRange = endRange - 1 + } + + w.Header().Set("Docker-Upload-UUID", buh.UUID) + w.Header().Set("Location", uploadURL) + + w.Header().Set("Content-Length", "0") + w.Header().Set("Range", fmt.Sprintf("0-%d", endRange)) + + return nil +} + +// mountBlob attempts to mount a blob from another repository by its digest. If +// successful, the blob is linked into the blob store and 201 Created is +// returned with the canonical url of the blob. +func (buh *blobUploadHandler) createBlobMountOption(fromRepo, mountDigest string) (distribution.BlobCreateOption, error) { + dgst, err := digest.Parse(mountDigest) + if err != nil { + return nil, err + } + + ref, err := reference.WithName(fromRepo) + if err != nil { + return nil, err + } + + canonical, err := reference.WithDigest(ref, dgst) + if err != nil { + return nil, err + } + + return storage.WithMountFrom(canonical), nil +} + +// writeBlobCreatedHeaders writes the standard headers describing a newly +// created blob. A 201 Created is written as well as the canonical URL and +// blob digest. +func (buh *blobUploadHandler) writeBlobCreatedHeaders(w http.ResponseWriter, desc distribution.Descriptor) error { + ref, err := reference.WithDigest(buh.Repository.Named(), desc.Digest) + if err != nil { + return err + } + blobURL, err := buh.urlBuilder.BuildBlobURL(ref) + if err != nil { + return err + } + + w.Header().Set("Location", blobURL) + w.Header().Set("Content-Length", "0") + w.Header().Set("Docker-Content-Digest", desc.Digest.String()) + w.WriteHeader(http.StatusCreated) + return nil +} diff --git a/vendor/github.com/docker/distribution/registry/handlers/catalog.go b/vendor/github.com/docker/distribution/registry/handlers/catalog.go new file mode 100644 index 000000000..83ec0a9c3 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/handlers/catalog.go @@ -0,0 +1,126 @@ +package handlers + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + + "github.com/docker/distribution/registry/api/errcode" + v2 "github.com/docker/distribution/registry/api/v2" + "github.com/docker/distribution/registry/storage/driver" + + "github.com/gorilla/handlers" +) + +const defaultReturnedEntries = 100 + +func catalogDispatcher(ctx *Context, r *http.Request) http.Handler { + catalogHandler := &catalogHandler{ + Context: ctx, + } + + return handlers.MethodHandler{ + "GET": http.HandlerFunc(catalogHandler.GetCatalog), + } +} + +type catalogHandler struct { + *Context +} + +type catalogAPIResponse struct { + Repositories []string `json:"repositories"` +} + +func (ch *catalogHandler) GetCatalog(w http.ResponseWriter, r *http.Request) { + var moreEntries = true + + q := r.URL.Query() + lastEntry := q.Get("last") + + entries := defaultReturnedEntries + maximumConfiguredEntries := ch.App.Config.Catalog.MaxEntries + + // parse n, if n unparseable, or negative assign it to defaultReturnedEntries + if n := q.Get("n"); n != "" { + parsedMax, err := strconv.Atoi(n) + if err == nil { + if parsedMax > maximumConfiguredEntries { + ch.Errors = append(ch.Errors, v2.ErrorCodePaginationNumberInvalid.WithDetail(map[string]int{"n": parsedMax})) + return + } else if parsedMax >= 0 { + entries = parsedMax + } + } + } + + // then enforce entries to be between 0 & maximumConfiguredEntries + // max(0, min(entries, maximumConfiguredEntries)) + if entries < 0 || entries > maximumConfiguredEntries { + entries = maximumConfiguredEntries + } + + repos := make([]string, entries) + filled := 0 + + // entries is guaranteed to be >= 0 and < maximumConfiguredEntries + if entries == 0 { + moreEntries = false + } else { + returnedRepositories, err := ch.App.registry.Repositories(ch.Context, repos, lastEntry) + if err != nil { + _, pathNotFound := err.(driver.PathNotFoundError) + if err != io.EOF && !pathNotFound { + ch.Errors = append(ch.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + return + } + // err is either io.EOF or not PathNotFoundError + moreEntries = false + } + filled = returnedRepositories + } + + w.Header().Set("Content-Type", "application/json; charset=utf-8") + + // Add a link header if there are more entries to retrieve + if moreEntries { + lastEntry = repos[filled-1] + urlStr, err := createLinkEntry(r.URL.String(), entries, lastEntry) + if err != nil { + ch.Errors = append(ch.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + return + } + w.Header().Set("Link", urlStr) + } + + enc := json.NewEncoder(w) + if err := enc.Encode(catalogAPIResponse{ + Repositories: repos[0:filled], + }); err != nil { + ch.Errors = append(ch.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + return + } +} + +// Use the original URL from the request to create a new URL for +// the link header +func createLinkEntry(origURL string, maxEntries int, lastEntry string) (string, error) { + calledURL, err := url.Parse(origURL) + if err != nil { + return "", err + } + + v := url.Values{} + v.Add("n", strconv.Itoa(maxEntries)) + v.Add("last", lastEntry) + + calledURL.RawQuery = v.Encode() + + calledURL.Fragment = "" + urlStr := fmt.Sprintf("<%s>; rel=\"next\"", calledURL.String()) + + return urlStr, nil +} diff --git a/vendor/github.com/docker/distribution/registry/handlers/context.go b/vendor/github.com/docker/distribution/registry/handlers/context.go new file mode 100644 index 000000000..b1b7a13d3 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/handlers/context.go @@ -0,0 +1,95 @@ +package handlers + +import ( + "context" + "fmt" + "net/http" + + "github.com/docker/distribution" + dcontext "github.com/docker/distribution/context" + "github.com/docker/distribution/registry/api/errcode" + v2 "github.com/docker/distribution/registry/api/v2" + "github.com/docker/distribution/registry/auth" + "github.com/opencontainers/go-digest" +) + +// Context should contain the request specific context for use in across +// handlers. Resources that don't need to be shared across handlers should not +// be on this object. +type Context struct { + // App points to the application structure that created this context. + *App + context.Context + + // Repository is the repository for the current request. All requests + // should be scoped to a single repository. This field may be nil. + Repository distribution.Repository + + // RepositoryRemover provides method to delete a repository + RepositoryRemover distribution.RepositoryRemover + + // Errors is a collection of errors encountered during the request to be + // returned to the client API. If errors are added to the collection, the + // handler *must not* start the response via http.ResponseWriter. + Errors errcode.Errors + + urlBuilder *v2.URLBuilder + + // TODO(stevvooe): The goal is too completely factor this context and + // dispatching out of the web application. Ideally, we should lean on + // context.Context for injection of these resources. +} + +// Value overrides context.Context.Value to ensure that calls are routed to +// correct context. +func (ctx *Context) Value(key interface{}) interface{} { + return ctx.Context.Value(key) +} + +func getName(ctx context.Context) (name string) { + return dcontext.GetStringValue(ctx, "vars.name") +} + +func getReference(ctx context.Context) (reference string) { + return dcontext.GetStringValue(ctx, "vars.reference") +} + +var errDigestNotAvailable = fmt.Errorf("digest not available in context") + +func getDigest(ctx context.Context) (dgst digest.Digest, err error) { + dgstStr := dcontext.GetStringValue(ctx, "vars.digest") + + if dgstStr == "" { + dcontext.GetLogger(ctx).Errorf("digest not available") + return "", errDigestNotAvailable + } + + d, err := digest.Parse(dgstStr) + if err != nil { + dcontext.GetLogger(ctx).Errorf("error parsing digest=%q: %v", dgstStr, err) + return "", err + } + + return d, nil +} + +func getUploadUUID(ctx context.Context) (uuid string) { + return dcontext.GetStringValue(ctx, "vars.uuid") +} + +// getUserName attempts to resolve a username from the context and request. If +// a username cannot be resolved, the empty string is returned. +func getUserName(ctx context.Context, r *http.Request) string { + username := dcontext.GetStringValue(ctx, auth.UserNameKey) + + // Fallback to request user with basic auth + if username == "" { + var ok bool + uname, _, ok := basicAuth(r) + if ok { + username = uname + } + } + + return username +} diff --git a/vendor/github.com/docker/distribution/registry/handlers/helpers.go b/vendor/github.com/docker/distribution/registry/handlers/helpers.go new file mode 100644 index 000000000..b02338e9b --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/handlers/helpers.go @@ -0,0 +1,66 @@ +package handlers + +import ( + "context" + "errors" + "io" + "net/http" + + dcontext "github.com/docker/distribution/context" +) + +// closeResources closes all the provided resources after running the target +// handler. +func closeResources(handler http.Handler, closers ...io.Closer) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + for _, closer := range closers { + defer closer.Close() + } + handler.ServeHTTP(w, r) + }) +} + +// copyFullPayload copies the payload of an HTTP request to destWriter. If it +// receives less content than expected, and the client disconnected during the +// upload, it avoids sending a 400 error to keep the logs cleaner. +// +// The copy will be limited to `limit` bytes, if limit is greater than zero. +func copyFullPayload(ctx context.Context, responseWriter http.ResponseWriter, r *http.Request, destWriter io.Writer, limit int64, action string) error { + // Get a channel that tells us if the client disconnects + clientClosed := r.Context().Done() + var body = r.Body + if limit > 0 { + body = http.MaxBytesReader(responseWriter, body, limit) + } + + // Read in the data, if any. + copied, err := io.Copy(destWriter, body) + if clientClosed != nil && (err != nil || (r.ContentLength > 0 && copied < r.ContentLength)) { + // Didn't receive as much content as expected. Did the client + // disconnect during the request? If so, avoid returning a 400 + // error to keep the logs cleaner. + select { + case <-clientClosed: + // Set the response code to "499 Client Closed Request" + // Even though the connection has already been closed, + // this causes the logger to pick up a 499 error + // instead of showing 0 for the HTTP status. + responseWriter.WriteHeader(499) + + dcontext.GetLoggerWithFields(ctx, map[interface{}]interface{}{ + "error": err, + "copied": copied, + "contentLength": r.ContentLength, + }, "error", "copied", "contentLength").Error("client disconnected during " + action) + return errors.New("client disconnected") + default: + } + } + + if err != nil { + dcontext.GetLogger(ctx).Errorf("unknown error reading request payload: %v", err) + return err + } + + return nil +} diff --git a/vendor/github.com/docker/distribution/registry/handlers/hmac.go b/vendor/github.com/docker/distribution/registry/handlers/hmac.go new file mode 100644 index 000000000..94ed9fda5 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/handlers/hmac.go @@ -0,0 +1,74 @@ +package handlers + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "time" +) + +// blobUploadState captures the state serializable state of the blob upload. +type blobUploadState struct { + // name is the primary repository under which the blob will be linked. + Name string + + // UUID identifies the upload. + UUID string + + // offset contains the current progress of the upload. + Offset int64 + + // StartedAt is the original start time of the upload. + StartedAt time.Time +} + +type hmacKey string + +var errInvalidSecret = fmt.Errorf("invalid secret") + +// unpackUploadState unpacks and validates the blob upload state from the +// token, using the hmacKey secret. +func (secret hmacKey) unpackUploadState(token string) (blobUploadState, error) { + var state blobUploadState + + tokenBytes, err := base64.URLEncoding.DecodeString(token) + if err != nil { + return state, err + } + mac := hmac.New(sha256.New, []byte(secret)) + + if len(tokenBytes) < mac.Size() { + return state, errInvalidSecret + } + + macBytes := tokenBytes[:mac.Size()] + messageBytes := tokenBytes[mac.Size():] + + mac.Write(messageBytes) + if !hmac.Equal(mac.Sum(nil), macBytes) { + return state, errInvalidSecret + } + + if err := json.Unmarshal(messageBytes, &state); err != nil { + return state, err + } + + return state, nil +} + +// packUploadState packs the upload state signed with and hmac digest using +// the hmacKey secret, encoding to url safe base64. The resulting token can be +// used to share data with minimized risk of external tampering. +func (secret hmacKey) packUploadState(lus blobUploadState) (string, error) { + mac := hmac.New(sha256.New, []byte(secret)) + p, err := json.Marshal(lus) + if err != nil { + return "", err + } + + mac.Write(p) + + return base64.URLEncoding.EncodeToString(append(mac.Sum(nil), p...)), nil +} diff --git a/vendor/github.com/docker/distribution/registry/handlers/hooks.go b/vendor/github.com/docker/distribution/registry/handlers/hooks.go new file mode 100644 index 000000000..4a2b1be84 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/handlers/hooks.go @@ -0,0 +1,53 @@ +package handlers + +import ( + "bytes" + "errors" + "fmt" + "strings" + "text/template" + + "github.com/sirupsen/logrus" +) + +// logHook is for hooking Panic in web application +type logHook struct { + LevelsParam []string + Mail *mailer +} + +// Fire forwards an error to LogHook +func (hook *logHook) Fire(entry *logrus.Entry) error { + addr := strings.Split(hook.Mail.Addr, ":") + if len(addr) != 2 { + return errors.New("invalid Mail Address") + } + host := addr[0] + subject := fmt.Sprintf("[%s] %s: %s", entry.Level, host, entry.Message) + + html := ` + {{.Message}} + + {{range $key, $value := .Data}} + {{$key}}: {{$value}} + {{end}} + ` + b := bytes.NewBuffer(make([]byte, 0)) + t := template.Must(template.New("mail body").Parse(html)) + if err := t.Execute(b, entry); err != nil { + return err + } + body := b.String() + + return hook.Mail.sendMail(subject, body) +} + +// Levels contains hook levels to be catched +func (hook *logHook) Levels() []logrus.Level { + levels := []logrus.Level{} + for _, v := range hook.LevelsParam { + lv, _ := logrus.ParseLevel(v) + levels = append(levels, lv) + } + return levels +} diff --git a/vendor/github.com/docker/distribution/registry/handlers/mail.go b/vendor/github.com/docker/distribution/registry/handlers/mail.go new file mode 100644 index 000000000..684b1b34c --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/handlers/mail.go @@ -0,0 +1,45 @@ +package handlers + +import ( + "errors" + "net/smtp" + "strings" +) + +// mailer provides fields of email configuration for sending. +type mailer struct { + Addr, Username, Password, From string + Insecure bool + To []string +} + +// sendMail allows users to send email, only if mail parameters is configured correctly. +func (mail *mailer) sendMail(subject, message string) error { + addr := strings.Split(mail.Addr, ":") + if len(addr) != 2 { + return errors.New("invalid Mail Address") + } + host := addr[0] + msg := []byte("To:" + strings.Join(mail.To, ";") + + "\r\nFrom: " + mail.From + + "\r\nSubject: " + subject + + "\r\nContent-Type: text/plain\r\n\r\n" + + message) + auth := smtp.PlainAuth( + "", + mail.Username, + mail.Password, + host, + ) + err := smtp.SendMail( + mail.Addr, + auth, + mail.From, + mail.To, + msg, + ) + if err != nil { + return err + } + return nil +} diff --git a/vendor/github.com/docker/distribution/registry/handlers/manifests.go b/vendor/github.com/docker/distribution/registry/handlers/manifests.go new file mode 100644 index 000000000..0d8b65886 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/handlers/manifests.go @@ -0,0 +1,531 @@ +package handlers + +import ( + "bytes" + "fmt" + "net/http" + "strings" + + "github.com/distribution/reference" + "github.com/docker/distribution" + dcontext "github.com/docker/distribution/context" + "github.com/docker/distribution/manifest/manifestlist" + "github.com/docker/distribution/manifest/ocischema" + "github.com/docker/distribution/manifest/schema1" + "github.com/docker/distribution/manifest/schema2" + "github.com/docker/distribution/registry/api/errcode" + v2 "github.com/docker/distribution/registry/api/v2" + "github.com/docker/distribution/registry/auth" + "github.com/gorilla/handlers" + "github.com/opencontainers/go-digest" + v1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +// These constants determine which architecture and OS to choose from a +// manifest list when downconverting it to a schema1 manifest. +const ( + defaultArch = "amd64" + defaultOS = "linux" + maxManifestBodySize = 4 << 20 + imageClass = "image" +) + +type storageType int + +const ( + manifestSchema1 storageType = iota // 0 + manifestSchema2 // 1 + manifestlistSchema // 2 + ociSchema // 3 + ociImageIndexSchema // 4 + numStorageTypes // 5 +) + +// manifestDispatcher takes the request context and builds the +// appropriate handler for handling manifest requests. +func manifestDispatcher(ctx *Context, r *http.Request) http.Handler { + manifestHandler := &manifestHandler{ + Context: ctx, + } + reference := getReference(ctx) + dgst, err := digest.Parse(reference) + if err != nil { + // We just have a tag + manifestHandler.Tag = reference + } else { + manifestHandler.Digest = dgst + } + + mhandler := handlers.MethodHandler{ + "GET": http.HandlerFunc(manifestHandler.GetManifest), + "HEAD": http.HandlerFunc(manifestHandler.GetManifest), + } + + if !ctx.readOnly { + mhandler["PUT"] = http.HandlerFunc(manifestHandler.PutManifest) + mhandler["DELETE"] = http.HandlerFunc(manifestHandler.DeleteManifest) + } + + return mhandler +} + +// manifestHandler handles http operations on image manifests. +type manifestHandler struct { + *Context + + // One of tag or digest gets set, depending on what is present in context. + Tag string + Digest digest.Digest +} + +// GetManifest fetches the image manifest from the storage backend, if it exists. +func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) { + dcontext.GetLogger(imh).Debug("GetImageManifest") + manifests, err := imh.Repository.Manifests(imh) + if err != nil { + imh.Errors = append(imh.Errors, err) + return + } + var supports [numStorageTypes]bool + + // this parsing of Accept headers is not quite as full-featured as godoc.org's parser, but we don't care about "q=" values + // https://github.com/golang/gddo/blob/e91d4165076d7474d20abda83f92d15c7ebc3e81/httputil/header/header.go#L165-L202 + for _, acceptHeader := range r.Header["Accept"] { + // r.Header[...] is a slice in case the request contains the same header more than once + // if the header isn't set, we'll get the zero value, which "range" will handle gracefully + + // we need to split each header value on "," to get the full list of "Accept" values (per RFC 2616) + // https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.1 + for _, mediaType := range strings.Split(acceptHeader, ",") { + // remove "; q=..." if present + if i := strings.Index(mediaType, ";"); i >= 0 { + mediaType = mediaType[:i] + } + + // it's common (but not required) for Accept values to be space separated ("a/b, c/d, e/f") + mediaType = strings.TrimSpace(mediaType) + + if mediaType == schema2.MediaTypeManifest { + supports[manifestSchema2] = true + } + if mediaType == manifestlist.MediaTypeManifestList { + supports[manifestlistSchema] = true + } + if mediaType == v1.MediaTypeImageManifest { + supports[ociSchema] = true + } + if mediaType == v1.MediaTypeImageIndex { + supports[ociImageIndexSchema] = true + } + } + } + + if imh.Tag != "" { + tags := imh.Repository.Tags(imh) + desc, err := tags.Get(imh, imh.Tag) + if err != nil { + if _, ok := err.(distribution.ErrTagUnknown); ok { + imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err)) + } else { + imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + } + return + } + imh.Digest = desc.Digest + } + + if etagMatch(r, imh.Digest.String()) { + w.WriteHeader(http.StatusNotModified) + return + } + + var options []distribution.ManifestServiceOption + if imh.Tag != "" { + options = append(options, distribution.WithTag(imh.Tag)) + } + manifest, err := manifests.Get(imh, imh.Digest, options...) + if err != nil { + if _, ok := err.(distribution.ErrManifestUnknownRevision); ok { + imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err)) + } else { + imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + } + return + } + // determine the type of the returned manifest + manifestType := manifestSchema1 + schema2Manifest, isSchema2 := manifest.(*schema2.DeserializedManifest) + manifestList, isManifestList := manifest.(*manifestlist.DeserializedManifestList) + if isSchema2 { + manifestType = manifestSchema2 + } else if _, isOCImanifest := manifest.(*ocischema.DeserializedManifest); isOCImanifest { + manifestType = ociSchema + } else if isManifestList { + if manifestList.MediaType == manifestlist.MediaTypeManifestList { + manifestType = manifestlistSchema + } else if manifestList.MediaType == v1.MediaTypeImageIndex { + manifestType = ociImageIndexSchema + } + } + + if manifestType == ociSchema && !supports[ociSchema] { + imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithMessage("OCI manifest found, but accept header does not support OCI manifests")) + return + } + if manifestType == ociImageIndexSchema && !supports[ociImageIndexSchema] { + imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithMessage("OCI index found, but accept header does not support OCI indexes")) + return + } + // Only rewrite schema2 manifests when they are being fetched by tag. + // If they are being fetched by digest, we can't return something not + // matching the digest. + if imh.Tag != "" && manifestType == manifestSchema2 && !supports[manifestSchema2] { + // Rewrite manifest in schema1 format + dcontext.GetLogger(imh).Infof("rewriting manifest %s in schema1 format to support old client", imh.Digest.String()) + + manifest, err = imh.convertSchema2Manifest(schema2Manifest) + if err != nil { + return + } + } else if imh.Tag != "" && manifestType == manifestlistSchema && !supports[manifestlistSchema] { + // Rewrite manifest in schema1 format + dcontext.GetLogger(imh).Infof("rewriting manifest list %s in schema1 format to support old client", imh.Digest.String()) + + // Find the image manifest corresponding to the default + // platform + var manifestDigest digest.Digest + for _, manifestDescriptor := range manifestList.Manifests { + if manifestDescriptor.Platform.Architecture == defaultArch && manifestDescriptor.Platform.OS == defaultOS { + manifestDigest = manifestDescriptor.Digest + break + } + } + + if manifestDigest == "" { + imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown) + return + } + + manifest, err = manifests.Get(imh, manifestDigest) + if err != nil { + if _, ok := err.(distribution.ErrManifestUnknownRevision); ok { + imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err)) + } else { + imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + } + return + } + + // If necessary, convert the image manifest + if schema2Manifest, isSchema2 := manifest.(*schema2.DeserializedManifest); isSchema2 && !supports[manifestSchema2] { + manifest, err = imh.convertSchema2Manifest(schema2Manifest) + if err != nil { + return + } + } else { + imh.Digest = manifestDigest + } + } + + ct, p, err := manifest.Payload() + if err != nil { + return + } + + w.Header().Set("Content-Type", ct) + w.Header().Set("Content-Length", fmt.Sprint(len(p))) + w.Header().Set("Docker-Content-Digest", imh.Digest.String()) + w.Header().Set("Etag", fmt.Sprintf(`"%s"`, imh.Digest)) + w.Write(p) +} + +func (imh *manifestHandler) convertSchema2Manifest(schema2Manifest *schema2.DeserializedManifest) (distribution.Manifest, error) { + targetDescriptor := schema2Manifest.Target() + blobs := imh.Repository.Blobs(imh) + configJSON, err := blobs.Get(imh, targetDescriptor.Digest) + if err != nil { + if err == distribution.ErrBlobUnknown { + imh.Errors = append(imh.Errors, v2.ErrorCodeManifestInvalid.WithDetail(err)) + } else { + imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + } + return nil, err + } + + ref := imh.Repository.Named() + + if imh.Tag != "" { + ref, err = reference.WithTag(ref, imh.Tag) + if err != nil { + imh.Errors = append(imh.Errors, v2.ErrorCodeTagInvalid.WithDetail(err)) + return nil, err + } + } + + builder := schema1.NewConfigManifestBuilder(imh.Repository.Blobs(imh), imh.Context.App.trustKey, ref, configJSON) + for _, d := range schema2Manifest.Layers { + if err := builder.AppendReference(d); err != nil { + imh.Errors = append(imh.Errors, v2.ErrorCodeManifestInvalid.WithDetail(err)) + return nil, err + } + } + manifest, err := builder.Build(imh) + if err != nil { + imh.Errors = append(imh.Errors, v2.ErrorCodeManifestInvalid.WithDetail(err)) + return nil, err + } + imh.Digest = digest.FromBytes(manifest.(*schema1.SignedManifest).Canonical) + + return manifest, nil +} + +func etagMatch(r *http.Request, etag string) bool { + for _, headerVal := range r.Header["If-None-Match"] { + if headerVal == etag || headerVal == fmt.Sprintf(`"%s"`, etag) { // allow quoted or unquoted + return true + } + } + return false +} + +// PutManifest validates and stores a manifest in the registry. +func (imh *manifestHandler) PutManifest(w http.ResponseWriter, r *http.Request) { + dcontext.GetLogger(imh).Debug("PutImageManifest") + manifests, err := imh.Repository.Manifests(imh) + if err != nil { + imh.Errors = append(imh.Errors, err) + return + } + + var jsonBuf bytes.Buffer + if err := copyFullPayload(imh, w, r, &jsonBuf, maxManifestBodySize, "image manifest PUT"); err != nil { + // copyFullPayload reports the error if necessary + imh.Errors = append(imh.Errors, v2.ErrorCodeManifestInvalid.WithDetail(err.Error())) + return + } + + mediaType := r.Header.Get("Content-Type") + manifest, desc, err := distribution.UnmarshalManifest(mediaType, jsonBuf.Bytes()) + if err != nil { + imh.Errors = append(imh.Errors, v2.ErrorCodeManifestInvalid.WithDetail(err)) + return + } + + if imh.Digest != "" { + if desc.Digest != imh.Digest { + dcontext.GetLogger(imh).Errorf("payload digest does match: %q != %q", desc.Digest, imh.Digest) + imh.Errors = append(imh.Errors, v2.ErrorCodeDigestInvalid) + return + } + } else if imh.Tag != "" { + imh.Digest = desc.Digest + } else { + imh.Errors = append(imh.Errors, v2.ErrorCodeTagInvalid.WithDetail("no tag or digest specified")) + return + } + + isAnOCIManifest := mediaType == v1.MediaTypeImageManifest || mediaType == v1.MediaTypeImageIndex + + if isAnOCIManifest { + dcontext.GetLogger(imh).Debug("Putting an OCI Manifest!") + } else { + dcontext.GetLogger(imh).Debug("Putting a Docker Manifest!") + } + + var options []distribution.ManifestServiceOption + if imh.Tag != "" { + options = append(options, distribution.WithTag(imh.Tag)) + } + + if err := imh.applyResourcePolicy(manifest); err != nil { + imh.Errors = append(imh.Errors, err) + return + } + + _, err = manifests.Put(imh, manifest, options...) + if err != nil { + // TODO(stevvooe): These error handling switches really need to be + // handled by an app global mapper. + if err == distribution.ErrUnsupported { + imh.Errors = append(imh.Errors, errcode.ErrorCodeUnsupported) + return + } + if err == distribution.ErrAccessDenied { + imh.Errors = append(imh.Errors, errcode.ErrorCodeDenied) + return + } + switch err := err.(type) { + case distribution.ErrManifestVerification: + for _, verificationError := range err { + switch verificationError := verificationError.(type) { + case distribution.ErrManifestBlobUnknown: + imh.Errors = append(imh.Errors, v2.ErrorCodeManifestBlobUnknown.WithDetail(verificationError.Digest)) + case distribution.ErrManifestNameInvalid: + imh.Errors = append(imh.Errors, v2.ErrorCodeNameInvalid.WithDetail(err)) + case distribution.ErrManifestUnverified: + imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnverified) + default: + if verificationError == digest.ErrDigestInvalidFormat { + imh.Errors = append(imh.Errors, v2.ErrorCodeDigestInvalid) + } else { + imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown, verificationError) + } + } + } + case errcode.Error: + imh.Errors = append(imh.Errors, err) + default: + imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + } + return + } + + // Tag this manifest + if imh.Tag != "" { + tags := imh.Repository.Tags(imh) + err = tags.Tag(imh, imh.Tag, desc) + if err != nil { + imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + return + } + + } + + // Construct a canonical url for the uploaded manifest. + ref, err := reference.WithDigest(imh.Repository.Named(), imh.Digest) + if err != nil { + imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + return + } + + location, err := imh.urlBuilder.BuildManifestURL(ref) + if err != nil { + // NOTE(stevvooe): Given the behavior above, this absurdly unlikely to + // happen. We'll log the error here but proceed as if it worked. Worst + // case, we set an empty location header. + dcontext.GetLogger(imh).Errorf("error building manifest url from digest: %v", err) + } + + w.Header().Set("Location", location) + w.Header().Set("Docker-Content-Digest", imh.Digest.String()) + w.WriteHeader(http.StatusCreated) + + dcontext.GetLogger(imh).Debug("Succeeded in putting manifest!") +} + +// applyResourcePolicy checks whether the resource class matches what has +// been authorized and allowed by the policy configuration. +func (imh *manifestHandler) applyResourcePolicy(manifest distribution.Manifest) error { + allowedClasses := imh.App.Config.Policy.Repository.Classes + if len(allowedClasses) == 0 { + return nil + } + + var class string + switch m := manifest.(type) { + case *schema1.SignedManifest: + class = imageClass + case *schema2.DeserializedManifest: + switch m.Config.MediaType { + case schema2.MediaTypeImageConfig: + class = imageClass + case schema2.MediaTypePluginConfig: + class = "plugin" + default: + return errcode.ErrorCodeDenied.WithMessage("unknown manifest class for " + m.Config.MediaType) + } + case *ocischema.DeserializedManifest: + switch m.Config.MediaType { + case v1.MediaTypeImageConfig: + class = imageClass + default: + return errcode.ErrorCodeDenied.WithMessage("unknown manifest class for " + m.Config.MediaType) + } + } + + if class == "" { + return nil + } + + // Check to see if class is allowed in registry + var allowedClass bool + for _, c := range allowedClasses { + if class == c { + allowedClass = true + break + } + } + if !allowedClass { + return errcode.ErrorCodeDenied.WithMessage(fmt.Sprintf("registry does not allow %s manifest", class)) + } + + resources := auth.AuthorizedResources(imh) + n := imh.Repository.Named().Name() + + var foundResource bool + for _, r := range resources { + if r.Name == n { + if r.Class == "" { + r.Class = imageClass + } + if r.Class == class { + return nil + } + foundResource = true + } + } + + // resource was found but no matching class was found + if foundResource { + return errcode.ErrorCodeDenied.WithMessage(fmt.Sprintf("repository not authorized for %s manifest", class)) + } + + return nil + +} + +// DeleteManifest removes the manifest with the given digest from the registry. +func (imh *manifestHandler) DeleteManifest(w http.ResponseWriter, r *http.Request) { + dcontext.GetLogger(imh).Debug("DeleteImageManifest") + + manifests, err := imh.Repository.Manifests(imh) + if err != nil { + imh.Errors = append(imh.Errors, err) + return + } + + err = manifests.Delete(imh, imh.Digest) + if err != nil { + switch err { + case digest.ErrDigestUnsupported: + case digest.ErrDigestInvalidFormat: + imh.Errors = append(imh.Errors, v2.ErrorCodeDigestInvalid) + return + case distribution.ErrBlobUnknown: + imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown) + return + case distribution.ErrUnsupported: + imh.Errors = append(imh.Errors, errcode.ErrorCodeUnsupported) + return + default: + imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown) + return + } + } + + tagService := imh.Repository.Tags(imh) + referencedTags, err := tagService.Lookup(imh, distribution.Descriptor{Digest: imh.Digest}) + if err != nil { + imh.Errors = append(imh.Errors, err) + return + } + + for _, tag := range referencedTags { + if err := tagService.Untag(imh, tag); err != nil { + imh.Errors = append(imh.Errors, err) + return + } + } + + w.WriteHeader(http.StatusAccepted) +} diff --git a/vendor/github.com/docker/distribution/registry/handlers/tags.go b/vendor/github.com/docker/distribution/registry/handlers/tags.go new file mode 100644 index 000000000..9d91cb9b1 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/handlers/tags.go @@ -0,0 +1,60 @@ +package handlers + +import ( + "encoding/json" + "net/http" + + "github.com/docker/distribution" + "github.com/docker/distribution/registry/api/errcode" + v2 "github.com/docker/distribution/registry/api/v2" + "github.com/gorilla/handlers" +) + +// tagsDispatcher constructs the tags handler api endpoint. +func tagsDispatcher(ctx *Context, r *http.Request) http.Handler { + tagsHandler := &tagsHandler{ + Context: ctx, + } + + return handlers.MethodHandler{ + "GET": http.HandlerFunc(tagsHandler.GetTags), + } +} + +// tagsHandler handles requests for lists of tags under a repository name. +type tagsHandler struct { + *Context +} + +type tagsAPIResponse struct { + Name string `json:"name"` + Tags []string `json:"tags"` +} + +// GetTags returns a json list of tags for a specific image name. +func (th *tagsHandler) GetTags(w http.ResponseWriter, r *http.Request) { + tagService := th.Repository.Tags(th) + tags, err := tagService.All(th) + if err != nil { + switch err := err.(type) { + case distribution.ErrRepositoryUnknown: + th.Errors = append(th.Errors, v2.ErrorCodeNameUnknown.WithDetail(map[string]string{"name": th.Repository.Named().Name()})) + case errcode.Error: + th.Errors = append(th.Errors, err) + default: + th.Errors = append(th.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + } + return + } + + w.Header().Set("Content-Type", "application/json; charset=utf-8") + + enc := json.NewEncoder(w) + if err := enc.Encode(tagsAPIResponse{ + Name: th.Repository.Named().Name(), + Tags: tags, + }); err != nil { + th.Errors = append(th.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + return + } +} diff --git a/vendor/github.com/docker/distribution/registry/listener/listener.go b/vendor/github.com/docker/distribution/registry/listener/listener.go new file mode 100644 index 000000000..b93a7a63f --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/listener/listener.go @@ -0,0 +1,74 @@ +package listener + +import ( + "fmt" + "net" + "os" + "time" +) + +// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted +// connections. It's used by ListenAndServe and ListenAndServeTLS so +// dead TCP connections (e.g. closing laptop mid-download) eventually +// go away. +// it is a plain copy-paste from net/http/server.go +type tcpKeepAliveListener struct { + *net.TCPListener +} + +func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) { + tc, err := ln.AcceptTCP() + if err != nil { + return + } + tc.SetKeepAlive(true) + tc.SetKeepAlivePeriod(3 * time.Minute) + return tc, nil +} + +// NewListener announces on laddr and net. Accepted values of the net are +// 'unix' and 'tcp' +func NewListener(net, laddr string) (net.Listener, error) { + switch net { + case "unix": + return newUnixListener(laddr) + case "tcp", "": // an empty net means tcp + return newTCPListener(laddr) + default: + return nil, fmt.Errorf("unknown address type %s", net) + } +} + +func newUnixListener(laddr string) (net.Listener, error) { + fi, err := os.Stat(laddr) + if err == nil { + // the file exists. + // try to remove it if it's a socket + if !isSocket(fi.Mode()) { + return nil, fmt.Errorf("file %s exists and is not a socket", laddr) + } + + if err := os.Remove(laddr); err != nil { + return nil, err + } + } else if !os.IsNotExist(err) { + // we can't do stat on the file. + // it means we can not remove it + return nil, err + } + + return net.Listen("unix", laddr) +} + +func isSocket(m os.FileMode) bool { + return m&os.ModeSocket != 0 +} + +func newTCPListener(laddr string) (net.Listener, error) { + ln, err := net.Listen("tcp", laddr) + if err != nil { + return nil, err + } + + return tcpKeepAliveListener{ln.(*net.TCPListener)}, nil +} diff --git a/vendor/github.com/docker/distribution/registry/middleware/registry/middleware.go b/vendor/github.com/docker/distribution/registry/middleware/registry/middleware.go new file mode 100644 index 000000000..49defd825 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/middleware/registry/middleware.go @@ -0,0 +1,54 @@ +package middleware + +import ( + "context" + "fmt" + + "github.com/docker/distribution" + "github.com/docker/distribution/registry/storage" +) + +// InitFunc is the type of a RegistryMiddleware factory function and is +// used to register the constructor for different RegistryMiddleware backends. +type InitFunc func(ctx context.Context, registry distribution.Namespace, options map[string]interface{}) (distribution.Namespace, error) + +var middlewares map[string]InitFunc +var registryoptions []storage.RegistryOption + +// Register is used to register an InitFunc for +// a RegistryMiddleware backend with the given name. +func Register(name string, initFunc InitFunc) error { + if middlewares == nil { + middlewares = make(map[string]InitFunc) + } + if _, exists := middlewares[name]; exists { + return fmt.Errorf("name already registered: %s", name) + } + + middlewares[name] = initFunc + + return nil +} + +// Get constructs a RegistryMiddleware with the given options using the named backend. +func Get(ctx context.Context, name string, options map[string]interface{}, registry distribution.Namespace) (distribution.Namespace, error) { + if middlewares != nil { + if initFunc, exists := middlewares[name]; exists { + return initFunc(ctx, registry, options) + } + } + + return nil, fmt.Errorf("no registry middleware registered with name: %s", name) +} + +// RegisterOptions adds more options to RegistryOption list. Options get applied before +// any other configuration-based options. +func RegisterOptions(options ...storage.RegistryOption) error { + registryoptions = append(registryoptions, options...) + return nil +} + +// GetRegistryOptions returns list of RegistryOption. +func GetRegistryOptions() []storage.RegistryOption { + return registryoptions +} diff --git a/vendor/github.com/docker/distribution/registry/middleware/repository/middleware.go b/vendor/github.com/docker/distribution/registry/middleware/repository/middleware.go new file mode 100644 index 000000000..f1554b709 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/middleware/repository/middleware.go @@ -0,0 +1,40 @@ +package middleware + +import ( + "context" + "fmt" + + "github.com/docker/distribution" +) + +// InitFunc is the type of a RepositoryMiddleware factory function and is +// used to register the constructor for different RepositoryMiddleware backends. +type InitFunc func(ctx context.Context, repository distribution.Repository, options map[string]interface{}) (distribution.Repository, error) + +var middlewares map[string]InitFunc + +// Register is used to register an InitFunc for +// a RepositoryMiddleware backend with the given name. +func Register(name string, initFunc InitFunc) error { + if middlewares == nil { + middlewares = make(map[string]InitFunc) + } + if _, exists := middlewares[name]; exists { + return fmt.Errorf("name already registered: %s", name) + } + + middlewares[name] = initFunc + + return nil +} + +// Get constructs a RepositoryMiddleware with the given options using the named backend. +func Get(ctx context.Context, name string, options map[string]interface{}, repository distribution.Repository) (distribution.Repository, error) { + if middlewares != nil { + if initFunc, exists := middlewares[name]; exists { + return initFunc(ctx, repository, options) + } + } + + return nil, fmt.Errorf("no repository middleware registered with name: %s", name) +} diff --git a/vendor/github.com/docker/distribution/registry/proxy/proxyauth.go b/vendor/github.com/docker/distribution/registry/proxy/proxyauth.go new file mode 100644 index 000000000..09a0973e6 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/proxy/proxyauth.go @@ -0,0 +1,83 @@ +package proxy + +import ( + "net/http" + "net/url" + "strings" + + "github.com/docker/distribution/context" + "github.com/docker/distribution/registry/client/auth" + "github.com/docker/distribution/registry/client/auth/challenge" +) + +const challengeHeader = "Docker-Distribution-Api-Version" + +type userpass struct { + username string + password string +} + +type credentials struct { + creds map[string]userpass +} + +func (c credentials) Basic(u *url.URL) (string, string) { + up := c.creds[u.String()] + + return up.username, up.password +} + +func (c credentials) RefreshToken(u *url.URL, service string) string { + return "" +} + +func (c credentials) SetRefreshToken(u *url.URL, service, token string) { +} + +// configureAuth stores credentials for challenge responses +func configureAuth(username, password, remoteURL string) (auth.CredentialStore, error) { + creds := map[string]userpass{} + + authURLs, err := getAuthURLs(remoteURL) + if err != nil { + return nil, err + } + + for _, url := range authURLs { + context.GetLogger(context.Background()).Infof("Discovered token authentication URL: %s", url) + creds[url] = userpass{ + username: username, + password: password, + } + } + + return credentials{creds: creds}, nil +} + +func getAuthURLs(remoteURL string) ([]string, error) { + authURLs := []string{} + + resp, err := http.Get(remoteURL + "/v2/") + if err != nil { + return nil, err + } + defer resp.Body.Close() + + for _, c := range challenge.ResponseChallenges(resp) { + if strings.EqualFold(c.Scheme, "bearer") { + authURLs = append(authURLs, c.Parameters["realm"]) + } + } + + return authURLs, nil +} + +func ping(manager challenge.Manager, endpoint, versionHeader string) error { + resp, err := http.Get(endpoint) + if err != nil { + return err + } + defer resp.Body.Close() + + return manager.AddResponse(resp) +} diff --git a/vendor/github.com/docker/distribution/registry/proxy/proxyblobstore.go b/vendor/github.com/docker/distribution/registry/proxy/proxyblobstore.go new file mode 100644 index 000000000..a9219957f --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/proxy/proxyblobstore.go @@ -0,0 +1,221 @@ +package proxy + +import ( + "context" + "io" + "net/http" + "strconv" + "sync" + + "github.com/distribution/reference" + "github.com/docker/distribution" + dcontext "github.com/docker/distribution/context" + "github.com/docker/distribution/registry/proxy/scheduler" + "github.com/opencontainers/go-digest" +) + +type proxyBlobStore struct { + localStore distribution.BlobStore + remoteStore distribution.BlobService + scheduler *scheduler.TTLExpirationScheduler + repositoryName reference.Named + authChallenger authChallenger +} + +var _ distribution.BlobStore = &proxyBlobStore{} + +// inflight tracks currently downloading blobs +var inflight = make(map[digest.Digest]struct{}) + +// mu protects inflight +var mu sync.Mutex + +func setResponseHeaders(w http.ResponseWriter, length int64, mediaType string, digest digest.Digest) { + w.Header().Set("Content-Length", strconv.FormatInt(length, 10)) + w.Header().Set("Content-Type", mediaType) + w.Header().Set("Docker-Content-Digest", digest.String()) + w.Header().Set("Etag", digest.String()) +} + +func (pbs *proxyBlobStore) copyContent(ctx context.Context, dgst digest.Digest, writer io.Writer) (distribution.Descriptor, error) { + desc, err := pbs.remoteStore.Stat(ctx, dgst) + if err != nil { + return distribution.Descriptor{}, err + } + + if w, ok := writer.(http.ResponseWriter); ok { + setResponseHeaders(w, desc.Size, desc.MediaType, dgst) + } + + remoteReader, err := pbs.remoteStore.Open(ctx, dgst) + if err != nil { + return distribution.Descriptor{}, err + } + + defer remoteReader.Close() + + _, err = io.CopyN(writer, remoteReader, desc.Size) + if err != nil { + return distribution.Descriptor{}, err + } + + proxyMetrics.BlobPush(uint64(desc.Size)) + + return desc, nil +} + +func (pbs *proxyBlobStore) serveLocal(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) (bool, error) { + localDesc, err := pbs.localStore.Stat(ctx, dgst) + if err != nil { + // Stat can report a zero sized file here if it's checked between creation + // and population. Return nil error, and continue + return false, nil + } + + if err == nil { + proxyMetrics.BlobPush(uint64(localDesc.Size)) + return true, pbs.localStore.ServeBlob(ctx, w, r, dgst) + } + + return false, nil + +} + +func (pbs *proxyBlobStore) storeLocal(ctx context.Context, dgst digest.Digest) error { + defer func() { + mu.Lock() + delete(inflight, dgst) + mu.Unlock() + }() + + var desc distribution.Descriptor + var err error + var bw distribution.BlobWriter + + bw, err = pbs.localStore.Create(ctx) + if err != nil { + return err + } + + desc, err = pbs.copyContent(ctx, dgst, bw) + if err != nil { + return err + } + + _, err = bw.Commit(ctx, desc) + if err != nil { + return err + } + + return nil +} + +func (pbs *proxyBlobStore) ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) error { + served, err := pbs.serveLocal(ctx, w, r, dgst) + if err != nil { + dcontext.GetLogger(ctx).Errorf("Error serving blob from local storage: %s", err.Error()) + return err + } + + if served { + return nil + } + + if err := pbs.authChallenger.tryEstablishChallenges(ctx); err != nil { + return err + } + + mu.Lock() + _, ok := inflight[dgst] + if ok { + mu.Unlock() + _, err := pbs.copyContent(ctx, dgst, w) + return err + } + inflight[dgst] = struct{}{} + mu.Unlock() + + go func(dgst digest.Digest) { + if err := pbs.storeLocal(ctx, dgst); err != nil { + dcontext.GetLogger(ctx).Errorf("Error committing to storage: %s", err.Error()) + } + + blobRef, err := reference.WithDigest(pbs.repositoryName, dgst) + if err != nil { + dcontext.GetLogger(ctx).Errorf("Error creating reference: %s", err) + return + } + + pbs.scheduler.AddBlob(blobRef, repositoryTTL) + }(dgst) + + _, err = pbs.copyContent(ctx, dgst, w) + if err != nil { + return err + } + return nil +} + +func (pbs *proxyBlobStore) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { + desc, err := pbs.localStore.Stat(ctx, dgst) + if err == nil { + return desc, err + } + + if err != distribution.ErrBlobUnknown { + return distribution.Descriptor{}, err + } + + if err := pbs.authChallenger.tryEstablishChallenges(ctx); err != nil { + return distribution.Descriptor{}, err + } + + return pbs.remoteStore.Stat(ctx, dgst) +} + +func (pbs *proxyBlobStore) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) { + blob, err := pbs.localStore.Get(ctx, dgst) + if err == nil { + return blob, nil + } + + if err := pbs.authChallenger.tryEstablishChallenges(ctx); err != nil { + return []byte{}, err + } + + blob, err = pbs.remoteStore.Get(ctx, dgst) + if err != nil { + return []byte{}, err + } + + _, err = pbs.localStore.Put(ctx, "", blob) + if err != nil { + return []byte{}, err + } + return blob, nil +} + +// Unsupported functions +func (pbs *proxyBlobStore) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) { + return distribution.Descriptor{}, distribution.ErrUnsupported +} + +func (pbs *proxyBlobStore) Create(ctx context.Context, options ...distribution.BlobCreateOption) (distribution.BlobWriter, error) { + return nil, distribution.ErrUnsupported +} + +func (pbs *proxyBlobStore) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) { + return nil, distribution.ErrUnsupported +} + +func (pbs *proxyBlobStore) Mount(ctx context.Context, sourceRepo reference.Named, dgst digest.Digest) (distribution.Descriptor, error) { + return distribution.Descriptor{}, distribution.ErrUnsupported +} + +func (pbs *proxyBlobStore) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) { + return nil, distribution.ErrUnsupported +} + +func (pbs *proxyBlobStore) Delete(ctx context.Context, dgst digest.Digest) error { + return distribution.ErrUnsupported +} diff --git a/vendor/github.com/docker/distribution/registry/proxy/proxymanifeststore.go b/vendor/github.com/docker/distribution/registry/proxy/proxymanifeststore.go new file mode 100644 index 000000000..30714fbec --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/proxy/proxymanifeststore.go @@ -0,0 +1,96 @@ +package proxy + +import ( + "context" + "time" + + "github.com/distribution/reference" + "github.com/docker/distribution" + dcontext "github.com/docker/distribution/context" + "github.com/docker/distribution/registry/proxy/scheduler" + "github.com/opencontainers/go-digest" +) + +// todo(richardscothern): from cache control header or config +const repositoryTTL = 24 * 7 * time.Hour + +type proxyManifestStore struct { + ctx context.Context + localManifests distribution.ManifestService + remoteManifests distribution.ManifestService + repositoryName reference.Named + scheduler *scheduler.TTLExpirationScheduler + authChallenger authChallenger +} + +var _ distribution.ManifestService = &proxyManifestStore{} + +func (pms proxyManifestStore) Exists(ctx context.Context, dgst digest.Digest) (bool, error) { + exists, err := pms.localManifests.Exists(ctx, dgst) + if err != nil { + return false, err + } + if exists { + return true, nil + } + if err := pms.authChallenger.tryEstablishChallenges(ctx); err != nil { + return false, err + } + return pms.remoteManifests.Exists(ctx, dgst) +} + +func (pms proxyManifestStore) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) { + // At this point `dgst` was either specified explicitly, or returned by the + // tagstore with the most recent association. + var fromRemote bool + manifest, err := pms.localManifests.Get(ctx, dgst, options...) + if err != nil { + if err := pms.authChallenger.tryEstablishChallenges(ctx); err != nil { + return nil, err + } + + manifest, err = pms.remoteManifests.Get(ctx, dgst, options...) + if err != nil { + return nil, err + } + fromRemote = true + } + + _, payload, err := manifest.Payload() + if err != nil { + return nil, err + } + + proxyMetrics.ManifestPush(uint64(len(payload))) + if fromRemote { + proxyMetrics.ManifestPull(uint64(len(payload))) + + _, err = pms.localManifests.Put(ctx, manifest) + if err != nil { + return nil, err + } + + // Schedule the manifest blob for removal + repoBlob, err := reference.WithDigest(pms.repositoryName, dgst) + if err != nil { + dcontext.GetLogger(ctx).Errorf("Error creating reference: %s", err) + return nil, err + } + + pms.scheduler.AddManifest(repoBlob, repositoryTTL) + // Ensure the manifest blob is cleaned up + //pms.scheduler.AddBlob(blobRef, repositoryTTL) + + } + + return manifest, err +} + +func (pms proxyManifestStore) Put(ctx context.Context, manifest distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) { + var d digest.Digest + return d, distribution.ErrUnsupported +} + +func (pms proxyManifestStore) Delete(ctx context.Context, dgst digest.Digest) error { + return distribution.ErrUnsupported +} diff --git a/vendor/github.com/docker/distribution/registry/proxy/proxymetrics.go b/vendor/github.com/docker/distribution/registry/proxy/proxymetrics.go new file mode 100644 index 000000000..d3d84d786 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/proxy/proxymetrics.go @@ -0,0 +1,74 @@ +package proxy + +import ( + "expvar" + "sync/atomic" +) + +// Metrics is used to hold metric counters +// related to the proxy +type Metrics struct { + Requests uint64 + Hits uint64 + Misses uint64 + BytesPulled uint64 + BytesPushed uint64 +} + +type proxyMetricsCollector struct { + blobMetrics Metrics + manifestMetrics Metrics +} + +// BlobPull tracks metrics about blobs pulled into the cache +func (pmc *proxyMetricsCollector) BlobPull(bytesPulled uint64) { + atomic.AddUint64(&pmc.blobMetrics.Misses, 1) + atomic.AddUint64(&pmc.blobMetrics.BytesPulled, bytesPulled) +} + +// BlobPush tracks metrics about blobs pushed to clients +func (pmc *proxyMetricsCollector) BlobPush(bytesPushed uint64) { + atomic.AddUint64(&pmc.blobMetrics.Requests, 1) + atomic.AddUint64(&pmc.blobMetrics.Hits, 1) + atomic.AddUint64(&pmc.blobMetrics.BytesPushed, bytesPushed) +} + +// ManifestPull tracks metrics related to Manifests pulled into the cache +func (pmc *proxyMetricsCollector) ManifestPull(bytesPulled uint64) { + atomic.AddUint64(&pmc.manifestMetrics.Misses, 1) + atomic.AddUint64(&pmc.manifestMetrics.BytesPulled, bytesPulled) +} + +// ManifestPush tracks metrics about manifests pushed to clients +func (pmc *proxyMetricsCollector) ManifestPush(bytesPushed uint64) { + atomic.AddUint64(&pmc.manifestMetrics.Requests, 1) + atomic.AddUint64(&pmc.manifestMetrics.Hits, 1) + atomic.AddUint64(&pmc.manifestMetrics.BytesPushed, bytesPushed) +} + +// proxyMetrics tracks metrics about the proxy cache. This is +// kept globally and made available via expvar. +var proxyMetrics = &proxyMetricsCollector{} + +func init() { + registry := expvar.Get("registry") + if registry == nil { + registry = expvar.NewMap("registry") + } + + pm := registry.(*expvar.Map).Get("proxy") + if pm == nil { + pm = &expvar.Map{} + pm.(*expvar.Map).Init() + registry.(*expvar.Map).Set("proxy", pm) + } + + pm.(*expvar.Map).Set("blobs", expvar.Func(func() interface{} { + return proxyMetrics.blobMetrics + })) + + pm.(*expvar.Map).Set("manifests", expvar.Func(func() interface{} { + return proxyMetrics.manifestMetrics + })) + +} diff --git a/vendor/github.com/docker/distribution/registry/proxy/proxyregistry.go b/vendor/github.com/docker/distribution/registry/proxy/proxyregistry.go new file mode 100644 index 000000000..8e5506eda --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/proxy/proxyregistry.go @@ -0,0 +1,263 @@ +package proxy + +import ( + "context" + "fmt" + "net/http" + "net/url" + "sync" + + "github.com/distribution/reference" + "github.com/docker/distribution" + "github.com/docker/distribution/configuration" + dcontext "github.com/docker/distribution/context" + "github.com/docker/distribution/registry/client" + "github.com/docker/distribution/registry/client/auth" + "github.com/docker/distribution/registry/client/auth/challenge" + "github.com/docker/distribution/registry/client/transport" + "github.com/docker/distribution/registry/proxy/scheduler" + "github.com/docker/distribution/registry/storage" + "github.com/docker/distribution/registry/storage/driver" +) + +// proxyingRegistry fetches content from a remote registry and caches it locally +type proxyingRegistry struct { + embedded distribution.Namespace // provides local registry functionality + scheduler *scheduler.TTLExpirationScheduler + remoteURL url.URL + authChallenger authChallenger +} + +// NewRegistryPullThroughCache creates a registry acting as a pull through cache +func NewRegistryPullThroughCache(ctx context.Context, registry distribution.Namespace, driver driver.StorageDriver, config configuration.Proxy) (distribution.Namespace, error) { + remoteURL, err := url.Parse(config.RemoteURL) + if err != nil { + return nil, err + } + + v := storage.NewVacuum(ctx, driver) + s := scheduler.New(ctx, driver, "/scheduler-state.json") + s.OnBlobExpire(func(ref reference.Reference) error { + var r reference.Canonical + var ok bool + if r, ok = ref.(reference.Canonical); !ok { + return fmt.Errorf("unexpected reference type : %T", ref) + } + + repo, err := registry.Repository(ctx, r) + if err != nil { + return err + } + + blobs := repo.Blobs(ctx) + + // Clear the repository reference and descriptor caches + err = blobs.Delete(ctx, r.Digest()) + if err != nil { + return err + } + + err = v.RemoveBlob(r.Digest().String()) + if err != nil { + return err + } + + return nil + }) + + s.OnManifestExpire(func(ref reference.Reference) error { + var r reference.Canonical + var ok bool + if r, ok = ref.(reference.Canonical); !ok { + return fmt.Errorf("unexpected reference type : %T", ref) + } + + repo, err := registry.Repository(ctx, r) + if err != nil { + return err + } + + manifests, err := repo.Manifests(ctx) + if err != nil { + return err + } + err = manifests.Delete(ctx, r.Digest()) + if err != nil { + return err + } + return nil + }) + + err = s.Start() + if err != nil { + return nil, err + } + + cs, err := configureAuth(config.Username, config.Password, config.RemoteURL) + if err != nil { + return nil, err + } + + return &proxyingRegistry{ + embedded: registry, + scheduler: s, + remoteURL: *remoteURL, + authChallenger: &remoteAuthChallenger{ + remoteURL: *remoteURL, + cm: challenge.NewSimpleManager(), + cs: cs, + }, + }, nil +} + +func (pr *proxyingRegistry) Scope() distribution.Scope { + return distribution.GlobalScope +} + +func (pr *proxyingRegistry) Repositories(ctx context.Context, repos []string, last string) (n int, err error) { + return pr.embedded.Repositories(ctx, repos, last) +} + +func (pr *proxyingRegistry) Repository(ctx context.Context, name reference.Named) (distribution.Repository, error) { + c := pr.authChallenger + + tkopts := auth.TokenHandlerOptions{ + Transport: http.DefaultTransport, + Credentials: c.credentialStore(), + Scopes: []auth.Scope{ + auth.RepositoryScope{ + Repository: name.Name(), + Actions: []string{"pull"}, + }, + }, + Logger: dcontext.GetLogger(ctx), + } + + tr := transport.NewTransport(http.DefaultTransport, + auth.NewAuthorizer(c.challengeManager(), + auth.NewTokenHandlerWithOptions(tkopts))) + + localRepo, err := pr.embedded.Repository(ctx, name) + if err != nil { + return nil, err + } + localManifests, err := localRepo.Manifests(ctx, storage.SkipLayerVerification()) + if err != nil { + return nil, err + } + + remoteRepo, err := client.NewRepository(name, pr.remoteURL.String(), tr) + if err != nil { + return nil, err + } + + remoteManifests, err := remoteRepo.Manifests(ctx) + if err != nil { + return nil, err + } + + return &proxiedRepository{ + blobStore: &proxyBlobStore{ + localStore: localRepo.Blobs(ctx), + remoteStore: remoteRepo.Blobs(ctx), + scheduler: pr.scheduler, + repositoryName: name, + authChallenger: pr.authChallenger, + }, + manifests: &proxyManifestStore{ + repositoryName: name, + localManifests: localManifests, // Options? + remoteManifests: remoteManifests, + ctx: ctx, + scheduler: pr.scheduler, + authChallenger: pr.authChallenger, + }, + name: name, + tags: &proxyTagService{ + localTags: localRepo.Tags(ctx), + remoteTags: remoteRepo.Tags(ctx), + authChallenger: pr.authChallenger, + }, + }, nil +} + +func (pr *proxyingRegistry) Blobs() distribution.BlobEnumerator { + return pr.embedded.Blobs() +} + +func (pr *proxyingRegistry) BlobStatter() distribution.BlobStatter { + return pr.embedded.BlobStatter() +} + +// authChallenger encapsulates a request to the upstream to establish credential challenges +type authChallenger interface { + tryEstablishChallenges(context.Context) error + challengeManager() challenge.Manager + credentialStore() auth.CredentialStore +} + +type remoteAuthChallenger struct { + remoteURL url.URL + sync.Mutex + cm challenge.Manager + cs auth.CredentialStore +} + +func (r *remoteAuthChallenger) credentialStore() auth.CredentialStore { + return r.cs +} + +func (r *remoteAuthChallenger) challengeManager() challenge.Manager { + return r.cm +} + +// tryEstablishChallenges will attempt to get a challenge type for the upstream if none currently exist +func (r *remoteAuthChallenger) tryEstablishChallenges(ctx context.Context) error { + r.Lock() + defer r.Unlock() + + remoteURL := r.remoteURL + remoteURL.Path = "/v2/" + challenges, err := r.cm.GetChallenges(remoteURL) + if err != nil { + return err + } + + if len(challenges) > 0 { + return nil + } + + // establish challenge type with upstream + if err := ping(r.cm, remoteURL.String(), challengeHeader); err != nil { + return err + } + + dcontext.GetLogger(ctx).Infof("Challenge established with upstream : %s %s", remoteURL, r.cm) + return nil +} + +// proxiedRepository uses proxying blob and manifest services to serve content +// locally, or pulling it through from a remote and caching it locally if it doesn't +// already exist +type proxiedRepository struct { + blobStore distribution.BlobStore + manifests distribution.ManifestService + name reference.Named + tags distribution.TagService +} + +func (pr *proxiedRepository) Manifests(ctx context.Context, options ...distribution.ManifestServiceOption) (distribution.ManifestService, error) { + return pr.manifests, nil +} + +func (pr *proxiedRepository) Blobs(ctx context.Context) distribution.BlobStore { + return pr.blobStore +} + +func (pr *proxiedRepository) Named() reference.Named { + return pr.name +} + +func (pr *proxiedRepository) Tags(ctx context.Context) distribution.TagService { + return pr.tags +} diff --git a/vendor/github.com/docker/distribution/registry/proxy/proxytagservice.go b/vendor/github.com/docker/distribution/registry/proxy/proxytagservice.go new file mode 100644 index 000000000..6a9256395 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/proxy/proxytagservice.go @@ -0,0 +1,66 @@ +package proxy + +import ( + "context" + + "github.com/docker/distribution" +) + +// proxyTagService supports local and remote lookup of tags. +type proxyTagService struct { + localTags distribution.TagService + remoteTags distribution.TagService + authChallenger authChallenger +} + +var _ distribution.TagService = proxyTagService{} + +// Get attempts to get the most recent digest for the tag by checking the remote +// tag service first and then caching it locally. If the remote is unavailable +// the local association is returned +func (pt proxyTagService) Get(ctx context.Context, tag string) (distribution.Descriptor, error) { + err := pt.authChallenger.tryEstablishChallenges(ctx) + if err == nil { + desc, err := pt.remoteTags.Get(ctx, tag) + if err == nil { + err := pt.localTags.Tag(ctx, tag, desc) + if err != nil { + return distribution.Descriptor{}, err + } + return desc, nil + } + } + + desc, err := pt.localTags.Get(ctx, tag) + if err != nil { + return distribution.Descriptor{}, err + } + return desc, nil +} + +func (pt proxyTagService) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error { + return distribution.ErrUnsupported +} + +func (pt proxyTagService) Untag(ctx context.Context, tag string) error { + err := pt.localTags.Untag(ctx, tag) + if err != nil { + return err + } + return nil +} + +func (pt proxyTagService) All(ctx context.Context) ([]string, error) { + err := pt.authChallenger.tryEstablishChallenges(ctx) + if err == nil { + tags, err := pt.remoteTags.All(ctx) + if err == nil { + return tags, err + } + } + return pt.localTags.All(ctx) +} + +func (pt proxyTagService) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) { + return []string{}, distribution.ErrUnsupported +} diff --git a/vendor/github.com/docker/distribution/registry/proxy/scheduler/scheduler.go b/vendor/github.com/docker/distribution/registry/proxy/scheduler/scheduler.go new file mode 100644 index 000000000..5673a1102 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/proxy/scheduler/scheduler.go @@ -0,0 +1,260 @@ +package scheduler + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "time" + + "github.com/distribution/reference" + dcontext "github.com/docker/distribution/context" + "github.com/docker/distribution/registry/storage/driver" +) + +// onTTLExpiryFunc is called when a repository's TTL expires +type expiryFunc func(reference.Reference) error + +const ( + entryTypeBlob = iota + entryTypeManifest + indexSaveFrequency = 5 * time.Second +) + +// schedulerEntry represents an entry in the scheduler +// fields are exported for serialization +type schedulerEntry struct { + Key string `json:"Key"` + Expiry time.Time `json:"ExpiryData"` + EntryType int `json:"EntryType"` + + timer *time.Timer +} + +// New returns a new instance of the scheduler +func New(ctx context.Context, driver driver.StorageDriver, path string) *TTLExpirationScheduler { + return &TTLExpirationScheduler{ + entries: make(map[string]*schedulerEntry), + driver: driver, + pathToStateFile: path, + ctx: ctx, + stopped: true, + doneChan: make(chan struct{}), + saveTimer: time.NewTicker(indexSaveFrequency), + } +} + +// TTLExpirationScheduler is a scheduler used to perform actions +// when TTLs expire +type TTLExpirationScheduler struct { + sync.Mutex + + entries map[string]*schedulerEntry + + driver driver.StorageDriver + ctx context.Context + pathToStateFile string + + stopped bool + + onBlobExpire expiryFunc + onManifestExpire expiryFunc + + indexDirty bool + saveTimer *time.Ticker + doneChan chan struct{} +} + +// OnBlobExpire is called when a scheduled blob's TTL expires +func (ttles *TTLExpirationScheduler) OnBlobExpire(f expiryFunc) { + ttles.Lock() + defer ttles.Unlock() + + ttles.onBlobExpire = f +} + +// OnManifestExpire is called when a scheduled manifest's TTL expires +func (ttles *TTLExpirationScheduler) OnManifestExpire(f expiryFunc) { + ttles.Lock() + defer ttles.Unlock() + + ttles.onManifestExpire = f +} + +// AddBlob schedules a blob cleanup after ttl expires +func (ttles *TTLExpirationScheduler) AddBlob(blobRef reference.Canonical, ttl time.Duration) error { + ttles.Lock() + defer ttles.Unlock() + + if ttles.stopped { + return fmt.Errorf("scheduler not started") + } + + ttles.add(blobRef, ttl, entryTypeBlob) + return nil +} + +// AddManifest schedules a manifest cleanup after ttl expires +func (ttles *TTLExpirationScheduler) AddManifest(manifestRef reference.Canonical, ttl time.Duration) error { + ttles.Lock() + defer ttles.Unlock() + + if ttles.stopped { + return fmt.Errorf("scheduler not started") + } + + ttles.add(manifestRef, ttl, entryTypeManifest) + return nil +} + +// Start starts the scheduler +func (ttles *TTLExpirationScheduler) Start() error { + ttles.Lock() + defer ttles.Unlock() + + err := ttles.readState() + if err != nil { + return err + } + + if !ttles.stopped { + return fmt.Errorf("scheduler already started") + } + + dcontext.GetLogger(ttles.ctx).Infof("Starting cached object TTL expiration scheduler...") + ttles.stopped = false + + // Start timer for each deserialized entry + for _, entry := range ttles.entries { + entry.timer = ttles.startTimer(entry, time.Until(entry.Expiry)) + } + + // Start a ticker to periodically save the entries index + + go func() { + for { + select { + case <-ttles.saveTimer.C: + ttles.Lock() + if !ttles.indexDirty { + ttles.Unlock() + continue + } + + err := ttles.writeState() + if err != nil { + dcontext.GetLogger(ttles.ctx).Errorf("Error writing scheduler state: %s", err) + } else { + ttles.indexDirty = false + } + ttles.Unlock() + + case <-ttles.doneChan: + return + } + } + }() + + return nil +} + +func (ttles *TTLExpirationScheduler) add(r reference.Reference, ttl time.Duration, eType int) { + entry := &schedulerEntry{ + Key: r.String(), + Expiry: time.Now().Add(ttl), + EntryType: eType, + } + dcontext.GetLogger(ttles.ctx).Infof("Adding new scheduler entry for %s with ttl=%s", entry.Key, time.Until(entry.Expiry)) + if oldEntry, present := ttles.entries[entry.Key]; present && oldEntry.timer != nil { + oldEntry.timer.Stop() + } + ttles.entries[entry.Key] = entry + entry.timer = ttles.startTimer(entry, ttl) + ttles.indexDirty = true +} + +func (ttles *TTLExpirationScheduler) startTimer(entry *schedulerEntry, ttl time.Duration) *time.Timer { + return time.AfterFunc(ttl, func() { + ttles.Lock() + defer ttles.Unlock() + + var f expiryFunc + + switch entry.EntryType { + case entryTypeBlob: + f = ttles.onBlobExpire + case entryTypeManifest: + f = ttles.onManifestExpire + default: + f = func(reference.Reference) error { + return fmt.Errorf("scheduler entry type") + } + } + + ref, err := reference.Parse(entry.Key) + if err == nil { + if err := f(ref); err != nil { + dcontext.GetLogger(ttles.ctx).Errorf("Scheduler error returned from OnExpire(%s): %s", entry.Key, err) + } + } else { + dcontext.GetLogger(ttles.ctx).Errorf("Error unpacking reference: %s", err) + } + + delete(ttles.entries, entry.Key) + ttles.indexDirty = true + }) +} + +// Stop stops the scheduler. +func (ttles *TTLExpirationScheduler) Stop() { + ttles.Lock() + defer ttles.Unlock() + + if err := ttles.writeState(); err != nil { + dcontext.GetLogger(ttles.ctx).Errorf("Error writing scheduler state: %s", err) + } + + for _, entry := range ttles.entries { + entry.timer.Stop() + } + + close(ttles.doneChan) + ttles.saveTimer.Stop() + ttles.stopped = true +} + +func (ttles *TTLExpirationScheduler) writeState() error { + jsonBytes, err := json.Marshal(ttles.entries) + if err != nil { + return err + } + + err = ttles.driver.PutContent(ttles.ctx, ttles.pathToStateFile, jsonBytes) + if err != nil { + return err + } + + return nil +} + +func (ttles *TTLExpirationScheduler) readState() error { + if _, err := ttles.driver.Stat(ttles.ctx, ttles.pathToStateFile); err != nil { + switch err := err.(type) { + case driver.PathNotFoundError: + return nil + default: + return err + } + } + + bytes, err := ttles.driver.GetContent(ttles.ctx, ttles.pathToStateFile) + if err != nil { + return err + } + + err = json.Unmarshal(bytes, &ttles.entries) + if err != nil { + return err + } + return nil +} diff --git a/vendor/github.com/docker/distribution/registry/storage/blobcachemetrics.go b/vendor/github.com/docker/distribution/registry/storage/blobcachemetrics.go new file mode 100644 index 000000000..238b5806c --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/blobcachemetrics.go @@ -0,0 +1,66 @@ +package storage + +import ( + "context" + "expvar" + "sync/atomic" + + dcontext "github.com/docker/distribution/context" + "github.com/docker/distribution/registry/storage/cache" +) + +type blobStatCollector struct { + metrics cache.Metrics +} + +func (bsc *blobStatCollector) Hit() { + atomic.AddUint64(&bsc.metrics.Requests, 1) + atomic.AddUint64(&bsc.metrics.Hits, 1) +} + +func (bsc *blobStatCollector) Miss() { + atomic.AddUint64(&bsc.metrics.Requests, 1) + atomic.AddUint64(&bsc.metrics.Misses, 1) +} + +func (bsc *blobStatCollector) Metrics() cache.Metrics { + return bsc.metrics +} + +func (bsc *blobStatCollector) Logger(ctx context.Context) cache.Logger { + return dcontext.GetLogger(ctx) +} + +// blobStatterCacheMetrics keeps track of cache metrics for blob descriptor +// cache requests. Note this is kept globally and made available via expvar. +// For more detailed metrics, its recommend to instrument a particular cache +// implementation. +var blobStatterCacheMetrics cache.MetricsTracker = &blobStatCollector{} + +func init() { + registry := expvar.Get("registry") + if registry == nil { + registry = expvar.NewMap("registry") + } + + cache := registry.(*expvar.Map).Get("cache") + if cache == nil { + cache = &expvar.Map{} + cache.(*expvar.Map).Init() + registry.(*expvar.Map).Set("cache", cache) + } + + storage := cache.(*expvar.Map).Get("storage") + if storage == nil { + storage = &expvar.Map{} + storage.(*expvar.Map).Init() + cache.(*expvar.Map).Set("storage", storage) + } + + storage.(*expvar.Map).Set("blobdescriptor", expvar.Func(func() interface{} { + // no need for synchronous access: the increments are atomic and + // during reading, we don't care if the data is up to date. The + // numbers will always *eventually* be reported correctly. + return blobStatterCacheMetrics + })) +} diff --git a/vendor/github.com/docker/distribution/registry/storage/blobserver.go b/vendor/github.com/docker/distribution/registry/storage/blobserver.go new file mode 100644 index 000000000..57d4f907b --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/blobserver.go @@ -0,0 +1,78 @@ +package storage + +import ( + "context" + "fmt" + "net/http" + "time" + + "github.com/docker/distribution" + "github.com/docker/distribution/registry/storage/driver" + "github.com/opencontainers/go-digest" +) + +// TODO(stevvooe): This should configurable in the future. +const blobCacheControlMaxAge = 365 * 24 * time.Hour + +// blobServer simply serves blobs from a driver instance using a path function +// to identify paths and a descriptor service to fill in metadata. +type blobServer struct { + driver driver.StorageDriver + statter distribution.BlobStatter + pathFn func(dgst digest.Digest) (string, error) + redirect bool // allows disabling URLFor redirects +} + +func (bs *blobServer) ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) error { + desc, err := bs.statter.Stat(ctx, dgst) + if err != nil { + return err + } + + path, err := bs.pathFn(desc.Digest) + if err != nil { + return err + } + + if bs.redirect { + redirectURL, err := bs.driver.URLFor(ctx, path, map[string]interface{}{"method": r.Method}) + switch err.(type) { + case nil: + // Redirect to storage URL. + http.Redirect(w, r, redirectURL, http.StatusTemporaryRedirect) + return err + + case driver.ErrUnsupportedMethod: + // Fallback to serving the content directly. + default: + // Some unexpected error. + return err + } + } + + br, err := newFileReader(ctx, bs.driver, path, desc.Size) + if err != nil { + return err + } + defer br.Close() + + w.Header().Set("ETag", fmt.Sprintf(`"%s"`, desc.Digest)) // If-None-Match handled by ServeContent + w.Header().Set("Cache-Control", fmt.Sprintf("max-age=%.f", blobCacheControlMaxAge.Seconds())) + + if w.Header().Get("Docker-Content-Digest") == "" { + w.Header().Set("Docker-Content-Digest", desc.Digest.String()) + } + + if w.Header().Get("Content-Type") == "" { + // Set the content type if not already set. + w.Header().Set("Content-Type", desc.MediaType) + } + + if w.Header().Get("Content-Length") == "" { + // Set the content length if not already set. + w.Header().Set("Content-Length", fmt.Sprint(desc.Size)) + } + + http.ServeContent(w, r, desc.Digest.String(), time.Time{}, br) + return nil +} diff --git a/vendor/github.com/docker/distribution/registry/storage/blobstore.go b/vendor/github.com/docker/distribution/registry/storage/blobstore.go new file mode 100644 index 000000000..1008aad8c --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/blobstore.go @@ -0,0 +1,212 @@ +package storage + +import ( + "context" + "path" + + "github.com/docker/distribution" + dcontext "github.com/docker/distribution/context" + "github.com/docker/distribution/registry/storage/driver" + "github.com/opencontainers/go-digest" +) + +// blobStore implements the read side of the blob store interface over a +// driver without enforcing per-repository membership. This object is +// intentionally a leaky abstraction, providing utility methods that support +// creating and traversing backend links. +type blobStore struct { + driver driver.StorageDriver + statter distribution.BlobStatter +} + +var _ distribution.BlobProvider = &blobStore{} + +// Get implements the BlobReadService.Get call. +func (bs *blobStore) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) { + bp, err := bs.path(dgst) + if err != nil { + return nil, err + } + + p, err := getContent(ctx, bs.driver, bp) + if err != nil { + switch err.(type) { + case driver.PathNotFoundError: + return nil, distribution.ErrBlobUnknown + } + + return nil, err + } + + return p, nil +} + +func (bs *blobStore) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) { + desc, err := bs.statter.Stat(ctx, dgst) + if err != nil { + return nil, err + } + + path, err := bs.path(desc.Digest) + if err != nil { + return nil, err + } + + return newFileReader(ctx, bs.driver, path, desc.Size) +} + +// Put stores the content p in the blob store, calculating the digest. If the +// content is already present, only the digest will be returned. This should +// only be used for small objects, such as manifests. This implemented as a convenience for other Put implementations +func (bs *blobStore) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) { + dgst := digest.FromBytes(p) + desc, err := bs.statter.Stat(ctx, dgst) + if err == nil { + // content already present + return desc, nil + } else if err != distribution.ErrBlobUnknown { + dcontext.GetLogger(ctx).Errorf("blobStore: error stating content (%v): %v", dgst, err) + // real error, return it + return distribution.Descriptor{}, err + } + + bp, err := bs.path(dgst) + if err != nil { + return distribution.Descriptor{}, err + } + + // TODO(stevvooe): Write out mediatype here, as well. + return distribution.Descriptor{ + Size: int64(len(p)), + + // NOTE(stevvooe): The central blob store firewalls media types from + // other users. The caller should look this up and override the value + // for the specific repository. + MediaType: "application/octet-stream", + Digest: dgst, + }, bs.driver.PutContent(ctx, bp, p) +} + +func (bs *blobStore) Enumerate(ctx context.Context, ingester func(dgst digest.Digest) error) error { + specPath, err := pathFor(blobsPathSpec{}) + if err != nil { + return err + } + + return bs.driver.Walk(ctx, specPath, func(fileInfo driver.FileInfo) error { + // skip directories + if fileInfo.IsDir() { + return nil + } + + currentPath := fileInfo.Path() + // we only want to parse paths that end with /data + _, fileName := path.Split(currentPath) + if fileName != "data" { + return nil + } + + digest, err := digestFromPath(currentPath) + if err != nil { + return err + } + + return ingester(digest) + }) +} + +// path returns the canonical path for the blob identified by digest. The blob +// may or may not exist. +func (bs *blobStore) path(dgst digest.Digest) (string, error) { + bp, err := pathFor(blobDataPathSpec{ + digest: dgst, + }) + + if err != nil { + return "", err + } + + return bp, nil +} + +// link links the path to the provided digest by writing the digest into the +// target file. Caller must ensure that the blob actually exists. +func (bs *blobStore) link(ctx context.Context, path string, dgst digest.Digest) error { + // The contents of the "link" file are the exact string contents of the + // digest, which is specified in that package. + return bs.driver.PutContent(ctx, path, []byte(dgst)) +} + +// readlink returns the linked digest at path. +func (bs *blobStore) readlink(ctx context.Context, path string) (digest.Digest, error) { + content, err := bs.driver.GetContent(ctx, path) + if err != nil { + return "", err + } + + linked, err := digest.Parse(string(content)) + if err != nil { + return "", err + } + + return linked, nil +} + +type blobStatter struct { + driver driver.StorageDriver +} + +var _ distribution.BlobDescriptorService = &blobStatter{} + +// Stat implements BlobStatter.Stat by returning the descriptor for the blob +// in the main blob store. If this method returns successfully, there is +// strong guarantee that the blob exists and is available. +func (bs *blobStatter) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { + path, err := pathFor(blobDataPathSpec{ + digest: dgst, + }) + + if err != nil { + return distribution.Descriptor{}, err + } + + fi, err := bs.driver.Stat(ctx, path) + if err != nil { + switch err := err.(type) { + case driver.PathNotFoundError: + return distribution.Descriptor{}, distribution.ErrBlobUnknown + default: + return distribution.Descriptor{}, err + } + } + + if fi.IsDir() { + // NOTE(stevvooe): This represents a corruption situation. Somehow, we + // calculated a blob path and then detected a directory. We log the + // error and then error on the side of not knowing about the blob. + dcontext.GetLogger(ctx).Warnf("blob path should not be a directory: %q", path) + return distribution.Descriptor{}, distribution.ErrBlobUnknown + } + + // TODO(stevvooe): Add method to resolve the mediatype. We can store and + // cache a "global" media type for the blob, even if a specific repo has a + // mediatype that overrides the main one. + + return distribution.Descriptor{ + Size: fi.Size(), + + // NOTE(stevvooe): The central blob store firewalls media types from + // other users. The caller should look this up and override the value + // for the specific repository. + MediaType: "application/octet-stream", + Digest: dgst, + }, nil +} + +func (bs *blobStatter) Clear(ctx context.Context, dgst digest.Digest) error { + return distribution.ErrUnsupported +} + +func (bs *blobStatter) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error { + return distribution.ErrUnsupported +} diff --git a/vendor/github.com/docker/distribution/registry/storage/blobwriter.go b/vendor/github.com/docker/distribution/registry/storage/blobwriter.go new file mode 100644 index 000000000..54c79b6f3 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/blobwriter.go @@ -0,0 +1,406 @@ +package storage + +import ( + "context" + "errors" + "fmt" + "io" + "path" + "time" + + "github.com/docker/distribution" + dcontext "github.com/docker/distribution/context" + storagedriver "github.com/docker/distribution/registry/storage/driver" + "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" +) + +var ( + errResumableDigestNotAvailable = errors.New("resumable digest not available") +) + +const ( + // digestSha256Empty is the canonical sha256 digest of empty data + digestSha256Empty = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" +) + +// blobWriter is used to control the various aspects of resumable +// blob upload. +type blobWriter struct { + ctx context.Context + blobStore *linkedBlobStore + + id string + startedAt time.Time + digester digest.Digester + written int64 // track the write to digester + + fileWriter storagedriver.FileWriter + driver storagedriver.StorageDriver + path string + + resumableDigestEnabled bool + committed bool +} + +var _ distribution.BlobWriter = &blobWriter{} + +// ID returns the identifier for this upload. +func (bw *blobWriter) ID() string { + return bw.id +} + +func (bw *blobWriter) StartedAt() time.Time { + return bw.startedAt +} + +// Commit marks the upload as completed, returning a valid descriptor. The +// final size and digest are checked against the first descriptor provided. +func (bw *blobWriter) Commit(ctx context.Context, desc distribution.Descriptor) (distribution.Descriptor, error) { + dcontext.GetLogger(ctx).Debug("(*blobWriter).Commit") + + if err := bw.fileWriter.Commit(); err != nil { + return distribution.Descriptor{}, err + } + + bw.Close() + desc.Size = bw.Size() + + canonical, err := bw.validateBlob(ctx, desc) + if err != nil { + return distribution.Descriptor{}, err + } + + if err := bw.moveBlob(ctx, canonical); err != nil { + return distribution.Descriptor{}, err + } + + if err := bw.blobStore.linkBlob(ctx, canonical, desc.Digest); err != nil { + return distribution.Descriptor{}, err + } + + if err := bw.removeResources(ctx); err != nil { + return distribution.Descriptor{}, err + } + + err = bw.blobStore.blobAccessController.SetDescriptor(ctx, canonical.Digest, canonical) + if err != nil { + return distribution.Descriptor{}, err + } + + bw.committed = true + return canonical, nil +} + +// Cancel the blob upload process, releasing any resources associated with +// the writer and canceling the operation. +func (bw *blobWriter) Cancel(ctx context.Context) error { + dcontext.GetLogger(ctx).Debug("(*blobWriter).Cancel") + if err := bw.fileWriter.Cancel(); err != nil { + return err + } + + if err := bw.Close(); err != nil { + dcontext.GetLogger(ctx).Errorf("error closing blobwriter: %s", err) + } + + return bw.removeResources(ctx) +} + +func (bw *blobWriter) Size() int64 { + return bw.fileWriter.Size() +} + +func (bw *blobWriter) Write(p []byte) (int, error) { + // Ensure that the current write offset matches how many bytes have been + // written to the digester. If not, we need to update the digest state to + // match the current write position. + if err := bw.resumeDigest(bw.blobStore.ctx); err != nil && err != errResumableDigestNotAvailable { + return 0, err + } + + _, err := bw.fileWriter.Write(p) + if err != nil { + return 0, err + } + + n, err := bw.digester.Hash().Write(p) + bw.written += int64(n) + + return n, err +} + +func (bw *blobWriter) ReadFrom(r io.Reader) (n int64, err error) { + // Ensure that the current write offset matches how many bytes have been + // written to the digester. If not, we need to update the digest state to + // match the current write position. + if err := bw.resumeDigest(bw.blobStore.ctx); err != nil && err != errResumableDigestNotAvailable { + return 0, err + } + + // Using a TeeReader instead of MultiWriter ensures Copy returns + // the amount written to the digester as well as ensuring that we + // write to the fileWriter first + tee := io.TeeReader(r, bw.fileWriter) + nn, err := io.Copy(bw.digester.Hash(), tee) + bw.written += nn + + return nn, err +} + +func (bw *blobWriter) Close() error { + if bw.committed { + return errors.New("blobwriter close after commit") + } + + if err := bw.storeHashState(bw.blobStore.ctx); err != nil && err != errResumableDigestNotAvailable { + return err + } + + return bw.fileWriter.Close() +} + +// validateBlob checks the data against the digest, returning an error if it +// does not match. The canonical descriptor is returned. +func (bw *blobWriter) validateBlob(ctx context.Context, desc distribution.Descriptor) (distribution.Descriptor, error) { + var ( + verified, fullHash bool + canonical digest.Digest + ) + + if desc.Digest == "" { + // if no descriptors are provided, we have nothing to validate + // against. We don't really want to support this for the registry. + return distribution.Descriptor{}, distribution.ErrBlobInvalidDigest{ + Reason: fmt.Errorf("cannot validate against empty digest"), + } + } + + var size int64 + + // Stat the on disk file + if fi, err := bw.driver.Stat(ctx, bw.path); err != nil { + switch err := err.(type) { + case storagedriver.PathNotFoundError: + // NOTE(stevvooe): We really don't care if the file is + // not actually present for the reader. We now assume + // that the desc length is zero. + desc.Size = 0 + default: + // Any other error we want propagated up the stack. + return distribution.Descriptor{}, err + } + } else { + if fi.IsDir() { + return distribution.Descriptor{}, fmt.Errorf("unexpected directory at upload location %q", bw.path) + } + + size = fi.Size() + } + + if desc.Size > 0 { + if desc.Size != size { + return distribution.Descriptor{}, distribution.ErrBlobInvalidLength + } + } else { + // if provided 0 or negative length, we can assume caller doesn't know or + // care about length. + desc.Size = size + } + + // TODO(stevvooe): This section is very meandering. Need to be broken down + // to be a lot more clear. + + if err := bw.resumeDigest(ctx); err == nil { + canonical = bw.digester.Digest() + + if canonical.Algorithm() == desc.Digest.Algorithm() { + // Common case: client and server prefer the same canonical digest + // algorithm - currently SHA256. + verified = desc.Digest == canonical + } else { + // The client wants to use a different digest algorithm. They'll just + // have to be patient and wait for us to download and re-hash the + // uploaded content using that digest algorithm. + fullHash = true + } + } else if err == errResumableDigestNotAvailable { + // Not using resumable digests, so we need to hash the entire layer. + fullHash = true + } else { + return distribution.Descriptor{}, err + } + + if fullHash { + // a fantastic optimization: if the the written data and the size are + // the same, we don't need to read the data from the backend. This is + // because we've written the entire file in the lifecycle of the + // current instance. + if bw.written == size && digest.Canonical == desc.Digest.Algorithm() { + canonical = bw.digester.Digest() + verified = desc.Digest == canonical + } + + // If the check based on size fails, we fall back to the slowest of + // paths. We may be able to make the size-based check a stronger + // guarantee, so this may be defensive. + if !verified { + digester := digest.Canonical.Digester() + verifier := desc.Digest.Verifier() + + // Read the file from the backend driver and validate it. + fr, err := newFileReader(ctx, bw.driver, bw.path, desc.Size) + if err != nil { + return distribution.Descriptor{}, err + } + defer fr.Close() + + tr := io.TeeReader(fr, digester.Hash()) + + if _, err := io.Copy(verifier, tr); err != nil { + return distribution.Descriptor{}, err + } + + canonical = digester.Digest() + verified = verifier.Verified() + } + } + + if !verified { + dcontext.GetLoggerWithFields(ctx, + map[interface{}]interface{}{ + "canonical": canonical, + "provided": desc.Digest, + }, "canonical", "provided"). + Errorf("canonical digest does match provided digest") + return distribution.Descriptor{}, distribution.ErrBlobInvalidDigest{ + Digest: desc.Digest, + Reason: fmt.Errorf("content does not match digest"), + } + } + + // update desc with canonical hash + desc.Digest = canonical + + if desc.MediaType == "" { + desc.MediaType = "application/octet-stream" + } + + return desc, nil +} + +// moveBlob moves the data into its final, hash-qualified destination, +// identified by dgst. The layer should be validated before commencing the +// move. +func (bw *blobWriter) moveBlob(ctx context.Context, desc distribution.Descriptor) error { + blobPath, err := pathFor(blobDataPathSpec{ + digest: desc.Digest, + }) + + if err != nil { + return err + } + + // Check for existence + if _, err := bw.blobStore.driver.Stat(ctx, blobPath); err != nil { + switch err := err.(type) { + case storagedriver.PathNotFoundError: + break // ensure that it doesn't exist. + default: + return err + } + } else { + // If the path exists, we can assume that the content has already + // been uploaded, since the blob storage is content-addressable. + // While it may be corrupted, detection of such corruption belongs + // elsewhere. + return nil + } + + // If no data was received, we may not actually have a file on disk. Check + // the size here and write a zero-length file to blobPath if this is the + // case. For the most part, this should only ever happen with zero-length + // blobs. + if _, err := bw.blobStore.driver.Stat(ctx, bw.path); err != nil { + switch err := err.(type) { + case storagedriver.PathNotFoundError: + // HACK(stevvooe): This is slightly dangerous: if we verify above, + // get a hash, then the underlying file is deleted, we risk moving + // a zero-length blob into a nonzero-length blob location. To + // prevent this horrid thing, we employ the hack of only allowing + // to this happen for the digest of an empty blob. + if desc.Digest == digestSha256Empty { + return bw.blobStore.driver.PutContent(ctx, blobPath, []byte{}) + } + + // We let this fail during the move below. + logrus. + WithField("upload.id", bw.ID()). + WithField("digest", desc.Digest).Warnf("attempted to move zero-length content with non-zero digest") + default: + return err // unrelated error + } + } + + // TODO(stevvooe): We should also write the mediatype when executing this move. + + return bw.blobStore.driver.Move(ctx, bw.path, blobPath) +} + +// removeResources should clean up all resources associated with the upload +// instance. An error will be returned if the clean up cannot proceed. If the +// resources are already not present, no error will be returned. +func (bw *blobWriter) removeResources(ctx context.Context) error { + dataPath, err := pathFor(uploadDataPathSpec{ + name: bw.blobStore.repository.Named().Name(), + id: bw.id, + }) + + if err != nil { + return err + } + + // Resolve and delete the containing directory, which should include any + // upload related files. + dirPath := path.Dir(dataPath) + if err := bw.blobStore.driver.Delete(ctx, dirPath); err != nil { + switch err := err.(type) { + case storagedriver.PathNotFoundError: + break // already gone! + default: + // This should be uncommon enough such that returning an error + // should be okay. At this point, the upload should be mostly + // complete, but perhaps the backend became unaccessible. + dcontext.GetLogger(ctx).Errorf("unable to delete layer upload resources %q: %v", dirPath, err) + return err + } + } + + return nil +} + +func (bw *blobWriter) Reader() (io.ReadCloser, error) { + // todo(richardscothern): Change to exponential backoff, i=0.5, e=2, n=4 + try := 1 + for try <= 5 { + _, err := bw.driver.Stat(bw.ctx, bw.path) + if err == nil { + break + } + switch err.(type) { + case storagedriver.PathNotFoundError: + dcontext.GetLogger(bw.ctx).Debugf("Nothing found on try %d, sleeping...", try) + time.Sleep(1 * time.Second) + try++ + default: + return nil, err + } + } + + readCloser, err := bw.driver.Reader(bw.ctx, bw.path, 0) + if err != nil { + return nil, err + } + + return readCloser, nil +} diff --git a/vendor/github.com/docker/distribution/registry/storage/blobwriter_nonresumable.go b/vendor/github.com/docker/distribution/registry/storage/blobwriter_nonresumable.go new file mode 100644 index 000000000..38267cf60 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/blobwriter_nonresumable.go @@ -0,0 +1,18 @@ +//go:build noresumabledigest +// +build noresumabledigest + +package storage + +import ( + "github.com/docker/distribution/context" +) + +// resumeHashAt is a noop when resumable digest support is disabled. +func (bw *blobWriter) resumeDigest(ctx context.Context) error { + return errResumableDigestNotAvailable +} + +// storeHashState is a noop when resumable digest support is disabled. +func (bw *blobWriter) storeHashState(ctx context.Context) error { + return errResumableDigestNotAvailable +} diff --git a/vendor/github.com/docker/distribution/registry/storage/blobwriter_resumable.go b/vendor/github.com/docker/distribution/registry/storage/blobwriter_resumable.go new file mode 100644 index 000000000..8406cfd1f --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/blobwriter_resumable.go @@ -0,0 +1,145 @@ +//go:build !noresumabledigest +// +build !noresumabledigest + +package storage + +import ( + "context" + "encoding" + "fmt" + "hash" + "path" + "strconv" + + storagedriver "github.com/docker/distribution/registry/storage/driver" + "github.com/sirupsen/logrus" +) + +// resumeDigest attempts to restore the state of the internal hash function +// by loading the most recent saved hash state equal to the current size of the blob. +func (bw *blobWriter) resumeDigest(ctx context.Context) error { + if !bw.resumableDigestEnabled { + return errResumableDigestNotAvailable + } + + h, ok := bw.digester.Hash().(encoding.BinaryUnmarshaler) + if !ok { + return errResumableDigestNotAvailable + } + + offset := bw.fileWriter.Size() + if offset == bw.written { + // State of digester is already at the requested offset. + return nil + } + + // List hash states from storage backend. + var hashStateMatch hashStateEntry + hashStates, err := bw.getStoredHashStates(ctx) + if err != nil { + return fmt.Errorf("unable to get stored hash states with offset %d: %s", offset, err) + } + + // Find the highest stored hashState with offset equal to + // the requested offset. + for _, hashState := range hashStates { + if hashState.offset == offset { + hashStateMatch = hashState + break // Found an exact offset match. + } + } + + if hashStateMatch.offset == 0 { + // No need to load any state, just reset the hasher. + h.(hash.Hash).Reset() + } else { + storedState, err := bw.driver.GetContent(ctx, hashStateMatch.path) + if err != nil { + return err + } + + if err = h.UnmarshalBinary(storedState); err != nil { + return err + } + bw.written = hashStateMatch.offset + } + + // Mind the gap. + if gapLen := offset - bw.written; gapLen > 0 { + return errResumableDigestNotAvailable + } + + return nil +} + +type hashStateEntry struct { + offset int64 + path string +} + +// getStoredHashStates returns a slice of hashStateEntries for this upload. +func (bw *blobWriter) getStoredHashStates(ctx context.Context) ([]hashStateEntry, error) { + uploadHashStatePathPrefix, err := pathFor(uploadHashStatePathSpec{ + name: bw.blobStore.repository.Named().String(), + id: bw.id, + alg: bw.digester.Digest().Algorithm(), + list: true, + }) + + if err != nil { + return nil, err + } + + paths, err := bw.blobStore.driver.List(ctx, uploadHashStatePathPrefix) + if err != nil { + if _, ok := err.(storagedriver.PathNotFoundError); !ok { + return nil, err + } + // Treat PathNotFoundError as no entries. + paths = nil + } + + hashStateEntries := make([]hashStateEntry, 0, len(paths)) + + for _, p := range paths { + pathSuffix := path.Base(p) + // The suffix should be the offset. + offset, err := strconv.ParseInt(pathSuffix, 0, 64) + if err != nil { + logrus.Errorf("unable to parse offset from upload state path %q: %s", p, err) + } + + hashStateEntries = append(hashStateEntries, hashStateEntry{offset: offset, path: p}) + } + + return hashStateEntries, nil +} + +func (bw *blobWriter) storeHashState(ctx context.Context) error { + if !bw.resumableDigestEnabled { + return errResumableDigestNotAvailable + } + + h, ok := bw.digester.Hash().(encoding.BinaryMarshaler) + if !ok { + return errResumableDigestNotAvailable + } + + state, err := h.MarshalBinary() + if err != nil { + return err + } + + uploadHashStatePath, err := pathFor(uploadHashStatePathSpec{ + name: bw.blobStore.repository.Named().String(), + id: bw.id, + alg: bw.digester.Digest().Algorithm(), + offset: bw.written, + }) + + if err != nil { + return err + } + + return bw.driver.PutContent(ctx, uploadHashStatePath, state) +} diff --git a/vendor/github.com/docker/distribution/registry/storage/cache/cache.go b/vendor/github.com/docker/distribution/registry/storage/cache/cache.go new file mode 100644 index 000000000..10a390919 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/cache/cache.go @@ -0,0 +1,35 @@ +// Package cache provides facilities to speed up access to the storage +// backend. +package cache + +import ( + "fmt" + + "github.com/docker/distribution" +) + +// BlobDescriptorCacheProvider provides repository scoped +// BlobDescriptorService cache instances and a global descriptor cache. +type BlobDescriptorCacheProvider interface { + distribution.BlobDescriptorService + + RepositoryScoped(repo string) (distribution.BlobDescriptorService, error) +} + +// ValidateDescriptor provides a helper function to ensure that caches have +// common criteria for admitting descriptors. +func ValidateDescriptor(desc distribution.Descriptor) error { + if err := desc.Digest.Validate(); err != nil { + return err + } + + if desc.Size < 0 { + return fmt.Errorf("cache: invalid length in descriptor: %v < 0", desc.Size) + } + + if desc.MediaType == "" { + return fmt.Errorf("cache: empty mediatype on descriptor: %v", desc) + } + + return nil +} diff --git a/vendor/github.com/docker/distribution/registry/storage/cache/cachedblobdescriptorstore.go b/vendor/github.com/docker/distribution/registry/storage/cache/cachedblobdescriptorstore.go new file mode 100644 index 000000000..ac4c45211 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/cache/cachedblobdescriptorstore.go @@ -0,0 +1,129 @@ +package cache + +import ( + "context" + + "github.com/docker/distribution" + prometheus "github.com/docker/distribution/metrics" + "github.com/opencontainers/go-digest" +) + +// Metrics is used to hold metric counters +// related to the number of times a cache was +// hit or missed. +type Metrics struct { + Requests uint64 + Hits uint64 + Misses uint64 +} + +// Logger can be provided on the MetricsTracker to log errors. +// +// Usually, this is just a proxy to dcontext.GetLogger. +type Logger interface { + Errorf(format string, args ...interface{}) +} + +// MetricsTracker represents a metric tracker +// which simply counts the number of hits and misses. +type MetricsTracker interface { + Hit() + Miss() + Metrics() Metrics + Logger(context.Context) Logger +} + +type cachedBlobStatter struct { + cache distribution.BlobDescriptorService + backend distribution.BlobDescriptorService + tracker MetricsTracker +} + +var ( + // cacheCount is the number of total cache request received/hits/misses + cacheCount = prometheus.StorageNamespace.NewLabeledCounter("cache", "The number of cache request received", "type") +) + +// NewCachedBlobStatter creates a new statter which prefers a cache and +// falls back to a backend. +func NewCachedBlobStatter(cache distribution.BlobDescriptorService, backend distribution.BlobDescriptorService) distribution.BlobDescriptorService { + return &cachedBlobStatter{ + cache: cache, + backend: backend, + } +} + +// NewCachedBlobStatterWithMetrics creates a new statter which prefers a cache and +// falls back to a backend. Hits and misses will send to the tracker. +func NewCachedBlobStatterWithMetrics(cache distribution.BlobDescriptorService, backend distribution.BlobDescriptorService, tracker MetricsTracker) distribution.BlobStatter { + return &cachedBlobStatter{ + cache: cache, + backend: backend, + tracker: tracker, + } +} + +func (cbds *cachedBlobStatter) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { + cacheCount.WithValues("Request").Inc(1) + desc, err := cbds.cache.Stat(ctx, dgst) + if err != nil { + if err != distribution.ErrBlobUnknown { + logErrorf(ctx, cbds.tracker, "error retrieving descriptor from cache: %v", err) + } + + goto fallback + } + cacheCount.WithValues("Hit").Inc(1) + if cbds.tracker != nil { + cbds.tracker.Hit() + } + return desc, nil +fallback: + cacheCount.WithValues("Miss").Inc(1) + if cbds.tracker != nil { + cbds.tracker.Miss() + } + desc, err = cbds.backend.Stat(ctx, dgst) + if err != nil { + return desc, err + } + + if err := cbds.cache.SetDescriptor(ctx, dgst, desc); err != nil { + logErrorf(ctx, cbds.tracker, "error adding descriptor %v to cache: %v", desc.Digest, err) + } + + return desc, err + +} + +func (cbds *cachedBlobStatter) Clear(ctx context.Context, dgst digest.Digest) error { + err := cbds.cache.Clear(ctx, dgst) + if err != nil { + return err + } + + err = cbds.backend.Clear(ctx, dgst) + if err != nil { + return err + } + return nil +} + +func (cbds *cachedBlobStatter) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error { + if err := cbds.cache.SetDescriptor(ctx, dgst, desc); err != nil { + logErrorf(ctx, cbds.tracker, "error adding descriptor %v to cache: %v", desc.Digest, err) + } + return nil +} + +func logErrorf(ctx context.Context, tracker MetricsTracker, format string, args ...interface{}) { + if tracker == nil { + return + } + + logger := tracker.Logger(ctx) + if logger == nil { + return + } + logger.Errorf(format, args...) +} diff --git a/vendor/github.com/docker/distribution/registry/storage/cache/memory/memory.go b/vendor/github.com/docker/distribution/registry/storage/cache/memory/memory.go new file mode 100644 index 000000000..f2953b02c --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/cache/memory/memory.go @@ -0,0 +1,179 @@ +package memory + +import ( + "context" + "sync" + + "github.com/distribution/reference" + "github.com/docker/distribution" + "github.com/docker/distribution/registry/storage/cache" + "github.com/opencontainers/go-digest" +) + +type inMemoryBlobDescriptorCacheProvider struct { + global *mapBlobDescriptorCache + repositories map[string]*mapBlobDescriptorCache + mu sync.RWMutex +} + +// NewInMemoryBlobDescriptorCacheProvider returns a new mapped-based cache for +// storing blob descriptor data. +func NewInMemoryBlobDescriptorCacheProvider() cache.BlobDescriptorCacheProvider { + return &inMemoryBlobDescriptorCacheProvider{ + global: newMapBlobDescriptorCache(), + repositories: make(map[string]*mapBlobDescriptorCache), + } +} + +func (imbdcp *inMemoryBlobDescriptorCacheProvider) RepositoryScoped(repo string) (distribution.BlobDescriptorService, error) { + if _, err := reference.ParseNormalizedNamed(repo); err != nil { + return nil, err + } + + imbdcp.mu.RLock() + defer imbdcp.mu.RUnlock() + + return &repositoryScopedInMemoryBlobDescriptorCache{ + repo: repo, + parent: imbdcp, + repository: imbdcp.repositories[repo], + }, nil +} + +func (imbdcp *inMemoryBlobDescriptorCacheProvider) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { + return imbdcp.global.Stat(ctx, dgst) +} + +func (imbdcp *inMemoryBlobDescriptorCacheProvider) Clear(ctx context.Context, dgst digest.Digest) error { + return imbdcp.global.Clear(ctx, dgst) +} + +func (imbdcp *inMemoryBlobDescriptorCacheProvider) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error { + _, err := imbdcp.Stat(ctx, dgst) + if err == distribution.ErrBlobUnknown { + + if dgst.Algorithm() != desc.Digest.Algorithm() && dgst != desc.Digest { + // if the digests differ, set the other canonical mapping + if err := imbdcp.global.SetDescriptor(ctx, desc.Digest, desc); err != nil { + return err + } + } + + // unknown, just set it + return imbdcp.global.SetDescriptor(ctx, dgst, desc) + } + + // we already know it, do nothing + return err +} + +// repositoryScopedInMemoryBlobDescriptorCache provides the request scoped +// repository cache. Instances are not thread-safe but the delegated +// operations are. +type repositoryScopedInMemoryBlobDescriptorCache struct { + repo string + parent *inMemoryBlobDescriptorCacheProvider // allows lazy allocation of repo's map + repository *mapBlobDescriptorCache +} + +func (rsimbdcp *repositoryScopedInMemoryBlobDescriptorCache) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { + rsimbdcp.parent.mu.Lock() + repo := rsimbdcp.repository + rsimbdcp.parent.mu.Unlock() + + if repo == nil { + return distribution.Descriptor{}, distribution.ErrBlobUnknown + } + + return repo.Stat(ctx, dgst) +} + +func (rsimbdcp *repositoryScopedInMemoryBlobDescriptorCache) Clear(ctx context.Context, dgst digest.Digest) error { + rsimbdcp.parent.mu.Lock() + repo := rsimbdcp.repository + rsimbdcp.parent.mu.Unlock() + + if repo == nil { + return distribution.ErrBlobUnknown + } + + return repo.Clear(ctx, dgst) +} + +func (rsimbdcp *repositoryScopedInMemoryBlobDescriptorCache) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error { + rsimbdcp.parent.mu.Lock() + repo := rsimbdcp.repository + if repo == nil { + // allocate map since we are setting it now. + var ok bool + // have to read back value since we may have allocated elsewhere. + repo, ok = rsimbdcp.parent.repositories[rsimbdcp.repo] + if !ok { + repo = newMapBlobDescriptorCache() + rsimbdcp.parent.repositories[rsimbdcp.repo] = repo + } + rsimbdcp.repository = repo + } + rsimbdcp.parent.mu.Unlock() + + if err := repo.SetDescriptor(ctx, dgst, desc); err != nil { + return err + } + + return rsimbdcp.parent.SetDescriptor(ctx, dgst, desc) +} + +// mapBlobDescriptorCache provides a simple map-based implementation of the +// descriptor cache. +type mapBlobDescriptorCache struct { + descriptors map[digest.Digest]distribution.Descriptor + mu sync.RWMutex +} + +var _ distribution.BlobDescriptorService = &mapBlobDescriptorCache{} + +func newMapBlobDescriptorCache() *mapBlobDescriptorCache { + return &mapBlobDescriptorCache{ + descriptors: make(map[digest.Digest]distribution.Descriptor), + } +} + +func (mbdc *mapBlobDescriptorCache) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { + if err := dgst.Validate(); err != nil { + return distribution.Descriptor{}, err + } + + mbdc.mu.RLock() + defer mbdc.mu.RUnlock() + + desc, ok := mbdc.descriptors[dgst] + if !ok { + return distribution.Descriptor{}, distribution.ErrBlobUnknown + } + + return desc, nil +} + +func (mbdc *mapBlobDescriptorCache) Clear(ctx context.Context, dgst digest.Digest) error { + mbdc.mu.Lock() + defer mbdc.mu.Unlock() + + delete(mbdc.descriptors, dgst) + return nil +} + +func (mbdc *mapBlobDescriptorCache) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error { + if err := dgst.Validate(); err != nil { + return err + } + + if err := cache.ValidateDescriptor(desc); err != nil { + return err + } + + mbdc.mu.Lock() + defer mbdc.mu.Unlock() + + mbdc.descriptors[dgst] = desc + return nil +} diff --git a/vendor/github.com/docker/distribution/registry/storage/cache/redis/redis.go b/vendor/github.com/docker/distribution/registry/storage/cache/redis/redis.go new file mode 100644 index 000000000..82fc873f6 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/cache/redis/redis.go @@ -0,0 +1,268 @@ +package redis + +import ( + "context" + "fmt" + + "github.com/distribution/reference" + "github.com/docker/distribution" + "github.com/docker/distribution/registry/storage/cache" + "github.com/garyburd/redigo/redis" + "github.com/opencontainers/go-digest" +) + +// redisBlobStatService provides an implementation of +// BlobDescriptorCacheProvider based on redis. Blob descriptors are stored in +// two parts. The first provide fast access to repository membership through a +// redis set for each repo. The second is a redis hash keyed by the digest of +// the layer, providing path, length and mediatype information. There is also +// a per-repository redis hash of the blob descriptor, allowing override of +// data. This is currently used to override the mediatype on a per-repository +// basis. +// +// Note that there is no implied relationship between these two caches. The +// layer may exist in one, both or none and the code must be written this way. +type redisBlobDescriptorService struct { + pool *redis.Pool + + // TODO(stevvooe): We use a pool because we don't have great control over + // the cache lifecycle to manage connections. A new connection if fetched + // for each operation. Once we have better lifecycle management of the + // request objects, we can change this to a connection. +} + +// NewRedisBlobDescriptorCacheProvider returns a new redis-based +// BlobDescriptorCacheProvider using the provided redis connection pool. +func NewRedisBlobDescriptorCacheProvider(pool *redis.Pool) cache.BlobDescriptorCacheProvider { + return &redisBlobDescriptorService{ + pool: pool, + } +} + +// RepositoryScoped returns the scoped cache. +func (rbds *redisBlobDescriptorService) RepositoryScoped(repo string) (distribution.BlobDescriptorService, error) { + if _, err := reference.ParseNormalizedNamed(repo); err != nil { + return nil, err + } + + return &repositoryScopedRedisBlobDescriptorService{ + repo: repo, + upstream: rbds, + }, nil +} + +// Stat retrieves the descriptor data from the redis hash entry. +func (rbds *redisBlobDescriptorService) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { + if err := dgst.Validate(); err != nil { + return distribution.Descriptor{}, err + } + + conn := rbds.pool.Get() + defer conn.Close() + + return rbds.stat(ctx, conn, dgst) +} + +func (rbds *redisBlobDescriptorService) Clear(ctx context.Context, dgst digest.Digest) error { + if err := dgst.Validate(); err != nil { + return err + } + + conn := rbds.pool.Get() + defer conn.Close() + + // Not atomic in redis <= 2.3 + reply, err := conn.Do("HDEL", rbds.blobDescriptorHashKey(dgst), "digest", "size", "mediatype") + if err != nil { + return err + } + + if reply == 0 { + return distribution.ErrBlobUnknown + } + + return nil +} + +// stat provides an internal stat call that takes a connection parameter. This +// allows some internal management of the connection scope. +func (rbds *redisBlobDescriptorService) stat(ctx context.Context, conn redis.Conn, dgst digest.Digest) (distribution.Descriptor, error) { + reply, err := redis.Values(conn.Do("HMGET", rbds.blobDescriptorHashKey(dgst), "digest", "size", "mediatype")) + if err != nil { + return distribution.Descriptor{}, err + } + + // NOTE(stevvooe): The "size" field used to be "length". We treat a + // missing "size" field here as an unknown blob, which causes a cache + // miss, effectively migrating the field. + if len(reply) < 3 || reply[0] == nil || reply[1] == nil { // don't care if mediatype is nil + return distribution.Descriptor{}, distribution.ErrBlobUnknown + } + + var desc distribution.Descriptor + if _, err := redis.Scan(reply, &desc.Digest, &desc.Size, &desc.MediaType); err != nil { + return distribution.Descriptor{}, err + } + + return desc, nil +} + +// SetDescriptor sets the descriptor data for the given digest using a redis +// hash. A hash is used here since we may store unrelated fields about a layer +// in the future. +func (rbds *redisBlobDescriptorService) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error { + if err := dgst.Validate(); err != nil { + return err + } + + if err := cache.ValidateDescriptor(desc); err != nil { + return err + } + + conn := rbds.pool.Get() + defer conn.Close() + + return rbds.setDescriptor(ctx, conn, dgst, desc) +} + +func (rbds *redisBlobDescriptorService) setDescriptor(ctx context.Context, conn redis.Conn, dgst digest.Digest, desc distribution.Descriptor) error { + if _, err := conn.Do("HMSET", rbds.blobDescriptorHashKey(dgst), + "digest", desc.Digest, + "size", desc.Size); err != nil { + return err + } + + // Only set mediatype if not already set. + if _, err := conn.Do("HSETNX", rbds.blobDescriptorHashKey(dgst), + "mediatype", desc.MediaType); err != nil { + return err + } + + return nil +} + +func (rbds *redisBlobDescriptorService) blobDescriptorHashKey(dgst digest.Digest) string { + return "blobs::" + dgst.String() +} + +type repositoryScopedRedisBlobDescriptorService struct { + repo string + upstream *redisBlobDescriptorService +} + +var _ distribution.BlobDescriptorService = &repositoryScopedRedisBlobDescriptorService{} + +// Stat ensures that the digest is a member of the specified repository and +// forwards the descriptor request to the global blob store. If the media type +// differs for the repository, we override it. +func (rsrbds *repositoryScopedRedisBlobDescriptorService) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { + if err := dgst.Validate(); err != nil { + return distribution.Descriptor{}, err + } + + conn := rsrbds.upstream.pool.Get() + defer conn.Close() + + // Check membership to repository first + member, err := redis.Bool(conn.Do("SISMEMBER", rsrbds.repositoryBlobSetKey(rsrbds.repo), dgst)) + if err != nil { + return distribution.Descriptor{}, err + } + + if !member { + return distribution.Descriptor{}, distribution.ErrBlobUnknown + } + + upstream, err := rsrbds.upstream.stat(ctx, conn, dgst) + if err != nil { + return distribution.Descriptor{}, err + } + + // We allow a per repository mediatype, let's look it up here. + mediatype, err := redis.String(conn.Do("HGET", rsrbds.blobDescriptorHashKey(dgst), "mediatype")) + if err != nil { + return distribution.Descriptor{}, err + } + + if mediatype != "" { + upstream.MediaType = mediatype + } + + return upstream, nil +} + +// Clear removes the descriptor from the cache and forwards to the upstream descriptor store +func (rsrbds *repositoryScopedRedisBlobDescriptorService) Clear(ctx context.Context, dgst digest.Digest) error { + if err := dgst.Validate(); err != nil { + return err + } + + conn := rsrbds.upstream.pool.Get() + defer conn.Close() + + // Check membership to repository first + member, err := redis.Bool(conn.Do("SISMEMBER", rsrbds.repositoryBlobSetKey(rsrbds.repo), dgst)) + if err != nil { + return err + } + + if !member { + return distribution.ErrBlobUnknown + } + + return rsrbds.upstream.Clear(ctx, dgst) +} + +func (rsrbds *repositoryScopedRedisBlobDescriptorService) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error { + if err := dgst.Validate(); err != nil { + return err + } + + if err := cache.ValidateDescriptor(desc); err != nil { + return err + } + + if dgst != desc.Digest { + if dgst.Algorithm() == desc.Digest.Algorithm() { + return fmt.Errorf("redis cache: digest for descriptors differ but algorithm does not: %q != %q", dgst, desc.Digest) + } + } + + conn := rsrbds.upstream.pool.Get() + defer conn.Close() + + return rsrbds.setDescriptor(ctx, conn, dgst, desc) +} + +func (rsrbds *repositoryScopedRedisBlobDescriptorService) setDescriptor(ctx context.Context, conn redis.Conn, dgst digest.Digest, desc distribution.Descriptor) error { + if _, err := conn.Do("SADD", rsrbds.repositoryBlobSetKey(rsrbds.repo), dgst); err != nil { + return err + } + + if err := rsrbds.upstream.setDescriptor(ctx, conn, dgst, desc); err != nil { + return err + } + + // Override repository mediatype. + if _, err := conn.Do("HSET", rsrbds.blobDescriptorHashKey(dgst), "mediatype", desc.MediaType); err != nil { + return err + } + + // Also set the values for the primary descriptor, if they differ by + // algorithm (ie sha256 vs sha512). + if desc.Digest != "" && dgst != desc.Digest && dgst.Algorithm() != desc.Digest.Algorithm() { + if err := rsrbds.setDescriptor(ctx, conn, desc.Digest, desc); err != nil { + return err + } + } + + return nil +} + +func (rsrbds *repositoryScopedRedisBlobDescriptorService) blobDescriptorHashKey(dgst digest.Digest) string { + return "repository::" + rsrbds.repo + "::blobs::" + dgst.String() +} + +func (rsrbds *repositoryScopedRedisBlobDescriptorService) repositoryBlobSetKey(repo string) string { + return "repository::" + rsrbds.repo + "::blobs" +} diff --git a/vendor/github.com/docker/distribution/registry/storage/catalog.go b/vendor/github.com/docker/distribution/registry/storage/catalog.go new file mode 100644 index 000000000..5104c9de4 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/catalog.go @@ -0,0 +1,159 @@ +package storage + +import ( + "context" + "errors" + "io" + "path" + "strings" + + "github.com/distribution/reference" + "github.com/docker/distribution/registry/storage/driver" +) + +// Returns a list, or partial list, of repositories in the registry. +// Because it's a quite expensive operation, it should only be used when building up +// an initial set of repositories. +func (reg *registry) Repositories(ctx context.Context, repos []string, last string) (n int, err error) { + var finishedWalk bool + var foundRepos []string + + if len(repos) == 0 { + return 0, errors.New("no space in slice") + } + + root, err := pathFor(repositoriesRootPathSpec{}) + if err != nil { + return 0, err + } + + err = reg.blobStore.driver.Walk(ctx, root, func(fileInfo driver.FileInfo) error { + err := handleRepository(fileInfo, root, last, func(repoPath string) error { + foundRepos = append(foundRepos, repoPath) + return nil + }) + if err != nil { + return err + } + + // if we've filled our array, no need to walk any further + if len(foundRepos) == len(repos) { + finishedWalk = true + return driver.ErrSkipDir + } + + return nil + }) + + n = copy(repos, foundRepos) + + if err != nil { + return n, err + } else if !finishedWalk { + // We didn't fill buffer. No more records are available. + return n, io.EOF + } + + return n, err +} + +// Enumerate applies ingester to each repository +func (reg *registry) Enumerate(ctx context.Context, ingester func(string) error) error { + root, err := pathFor(repositoriesRootPathSpec{}) + if err != nil { + return err + } + + err = reg.blobStore.driver.Walk(ctx, root, func(fileInfo driver.FileInfo) error { + return handleRepository(fileInfo, root, "", ingester) + }) + + return err +} + +// Remove removes a repository from storage +func (reg *registry) Remove(ctx context.Context, name reference.Named) error { + root, err := pathFor(repositoriesRootPathSpec{}) + if err != nil { + return err + } + repoDir := path.Join(root, name.Name()) + return reg.driver.Delete(ctx, repoDir) +} + +// lessPath returns true if one path a is less than path b. +// +// A component-wise comparison is done, rather than the lexical comparison of +// strings. +func lessPath(a, b string) bool { + // we provide this behavior by making separator always sort first. + return compareReplaceInline(a, b, '/', '\x00') < 0 +} + +// compareReplaceInline modifies runtime.cmpstring to replace old with new +// during a byte-wise comparison. +func compareReplaceInline(s1, s2 string, old, new byte) int { + // TODO(stevvooe): We are missing an optimization when the s1 and s2 have + // the exact same slice header. It will make the code unsafe but can + // provide some extra performance. + + l := len(s1) + if len(s2) < l { + l = len(s2) + } + + for i := 0; i < l; i++ { + c1, c2 := s1[i], s2[i] + if c1 == old { + c1 = new + } + + if c2 == old { + c2 = new + } + + if c1 < c2 { + return -1 + } + + if c1 > c2 { + return +1 + } + } + + if len(s1) < len(s2) { + return -1 + } + + if len(s1) > len(s2) { + return +1 + } + + return 0 +} + +// handleRepository calls function fn with a repository path if fileInfo +// has a path of a repository under root and that it is lexographically +// after last. Otherwise, it will return ErrSkipDir. This should be used +// with Walk to do handling with repositories in a storage. +func handleRepository(fileInfo driver.FileInfo, root, last string, fn func(repoPath string) error) error { + filePath := fileInfo.Path() + + // lop the base path off + repo := filePath[len(root)+1:] + + _, file := path.Split(repo) + if file == "_layers" { + repo = strings.TrimSuffix(repo, "/_layers") + if lessPath(last, repo) { + if err := fn(repo); err != nil { + return err + } + } + return driver.ErrSkipDir + } else if strings.HasPrefix(file, "_") { + return driver.ErrSkipDir + } + + return nil +} diff --git a/vendor/github.com/docker/distribution/registry/storage/doc.go b/vendor/github.com/docker/distribution/registry/storage/doc.go new file mode 100644 index 000000000..387d92348 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/doc.go @@ -0,0 +1,3 @@ +// Package storage contains storage services for use in the registry +// application. It should be considered an internal package, as of Go 1.4. +package storage diff --git a/vendor/github.com/docker/distribution/registry/storage/driver/base/base.go b/vendor/github.com/docker/distribution/registry/storage/driver/base/base.go new file mode 100644 index 000000000..83b8c423b --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/driver/base/base.go @@ -0,0 +1,240 @@ +// Package base provides a base implementation of the storage driver that can +// be used to implement common checks. The goal is to increase the amount of +// code sharing. +// +// The canonical approach to use this class is to embed in the exported driver +// struct such that calls are proxied through this implementation. First, +// declare the internal driver, as follows: +// +// type driver struct { ... internal ...} +// +// The resulting type should implement StorageDriver such that it can be the +// target of a Base struct. The exported type can then be declared as follows: +// +// type Driver struct { +// Base +// } +// +// Because Driver embeds Base, it effectively implements Base. If the driver +// needs to intercept a call, before going to base, Driver should implement +// that method. Effectively, Driver can intercept calls before coming in and +// driver implements the actual logic. +// +// To further shield the embed from other packages, it is recommended to +// employ a private embed struct: +// +// type baseEmbed struct { +// base.Base +// } +// +// Then, declare driver to embed baseEmbed, rather than Base directly: +// +// type Driver struct { +// baseEmbed +// } +// +// The type now implements StorageDriver, proxying through Base, without +// exporting an unnecessary field. +package base + +import ( + "context" + "io" + "time" + + dcontext "github.com/docker/distribution/context" + prometheus "github.com/docker/distribution/metrics" + storagedriver "github.com/docker/distribution/registry/storage/driver" + "github.com/docker/go-metrics" +) + +var ( + // storageAction is the metrics of blob related operations + storageAction = prometheus.StorageNamespace.NewLabeledTimer("action", "The number of seconds that the storage action takes", "driver", "action") +) + +func init() { + metrics.Register(prometheus.StorageNamespace) +} + +// Base provides a wrapper around a storagedriver implementation that provides +// common path and bounds checking. +type Base struct { + storagedriver.StorageDriver +} + +// Format errors received from the storage driver +func (base *Base) setDriverName(e error) error { + switch actual := e.(type) { + case nil: + return nil + case storagedriver.ErrUnsupportedMethod: + actual.DriverName = base.StorageDriver.Name() + return actual + case storagedriver.PathNotFoundError: + actual.DriverName = base.StorageDriver.Name() + return actual + case storagedriver.InvalidPathError: + actual.DriverName = base.StorageDriver.Name() + return actual + case storagedriver.InvalidOffsetError: + actual.DriverName = base.StorageDriver.Name() + return actual + default: + storageError := storagedriver.Error{ + DriverName: base.StorageDriver.Name(), + Enclosed: e, + } + + return storageError + } +} + +// GetContent wraps GetContent of underlying storage driver. +func (base *Base) GetContent(ctx context.Context, path string) ([]byte, error) { + ctx, done := dcontext.WithTrace(ctx) + defer done("%s.GetContent(%q)", base.Name(), path) + + if !storagedriver.PathRegexp.MatchString(path) { + return nil, storagedriver.InvalidPathError{Path: path, DriverName: base.StorageDriver.Name()} + } + + start := time.Now() + b, e := base.StorageDriver.GetContent(ctx, path) + storageAction.WithValues(base.Name(), "GetContent").UpdateSince(start) + return b, base.setDriverName(e) +} + +// PutContent wraps PutContent of underlying storage driver. +func (base *Base) PutContent(ctx context.Context, path string, content []byte) error { + ctx, done := dcontext.WithTrace(ctx) + defer done("%s.PutContent(%q)", base.Name(), path) + + if !storagedriver.PathRegexp.MatchString(path) { + return storagedriver.InvalidPathError{Path: path, DriverName: base.StorageDriver.Name()} + } + + start := time.Now() + err := base.setDriverName(base.StorageDriver.PutContent(ctx, path, content)) + storageAction.WithValues(base.Name(), "PutContent").UpdateSince(start) + return err +} + +// Reader wraps Reader of underlying storage driver. +func (base *Base) Reader(ctx context.Context, path string, offset int64) (io.ReadCloser, error) { + ctx, done := dcontext.WithTrace(ctx) + defer done("%s.Reader(%q, %d)", base.Name(), path, offset) + + if offset < 0 { + return nil, storagedriver.InvalidOffsetError{Path: path, Offset: offset, DriverName: base.StorageDriver.Name()} + } + + if !storagedriver.PathRegexp.MatchString(path) { + return nil, storagedriver.InvalidPathError{Path: path, DriverName: base.StorageDriver.Name()} + } + + rc, e := base.StorageDriver.Reader(ctx, path, offset) + return rc, base.setDriverName(e) +} + +// Writer wraps Writer of underlying storage driver. +func (base *Base) Writer(ctx context.Context, path string, append bool) (storagedriver.FileWriter, error) { + ctx, done := dcontext.WithTrace(ctx) + defer done("%s.Writer(%q, %v)", base.Name(), path, append) + + if !storagedriver.PathRegexp.MatchString(path) { + return nil, storagedriver.InvalidPathError{Path: path, DriverName: base.StorageDriver.Name()} + } + + writer, e := base.StorageDriver.Writer(ctx, path, append) + return writer, base.setDriverName(e) +} + +// Stat wraps Stat of underlying storage driver. +func (base *Base) Stat(ctx context.Context, path string) (storagedriver.FileInfo, error) { + ctx, done := dcontext.WithTrace(ctx) + defer done("%s.Stat(%q)", base.Name(), path) + + if !storagedriver.PathRegexp.MatchString(path) && path != "/" { + return nil, storagedriver.InvalidPathError{Path: path, DriverName: base.StorageDriver.Name()} + } + + start := time.Now() + fi, e := base.StorageDriver.Stat(ctx, path) + storageAction.WithValues(base.Name(), "Stat").UpdateSince(start) + return fi, base.setDriverName(e) +} + +// List wraps List of underlying storage driver. +func (base *Base) List(ctx context.Context, path string) ([]string, error) { + ctx, done := dcontext.WithTrace(ctx) + defer done("%s.List(%q)", base.Name(), path) + + if !storagedriver.PathRegexp.MatchString(path) && path != "/" { + return nil, storagedriver.InvalidPathError{Path: path, DriverName: base.StorageDriver.Name()} + } + + start := time.Now() + str, e := base.StorageDriver.List(ctx, path) + storageAction.WithValues(base.Name(), "List").UpdateSince(start) + return str, base.setDriverName(e) +} + +// Move wraps Move of underlying storage driver. +func (base *Base) Move(ctx context.Context, sourcePath string, destPath string) error { + ctx, done := dcontext.WithTrace(ctx) + defer done("%s.Move(%q, %q", base.Name(), sourcePath, destPath) + + if !storagedriver.PathRegexp.MatchString(sourcePath) { + return storagedriver.InvalidPathError{Path: sourcePath, DriverName: base.StorageDriver.Name()} + } else if !storagedriver.PathRegexp.MatchString(destPath) { + return storagedriver.InvalidPathError{Path: destPath, DriverName: base.StorageDriver.Name()} + } + + start := time.Now() + err := base.setDriverName(base.StorageDriver.Move(ctx, sourcePath, destPath)) + storageAction.WithValues(base.Name(), "Move").UpdateSince(start) + return err +} + +// Delete wraps Delete of underlying storage driver. +func (base *Base) Delete(ctx context.Context, path string) error { + ctx, done := dcontext.WithTrace(ctx) + defer done("%s.Delete(%q)", base.Name(), path) + + if !storagedriver.PathRegexp.MatchString(path) { + return storagedriver.InvalidPathError{Path: path, DriverName: base.StorageDriver.Name()} + } + + start := time.Now() + err := base.setDriverName(base.StorageDriver.Delete(ctx, path)) + storageAction.WithValues(base.Name(), "Delete").UpdateSince(start) + return err +} + +// URLFor wraps URLFor of underlying storage driver. +func (base *Base) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) { + ctx, done := dcontext.WithTrace(ctx) + defer done("%s.URLFor(%q)", base.Name(), path) + + if !storagedriver.PathRegexp.MatchString(path) { + return "", storagedriver.InvalidPathError{Path: path, DriverName: base.StorageDriver.Name()} + } + + start := time.Now() + str, e := base.StorageDriver.URLFor(ctx, path, options) + storageAction.WithValues(base.Name(), "URLFor").UpdateSince(start) + return str, base.setDriverName(e) +} + +// Walk wraps Walk of underlying storage driver. +func (base *Base) Walk(ctx context.Context, path string, f storagedriver.WalkFn) error { + ctx, done := dcontext.WithTrace(ctx) + defer done("%s.Walk(%q)", base.Name(), path) + + if !storagedriver.PathRegexp.MatchString(path) && path != "/" { + return storagedriver.InvalidPathError{Path: path, DriverName: base.StorageDriver.Name()} + } + + return base.setDriverName(base.StorageDriver.Walk(ctx, path, f)) +} diff --git a/vendor/github.com/docker/distribution/registry/storage/driver/base/regulator.go b/vendor/github.com/docker/distribution/registry/storage/driver/base/regulator.go new file mode 100644 index 000000000..df6e50385 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/driver/base/regulator.go @@ -0,0 +1,184 @@ +package base + +import ( + "context" + "fmt" + "io" + "reflect" + "strconv" + "sync" + + storagedriver "github.com/docker/distribution/registry/storage/driver" +) + +type regulator struct { + storagedriver.StorageDriver + *sync.Cond + + available uint64 +} + +// GetLimitFromParameter takes an interface type as decoded from the YAML +// configuration and returns a uint64 representing the maximum number of +// concurrent calls given a minimum limit and default. +// +// If the parameter supplied is of an invalid type this returns an error. +func GetLimitFromParameter(param interface{}, min, def uint64) (uint64, error) { + limit := def + + switch v := param.(type) { + case string: + var err error + if limit, err = strconv.ParseUint(v, 0, 64); err != nil { + return limit, fmt.Errorf("parameter must be an integer, '%v' invalid", param) + } + case uint64: + limit = v + case int, int32, int64: + val := reflect.ValueOf(v).Convert(reflect.TypeOf(param)).Int() + // if param is negative casting to uint64 will wrap around and + // give you the hugest thread limit ever. Let's be sensible, here + if val > 0 { + limit = uint64(val) + } else { + limit = min + } + case uint, uint32: + limit = reflect.ValueOf(v).Convert(reflect.TypeOf(param)).Uint() + case nil: + // use the default + default: + return 0, fmt.Errorf("invalid value '%#v'", param) + } + + if limit < min { + return min, nil + } + + return limit, nil +} + +// NewRegulator wraps the given driver and is used to regulate concurrent calls +// to the given storage driver to a maximum of the given limit. This is useful +// for storage drivers that would otherwise create an unbounded number of OS +// threads if allowed to be called unregulated. +func NewRegulator(driver storagedriver.StorageDriver, limit uint64) storagedriver.StorageDriver { + return ®ulator{ + StorageDriver: driver, + Cond: sync.NewCond(&sync.Mutex{}), + available: limit, + } +} + +func (r *regulator) enter() { + r.L.Lock() + for r.available == 0 { + r.Wait() + } + r.available-- + r.L.Unlock() +} + +func (r *regulator) exit() { + r.L.Lock() + r.Signal() + r.available++ + r.L.Unlock() +} + +// Name returns the human-readable "name" of the driver, useful in error +// messages and logging. By convention, this will just be the registration +// name, but drivers may provide other information here. +func (r *regulator) Name() string { + r.enter() + defer r.exit() + + return r.StorageDriver.Name() +} + +// GetContent retrieves the content stored at "path" as a []byte. +// This should primarily be used for small objects. +func (r *regulator) GetContent(ctx context.Context, path string) ([]byte, error) { + r.enter() + defer r.exit() + + return r.StorageDriver.GetContent(ctx, path) +} + +// PutContent stores the []byte content at a location designated by "path". +// This should primarily be used for small objects. +func (r *regulator) PutContent(ctx context.Context, path string, content []byte) error { + r.enter() + defer r.exit() + + return r.StorageDriver.PutContent(ctx, path, content) +} + +// Reader retrieves an io.ReadCloser for the content stored at "path" +// with a given byte offset. +// May be used to resume reading a stream by providing a nonzero offset. +func (r *regulator) Reader(ctx context.Context, path string, offset int64) (io.ReadCloser, error) { + r.enter() + defer r.exit() + + return r.StorageDriver.Reader(ctx, path, offset) +} + +// Writer stores the contents of the provided io.ReadCloser at a +// location designated by the given path. +// May be used to resume writing a stream by providing a nonzero offset. +// The offset must be no larger than the CurrentSize for this path. +func (r *regulator) Writer(ctx context.Context, path string, append bool) (storagedriver.FileWriter, error) { + r.enter() + defer r.exit() + + return r.StorageDriver.Writer(ctx, path, append) +} + +// Stat retrieves the FileInfo for the given path, including the current +// size in bytes and the creation time. +func (r *regulator) Stat(ctx context.Context, path string) (storagedriver.FileInfo, error) { + r.enter() + defer r.exit() + + return r.StorageDriver.Stat(ctx, path) +} + +// List returns a list of the objects that are direct descendants of the +// given path. +func (r *regulator) List(ctx context.Context, path string) ([]string, error) { + r.enter() + defer r.exit() + + return r.StorageDriver.List(ctx, path) +} + +// Move moves an object stored at sourcePath to destPath, removing the +// original object. +// Note: This may be no more efficient than a copy followed by a delete for +// many implementations. +func (r *regulator) Move(ctx context.Context, sourcePath string, destPath string) error { + r.enter() + defer r.exit() + + return r.StorageDriver.Move(ctx, sourcePath, destPath) +} + +// Delete recursively deletes all objects stored at "path" and its subpaths. +func (r *regulator) Delete(ctx context.Context, path string) error { + r.enter() + defer r.exit() + + return r.StorageDriver.Delete(ctx, path) +} + +// URLFor returns a URL which may be used to retrieve the content stored at +// the given path, possibly using the given options. +// May return an ErrUnsupportedMethod in certain StorageDriver +// implementations. +func (r *regulator) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) { + r.enter() + defer r.exit() + + return r.StorageDriver.URLFor(ctx, path, options) +} diff --git a/vendor/github.com/docker/distribution/registry/storage/driver/factory/factory.go b/vendor/github.com/docker/distribution/registry/storage/driver/factory/factory.go new file mode 100644 index 000000000..a9c04ec59 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/driver/factory/factory.go @@ -0,0 +1,64 @@ +package factory + +import ( + "fmt" + + storagedriver "github.com/docker/distribution/registry/storage/driver" +) + +// driverFactories stores an internal mapping between storage driver names and their respective +// factories +var driverFactories = make(map[string]StorageDriverFactory) + +// StorageDriverFactory is a factory interface for creating storagedriver.StorageDriver interfaces +// Storage drivers should call Register() with a factory to make the driver available by name. +// Individual StorageDriver implementations generally register with the factory via the Register +// func (below) in their init() funcs, and as such they should be imported anonymously before use. +// See below for an example of how to register and get a StorageDriver for S3 +// +// import _ "github.com/docker/distribution/registry/storage/driver/s3-aws" +// s3Driver, err = factory.Create("s3", storageParams) +// // assuming no error, s3Driver is the StorageDriver that communicates with S3 according to storageParams +type StorageDriverFactory interface { + // Create returns a new storagedriver.StorageDriver with the given parameters + // Parameters will vary by driver and may be ignored + // Each parameter key must only consist of lowercase letters and numbers + Create(parameters map[string]interface{}) (storagedriver.StorageDriver, error) +} + +// Register makes a storage driver available by the provided name. +// If Register is called twice with the same name or if driver factory is nil, it panics. +// Additionally, it is not concurrency safe. Most Storage Drivers call this function +// in their init() functions. See the documentation for StorageDriverFactory for more. +func Register(name string, factory StorageDriverFactory) { + if factory == nil { + panic("Must not provide nil StorageDriverFactory") + } + _, registered := driverFactories[name] + if registered { + panic(fmt.Sprintf("StorageDriverFactory named %s already registered", name)) + } + + driverFactories[name] = factory +} + +// Create a new storagedriver.StorageDriver with the given name and +// parameters. To use a driver, the StorageDriverFactory must first be +// registered with the given name. If no drivers are found, an +// InvalidStorageDriverError is returned +func Create(name string, parameters map[string]interface{}) (storagedriver.StorageDriver, error) { + driverFactory, ok := driverFactories[name] + if !ok { + return nil, InvalidStorageDriverError{name} + } + return driverFactory.Create(parameters) +} + +// InvalidStorageDriverError records an attempt to construct an unregistered storage driver +type InvalidStorageDriverError struct { + Name string +} + +func (err InvalidStorageDriverError) Error() string { + return fmt.Sprintf("StorageDriver not registered: %s", err.Name) +} diff --git a/vendor/github.com/docker/distribution/registry/storage/driver/fileinfo.go b/vendor/github.com/docker/distribution/registry/storage/driver/fileinfo.go new file mode 100644 index 000000000..e5064029a --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/driver/fileinfo.go @@ -0,0 +1,79 @@ +package driver + +import "time" + +// FileInfo returns information about a given path. Inspired by os.FileInfo, +// it elides the base name method for a full path instead. +type FileInfo interface { + // Path provides the full path of the target of this file info. + Path() string + + // Size returns current length in bytes of the file. The return value can + // be used to write to the end of the file at path. The value is + // meaningless if IsDir returns true. + Size() int64 + + // ModTime returns the modification time for the file. For backends that + // don't have a modification time, the creation time should be returned. + ModTime() time.Time + + // IsDir returns true if the path is a directory. + IsDir() bool +} + +// NOTE(stevvooe): The next two types, FileInfoFields and FileInfoInternal +// should only be used by storagedriver implementations. They should moved to +// a "driver" package, similar to database/sql. + +// FileInfoFields provides the exported fields for implementing FileInfo +// interface in storagedriver implementations. It should be used with +// InternalFileInfo. +type FileInfoFields struct { + // Path provides the full path of the target of this file info. + Path string + + // Size is current length in bytes of the file. The value of this field + // can be used to write to the end of the file at path. The value is + // meaningless if IsDir is set to true. + Size int64 + + // ModTime returns the modification time for the file. For backends that + // don't have a modification time, the creation time should be returned. + ModTime time.Time + + // IsDir returns true if the path is a directory. + IsDir bool +} + +// FileInfoInternal implements the FileInfo interface. This should only be +// used by storagedriver implementations that don't have a specialized +// FileInfo type. +type FileInfoInternal struct { + FileInfoFields +} + +var _ FileInfo = FileInfoInternal{} +var _ FileInfo = &FileInfoInternal{} + +// Path provides the full path of the target of this file info. +func (fi FileInfoInternal) Path() string { + return fi.FileInfoFields.Path +} + +// Size returns current length in bytes of the file. The return value can +// be used to write to the end of the file at path. The value is +// meaningless if IsDir returns true. +func (fi FileInfoInternal) Size() int64 { + return fi.FileInfoFields.Size +} + +// ModTime returns the modification time for the file. For backends that +// don't have a modification time, the creation time should be returned. +func (fi FileInfoInternal) ModTime() time.Time { + return fi.FileInfoFields.ModTime +} + +// IsDir returns true if the path is a directory. +func (fi FileInfoInternal) IsDir() bool { + return fi.FileInfoFields.IsDir +} diff --git a/vendor/github.com/docker/distribution/registry/storage/driver/middleware/storagemiddleware.go b/vendor/github.com/docker/distribution/registry/storage/driver/middleware/storagemiddleware.go new file mode 100644 index 000000000..7e40a8dd9 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/driver/middleware/storagemiddleware.go @@ -0,0 +1,39 @@ +package storagemiddleware + +import ( + "fmt" + + storagedriver "github.com/docker/distribution/registry/storage/driver" +) + +// InitFunc is the type of a StorageMiddleware factory function and is +// used to register the constructor for different StorageMiddleware backends. +type InitFunc func(storageDriver storagedriver.StorageDriver, options map[string]interface{}) (storagedriver.StorageDriver, error) + +var storageMiddlewares map[string]InitFunc + +// Register is used to register an InitFunc for +// a StorageMiddleware backend with the given name. +func Register(name string, initFunc InitFunc) error { + if storageMiddlewares == nil { + storageMiddlewares = make(map[string]InitFunc) + } + if _, exists := storageMiddlewares[name]; exists { + return fmt.Errorf("name already registered: %s", name) + } + + storageMiddlewares[name] = initFunc + + return nil +} + +// Get constructs a StorageMiddleware with the given options using the named backend. +func Get(name string, options map[string]interface{}, storageDriver storagedriver.StorageDriver) (storagedriver.StorageDriver, error) { + if storageMiddlewares != nil { + if initFunc, exists := storageMiddlewares[name]; exists { + return initFunc(storageDriver, options) + } + } + + return nil, fmt.Errorf("no storage middleware registered with name: %s", name) +} diff --git a/vendor/github.com/docker/distribution/registry/storage/driver/storagedriver.go b/vendor/github.com/docker/distribution/registry/storage/driver/storagedriver.go new file mode 100644 index 000000000..b220713f2 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/driver/storagedriver.go @@ -0,0 +1,171 @@ +package driver + +import ( + "context" + "fmt" + "io" + "regexp" + "strconv" + "strings" +) + +// Version is a string representing the storage driver version, of the form +// Major.Minor. +// The registry must accept storage drivers with equal major version and greater +// minor version, but may not be compatible with older storage driver versions. +type Version string + +// Major returns the major (primary) component of a version. +func (version Version) Major() uint { + majorPart := strings.Split(string(version), ".")[0] + major, _ := strconv.ParseUint(majorPart, 10, 0) + return uint(major) +} + +// Minor returns the minor (secondary) component of a version. +func (version Version) Minor() uint { + minorPart := strings.Split(string(version), ".")[1] + minor, _ := strconv.ParseUint(minorPart, 10, 0) + return uint(minor) +} + +// CurrentVersion is the current storage driver Version. +const CurrentVersion Version = "0.1" + +// StorageDriver defines methods that a Storage Driver must implement for a +// filesystem-like key/value object storage. Storage Drivers are automatically +// registered via an internal registration mechanism, and generally created +// via the StorageDriverFactory interface (https://godoc.org/github.com/docker/distribution/registry/storage/driver/factory). +// Please see the aforementioned factory package for example code showing how to get an instance +// of a StorageDriver +type StorageDriver interface { + // Name returns the human-readable "name" of the driver, useful in error + // messages and logging. By convention, this will just be the registration + // name, but drivers may provide other information here. + Name() string + + // GetContent retrieves the content stored at "path" as a []byte. + // This should primarily be used for small objects. + GetContent(ctx context.Context, path string) ([]byte, error) + + // PutContent stores the []byte content at a location designated by "path". + // This should primarily be used for small objects. + PutContent(ctx context.Context, path string, content []byte) error + + // Reader retrieves an io.ReadCloser for the content stored at "path" + // with a given byte offset. + // May be used to resume reading a stream by providing a nonzero offset. + Reader(ctx context.Context, path string, offset int64) (io.ReadCloser, error) + + // Writer returns a FileWriter which will store the content written to it + // at the location designated by "path" after the call to Commit. + Writer(ctx context.Context, path string, append bool) (FileWriter, error) + + // Stat retrieves the FileInfo for the given path, including the current + // size in bytes and the creation time. + Stat(ctx context.Context, path string) (FileInfo, error) + + // List returns a list of the objects that are direct descendants of the + //given path. + List(ctx context.Context, path string) ([]string, error) + + // Move moves an object stored at sourcePath to destPath, removing the + // original object. + // Note: This may be no more efficient than a copy followed by a delete for + // many implementations. + Move(ctx context.Context, sourcePath string, destPath string) error + + // Delete recursively deletes all objects stored at "path" and its subpaths. + Delete(ctx context.Context, path string) error + + // URLFor returns a URL which may be used to retrieve the content stored at + // the given path, possibly using the given options. + // May return an ErrUnsupportedMethod in certain StorageDriver + // implementations. + URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) + + // Walk traverses a filesystem defined within driver, starting + // from the given path, calling f on each file. + // If the returned error from the WalkFn is ErrSkipDir and fileInfo refers + // to a directory, the directory will not be entered and Walk + // will continue the traversal. If fileInfo refers to a normal file, processing stops + Walk(ctx context.Context, path string, f WalkFn) error +} + +// FileWriter provides an abstraction for an opened writable file-like object in +// the storage backend. The FileWriter must flush all content written to it on +// the call to Close, but is only required to make its content readable on a +// call to Commit. +type FileWriter interface { + io.WriteCloser + + // Size returns the number of bytes written to this FileWriter. + Size() int64 + + // Cancel removes any written content from this FileWriter. + Cancel() error + + // Commit flushes all content written to this FileWriter and makes it + // available for future calls to StorageDriver.GetContent and + // StorageDriver.Reader. + Commit() error +} + +// PathRegexp is the regular expression which each file path must match. A +// file path is absolute, beginning with a slash and containing a positive +// number of path components separated by slashes, where each component is +// restricted to alphanumeric characters or a period, underscore, or +// hyphen. +var PathRegexp = regexp.MustCompile(`^(/[A-Za-z0-9._-]+)+$`) + +// ErrUnsupportedMethod may be returned in the case where a StorageDriver implementation does not support an optional method. +type ErrUnsupportedMethod struct { + DriverName string +} + +func (err ErrUnsupportedMethod) Error() string { + return fmt.Sprintf("%s: unsupported method", err.DriverName) +} + +// PathNotFoundError is returned when operating on a nonexistent path. +type PathNotFoundError struct { + Path string + DriverName string +} + +func (err PathNotFoundError) Error() string { + return fmt.Sprintf("%s: Path not found: %s", err.DriverName, err.Path) +} + +// InvalidPathError is returned when the provided path is malformed. +type InvalidPathError struct { + Path string + DriverName string +} + +func (err InvalidPathError) Error() string { + return fmt.Sprintf("%s: invalid path: %s", err.DriverName, err.Path) +} + +// InvalidOffsetError is returned when attempting to read or write from an +// invalid offset. +type InvalidOffsetError struct { + Path string + Offset int64 + DriverName string +} + +func (err InvalidOffsetError) Error() string { + return fmt.Sprintf("%s: invalid offset: %d for path: %s", err.DriverName, err.Offset, err.Path) +} + +// Error is a catch-all error type which captures an error string and +// the driver type on which it occurred. +type Error struct { + DriverName string + Enclosed error +} + +func (err Error) Error() string { + return fmt.Sprintf("%s: %s", err.DriverName, err.Enclosed) +} diff --git a/vendor/github.com/docker/distribution/registry/storage/driver/walk.go b/vendor/github.com/docker/distribution/registry/storage/driver/walk.go new file mode 100644 index 000000000..8ded5b8be --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/driver/walk.go @@ -0,0 +1,61 @@ +package driver + +import ( + "context" + "errors" + "sort" + + "github.com/sirupsen/logrus" +) + +// ErrSkipDir is used as a return value from onFileFunc to indicate that +// the directory named in the call is to be skipped. It is not returned +// as an error by any function. +var ErrSkipDir = errors.New("skip this directory") + +// WalkFn is called once per file by Walk +type WalkFn func(fileInfo FileInfo) error + +// WalkFallback traverses a filesystem defined within driver, starting +// from the given path, calling f on each file. It uses the List method and Stat to drive itself. +// If the returned error from the WalkFn is ErrSkipDir and fileInfo refers +// to a directory, the directory will not be entered and Walk +// will continue the traversal. If fileInfo refers to a normal file, processing stops +func WalkFallback(ctx context.Context, driver StorageDriver, from string, f WalkFn) error { + children, err := driver.List(ctx, from) + if err != nil { + return err + } + sort.Stable(sort.StringSlice(children)) + for _, child := range children { + // TODO(stevvooe): Calling driver.Stat for every entry is quite + // expensive when running against backends with a slow Stat + // implementation, such as s3. This is very likely a serious + // performance bottleneck. + fileInfo, err := driver.Stat(ctx, child) + if err != nil { + switch err.(type) { + case PathNotFoundError: + // repository was removed in between listing and enumeration. Ignore it. + logrus.WithField("path", child).Infof("ignoring deleted path") + continue + default: + return err + } + } + err = f(fileInfo) + if err == nil && fileInfo.IsDir() { + if err := WalkFallback(ctx, driver, child, f); err != nil { + return err + } + } else if err == ErrSkipDir { + // Stop iteration if it's a file, otherwise noop if it's a directory + if !fileInfo.IsDir() { + return nil + } + } else if err != nil { + return err + } + } + return nil +} diff --git a/vendor/github.com/docker/distribution/registry/storage/error.go b/vendor/github.com/docker/distribution/registry/storage/error.go new file mode 100644 index 000000000..109136586 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/error.go @@ -0,0 +1,9 @@ +package storage + +import "fmt" + +// pushError formats an error type given a path and an error +// and pushes it to a slice of errors +func pushError(errors []error, path string, err error) []error { + return append(errors, fmt.Errorf("%s: %s", path, err)) +} diff --git a/vendor/github.com/docker/distribution/registry/storage/filereader.go b/vendor/github.com/docker/distribution/registry/storage/filereader.go new file mode 100644 index 000000000..90afaad0a --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/filereader.go @@ -0,0 +1,176 @@ +package storage + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io" + "io/ioutil" + + storagedriver "github.com/docker/distribution/registry/storage/driver" +) + +// TODO(stevvooe): Set an optimal buffer size here. We'll have to +// understand the latency characteristics of the underlying network to +// set this correctly, so we may want to leave it to the driver. For +// out of process drivers, we'll have to optimize this buffer size for +// local communication. +const fileReaderBufferSize = 4 << 20 + +// remoteFileReader provides a read seeker interface to files stored in +// storagedriver. Used to implement part of layer interface and will be used +// to implement read side of LayerUpload. +type fileReader struct { + driver storagedriver.StorageDriver + + ctx context.Context + + // identifying fields + path string + size int64 // size is the total size, must be set. + + // mutable fields + rc io.ReadCloser // remote read closer + brd *bufio.Reader // internal buffered io + offset int64 // offset is the current read offset + err error // terminal error, if set, reader is closed +} + +// newFileReader initializes a file reader for the remote file. The reader +// takes on the size and path that must be determined externally with a stat +// call. The reader operates optimistically, assuming that the file is already +// there. +func newFileReader(ctx context.Context, driver storagedriver.StorageDriver, path string, size int64) (*fileReader, error) { + return &fileReader{ + ctx: ctx, + driver: driver, + path: path, + size: size, + }, nil +} + +func (fr *fileReader) Read(p []byte) (n int, err error) { + if fr.err != nil { + return 0, fr.err + } + + rd, err := fr.reader() + if err != nil { + return 0, err + } + + n, err = rd.Read(p) + fr.offset += int64(n) + + // Simulate io.EOR error if we reach filesize. + if err == nil && fr.offset >= fr.size { + err = io.EOF + } + + return n, err +} + +func (fr *fileReader) Seek(offset int64, whence int) (int64, error) { + if fr.err != nil { + return 0, fr.err + } + + var err error + newOffset := fr.offset + + switch whence { + case io.SeekCurrent: + newOffset += offset + case io.SeekEnd: + newOffset = fr.size + offset + case io.SeekStart: + newOffset = offset + } + + if newOffset < 0 { + err = fmt.Errorf("cannot seek to negative position") + } else { + if fr.offset != newOffset { + fr.reset() + } + + // No problems, set the offset. + fr.offset = newOffset + } + + return fr.offset, err +} + +func (fr *fileReader) Close() error { + return fr.closeWithErr(fmt.Errorf("fileReader: closed")) +} + +// reader prepares the current reader at the lrs offset, ensuring its buffered +// and ready to go. +func (fr *fileReader) reader() (io.Reader, error) { + if fr.err != nil { + return nil, fr.err + } + + if fr.rc != nil { + return fr.brd, nil + } + + // If we don't have a reader, open one up. + rc, err := fr.driver.Reader(fr.ctx, fr.path, fr.offset) + if err != nil { + switch err := err.(type) { + case storagedriver.PathNotFoundError: + // NOTE(stevvooe): If the path is not found, we simply return a + // reader that returns io.EOF. However, we do not set fr.rc, + // allowing future attempts at getting a reader to possibly + // succeed if the file turns up later. + return ioutil.NopCloser(bytes.NewReader([]byte{})), nil + default: + return nil, err + } + } + + fr.rc = rc + + if fr.brd == nil { + fr.brd = bufio.NewReaderSize(fr.rc, fileReaderBufferSize) + } else { + fr.brd.Reset(fr.rc) + } + + return fr.brd, nil +} + +// resetReader resets the reader, forcing the read method to open up a new +// connection and rebuild the buffered reader. This should be called when the +// offset and the reader will become out of sync, such as during a seek +// operation. +func (fr *fileReader) reset() { + if fr.err != nil { + return + } + if fr.rc != nil { + fr.rc.Close() + fr.rc = nil + } +} + +func (fr *fileReader) closeWithErr(err error) error { + if fr.err != nil { + return fr.err + } + + fr.err = err + + // close and release reader chain + if fr.rc != nil { + fr.rc.Close() + } + + fr.rc = nil + fr.brd = nil + + return fr.err +} diff --git a/vendor/github.com/docker/distribution/registry/storage/garbagecollect.go b/vendor/github.com/docker/distribution/registry/storage/garbagecollect.go new file mode 100644 index 000000000..5fd99b15b --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/garbagecollect.go @@ -0,0 +1,152 @@ +package storage + +import ( + "context" + "fmt" + + "github.com/distribution/reference" + "github.com/docker/distribution" + "github.com/docker/distribution/registry/storage/driver" + "github.com/opencontainers/go-digest" +) + +func emit(format string, a ...interface{}) { + fmt.Printf(format+"\n", a...) +} + +// GCOpts contains options for garbage collector +type GCOpts struct { + DryRun bool + RemoveUntagged bool +} + +// ManifestDel contains manifest structure which will be deleted +type ManifestDel struct { + Name string + Digest digest.Digest + Tags []string +} + +// MarkAndSweep performs a mark and sweep of registry data +func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace, opts GCOpts) error { + repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator) + if !ok { + return fmt.Errorf("unable to convert Namespace to RepositoryEnumerator") + } + + // mark + markSet := make(map[digest.Digest]struct{}) + manifestArr := make([]ManifestDel, 0) + err := repositoryEnumerator.Enumerate(ctx, func(repoName string) error { + emit(repoName) + + var err error + named, err := reference.WithName(repoName) + if err != nil { + return fmt.Errorf("failed to parse repo name %s: %v", repoName, err) + } + repository, err := registry.Repository(ctx, named) + if err != nil { + return fmt.Errorf("failed to construct repository: %v", err) + } + + manifestService, err := repository.Manifests(ctx) + if err != nil { + return fmt.Errorf("failed to construct manifest service: %v", err) + } + + manifestEnumerator, ok := manifestService.(distribution.ManifestEnumerator) + if !ok { + return fmt.Errorf("unable to convert ManifestService into ManifestEnumerator") + } + + err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error { + if opts.RemoveUntagged { + // fetch all tags where this manifest is the latest one + tags, err := repository.Tags(ctx).Lookup(ctx, distribution.Descriptor{Digest: dgst}) + if err != nil { + return fmt.Errorf("failed to retrieve tags for digest %v: %v", dgst, err) + } + if len(tags) == 0 { + emit("manifest eligible for deletion: %s", dgst) + // fetch all tags from repository + // all of these tags could contain manifest in history + // which means that we need check (and delete) those references when deleting manifest + allTags, err := repository.Tags(ctx).All(ctx) + if err != nil { + return fmt.Errorf("failed to retrieve tags %v", err) + } + manifestArr = append(manifestArr, ManifestDel{Name: repoName, Digest: dgst, Tags: allTags}) + return nil + } + } + // Mark the manifest's blob + emit("%s: marking manifest %s ", repoName, dgst) + markSet[dgst] = struct{}{} + + manifest, err := manifestService.Get(ctx, dgst) + if err != nil { + return fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err) + } + + descriptors := manifest.References() + for _, descriptor := range descriptors { + markSet[descriptor.Digest] = struct{}{} + emit("%s: marking blob %s", repoName, descriptor.Digest) + } + + return nil + }) + + // In certain situations such as unfinished uploads, deleting all + // tags in S3 or removing the _manifests folder manually, this + // error may be of type PathNotFound. + // + // In these cases we can continue marking other manifests safely. + if _, ok := err.(driver.PathNotFoundError); ok { + return nil + } + + return err + }) + + if err != nil { + return fmt.Errorf("failed to mark: %v", err) + } + + // sweep + vacuum := NewVacuum(ctx, storageDriver) + if !opts.DryRun { + for _, obj := range manifestArr { + err = vacuum.RemoveManifest(obj.Name, obj.Digest, obj.Tags) + if err != nil { + return fmt.Errorf("failed to delete manifest %s: %v", obj.Digest, err) + } + } + } + blobService := registry.Blobs() + deleteSet := make(map[digest.Digest]struct{}) + err = blobService.Enumerate(ctx, func(dgst digest.Digest) error { + // check if digest is in markSet. If not, delete it! + if _, ok := markSet[dgst]; !ok { + deleteSet[dgst] = struct{}{} + } + return nil + }) + if err != nil { + return fmt.Errorf("error enumerating blobs: %v", err) + } + emit("\n%d blobs marked, %d blobs and %d manifests eligible for deletion", len(markSet), len(deleteSet), len(manifestArr)) + for dgst := range deleteSet { + emit("blob eligible for deletion: %s", dgst) + if opts.DryRun { + continue + } + err = vacuum.RemoveBlob(string(dgst)) + if err != nil { + return fmt.Errorf("failed to delete blob %s: %v", dgst, err) + } + } + + return err +} diff --git a/vendor/github.com/docker/distribution/registry/storage/io.go b/vendor/github.com/docker/distribution/registry/storage/io.go new file mode 100644 index 000000000..7cde6a334 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/io.go @@ -0,0 +1,72 @@ +package storage + +import ( + "context" + "errors" + "io" + "io/ioutil" + + "github.com/docker/distribution/registry/storage/driver" +) + +const ( + maxBlobGetSize = 4 << 20 +) + +func getContent(ctx context.Context, driver driver.StorageDriver, p string) ([]byte, error) { + r, err := driver.Reader(ctx, p, 0) + if err != nil { + return nil, err + } + defer r.Close() + + return readAllLimited(r, maxBlobGetSize) +} + +func readAllLimited(r io.Reader, limit int64) ([]byte, error) { + r = limitReader(r, limit) + return ioutil.ReadAll(r) +} + +// limitReader returns a new reader limited to n bytes. Unlike io.LimitReader, +// this returns an error when the limit reached. +func limitReader(r io.Reader, n int64) io.Reader { + return &limitedReader{r: r, n: n} +} + +// limitedReader implements a reader that errors when the limit is reached. +// +// Partially cribbed from net/http.MaxBytesReader. +type limitedReader struct { + r io.Reader // underlying reader + n int64 // max bytes remaining + err error // sticky error +} + +func (l *limitedReader) Read(p []byte) (n int, err error) { + if l.err != nil { + return 0, l.err + } + if len(p) == 0 { + return 0, nil + } + // If they asked for a 32KB byte read but only 5 bytes are + // remaining, no need to read 32KB. 6 bytes will answer the + // question of the whether we hit the limit or go past it. + if int64(len(p)) > l.n+1 { + p = p[:l.n+1] + } + n, err = l.r.Read(p) + + if int64(n) <= l.n { + l.n -= int64(n) + l.err = err + return n, err + } + + n = int(l.n) + l.n = 0 + + l.err = errors.New("storage: read exceeds limit") + return n, l.err +} diff --git a/vendor/github.com/docker/distribution/registry/storage/linkedblobstore.go b/vendor/github.com/docker/distribution/registry/storage/linkedblobstore.go new file mode 100644 index 000000000..32e9a4898 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/linkedblobstore.go @@ -0,0 +1,465 @@ +package storage + +import ( + "context" + "fmt" + "net/http" + "path" + "time" + + "github.com/distribution/reference" + "github.com/docker/distribution" + dcontext "github.com/docker/distribution/context" + "github.com/docker/distribution/registry/storage/driver" + "github.com/docker/distribution/uuid" + "github.com/opencontainers/go-digest" +) + +// linkPathFunc describes a function that can resolve a link based on the +// repository name and digest. +type linkPathFunc func(name string, dgst digest.Digest) (string, error) + +// linkedBlobStore provides a full BlobService that namespaces the blobs to a +// given repository. Effectively, it manages the links in a given repository +// that grant access to the global blob store. +type linkedBlobStore struct { + *blobStore + registry *registry + blobServer distribution.BlobServer + blobAccessController distribution.BlobDescriptorService + repository distribution.Repository + ctx context.Context // only to be used where context can't come through method args + deleteEnabled bool + resumableDigestEnabled bool + + // linkPathFns specifies one or more path functions allowing one to + // control the repository blob link set to which the blob store + // dispatches. This is required because manifest and layer blobs have not + // yet been fully merged. At some point, this functionality should be + // removed the blob links folder should be merged. The first entry is + // treated as the "canonical" link location and will be used for writes. + linkPathFns []linkPathFunc + + // linkDirectoryPathSpec locates the root directories in which one might find links + linkDirectoryPathSpec pathSpec +} + +var _ distribution.BlobStore = &linkedBlobStore{} + +func (lbs *linkedBlobStore) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { + return lbs.blobAccessController.Stat(ctx, dgst) +} + +func (lbs *linkedBlobStore) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) { + canonical, err := lbs.Stat(ctx, dgst) // access check + if err != nil { + return nil, err + } + + return lbs.blobStore.Get(ctx, canonical.Digest) +} + +func (lbs *linkedBlobStore) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) { + canonical, err := lbs.Stat(ctx, dgst) // access check + if err != nil { + return nil, err + } + + return lbs.blobStore.Open(ctx, canonical.Digest) +} + +func (lbs *linkedBlobStore) ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) error { + canonical, err := lbs.Stat(ctx, dgst) // access check + if err != nil { + return err + } + + if canonical.MediaType != "" { + // Set the repository local content type. + w.Header().Set("Content-Type", canonical.MediaType) + } + + return lbs.blobServer.ServeBlob(ctx, w, r, canonical.Digest) +} + +func (lbs *linkedBlobStore) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) { + dgst := digest.FromBytes(p) + // Place the data in the blob store first. + desc, err := lbs.blobStore.Put(ctx, mediaType, p) + if err != nil { + dcontext.GetLogger(ctx).Errorf("error putting into main store: %v", err) + return distribution.Descriptor{}, err + } + + if err := lbs.blobAccessController.SetDescriptor(ctx, dgst, desc); err != nil { + return distribution.Descriptor{}, err + } + + // TODO(stevvooe): Write out mediatype if incoming differs from what is + // returned by Put above. Note that we should allow updates for a given + // repository. + + return desc, lbs.linkBlob(ctx, desc) +} + +type optionFunc func(interface{}) error + +func (f optionFunc) Apply(v interface{}) error { + return f(v) +} + +// WithMountFrom returns a BlobCreateOption which designates that the blob should be +// mounted from the given canonical reference. +func WithMountFrom(ref reference.Canonical) distribution.BlobCreateOption { + return optionFunc(func(v interface{}) error { + opts, ok := v.(*distribution.CreateOptions) + if !ok { + return fmt.Errorf("unexpected options type: %T", v) + } + + opts.Mount.ShouldMount = true + opts.Mount.From = ref + + return nil + }) +} + +// Writer begins a blob write session, returning a handle. +func (lbs *linkedBlobStore) Create(ctx context.Context, options ...distribution.BlobCreateOption) (distribution.BlobWriter, error) { + dcontext.GetLogger(ctx).Debug("(*linkedBlobStore).Writer") + + var opts distribution.CreateOptions + + for _, option := range options { + err := option.Apply(&opts) + if err != nil { + return nil, err + } + } + + if opts.Mount.ShouldMount { + desc, err := lbs.mount(ctx, opts.Mount.From, opts.Mount.From.Digest(), opts.Mount.Stat) + if err == nil { + // Mount successful, no need to initiate an upload session + return nil, distribution.ErrBlobMounted{From: opts.Mount.From, Descriptor: desc} + } + } + + uuid := uuid.Generate().String() + startedAt := time.Now().UTC() + + path, err := pathFor(uploadDataPathSpec{ + name: lbs.repository.Named().Name(), + id: uuid, + }) + + if err != nil { + return nil, err + } + + startedAtPath, err := pathFor(uploadStartedAtPathSpec{ + name: lbs.repository.Named().Name(), + id: uuid, + }) + + if err != nil { + return nil, err + } + + // Write a startedat file for this upload + if err := lbs.blobStore.driver.PutContent(ctx, startedAtPath, []byte(startedAt.Format(time.RFC3339))); err != nil { + return nil, err + } + + return lbs.newBlobUpload(ctx, uuid, path, startedAt, false) +} + +func (lbs *linkedBlobStore) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) { + dcontext.GetLogger(ctx).Debug("(*linkedBlobStore).Resume") + + startedAtPath, err := pathFor(uploadStartedAtPathSpec{ + name: lbs.repository.Named().Name(), + id: id, + }) + + if err != nil { + return nil, err + } + + startedAtBytes, err := lbs.blobStore.driver.GetContent(ctx, startedAtPath) + if err != nil { + switch err := err.(type) { + case driver.PathNotFoundError: + return nil, distribution.ErrBlobUploadUnknown + default: + return nil, err + } + } + + startedAt, err := time.Parse(time.RFC3339, string(startedAtBytes)) + if err != nil { + return nil, err + } + + path, err := pathFor(uploadDataPathSpec{ + name: lbs.repository.Named().Name(), + id: id, + }) + + if err != nil { + return nil, err + } + + return lbs.newBlobUpload(ctx, id, path, startedAt, true) +} + +func (lbs *linkedBlobStore) Delete(ctx context.Context, dgst digest.Digest) error { + if !lbs.deleteEnabled { + return distribution.ErrUnsupported + } + + // Ensure the blob is available for deletion + _, err := lbs.blobAccessController.Stat(ctx, dgst) + if err != nil { + return err + } + + err = lbs.blobAccessController.Clear(ctx, dgst) + if err != nil { + return err + } + + return nil +} + +func (lbs *linkedBlobStore) Enumerate(ctx context.Context, ingestor func(digest.Digest) error) error { + rootPath, err := pathFor(lbs.linkDirectoryPathSpec) + if err != nil { + return err + } + return lbs.driver.Walk(ctx, rootPath, func(fileInfo driver.FileInfo) error { + // exit early if directory... + if fileInfo.IsDir() { + return nil + } + filePath := fileInfo.Path() + + // check if it's a link + _, fileName := path.Split(filePath) + if fileName != "link" { + return nil + } + + // read the digest found in link + digest, err := lbs.blobStore.readlink(ctx, filePath) + if err != nil { + return err + } + + // ensure this conforms to the linkPathFns + _, err = lbs.Stat(ctx, digest) + if err != nil { + // we expect this error to occur so we move on + if err == distribution.ErrBlobUnknown { + return nil + } + return err + } + + err = ingestor(digest) + if err != nil { + return err + } + + return nil + }) +} + +func (lbs *linkedBlobStore) mount(ctx context.Context, sourceRepo reference.Named, dgst digest.Digest, sourceStat *distribution.Descriptor) (distribution.Descriptor, error) { + var stat distribution.Descriptor + if sourceStat == nil { + // look up the blob info from the sourceRepo if not already provided + repo, err := lbs.registry.Repository(ctx, sourceRepo) + if err != nil { + return distribution.Descriptor{}, err + } + stat, err = repo.Blobs(ctx).Stat(ctx, dgst) + if err != nil { + return distribution.Descriptor{}, err + } + } else { + // use the provided blob info + stat = *sourceStat + } + + desc := distribution.Descriptor{ + Size: stat.Size, + + // NOTE(stevvooe): The central blob store firewalls media types from + // other users. The caller should look this up and override the value + // for the specific repository. + MediaType: "application/octet-stream", + Digest: dgst, + } + return desc, lbs.linkBlob(ctx, desc) +} + +// newBlobUpload allocates a new upload controller with the given state. +func (lbs *linkedBlobStore) newBlobUpload(ctx context.Context, uuid, path string, startedAt time.Time, append bool) (distribution.BlobWriter, error) { + fw, err := lbs.driver.Writer(ctx, path, append) + if err != nil { + return nil, err + } + + bw := &blobWriter{ + ctx: ctx, + blobStore: lbs, + id: uuid, + startedAt: startedAt, + digester: digest.Canonical.Digester(), + fileWriter: fw, + driver: lbs.driver, + path: path, + resumableDigestEnabled: lbs.resumableDigestEnabled, + } + + return bw, nil +} + +// linkBlob links a valid, written blob into the registry under the named +// repository for the upload controller. +func (lbs *linkedBlobStore) linkBlob(ctx context.Context, canonical distribution.Descriptor, aliases ...digest.Digest) error { + dgsts := append([]digest.Digest{canonical.Digest}, aliases...) + + // TODO(stevvooe): Need to write out mediatype for only canonical hash + // since we don't care about the aliases. They are generally unused except + // for tarsum but those versions don't care about mediatype. + + // Don't make duplicate links. + seenDigests := make(map[digest.Digest]struct{}, len(dgsts)) + + // only use the first link + linkPathFn := lbs.linkPathFns[0] + + for _, dgst := range dgsts { + if _, seen := seenDigests[dgst]; seen { + continue + } + seenDigests[dgst] = struct{}{} + + blobLinkPath, err := linkPathFn(lbs.repository.Named().Name(), dgst) + if err != nil { + return err + } + + if err := lbs.blobStore.link(ctx, blobLinkPath, canonical.Digest); err != nil { + return err + } + } + + return nil +} + +type linkedBlobStatter struct { + *blobStore + repository distribution.Repository + + // linkPathFns specifies one or more path functions allowing one to + // control the repository blob link set to which the blob store + // dispatches. This is required because manifest and layer blobs have not + // yet been fully merged. At some point, this functionality should be + // removed an the blob links folder should be merged. The first entry is + // treated as the "canonical" link location and will be used for writes. + linkPathFns []linkPathFunc +} + +var _ distribution.BlobDescriptorService = &linkedBlobStatter{} + +func (lbs *linkedBlobStatter) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { + var ( + found bool + target digest.Digest + ) + + // try the many link path functions until we get success or an error that + // is not PathNotFoundError. + for _, linkPathFn := range lbs.linkPathFns { + var err error + target, err = lbs.resolveWithLinkFunc(ctx, dgst, linkPathFn) + + if err == nil { + found = true + break // success! + } + + switch err := err.(type) { + case driver.PathNotFoundError: + // do nothing, just move to the next linkPathFn + default: + return distribution.Descriptor{}, err + } + } + + if !found { + return distribution.Descriptor{}, distribution.ErrBlobUnknown + } + + if target != dgst { + // Track when we are doing cross-digest domain lookups. ie, sha512 to sha256. + dcontext.GetLogger(ctx).Warnf("looking up blob with canonical target: %v -> %v", dgst, target) + } + + // TODO(stevvooe): Look up repository local mediatype and replace that on + // the returned descriptor. + + return lbs.blobStore.statter.Stat(ctx, target) +} + +func (lbs *linkedBlobStatter) Clear(ctx context.Context, dgst digest.Digest) (err error) { + // clear any possible existence of a link described in linkPathFns + for _, linkPathFn := range lbs.linkPathFns { + blobLinkPath, err := linkPathFn(lbs.repository.Named().Name(), dgst) + if err != nil { + return err + } + + err = lbs.blobStore.driver.Delete(ctx, blobLinkPath) + if err != nil { + switch err := err.(type) { + case driver.PathNotFoundError: + continue // just ignore this error and continue + default: + return err + } + } + } + + return nil +} + +// resolveTargetWithFunc allows us to read a link to a resource with different +// linkPathFuncs to let us try a few different paths before returning not +// found. +func (lbs *linkedBlobStatter) resolveWithLinkFunc(ctx context.Context, dgst digest.Digest, linkPathFn linkPathFunc) (digest.Digest, error) { + blobLinkPath, err := linkPathFn(lbs.repository.Named().Name(), dgst) + if err != nil { + return "", err + } + + return lbs.blobStore.readlink(ctx, blobLinkPath) +} + +func (lbs *linkedBlobStatter) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error { + // The canonical descriptor for a blob is set at the commit phase of upload + return nil +} + +// blobLinkPath provides the path to the blob link, also known as layers. +func blobLinkPath(name string, dgst digest.Digest) (string, error) { + return pathFor(layerLinkPathSpec{name: name, digest: dgst}) +} + +// manifestRevisionLinkPath provides the path to the manifest revision link. +func manifestRevisionLinkPath(name string, dgst digest.Digest) (string, error) { + return pathFor(manifestRevisionLinkPathSpec{name: name, revision: dgst}) +} diff --git a/vendor/github.com/docker/distribution/registry/storage/manifestlisthandler.go b/vendor/github.com/docker/distribution/registry/storage/manifestlisthandler.go new file mode 100644 index 000000000..ffc391617 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/manifestlisthandler.go @@ -0,0 +1,96 @@ +package storage + +import ( + "context" + "fmt" + + "github.com/docker/distribution" + dcontext "github.com/docker/distribution/context" + "github.com/docker/distribution/manifest/manifestlist" + "github.com/opencontainers/go-digest" +) + +// manifestListHandler is a ManifestHandler that covers schema2 manifest lists. +type manifestListHandler struct { + repository distribution.Repository + blobStore distribution.BlobStore + ctx context.Context +} + +var _ ManifestHandler = &manifestListHandler{} + +func (ms *manifestListHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) { + dcontext.GetLogger(ms.ctx).Debug("(*manifestListHandler).Unmarshal") + + m := &manifestlist.DeserializedManifestList{} + if err := m.UnmarshalJSON(content); err != nil { + return nil, err + } + + return m, nil +} + +func (ms *manifestListHandler) Put(ctx context.Context, manifestList distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error) { + dcontext.GetLogger(ms.ctx).Debug("(*manifestListHandler).Put") + + m, ok := manifestList.(*manifestlist.DeserializedManifestList) + if !ok { + return "", fmt.Errorf("wrong type put to manifestListHandler: %T", manifestList) + } + + if err := ms.verifyManifest(ms.ctx, *m, skipDependencyVerification); err != nil { + return "", err + } + + mt, payload, err := m.Payload() + if err != nil { + return "", err + } + + revision, err := ms.blobStore.Put(ctx, mt, payload) + if err != nil { + dcontext.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err) + return "", err + } + + return revision.Digest, nil +} + +// verifyManifest ensures that the manifest content is valid from the +// perspective of the registry. As a policy, the registry only tries to +// store valid content, leaving trust policies of that content up to +// consumers. +func (ms *manifestListHandler) verifyManifest(ctx context.Context, mnfst manifestlist.DeserializedManifestList, skipDependencyVerification bool) error { + var errs distribution.ErrManifestVerification + + if mnfst.SchemaVersion != 2 { + return fmt.Errorf("unrecognized manifest list schema version %d", mnfst.SchemaVersion) + } + + if !skipDependencyVerification { + // This manifest service is different from the blob service + // returned by Blob. It uses a linked blob store to ensure that + // only manifests are accessible. + + manifestService, err := ms.repository.Manifests(ctx) + if err != nil { + return err + } + + for _, manifestDescriptor := range mnfst.References() { + exists, err := manifestService.Exists(ctx, manifestDescriptor.Digest) + if err != nil && err != distribution.ErrBlobUnknown { + errs = append(errs, err) + } + if err != nil || !exists { + // On error here, we always append unknown blob errors. + errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: manifestDescriptor.Digest}) + } + } + } + if len(errs) != 0 { + return errs + } + + return nil +} diff --git a/vendor/github.com/docker/distribution/registry/storage/manifeststore.go b/vendor/github.com/docker/distribution/registry/storage/manifeststore.go new file mode 100644 index 000000000..7daa9a729 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/manifeststore.go @@ -0,0 +1,161 @@ +package storage + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/docker/distribution" + dcontext "github.com/docker/distribution/context" + "github.com/docker/distribution/manifest" + "github.com/docker/distribution/manifest/manifestlist" + "github.com/docker/distribution/manifest/ocischema" + "github.com/docker/distribution/manifest/schema1" + "github.com/docker/distribution/manifest/schema2" + "github.com/opencontainers/go-digest" + v1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +// A ManifestHandler gets and puts manifests of a particular type. +type ManifestHandler interface { + // Unmarshal unmarshals the manifest from a byte slice. + Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) + + // Put creates or updates the given manifest returning the manifest digest. + Put(ctx context.Context, manifest distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error) +} + +// SkipLayerVerification allows a manifest to be Put before its +// layers are on the filesystem +func SkipLayerVerification() distribution.ManifestServiceOption { + return skipLayerOption{} +} + +type skipLayerOption struct{} + +func (o skipLayerOption) Apply(m distribution.ManifestService) error { + if ms, ok := m.(*manifestStore); ok { + ms.skipDependencyVerification = true + return nil + } + return fmt.Errorf("skip layer verification only valid for manifestStore") +} + +type manifestStore struct { + repository *repository + blobStore *linkedBlobStore + ctx context.Context + + skipDependencyVerification bool + + schema1Handler ManifestHandler + schema2Handler ManifestHandler + ocischemaHandler ManifestHandler + manifestListHandler ManifestHandler +} + +var _ distribution.ManifestService = &manifestStore{} + +func (ms *manifestStore) Exists(ctx context.Context, dgst digest.Digest) (bool, error) { + dcontext.GetLogger(ms.ctx).Debug("(*manifestStore).Exists") + + _, err := ms.blobStore.Stat(ms.ctx, dgst) + if err != nil { + if err == distribution.ErrBlobUnknown { + return false, nil + } + + return false, err + } + + return true, nil +} + +func (ms *manifestStore) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) { + dcontext.GetLogger(ms.ctx).Debug("(*manifestStore).Get") + + // TODO(stevvooe): Need to check descriptor from above to ensure that the + // mediatype is as we expect for the manifest store. + + content, err := ms.blobStore.Get(ctx, dgst) + if err != nil { + if err == distribution.ErrBlobUnknown { + return nil, distribution.ErrManifestUnknownRevision{ + Name: ms.repository.Named().Name(), + Revision: dgst, + } + } + + return nil, err + } + + var versioned manifest.Versioned + if err = json.Unmarshal(content, &versioned); err != nil { + return nil, err + } + + switch versioned.SchemaVersion { + case 1: + return ms.schema1Handler.Unmarshal(ctx, dgst, content) + case 2: + // This can be an image manifest or a manifest list + switch versioned.MediaType { + case schema2.MediaTypeManifest: + return ms.schema2Handler.Unmarshal(ctx, dgst, content) + case v1.MediaTypeImageManifest: + return ms.ocischemaHandler.Unmarshal(ctx, dgst, content) + case manifestlist.MediaTypeManifestList, v1.MediaTypeImageIndex: + return ms.manifestListHandler.Unmarshal(ctx, dgst, content) + case "": + // OCI image or image index - no media type in the content + + // First see if it looks like an image index + res, err := ms.manifestListHandler.Unmarshal(ctx, dgst, content) + resIndex := res.(*manifestlist.DeserializedManifestList) + if err == nil && resIndex.Manifests != nil { + return resIndex, nil + } + + // Otherwise, assume it must be an image manifest + return ms.ocischemaHandler.Unmarshal(ctx, dgst, content) + default: + return nil, distribution.ErrManifestVerification{fmt.Errorf("unrecognized manifest content type %s", versioned.MediaType)} + } + } + + return nil, fmt.Errorf("unrecognized manifest schema version %d", versioned.SchemaVersion) +} + +func (ms *manifestStore) Put(ctx context.Context, manifest distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) { + dcontext.GetLogger(ms.ctx).Debug("(*manifestStore).Put") + + switch manifest.(type) { + case *schema1.SignedManifest: + return ms.schema1Handler.Put(ctx, manifest, ms.skipDependencyVerification) + case *schema2.DeserializedManifest: + return ms.schema2Handler.Put(ctx, manifest, ms.skipDependencyVerification) + case *ocischema.DeserializedManifest: + return ms.ocischemaHandler.Put(ctx, manifest, ms.skipDependencyVerification) + case *manifestlist.DeserializedManifestList: + return ms.manifestListHandler.Put(ctx, manifest, ms.skipDependencyVerification) + } + + return "", fmt.Errorf("unrecognized manifest type %T", manifest) +} + +// Delete removes the revision of the specified manifest. +func (ms *manifestStore) Delete(ctx context.Context, dgst digest.Digest) error { + dcontext.GetLogger(ms.ctx).Debug("(*manifestStore).Delete") + return ms.blobStore.Delete(ctx, dgst) +} + +func (ms *manifestStore) Enumerate(ctx context.Context, ingester func(digest.Digest) error) error { + err := ms.blobStore.Enumerate(ctx, func(dgst digest.Digest) error { + err := ingester(dgst) + if err != nil { + return err + } + return nil + }) + return err +} diff --git a/vendor/github.com/docker/distribution/registry/storage/ocimanifesthandler.go b/vendor/github.com/docker/distribution/registry/storage/ocimanifesthandler.go new file mode 100644 index 000000000..6fa499fe7 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/ocimanifesthandler.go @@ -0,0 +1,133 @@ +package storage + +import ( + "context" + "fmt" + "net/url" + + "github.com/docker/distribution" + dcontext "github.com/docker/distribution/context" + "github.com/docker/distribution/manifest/ocischema" + "github.com/opencontainers/go-digest" + v1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +// ocischemaManifestHandler is a ManifestHandler that covers ocischema manifests. +type ocischemaManifestHandler struct { + repository distribution.Repository + blobStore distribution.BlobStore + ctx context.Context + manifestURLs manifestURLs +} + +var _ ManifestHandler = &ocischemaManifestHandler{} + +func (ms *ocischemaManifestHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) { + dcontext.GetLogger(ms.ctx).Debug("(*ocischemaManifestHandler).Unmarshal") + + m := &ocischema.DeserializedManifest{} + if err := m.UnmarshalJSON(content); err != nil { + return nil, err + } + + return m, nil +} + +func (ms *ocischemaManifestHandler) Put(ctx context.Context, manifest distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error) { + dcontext.GetLogger(ms.ctx).Debug("(*ocischemaManifestHandler).Put") + + m, ok := manifest.(*ocischema.DeserializedManifest) + if !ok { + return "", fmt.Errorf("non-ocischema manifest put to ocischemaManifestHandler: %T", manifest) + } + + if err := ms.verifyManifest(ms.ctx, *m, skipDependencyVerification); err != nil { + return "", err + } + + mt, payload, err := m.Payload() + if err != nil { + return "", err + } + + revision, err := ms.blobStore.Put(ctx, mt, payload) + if err != nil { + dcontext.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err) + return "", err + } + + return revision.Digest, nil +} + +// verifyManifest ensures that the manifest content is valid from the +// perspective of the registry. As a policy, the registry only tries to store +// valid content, leaving trust policies of that content up to consumers. +func (ms *ocischemaManifestHandler) verifyManifest(ctx context.Context, mnfst ocischema.DeserializedManifest, skipDependencyVerification bool) error { + var errs distribution.ErrManifestVerification + + if mnfst.Manifest.SchemaVersion != 2 { + return fmt.Errorf("unrecognized manifest schema version %d", mnfst.Manifest.SchemaVersion) + } + + if skipDependencyVerification { + return nil + } + + manifestService, err := ms.repository.Manifests(ctx) + if err != nil { + return err + } + + blobsService := ms.repository.Blobs(ctx) + + for _, descriptor := range mnfst.References() { + var err error + + switch descriptor.MediaType { + case v1.MediaTypeImageLayer, v1.MediaTypeImageLayerGzip, v1.MediaTypeImageLayerNonDistributable, v1.MediaTypeImageLayerNonDistributableGzip: + allow := ms.manifestURLs.allow + deny := ms.manifestURLs.deny + for _, u := range descriptor.URLs { + var pu *url.URL + pu, err = url.Parse(u) + if err != nil || (pu.Scheme != "http" && pu.Scheme != "https") || pu.Fragment != "" || (allow != nil && !allow.MatchString(u)) || (deny != nil && deny.MatchString(u)) { + err = errInvalidURL + break + } + } + if err == nil && len(descriptor.URLs) == 0 { + // If no URLs, require that the blob exists + _, err = blobsService.Stat(ctx, descriptor.Digest) + } + + case v1.MediaTypeImageManifest: + var exists bool + exists, err = manifestService.Exists(ctx, descriptor.Digest) + if err != nil || !exists { + err = distribution.ErrBlobUnknown // just coerce to unknown. + } + + fallthrough // double check the blob store. + default: + // forward all else to blob storage + if len(descriptor.URLs) == 0 { + _, err = blobsService.Stat(ctx, descriptor.Digest) + } + } + + if err != nil { + if err != distribution.ErrBlobUnknown { + errs = append(errs, err) + } + + // On error here, we always append unknown blob errors. + errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: descriptor.Digest}) + } + } + + if len(errs) != 0 { + return errs + } + + return nil +} diff --git a/vendor/github.com/docker/distribution/registry/storage/paths.go b/vendor/github.com/docker/distribution/registry/storage/paths.go new file mode 100644 index 000000000..b8f327267 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/paths.go @@ -0,0 +1,476 @@ +package storage + +import ( + "fmt" + "path" + "strings" + + "github.com/opencontainers/go-digest" +) + +const ( + storagePathVersion = "v2" // fixed storage layout version + storagePathRoot = "/docker/registry/" // all driver paths have a prefix + + // TODO(stevvooe): Get rid of the "storagePathRoot". Initially, we though + // storage path root would configurable for all drivers through this + // package. In reality, we've found it simpler to do this on a per driver + // basis. +) + +// pathFor maps paths based on "object names" and their ids. The "object +// names" mapped by are internal to the storage system. +// +// The path layout in the storage backend is roughly as follows: +// +// /v2 +// -> repositories/ +// ->/ +// -> _manifests/ +// revisions +// -> +// -> link +// tags/ +// -> current/link +// -> index +// -> //link +// -> _layers/ +// +// -> _uploads/ +// data +// startedat +// hashstates// +// -> blob/ +// +// +// The storage backend layout is broken up into a content-addressable blob +// store and repositories. The content-addressable blob store holds most data +// throughout the backend, keyed by algorithm and digests of the underlying +// content. Access to the blob store is controlled through links from the +// repository to blobstore. +// +// A repository is made up of layers, manifests and tags. The layers component +// is just a directory of layers which are "linked" into a repository. A layer +// can only be accessed through a qualified repository name if it is linked in +// the repository. Uploads of layers are managed in the uploads directory, +// which is key by upload id. When all data for an upload is received, the +// data is moved into the blob store and the upload directory is deleted. +// Abandoned uploads can be garbage collected by reading the startedat file +// and removing uploads that have been active for longer than a certain time. +// +// The third component of the repository directory is the manifests store, +// which is made up of a revision store and tag store. Manifests are stored in +// the blob store and linked into the revision store. +// While the registry can save all revisions of a manifest, no relationship is +// implied as to the ordering of changes to a manifest. The tag store provides +// support for name, tag lookups of manifests, using "current/link" under a +// named tag directory. An index is maintained to support deletions of all +// revisions of a given manifest tag. +// +// We cover the path formats implemented by this path mapper below. +// +// Manifests: +// +// manifestRevisionsPathSpec: /v2/repositories//_manifests/revisions/ +// manifestRevisionPathSpec: /v2/repositories//_manifests/revisions/// +// manifestRevisionLinkPathSpec: /v2/repositories//_manifests/revisions///link +// +// Tags: +// +// manifestTagsPathSpec: /v2/repositories//_manifests/tags/ +// manifestTagPathSpec: /v2/repositories//_manifests/tags// +// manifestTagCurrentPathSpec: /v2/repositories//_manifests/tags//current/link +// manifestTagIndexPathSpec: /v2/repositories//_manifests/tags//index/ +// manifestTagIndexEntryPathSpec: /v2/repositories//_manifests/tags//index/// +// manifestTagIndexEntryLinkPathSpec: /v2/repositories//_manifests/tags//index///link +// +// Blobs: +// +// layerLinkPathSpec: /v2/repositories//_layers///link +// +// Uploads: +// +// uploadDataPathSpec: /v2/repositories//_uploads//data +// uploadStartedAtPathSpec: /v2/repositories//_uploads//startedat +// uploadHashStatePathSpec: /v2/repositories//_uploads//hashstates// +// +// Blob Store: +// +// blobsPathSpec: /v2/blobs/ +// blobPathSpec: /v2/blobs/// +// blobDataPathSpec: /v2/blobs////data +// blobMediaTypePathSpec: /v2/blobs////data +// +// For more information on the semantic meaning of each path and their +// contents, please see the path spec documentation. +func pathFor(spec pathSpec) (string, error) { + + // Switch on the path object type and return the appropriate path. At + // first glance, one may wonder why we don't use an interface to + // accomplish this. By keep the formatting separate from the pathSpec, we + // keep separate the path generation componentized. These specs could be + // passed to a completely different mapper implementation and generate a + // different set of paths. + // + // For example, imagine migrating from one backend to the other: one could + // build a filesystem walker that converts a string path in one version, + // to an intermediate path object, than can be consumed and mapped by the + // other version. + + rootPrefix := []string{storagePathRoot, storagePathVersion} + repoPrefix := append(rootPrefix, "repositories") + + switch v := spec.(type) { + + case manifestRevisionsPathSpec: + return path.Join(append(repoPrefix, v.name, "_manifests", "revisions")...), nil + + case manifestRevisionPathSpec: + components, err := digestPathComponents(v.revision, false) + if err != nil { + return "", err + } + + return path.Join(append(append(repoPrefix, v.name, "_manifests", "revisions"), components...)...), nil + case manifestRevisionLinkPathSpec: + root, err := pathFor(manifestRevisionPathSpec(v)) + + if err != nil { + return "", err + } + + return path.Join(root, "link"), nil + case manifestTagsPathSpec: + return path.Join(append(repoPrefix, v.name, "_manifests", "tags")...), nil + case manifestTagPathSpec: + root, err := pathFor(manifestTagsPathSpec{ + name: v.name, + }) + + if err != nil { + return "", err + } + + return path.Join(root, v.tag), nil + case manifestTagCurrentPathSpec: + root, err := pathFor(manifestTagPathSpec(v)) + + if err != nil { + return "", err + } + + return path.Join(root, "current", "link"), nil + case manifestTagIndexPathSpec: + root, err := pathFor(manifestTagPathSpec(v)) + + if err != nil { + return "", err + } + + return path.Join(root, "index"), nil + case manifestTagIndexEntryLinkPathSpec: + root, err := pathFor(manifestTagIndexEntryPathSpec(v)) + + if err != nil { + return "", err + } + + return path.Join(root, "link"), nil + case manifestTagIndexEntryPathSpec: + root, err := pathFor(manifestTagIndexPathSpec{ + name: v.name, + tag: v.tag, + }) + + if err != nil { + return "", err + } + + components, err := digestPathComponents(v.revision, false) + if err != nil { + return "", err + } + + return path.Join(root, path.Join(components...)), nil + case layerLinkPathSpec: + components, err := digestPathComponents(v.digest, false) + if err != nil { + return "", err + } + + // TODO(stevvooe): Right now, all blobs are linked under "_layers". If + // we have future migrations, we may want to rename this to "_blobs". + // A migration strategy would simply leave existing items in place and + // write the new paths, commit a file then delete the old files. + + blobLinkPathComponents := append(repoPrefix, v.name, "_layers") + + return path.Join(path.Join(append(blobLinkPathComponents, components...)...), "link"), nil + case blobsPathSpec: + blobsPathPrefix := append(rootPrefix, "blobs") + return path.Join(blobsPathPrefix...), nil + case blobPathSpec: + components, err := digestPathComponents(v.digest, true) + if err != nil { + return "", err + } + + blobPathPrefix := append(rootPrefix, "blobs") + return path.Join(append(blobPathPrefix, components...)...), nil + case blobDataPathSpec: + components, err := digestPathComponents(v.digest, true) + if err != nil { + return "", err + } + + components = append(components, "data") + blobPathPrefix := append(rootPrefix, "blobs") + return path.Join(append(blobPathPrefix, components...)...), nil + + case uploadDataPathSpec: + return path.Join(append(repoPrefix, v.name, "_uploads", v.id, "data")...), nil + case uploadStartedAtPathSpec: + return path.Join(append(repoPrefix, v.name, "_uploads", v.id, "startedat")...), nil + case uploadHashStatePathSpec: + offset := fmt.Sprintf("%d", v.offset) + if v.list { + offset = "" // Limit to the prefix for listing offsets. + } + return path.Join(append(repoPrefix, v.name, "_uploads", v.id, "hashstates", string(v.alg), offset)...), nil + case repositoriesRootPathSpec: + return path.Join(repoPrefix...), nil + default: + // TODO(sday): This is an internal error. Ensure it doesn't escape (panic?). + return "", fmt.Errorf("unknown path spec: %#v", v) + } +} + +// pathSpec is a type to mark structs as path specs. There is no +// implementation because we'd like to keep the specs and the mappers +// decoupled. +type pathSpec interface { + pathSpec() +} + +// manifestRevisionsPathSpec describes the directory path for +// a manifest revision. +type manifestRevisionsPathSpec struct { + name string +} + +func (manifestRevisionsPathSpec) pathSpec() {} + +// manifestRevisionPathSpec describes the components of the directory path for +// a manifest revision. +type manifestRevisionPathSpec struct { + name string + revision digest.Digest +} + +func (manifestRevisionPathSpec) pathSpec() {} + +// manifestRevisionLinkPathSpec describes the path components required to look +// up the data link for a revision of a manifest. If this file is not present, +// the manifest blob is not available in the given repo. The contents of this +// file should just be the digest. +type manifestRevisionLinkPathSpec struct { + name string + revision digest.Digest +} + +func (manifestRevisionLinkPathSpec) pathSpec() {} + +// manifestTagsPathSpec describes the path elements required to point to the +// manifest tags directory. +type manifestTagsPathSpec struct { + name string +} + +func (manifestTagsPathSpec) pathSpec() {} + +// manifestTagPathSpec describes the path elements required to point to the +// manifest tag links files under a repository. These contain a blob id that +// can be used to look up the data and signatures. +type manifestTagPathSpec struct { + name string + tag string +} + +func (manifestTagPathSpec) pathSpec() {} + +// manifestTagCurrentPathSpec describes the link to the current revision for a +// given tag. +type manifestTagCurrentPathSpec struct { + name string + tag string +} + +func (manifestTagCurrentPathSpec) pathSpec() {} + +// manifestTagCurrentPathSpec describes the link to the index of revisions +// with the given tag. +type manifestTagIndexPathSpec struct { + name string + tag string +} + +func (manifestTagIndexPathSpec) pathSpec() {} + +// manifestTagIndexEntryPathSpec contains the entries of the index by revision. +type manifestTagIndexEntryPathSpec struct { + name string + tag string + revision digest.Digest +} + +func (manifestTagIndexEntryPathSpec) pathSpec() {} + +// manifestTagIndexEntryLinkPathSpec describes the link to a revisions of a +// manifest with given tag within the index. +type manifestTagIndexEntryLinkPathSpec struct { + name string + tag string + revision digest.Digest +} + +func (manifestTagIndexEntryLinkPathSpec) pathSpec() {} + +// blobLinkPathSpec specifies a path for a blob link, which is a file with a +// blob id. The blob link will contain a content addressable blob id reference +// into the blob store. The format of the contents is as follows: +// +// : +// +// The following example of the file contents is more illustrative: +// +// sha256:96443a84ce518ac22acb2e985eda402b58ac19ce6f91980bde63726a79d80b36 +// +// This indicates that there is a blob with the id/digest, calculated via +// sha256 that can be fetched from the blob store. +type layerLinkPathSpec struct { + name string + digest digest.Digest +} + +func (layerLinkPathSpec) pathSpec() {} + +// blobAlgorithmReplacer does some very simple path sanitization for user +// input. Paths should be "safe" before getting this far due to strict digest +// requirements but we can add further path conversion here, if needed. +var blobAlgorithmReplacer = strings.NewReplacer( + "+", "/", + ".", "/", + ";", "/", +) + +// blobsPathSpec contains the path for the blobs directory +type blobsPathSpec struct{} + +func (blobsPathSpec) pathSpec() {} + +// blobPathSpec contains the path for the registry global blob store. +type blobPathSpec struct { + digest digest.Digest +} + +func (blobPathSpec) pathSpec() {} + +// blobDataPathSpec contains the path for the registry global blob store. For +// now, this contains layer data, exclusively. +type blobDataPathSpec struct { + digest digest.Digest +} + +func (blobDataPathSpec) pathSpec() {} + +// uploadDataPathSpec defines the path parameters of the data file for +// uploads. +type uploadDataPathSpec struct { + name string + id string +} + +func (uploadDataPathSpec) pathSpec() {} + +// uploadDataPathSpec defines the path parameters for the file that stores the +// start time of an uploads. If it is missing, the upload is considered +// unknown. Admittedly, the presence of this file is an ugly hack to make sure +// we have a way to cleanup old or stalled uploads that doesn't rely on driver +// FileInfo behavior. If we come up with a more clever way to do this, we +// should remove this file immediately and rely on the startetAt field from +// the client to enforce time out policies. +type uploadStartedAtPathSpec struct { + name string + id string +} + +func (uploadStartedAtPathSpec) pathSpec() {} + +// uploadHashStatePathSpec defines the path parameters for the file that stores +// the hash function state of an upload at a specific byte offset. If `list` is +// set, then the path mapper will generate a list prefix for all hash state +// offsets for the upload identified by the name, id, and alg. +type uploadHashStatePathSpec struct { + name string + id string + alg digest.Algorithm + offset int64 + list bool +} + +func (uploadHashStatePathSpec) pathSpec() {} + +// repositoriesRootPathSpec returns the root of repositories +type repositoriesRootPathSpec struct { +} + +func (repositoriesRootPathSpec) pathSpec() {} + +// digestPathComponents provides a consistent path breakdown for a given +// digest. For a generic digest, it will be as follows: +// +// / +// +// If multilevel is true, the first two bytes of the digest will separate +// groups of digest folder. It will be as follows: +// +// // +func digestPathComponents(dgst digest.Digest, multilevel bool) ([]string, error) { + if err := dgst.Validate(); err != nil { + return nil, err + } + + algorithm := blobAlgorithmReplacer.Replace(string(dgst.Algorithm())) + hex := dgst.Hex() + prefix := []string{algorithm} + + var suffix []string + + if multilevel { + suffix = append(suffix, hex[:2]) + } + + suffix = append(suffix, hex) + + return append(prefix, suffix...), nil +} + +// Reconstructs a digest from a path +func digestFromPath(digestPath string) (digest.Digest, error) { + + digestPath = strings.TrimSuffix(digestPath, "/data") + dir, hex := path.Split(digestPath) + dir = path.Dir(dir) + dir, next := path.Split(dir) + + // next is either the algorithm OR the first two characters in the hex string + var algo string + if next == hex[:2] { + algo = path.Base(dir) + } else { + algo = next + } + + dgst := digest.NewDigestFromHex(algo, hex) + return dgst, dgst.Validate() +} diff --git a/vendor/github.com/docker/distribution/registry/storage/purgeuploads.go b/vendor/github.com/docker/distribution/registry/storage/purgeuploads.go new file mode 100644 index 000000000..cac921210 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/purgeuploads.go @@ -0,0 +1,139 @@ +package storage + +import ( + "context" + "path" + "strings" + "time" + + storageDriver "github.com/docker/distribution/registry/storage/driver" + "github.com/docker/distribution/uuid" + "github.com/sirupsen/logrus" +) + +// uploadData stored the location of temporary files created during a layer upload +// along with the date the upload was started +type uploadData struct { + containingDir string + startedAt time.Time +} + +func newUploadData() uploadData { + return uploadData{ + containingDir: "", + // default to far in future to protect against missing startedat + startedAt: time.Now().Add(10000 * time.Hour), + } +} + +// PurgeUploads deletes files from the upload directory +// created before olderThan. The list of files deleted and errors +// encountered are returned +func PurgeUploads(ctx context.Context, driver storageDriver.StorageDriver, olderThan time.Time, actuallyDelete bool) ([]string, []error) { + logrus.Infof("PurgeUploads starting: olderThan=%s, actuallyDelete=%t", olderThan, actuallyDelete) + uploadData, errors := getOutstandingUploads(ctx, driver) + var deleted []string + for _, uploadData := range uploadData { + if uploadData.startedAt.Before(olderThan) { + var err error + logrus.Infof("Upload files in %s have older date (%s) than purge date (%s). Removing upload directory.", + uploadData.containingDir, uploadData.startedAt, olderThan) + if actuallyDelete { + err = driver.Delete(ctx, uploadData.containingDir) + } + if err == nil { + deleted = append(deleted, uploadData.containingDir) + } else { + errors = append(errors, err) + } + } + } + + logrus.Infof("Purge uploads finished. Num deleted=%d, num errors=%d", len(deleted), len(errors)) + return deleted, errors +} + +// getOutstandingUploads walks the upload directory, collecting files +// which could be eligible for deletion. The only reliable way to +// classify the age of a file is with the date stored in the startedAt +// file, so gather files by UUID with a date from startedAt. +func getOutstandingUploads(ctx context.Context, driver storageDriver.StorageDriver) (map[string]uploadData, []error) { + var errors []error + uploads := make(map[string]uploadData) + + inUploadDir := false + root, err := pathFor(repositoriesRootPathSpec{}) + if err != nil { + return uploads, append(errors, err) + } + + err = driver.Walk(ctx, root, func(fileInfo storageDriver.FileInfo) error { + filePath := fileInfo.Path() + _, file := path.Split(filePath) + if file[0] == '_' { + // Reserved directory + inUploadDir = (file == "_uploads") + + if fileInfo.IsDir() && !inUploadDir { + return storageDriver.ErrSkipDir + } + + } + + uuid, isContainingDir := uuidFromPath(filePath) + if uuid == "" { + // Cannot reliably delete + return nil + } + ud, ok := uploads[uuid] + if !ok { + ud = newUploadData() + } + if isContainingDir { + ud.containingDir = filePath + } + if file == "startedat" { + if t, err := readStartedAtFile(driver, filePath); err == nil { + ud.startedAt = t + } else { + errors = pushError(errors, filePath, err) + } + + } + + uploads[uuid] = ud + return nil + }) + + if err != nil { + errors = pushError(errors, root, err) + } + return uploads, errors +} + +// uuidFromPath extracts the upload UUID from a given path +// If the UUID is the last path component, this is the containing +// directory for all upload files +func uuidFromPath(path string) (string, bool) { + components := strings.Split(path, "/") + for i := len(components) - 1; i >= 0; i-- { + if u, err := uuid.Parse(components[i]); err == nil { + return u.String(), i == len(components)-1 + } + } + return "", false +} + +// readStartedAtFile reads the date from an upload's startedAtFile +func readStartedAtFile(driver storageDriver.StorageDriver, path string) (time.Time, error) { + // todo:(richardscothern) - pass in a context + startedAtBytes, err := driver.GetContent(context.Background(), path) + if err != nil { + return time.Now(), err + } + startedAt, err := time.Parse(time.RFC3339, string(startedAtBytes)) + if err != nil { + return time.Now(), err + } + return startedAt, nil +} diff --git a/vendor/github.com/docker/distribution/registry/storage/registry.go b/vendor/github.com/docker/distribution/registry/storage/registry.go new file mode 100644 index 000000000..102abb6e5 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/registry.go @@ -0,0 +1,336 @@ +package storage + +import ( + "context" + "regexp" + + "github.com/distribution/reference" + "github.com/docker/distribution" + "github.com/docker/distribution/registry/storage/cache" + storagedriver "github.com/docker/distribution/registry/storage/driver" + "github.com/docker/libtrust" +) + +// registry is the top-level implementation of Registry for use in the storage +// package. All instances should descend from this object. +type registry struct { + blobStore *blobStore + blobServer *blobServer + statter *blobStatter // global statter service. + blobDescriptorCacheProvider cache.BlobDescriptorCacheProvider + deleteEnabled bool + schema1Enabled bool + resumableDigestEnabled bool + schema1SigningKey libtrust.PrivateKey + blobDescriptorServiceFactory distribution.BlobDescriptorServiceFactory + manifestURLs manifestURLs + driver storagedriver.StorageDriver +} + +// manifestURLs holds regular expressions for controlling manifest URL whitelisting +type manifestURLs struct { + allow *regexp.Regexp + deny *regexp.Regexp +} + +// RegistryOption is the type used for functional options for NewRegistry. +type RegistryOption func(*registry) error + +// EnableRedirect is a functional option for NewRegistry. It causes the backend +// blob server to attempt using (StorageDriver).URLFor to serve all blobs. +func EnableRedirect(registry *registry) error { + registry.blobServer.redirect = true + return nil +} + +// EnableDelete is a functional option for NewRegistry. It enables deletion on +// the registry. +func EnableDelete(registry *registry) error { + registry.deleteEnabled = true + return nil +} + +// EnableSchema1 is a functional option for NewRegistry. It enables pushing of +// schema1 manifests. +func EnableSchema1(registry *registry) error { + registry.schema1Enabled = true + return nil +} + +// DisableDigestResumption is a functional option for NewRegistry. It should be +// used if the registry is acting as a caching proxy. +func DisableDigestResumption(registry *registry) error { + registry.resumableDigestEnabled = false + return nil +} + +// ManifestURLsAllowRegexp is a functional option for NewRegistry. +func ManifestURLsAllowRegexp(r *regexp.Regexp) RegistryOption { + return func(registry *registry) error { + registry.manifestURLs.allow = r + return nil + } +} + +// ManifestURLsDenyRegexp is a functional option for NewRegistry. +func ManifestURLsDenyRegexp(r *regexp.Regexp) RegistryOption { + return func(registry *registry) error { + registry.manifestURLs.deny = r + return nil + } +} + +// Schema1SigningKey returns a functional option for NewRegistry. It sets the +// key for signing all schema1 manifests. +func Schema1SigningKey(key libtrust.PrivateKey) RegistryOption { + return func(registry *registry) error { + registry.schema1SigningKey = key + return nil + } +} + +// BlobDescriptorServiceFactory returns a functional option for NewRegistry. It sets the +// factory to create BlobDescriptorServiceFactory middleware. +func BlobDescriptorServiceFactory(factory distribution.BlobDescriptorServiceFactory) RegistryOption { + return func(registry *registry) error { + registry.blobDescriptorServiceFactory = factory + return nil + } +} + +// BlobDescriptorCacheProvider returns a functional option for +// NewRegistry. It creates a cached blob statter for use by the +// registry. +func BlobDescriptorCacheProvider(blobDescriptorCacheProvider cache.BlobDescriptorCacheProvider) RegistryOption { + // TODO(aaronl): The duplication of statter across several objects is + // ugly, and prevents us from using interface types in the registry + // struct. Ideally, blobStore and blobServer should be lazily + // initialized, and use the current value of + // blobDescriptorCacheProvider. + return func(registry *registry) error { + if blobDescriptorCacheProvider != nil { + statter := cache.NewCachedBlobStatter(blobDescriptorCacheProvider, registry.statter) + registry.blobStore.statter = statter + registry.blobServer.statter = statter + registry.blobDescriptorCacheProvider = blobDescriptorCacheProvider + } + return nil + } +} + +// NewRegistry creates a new registry instance from the provided driver. The +// resulting registry may be shared by multiple goroutines but is cheap to +// allocate. If the Redirect option is specified, the backend blob server will +// attempt to use (StorageDriver).URLFor to serve all blobs. +func NewRegistry(ctx context.Context, driver storagedriver.StorageDriver, options ...RegistryOption) (distribution.Namespace, error) { + // create global statter + statter := &blobStatter{ + driver: driver, + } + + bs := &blobStore{ + driver: driver, + statter: statter, + } + + registry := ®istry{ + blobStore: bs, + blobServer: &blobServer{ + driver: driver, + statter: statter, + pathFn: bs.path, + }, + statter: statter, + resumableDigestEnabled: true, + driver: driver, + } + + for _, option := range options { + if err := option(registry); err != nil { + return nil, err + } + } + + return registry, nil +} + +// Scope returns the namespace scope for a registry. The registry +// will only serve repositories contained within this scope. +func (reg *registry) Scope() distribution.Scope { + return distribution.GlobalScope +} + +// Repository returns an instance of the repository tied to the registry. +// Instances should not be shared between goroutines but are cheap to +// allocate. In general, they should be request scoped. +func (reg *registry) Repository(ctx context.Context, canonicalName reference.Named) (distribution.Repository, error) { + var descriptorCache distribution.BlobDescriptorService + if reg.blobDescriptorCacheProvider != nil { + var err error + descriptorCache, err = reg.blobDescriptorCacheProvider.RepositoryScoped(canonicalName.Name()) + if err != nil { + return nil, err + } + } + + return &repository{ + ctx: ctx, + registry: reg, + name: canonicalName, + descriptorCache: descriptorCache, + }, nil +} + +func (reg *registry) Blobs() distribution.BlobEnumerator { + return reg.blobStore +} + +func (reg *registry) BlobStatter() distribution.BlobStatter { + return reg.statter +} + +// repository provides name-scoped access to various services. +type repository struct { + *registry + ctx context.Context + name reference.Named + descriptorCache distribution.BlobDescriptorService +} + +// Name returns the name of the repository. +func (repo *repository) Named() reference.Named { + return repo.name +} + +func (repo *repository) Tags(ctx context.Context) distribution.TagService { + tags := &tagStore{ + repository: repo, + blobStore: repo.registry.blobStore, + } + + return tags +} + +// Manifests returns an instance of ManifestService. Instantiation is cheap and +// may be context sensitive in the future. The instance should be used similar +// to a request local. +func (repo *repository) Manifests(ctx context.Context, options ...distribution.ManifestServiceOption) (distribution.ManifestService, error) { + manifestLinkPathFns := []linkPathFunc{ + // NOTE(stevvooe): Need to search through multiple locations since + // 2.1.0 unintentionally linked into _layers. + manifestRevisionLinkPath, + blobLinkPath, + } + + manifestDirectoryPathSpec := manifestRevisionsPathSpec{name: repo.name.Name()} + + var statter distribution.BlobDescriptorService = &linkedBlobStatter{ + blobStore: repo.blobStore, + repository: repo, + linkPathFns: manifestLinkPathFns, + } + + if repo.registry.blobDescriptorServiceFactory != nil { + statter = repo.registry.blobDescriptorServiceFactory.BlobAccessController(statter) + } + + blobStore := &linkedBlobStore{ + ctx: ctx, + blobStore: repo.blobStore, + repository: repo, + deleteEnabled: repo.registry.deleteEnabled, + blobAccessController: statter, + + // TODO(stevvooe): linkPath limits this blob store to only + // manifests. This instance cannot be used for blob checks. + linkPathFns: manifestLinkPathFns, + linkDirectoryPathSpec: manifestDirectoryPathSpec, + } + + var v1Handler ManifestHandler + if repo.schema1Enabled { + v1Handler = &signedManifestHandler{ + ctx: ctx, + schema1SigningKey: repo.schema1SigningKey, + repository: repo, + blobStore: blobStore, + } + } else { + v1Handler = &v1UnsupportedHandler{ + innerHandler: &signedManifestHandler{ + ctx: ctx, + schema1SigningKey: repo.schema1SigningKey, + repository: repo, + blobStore: blobStore, + }, + } + } + + ms := &manifestStore{ + ctx: ctx, + repository: repo, + blobStore: blobStore, + schema1Handler: v1Handler, + schema2Handler: &schema2ManifestHandler{ + ctx: ctx, + repository: repo, + blobStore: blobStore, + manifestURLs: repo.registry.manifestURLs, + }, + manifestListHandler: &manifestListHandler{ + ctx: ctx, + repository: repo, + blobStore: blobStore, + }, + ocischemaHandler: &ocischemaManifestHandler{ + ctx: ctx, + repository: repo, + blobStore: blobStore, + manifestURLs: repo.registry.manifestURLs, + }, + } + + // Apply options + for _, option := range options { + err := option.Apply(ms) + if err != nil { + return nil, err + } + } + + return ms, nil +} + +// Blobs returns an instance of the BlobStore. Instantiation is cheap and +// may be context sensitive in the future. The instance should be used similar +// to a request local. +func (repo *repository) Blobs(ctx context.Context) distribution.BlobStore { + var statter distribution.BlobDescriptorService = &linkedBlobStatter{ + blobStore: repo.blobStore, + repository: repo, + linkPathFns: []linkPathFunc{blobLinkPath}, + } + + if repo.descriptorCache != nil { + statter = cache.NewCachedBlobStatter(repo.descriptorCache, statter) + } + + if repo.registry.blobDescriptorServiceFactory != nil { + statter = repo.registry.blobDescriptorServiceFactory.BlobAccessController(statter) + } + + return &linkedBlobStore{ + registry: repo.registry, + blobStore: repo.blobStore, + blobServer: repo.blobServer, + blobAccessController: statter, + repository: repo, + ctx: ctx, + + // TODO(stevvooe): linkPath limits this blob store to only layers. + // This instance cannot be used for manifest checks. + linkPathFns: []linkPathFunc{blobLinkPath}, + deleteEnabled: repo.registry.deleteEnabled, + resumableDigestEnabled: repo.resumableDigestEnabled, + } +} diff --git a/vendor/github.com/docker/distribution/registry/storage/schema2manifesthandler.go b/vendor/github.com/docker/distribution/registry/storage/schema2manifesthandler.go new file mode 100644 index 000000000..d9339403a --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/schema2manifesthandler.go @@ -0,0 +1,139 @@ +package storage + +import ( + "context" + "errors" + "fmt" + "net/url" + + "github.com/docker/distribution" + dcontext "github.com/docker/distribution/context" + "github.com/docker/distribution/manifest/schema1" + "github.com/docker/distribution/manifest/schema2" + "github.com/opencontainers/go-digest" +) + +var ( + errMissingURL = errors.New("missing URL on layer") + errInvalidURL = errors.New("invalid URL on layer") +) + +// schema2ManifestHandler is a ManifestHandler that covers schema2 manifests. +type schema2ManifestHandler struct { + repository distribution.Repository + blobStore distribution.BlobStore + ctx context.Context + manifestURLs manifestURLs +} + +var _ ManifestHandler = &schema2ManifestHandler{} + +func (ms *schema2ManifestHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) { + dcontext.GetLogger(ms.ctx).Debug("(*schema2ManifestHandler).Unmarshal") + + m := &schema2.DeserializedManifest{} + if err := m.UnmarshalJSON(content); err != nil { + return nil, err + } + + return m, nil +} + +func (ms *schema2ManifestHandler) Put(ctx context.Context, manifest distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error) { + dcontext.GetLogger(ms.ctx).Debug("(*schema2ManifestHandler).Put") + + m, ok := manifest.(*schema2.DeserializedManifest) + if !ok { + return "", fmt.Errorf("non-schema2 manifest put to schema2ManifestHandler: %T", manifest) + } + + if err := ms.verifyManifest(ms.ctx, *m, skipDependencyVerification); err != nil { + return "", err + } + + mt, payload, err := m.Payload() + if err != nil { + return "", err + } + + revision, err := ms.blobStore.Put(ctx, mt, payload) + if err != nil { + dcontext.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err) + return "", err + } + + return revision.Digest, nil +} + +// verifyManifest ensures that the manifest content is valid from the +// perspective of the registry. As a policy, the registry only tries to store +// valid content, leaving trust policies of that content up to consumers. +func (ms *schema2ManifestHandler) verifyManifest(ctx context.Context, mnfst schema2.DeserializedManifest, skipDependencyVerification bool) error { + var errs distribution.ErrManifestVerification + + if mnfst.Manifest.SchemaVersion != 2 { + return fmt.Errorf("unrecognized manifest schema version %d", mnfst.Manifest.SchemaVersion) + } + + if skipDependencyVerification { + return nil + } + + manifestService, err := ms.repository.Manifests(ctx) + if err != nil { + return err + } + + blobsService := ms.repository.Blobs(ctx) + + for _, descriptor := range mnfst.References() { + var err error + + switch descriptor.MediaType { + case schema2.MediaTypeForeignLayer: + // Clients download this layer from an external URL, so do not check for + // its presence. + if len(descriptor.URLs) == 0 { + err = errMissingURL + } + allow := ms.manifestURLs.allow + deny := ms.manifestURLs.deny + for _, u := range descriptor.URLs { + var pu *url.URL + pu, err = url.Parse(u) + if err != nil || (pu.Scheme != "http" && pu.Scheme != "https") || pu.Fragment != "" || (allow != nil && !allow.MatchString(u)) || (deny != nil && deny.MatchString(u)) { + err = errInvalidURL + break + } + } + case schema2.MediaTypeManifest, schema1.MediaTypeManifest: + var exists bool + exists, err = manifestService.Exists(ctx, descriptor.Digest) + if err != nil || !exists { + err = distribution.ErrBlobUnknown // just coerce to unknown. + } + + fallthrough // double check the blob store. + default: + // forward all else to blob storage + if len(descriptor.URLs) == 0 { + _, err = blobsService.Stat(ctx, descriptor.Digest) + } + } + + if err != nil { + if err != distribution.ErrBlobUnknown { + errs = append(errs, err) + } + + // On error here, we always append unknown blob errors. + errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: descriptor.Digest}) + } + } + + if len(errs) != 0 { + return errs + } + + return nil +} diff --git a/vendor/github.com/docker/distribution/registry/storage/signedmanifesthandler.go b/vendor/github.com/docker/distribution/registry/storage/signedmanifesthandler.go new file mode 100644 index 000000000..3bf9dd686 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/signedmanifesthandler.go @@ -0,0 +1,142 @@ +package storage + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/distribution/reference" + "github.com/docker/distribution" + dcontext "github.com/docker/distribution/context" + "github.com/docker/distribution/manifest/schema1" + "github.com/docker/libtrust" + "github.com/opencontainers/go-digest" +) + +// signedManifestHandler is a ManifestHandler that covers schema1 manifests. It +// can unmarshal and put schema1 manifests that have been signed by libtrust. +type signedManifestHandler struct { + repository distribution.Repository + schema1SigningKey libtrust.PrivateKey + blobStore distribution.BlobStore + ctx context.Context +} + +var _ ManifestHandler = &signedManifestHandler{} + +func (ms *signedManifestHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) { + dcontext.GetLogger(ms.ctx).Debug("(*signedManifestHandler).Unmarshal") + + var ( + signatures [][]byte + err error + ) + + jsig, err := libtrust.NewJSONSignature(content, signatures...) + if err != nil { + return nil, err + } + + if ms.schema1SigningKey != nil { + if err := jsig.Sign(ms.schema1SigningKey); err != nil { + return nil, err + } + } + + // Extract the pretty JWS + raw, err := jsig.PrettySignature("signatures") + if err != nil { + return nil, err + } + + var sm schema1.SignedManifest + if err := json.Unmarshal(raw, &sm); err != nil { + return nil, err + } + return &sm, nil +} + +func (ms *signedManifestHandler) Put(ctx context.Context, manifest distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error) { + dcontext.GetLogger(ms.ctx).Debug("(*signedManifestHandler).Put") + + sm, ok := manifest.(*schema1.SignedManifest) + if !ok { + return "", fmt.Errorf("non-schema1 manifest put to signedManifestHandler: %T", manifest) + } + + if err := ms.verifyManifest(ms.ctx, *sm, skipDependencyVerification); err != nil { + return "", err + } + + mt := schema1.MediaTypeManifest + payload := sm.Canonical + + revision, err := ms.blobStore.Put(ctx, mt, payload) + if err != nil { + dcontext.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err) + return "", err + } + + return revision.Digest, nil +} + +// verifyManifest ensures that the manifest content is valid from the +// perspective of the registry. It ensures that the signature is valid for the +// enclosed payload. As a policy, the registry only tries to store valid +// content, leaving trust policies of that content up to consumers. +func (ms *signedManifestHandler) verifyManifest(ctx context.Context, mnfst schema1.SignedManifest, skipDependencyVerification bool) error { + var errs distribution.ErrManifestVerification + + if len(mnfst.Name) > reference.NameTotalLengthMax { + errs = append(errs, + distribution.ErrManifestNameInvalid{ + Name: mnfst.Name, + Reason: fmt.Errorf("manifest name must not be more than %v characters", reference.NameTotalLengthMax), + }) + } + + if !reference.NameRegexp.MatchString(mnfst.Name) { + errs = append(errs, + distribution.ErrManifestNameInvalid{ + Name: mnfst.Name, + Reason: fmt.Errorf("invalid manifest name format"), + }) + } + + if len(mnfst.History) != len(mnfst.FSLayers) { + errs = append(errs, fmt.Errorf("mismatched history and fslayer cardinality %d != %d", + len(mnfst.History), len(mnfst.FSLayers))) + } + + if _, err := schema1.Verify(&mnfst); err != nil { + switch err { + case libtrust.ErrMissingSignatureKey, libtrust.ErrInvalidJSONContent, libtrust.ErrMissingSignatureKey: + errs = append(errs, distribution.ErrManifestUnverified{}) + default: + if err.Error() == "invalid signature" { // TODO(stevvooe): This should be exported by libtrust + errs = append(errs, distribution.ErrManifestUnverified{}) + } else { + errs = append(errs, err) + } + } + } + + if !skipDependencyVerification { + for _, fsLayer := range mnfst.References() { + _, err := ms.repository.Blobs(ctx).Stat(ctx, fsLayer.Digest) + if err != nil { + if err != distribution.ErrBlobUnknown { + errs = append(errs, err) + } + + // On error here, we always append unknown blob errors. + errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: fsLayer.Digest}) + } + } + } + if len(errs) != 0 { + return errs + } + + return nil +} diff --git a/vendor/github.com/docker/distribution/registry/storage/tagstore.go b/vendor/github.com/docker/distribution/registry/storage/tagstore.go new file mode 100644 index 000000000..a3c766b4f --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/tagstore.go @@ -0,0 +1,179 @@ +package storage + +import ( + "context" + "path" + + "github.com/docker/distribution" + storagedriver "github.com/docker/distribution/registry/storage/driver" + "github.com/opencontainers/go-digest" +) + +var _ distribution.TagService = &tagStore{} + +// tagStore provides methods to manage manifest tags in a backend storage driver. +// This implementation uses the same on-disk layout as the (now deleted) tag +// store. This provides backward compatibility with current registry deployments +// which only makes use of the Digest field of the returned distribution.Descriptor +// but does not enable full roundtripping of Descriptor objects +type tagStore struct { + repository *repository + blobStore *blobStore +} + +// All returns all tags +func (ts *tagStore) All(ctx context.Context) ([]string, error) { + var tags []string + + pathSpec, err := pathFor(manifestTagPathSpec{ + name: ts.repository.Named().Name(), + }) + if err != nil { + return tags, err + } + + entries, err := ts.blobStore.driver.List(ctx, pathSpec) + if err != nil { + switch err := err.(type) { + case storagedriver.PathNotFoundError: + return tags, distribution.ErrRepositoryUnknown{Name: ts.repository.Named().Name()} + default: + return tags, err + } + } + + for _, entry := range entries { + _, filename := path.Split(entry) + tags = append(tags, filename) + } + + return tags, nil +} + +// Tag tags the digest with the given tag, updating the the store to point at +// the current tag. The digest must point to a manifest. +func (ts *tagStore) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error { + currentPath, err := pathFor(manifestTagCurrentPathSpec{ + name: ts.repository.Named().Name(), + tag: tag, + }) + + if err != nil { + return err + } + + lbs := ts.linkedBlobStore(ctx, tag) + + // Link into the index + if err := lbs.linkBlob(ctx, desc); err != nil { + return err + } + + // Overwrite the current link + return ts.blobStore.link(ctx, currentPath, desc.Digest) +} + +// resolve the current revision for name and tag. +func (ts *tagStore) Get(ctx context.Context, tag string) (distribution.Descriptor, error) { + currentPath, err := pathFor(manifestTagCurrentPathSpec{ + name: ts.repository.Named().Name(), + tag: tag, + }) + + if err != nil { + return distribution.Descriptor{}, err + } + + revision, err := ts.blobStore.readlink(ctx, currentPath) + if err != nil { + switch err.(type) { + case storagedriver.PathNotFoundError: + return distribution.Descriptor{}, distribution.ErrTagUnknown{Tag: tag} + } + + return distribution.Descriptor{}, err + } + + return distribution.Descriptor{Digest: revision}, nil +} + +// Untag removes the tag association +func (ts *tagStore) Untag(ctx context.Context, tag string) error { + tagPath, err := pathFor(manifestTagPathSpec{ + name: ts.repository.Named().Name(), + tag: tag, + }) + if err != nil { + return err + } + + if err := ts.blobStore.driver.Delete(ctx, tagPath); err != nil { + switch err.(type) { + case storagedriver.PathNotFoundError: + return nil // Untag is idempotent, we don't care if it didn't exist + default: + return err + } + } + + return nil +} + +// linkedBlobStore returns the linkedBlobStore for the named tag, allowing one +// to index manifest blobs by tag name. While the tag store doesn't map +// precisely to the linked blob store, using this ensures the links are +// managed via the same code path. +func (ts *tagStore) linkedBlobStore(ctx context.Context, tag string) *linkedBlobStore { + return &linkedBlobStore{ + blobStore: ts.blobStore, + repository: ts.repository, + ctx: ctx, + linkPathFns: []linkPathFunc{func(name string, dgst digest.Digest) (string, error) { + return pathFor(manifestTagIndexEntryLinkPathSpec{ + name: name, + tag: tag, + revision: dgst, + }) + + }}, + } +} + +// Lookup recovers a list of tags which refer to this digest. When a manifest is deleted by +// digest, tag entries which point to it need to be recovered to avoid dangling tags. +func (ts *tagStore) Lookup(ctx context.Context, desc distribution.Descriptor) ([]string, error) { + allTags, err := ts.All(ctx) + switch err.(type) { + case distribution.ErrRepositoryUnknown: + // This tag store has been initialized but not yet populated + break + case nil: + break + default: + return nil, err + } + + var tags []string + for _, tag := range allTags { + tagLinkPathSpec := manifestTagCurrentPathSpec{ + name: ts.repository.Named().Name(), + tag: tag, + } + + tagLinkPath, _ := pathFor(tagLinkPathSpec) + tagDigest, err := ts.blobStore.readlink(ctx, tagLinkPath) + if err != nil { + switch err.(type) { + case storagedriver.PathNotFoundError: + continue + } + return nil, err + } + + if tagDigest == desc.Digest { + tags = append(tags, tag) + } + } + + return tags, nil +} diff --git a/vendor/github.com/docker/distribution/registry/storage/v1unsupportedhandler.go b/vendor/github.com/docker/distribution/registry/storage/v1unsupportedhandler.go new file mode 100644 index 000000000..fb6ca14e3 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/v1unsupportedhandler.go @@ -0,0 +1,23 @@ +package storage + +import ( + "context" + + "github.com/docker/distribution" + digest "github.com/opencontainers/go-digest" +) + +// signedManifestHandler is a ManifestHandler that unmarshals v1 manifests but +// refuses to Put v1 manifests +type v1UnsupportedHandler struct { + innerHandler ManifestHandler +} + +var _ ManifestHandler = &v1UnsupportedHandler{} + +func (v *v1UnsupportedHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) { + return v.innerHandler.Unmarshal(ctx, dgst, content) +} +func (v *v1UnsupportedHandler) Put(ctx context.Context, manifest distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error) { + return digest.Digest(""), distribution.ErrSchemaV1Unsupported +} diff --git a/vendor/github.com/docker/distribution/registry/storage/vacuum.go b/vendor/github.com/docker/distribution/registry/storage/vacuum.go new file mode 100644 index 000000000..a43db17a4 --- /dev/null +++ b/vendor/github.com/docker/distribution/registry/storage/vacuum.go @@ -0,0 +1,102 @@ +package storage + +import ( + "context" + "path" + + dcontext "github.com/docker/distribution/context" + "github.com/docker/distribution/registry/storage/driver" + "github.com/opencontainers/go-digest" +) + +// vacuum contains functions for cleaning up repositories and blobs +// These functions will only reliably work on strongly consistent +// storage systems. +// https://en.wikipedia.org/wiki/Consistency_model + +// NewVacuum creates a new Vacuum +func NewVacuum(ctx context.Context, driver driver.StorageDriver) Vacuum { + return Vacuum{ + ctx: ctx, + driver: driver, + } +} + +// Vacuum removes content from the filesystem +type Vacuum struct { + driver driver.StorageDriver + ctx context.Context +} + +// RemoveBlob removes a blob from the filesystem +func (v Vacuum) RemoveBlob(dgst string) error { + d, err := digest.Parse(dgst) + if err != nil { + return err + } + + blobPath, err := pathFor(blobPathSpec{digest: d}) + if err != nil { + return err + } + + dcontext.GetLogger(v.ctx).Infof("Deleting blob: %s", blobPath) + + err = v.driver.Delete(v.ctx, blobPath) + if err != nil { + return err + } + + return nil +} + +// RemoveManifest removes a manifest from the filesystem +func (v Vacuum) RemoveManifest(name string, dgst digest.Digest, tags []string) error { + // remove a tag manifest reference, in case of not found continue to next one + for _, tag := range tags { + + tagsPath, err := pathFor(manifestTagIndexEntryPathSpec{name: name, revision: dgst, tag: tag}) + if err != nil { + return err + } + + _, err = v.driver.Stat(v.ctx, tagsPath) + if err != nil { + switch err := err.(type) { + case driver.PathNotFoundError: + continue + default: + return err + } + } + dcontext.GetLogger(v.ctx).Infof("deleting manifest tag reference: %s", tagsPath) + err = v.driver.Delete(v.ctx, tagsPath) + if err != nil { + return err + } + } + + manifestPath, err := pathFor(manifestRevisionPathSpec{name: name, revision: dgst}) + if err != nil { + return err + } + dcontext.GetLogger(v.ctx).Infof("deleting manifest: %s", manifestPath) + return v.driver.Delete(v.ctx, manifestPath) +} + +// RemoveRepository removes a repository directory from the +// filesystem +func (v Vacuum) RemoveRepository(repoName string) error { + rootForRepository, err := pathFor(repositoriesRootPathSpec{}) + if err != nil { + return err + } + repoDir := path.Join(rootForRepository, repoName) + dcontext.GetLogger(v.ctx).Infof("Deleting repo: %s", repoDir) + err = v.driver.Delete(v.ctx, repoDir) + if err != nil { + return err + } + + return nil +} diff --git a/vendor/github.com/docker/distribution/tags.go b/vendor/github.com/docker/distribution/tags.go new file mode 100644 index 000000000..f22df2b85 --- /dev/null +++ b/vendor/github.com/docker/distribution/tags.go @@ -0,0 +1,27 @@ +package distribution + +import ( + "context" +) + +// TagService provides access to information about tagged objects. +type TagService interface { + // Get retrieves the descriptor identified by the tag. Some + // implementations may differentiate between "trusted" tags and + // "untrusted" tags. If a tag is "untrusted", the mapping will be returned + // as an ErrTagUntrusted error, with the target descriptor. + Get(ctx context.Context, tag string) (Descriptor, error) + + // Tag associates the tag with the provided descriptor, updating the + // current association, if needed. + Tag(ctx context.Context, tag string, desc Descriptor) error + + // Untag removes the given tag association + Untag(ctx context.Context, tag string) error + + // All returns the set of tags managed by this tag service + All(ctx context.Context) ([]string, error) + + // Lookup returns the set of tags referencing the given digest. + Lookup(ctx context.Context, digest Descriptor) ([]string, error) +} diff --git a/vendor/github.com/docker/distribution/uuid/uuid.go b/vendor/github.com/docker/distribution/uuid/uuid.go new file mode 100644 index 000000000..d433ccaf5 --- /dev/null +++ b/vendor/github.com/docker/distribution/uuid/uuid.go @@ -0,0 +1,126 @@ +// Package uuid provides simple UUID generation. Only version 4 style UUIDs +// can be generated. +// +// Please see http://tools.ietf.org/html/rfc4122 for details on UUIDs. +package uuid + +import ( + "crypto/rand" + "fmt" + "io" + "os" + "syscall" + "time" +) + +const ( + // Bits is the number of bits in a UUID + Bits = 128 + + // Size is the number of bytes in a UUID + Size = Bits / 8 + + format = "%08x-%04x-%04x-%04x-%012x" +) + +var ( + // ErrUUIDInvalid indicates a parsed string is not a valid uuid. + ErrUUIDInvalid = fmt.Errorf("invalid uuid") + + // Loggerf can be used to override the default logging destination. Such + // log messages in this library should be logged at warning or higher. + Loggerf = func(format string, args ...interface{}) {} +) + +// UUID represents a UUID value. UUIDs can be compared and set to other values +// and accessed by byte. +type UUID [Size]byte + +// Generate creates a new, version 4 uuid. +func Generate() (u UUID) { + const ( + // ensures we backoff for less than 450ms total. Use the following to + // select new value, in units of 10ms: + // n*(n+1)/2 = d -> n^2 + n - 2d -> n = (sqrt(8d + 1) - 1)/2 + maxretries = 9 + backoff = time.Millisecond * 10 + ) + + var ( + totalBackoff time.Duration + count int + retries int + ) + + for { + // This should never block but the read may fail. Because of this, + // we just try to read the random number generator until we get + // something. This is a very rare condition but may happen. + b := time.Duration(retries) * backoff + time.Sleep(b) + totalBackoff += b + + n, err := io.ReadFull(rand.Reader, u[count:]) + if err != nil { + if retryOnError(err) && retries < maxretries { + count += n + retries++ + Loggerf("error generating version 4 uuid, retrying: %v", err) + continue + } + + // Any other errors represent a system problem. What did someone + // do to /dev/urandom? + panic(fmt.Errorf("error reading random number generator, retried for %v: %v", totalBackoff.String(), err)) + } + + break + } + + u[6] = (u[6] & 0x0f) | 0x40 // set version byte + u[8] = (u[8] & 0x3f) | 0x80 // set high order byte 0b10{8,9,a,b} + + return u +} + +// Parse attempts to extract a uuid from the string or returns an error. +func Parse(s string) (u UUID, err error) { + if len(s) != 36 { + return UUID{}, ErrUUIDInvalid + } + + // create stack addresses for each section of the uuid. + p := make([][]byte, 5) + + if _, err := fmt.Sscanf(s, format, &p[0], &p[1], &p[2], &p[3], &p[4]); err != nil { + return u, err + } + + copy(u[0:4], p[0]) + copy(u[4:6], p[1]) + copy(u[6:8], p[2]) + copy(u[8:10], p[3]) + copy(u[10:16], p[4]) + + return +} + +func (u UUID) String() string { + return fmt.Sprintf(format, u[:4], u[4:6], u[6:8], u[8:10], u[10:]) +} + +// retryOnError tries to detect whether or not retrying would be fruitful. +func retryOnError(err error) bool { + switch err := err.(type) { + case *os.PathError: + return retryOnError(err.Err) // unpack the target error + case syscall.Errno: + if err == syscall.EPERM { + // EPERM represents an entropy pool exhaustion, a condition under + // which we backoff and retry. + return true + } + } + + return false +} diff --git a/vendor/github.com/docker/distribution/vendor.conf b/vendor/github.com/docker/distribution/vendor.conf new file mode 100644 index 000000000..20818428f --- /dev/null +++ b/vendor/github.com/docker/distribution/vendor.conf @@ -0,0 +1,52 @@ +github.com/Azure/azure-sdk-for-go 4650843026a7fdec254a8d9cf893693a254edd0b +github.com/Azure/go-autorest eaa7994b2278094c904d31993d26f56324db3052 +github.com/sirupsen/logrus 3d4380f53a34dcdc95f0c1db702615992b38d9a4 +github.com/aws/aws-sdk-go f831d5a0822a1ad72420ab18c6269bca1ddaf490 +github.com/bshuster-repo/logrus-logstash-hook d2c0ecc1836d91814e15e23bb5dc309c3ef51f4a +github.com/beorn7/perks 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9 +github.com/bugsnag/bugsnag-go b1d153021fcd90ca3f080db36bec96dc690fb274 +github.com/bugsnag/osext 0dd3f918b21bec95ace9dc86c7e70266cfc5c702 +github.com/bugsnag/panicwrap e2c28503fcd0675329da73bf48b33404db873782 +github.com/denverdino/aliyungo afedced274aa9a7fcdd47ac97018f0f8db4e5de2 +github.com/dgrijalva/jwt-go 4bbdd8ac624fc7a9ef7aec841c43d99b5fe65a29 https://github.com/golang-jwt/jwt.git # v3.2.2 +github.com/distribution/reference 49c28499d219290c3226822e9cfcd4ede6d75379 # v0.5.0 +github.com/docker/go-metrics 399ea8c73916000c64c2c76e8da00ca82f8387ab +github.com/docker/libtrust fa567046d9b14f6aa788882a950d69651d230b21 +github.com/garyburd/redigo 535138d7bcd717d6531c701ef5933d98b1866257 +github.com/go-ini/ini 2ba15ac2dc9cdf88c110ec2dc0ced7fa45f5678c +github.com/golang/protobuf 8d92cf5fc15a4382f8964b08e1f42a75c0591aa3 +github.com/gorilla/handlers 60c7bfde3e33c201519a200a4507a158cc03a17b +github.com/gorilla/mux 599cba5e7b6137d46ddf58fb1765f5d928e69604 +github.com/inconshreveable/mousetrap 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 +github.com/jmespath/go-jmespath bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d +github.com/marstr/guid 8bd9a64bf37eb297b492a4101fb28e80ac0b290f +github.com/satori/go.uuid f58768cc1a7a7e77a3bd49e98cdd21419399b6a3 +github.com/matttproud/golang_protobuf_extensions c12348ce28de40eed0136aa2b644d0ee0650e56c +github.com/miekg/dns 271c58e0c14f552178ea321a545ff9af38930f39 +github.com/mitchellh/mapstructure 482a9fd5fa83e8c4e7817413b80f3eb8feec03ef +github.com/ncw/swift a0320860b16212c2b59b4912bb6508cda1d7cee6 +github.com/prometheus/client_golang c332b6f63c0658a65eca15c0e5247ded801cf564 +github.com/prometheus/client_model 99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c +github.com/prometheus/common 89604d197083d4781071d3c65855d24ecfb0a563 +github.com/prometheus/procfs cb4147076ac75738c9a7d279075a253c0cc5acbd +github.com/Shopify/logrus-bugsnag 577dee27f20dd8f1a529f82210094af593be12bd +github.com/spf13/cobra 312092086bed4968099259622145a0c9ae280064 +github.com/spf13/pflag 5644820622454e71517561946e3d94b9f9db6842 +github.com/xenolf/lego a9d8cec0e6563575e5868a005359ac97911b5985 +github.com/yvasiyarov/go-metrics 57bccd1ccd43f94bb17fdd8bf3007059b802f85e +github.com/yvasiyarov/gorelic a9bba5b9ab508a086f9a12b8c51fab68478e2128 +github.com/yvasiyarov/newrelic_platform_go b21fdbd4370f3717f3bbd2bf41c223bc273068e6 +golang.org/x/crypto c10c31b5e94b6f7a0283272dc2bb27163dcea24b +golang.org/x/net 4876518f9e71663000c348837735820161a42df7 +golang.org/x/oauth2 045497edb6234273d67dbc25da3f2ddbc4c4cacf +golang.org/x/time a4bde12657593d5e90d0533a3e4fd95e635124cb +google.golang.org/api 9bf6e6e569ff057f75d9604a46c52928f17d2b54 +google.golang.org/appengine 12d5545dc1cfa6047a286d5e853841b6471f4c19 +google.golang.org/cloud 975617b05ea8a58727e6c1a06b6161ff4185a9f2 +google.golang.org/grpc d3ddb4469d5a1b949fc7a7da7c1d6a0d1b6de994 +gopkg.in/check.v1 64131543e7896d5bcc6bd5a76287eb75ea96c673 +gopkg.in/square/go-jose.v1 40d457b439244b546f023d056628e5184136899b +gopkg.in/yaml.v2 v2.2.1 +rsc.io/letsencrypt e770c10b0f1a64775ae91d240407ce00d1a5bdeb https://github.com/dmcgowan/letsencrypt.git +github.com/opencontainers/go-digest ea51bea511f75cfa3ef6098cc253c5c3609b037a # v1.0.0 +github.com/opencontainers/image-spec 67d2d5658fe0476ab9bf414cec164077ebff3920 # v1.0.2 diff --git a/vendor/github.com/docker/distribution/version/print.go b/vendor/github.com/docker/distribution/version/print.go new file mode 100644 index 000000000..318ac6953 --- /dev/null +++ b/vendor/github.com/docker/distribution/version/print.go @@ -0,0 +1,25 @@ +package version + +import ( + "fmt" + "io" + "os" +) + +// FprintVersion outputs the version string to the writer, in the following +// format, followed by a newline: +// +// +// +// For example, a binary "registry" built from github.com/docker/distribution +// with version "v2.0" would print the following: +// +// registry github.com/docker/distribution v2.0 +func FprintVersion(w io.Writer) { + fmt.Fprintln(w, os.Args[0], Package, Version) +} + +// PrintVersion outputs the version information, from Fprint, to stdout. +func PrintVersion() { + FprintVersion(os.Stdout) +} diff --git a/vendor/github.com/docker/distribution/version/version.go b/vendor/github.com/docker/distribution/version/version.go new file mode 100644 index 000000000..182b300e8 --- /dev/null +++ b/vendor/github.com/docker/distribution/version/version.go @@ -0,0 +1,15 @@ +package version + +// Package is the overall, canonical project import path under which the +// package was built. +var Package = "github.com/docker/distribution" + +// Version indicates which version of the binary is running. This is set to +// the latest release tag by hand, always suffixed by "+unknown". During +// build, it will be replaced by the actual version. The value here will be +// used if the registry is run after a go get based install. +var Version = "v2.8.3+unknown" + +// Revision is filled with the VCS (e.g. git) revision being used to build +// the program at linking time. +var Revision = "" diff --git a/vendor/github.com/docker/distribution/version/version.sh b/vendor/github.com/docker/distribution/version/version.sh new file mode 100644 index 000000000..dd8296968 --- /dev/null +++ b/vendor/github.com/docker/distribution/version/version.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +# This bash script outputs the current, desired content of version.go, using +# git describe. For best effect, pipe this to the target file. Generally, this +# only needs to updated for releases. The actual value of will be replaced +# during build time if the makefile is used. + +set -e + +cat < +Josh Hawn (github: jlhawn) +Derek McGowan (github: dmcgowan) diff --git a/vendor/github.com/docker/libtrust/README.md b/vendor/github.com/docker/libtrust/README.md new file mode 100644 index 000000000..dcffb31ae --- /dev/null +++ b/vendor/github.com/docker/libtrust/README.md @@ -0,0 +1,22 @@ +# libtrust + +> **WARNING** this library is no longer actively developed, and will be integrated +> in the [docker/distribution][https://www.github.com/docker/distribution] +> repository in future. + +Libtrust is library for managing authentication and authorization using public key cryptography. + +Authentication is handled using the identity attached to the public key. +Libtrust provides multiple methods to prove possession of the private key associated with an identity. + - TLS x509 certificates + - Signature verification + - Key Challenge + +Authorization and access control is managed through a distributed trust graph. +Trust servers are used as the authorities of the trust graph and allow caching portions of the graph for faster access. + +## Copyright and license + +Code and documentation copyright 2014 Docker, inc. Code released under the Apache 2.0 license. +Docs released under Creative commons. + diff --git a/vendor/github.com/docker/libtrust/certificates.go b/vendor/github.com/docker/libtrust/certificates.go new file mode 100644 index 000000000..3dcca33cb --- /dev/null +++ b/vendor/github.com/docker/libtrust/certificates.go @@ -0,0 +1,175 @@ +package libtrust + +import ( + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "io/ioutil" + "math/big" + "net" + "time" +) + +type certTemplateInfo struct { + commonName string + domains []string + ipAddresses []net.IP + isCA bool + clientAuth bool + serverAuth bool +} + +func generateCertTemplate(info *certTemplateInfo) *x509.Certificate { + // Generate a certificate template which is valid from the past week to + // 10 years from now. The usage of the certificate depends on the + // specified fields in the given certTempInfo object. + var ( + keyUsage x509.KeyUsage + extKeyUsage []x509.ExtKeyUsage + ) + + if info.isCA { + keyUsage = x509.KeyUsageCertSign + } + + if info.clientAuth { + extKeyUsage = append(extKeyUsage, x509.ExtKeyUsageClientAuth) + } + + if info.serverAuth { + extKeyUsage = append(extKeyUsage, x509.ExtKeyUsageServerAuth) + } + + return &x509.Certificate{ + SerialNumber: big.NewInt(0), + Subject: pkix.Name{ + CommonName: info.commonName, + }, + NotBefore: time.Now().Add(-time.Hour * 24 * 7), + NotAfter: time.Now().Add(time.Hour * 24 * 365 * 10), + DNSNames: info.domains, + IPAddresses: info.ipAddresses, + IsCA: info.isCA, + KeyUsage: keyUsage, + ExtKeyUsage: extKeyUsage, + BasicConstraintsValid: info.isCA, + } +} + +func generateCert(pub PublicKey, priv PrivateKey, subInfo, issInfo *certTemplateInfo) (cert *x509.Certificate, err error) { + pubCertTemplate := generateCertTemplate(subInfo) + privCertTemplate := generateCertTemplate(issInfo) + + certDER, err := x509.CreateCertificate( + rand.Reader, pubCertTemplate, privCertTemplate, + pub.CryptoPublicKey(), priv.CryptoPrivateKey(), + ) + if err != nil { + return nil, fmt.Errorf("failed to create certificate: %s", err) + } + + cert, err = x509.ParseCertificate(certDER) + if err != nil { + return nil, fmt.Errorf("failed to parse certificate: %s", err) + } + + return +} + +// GenerateSelfSignedServerCert creates a self-signed certificate for the +// given key which is to be used for TLS servers with the given domains and +// IP addresses. +func GenerateSelfSignedServerCert(key PrivateKey, domains []string, ipAddresses []net.IP) (*x509.Certificate, error) { + info := &certTemplateInfo{ + commonName: key.KeyID(), + domains: domains, + ipAddresses: ipAddresses, + serverAuth: true, + } + + return generateCert(key.PublicKey(), key, info, info) +} + +// GenerateSelfSignedClientCert creates a self-signed certificate for the +// given key which is to be used for TLS clients. +func GenerateSelfSignedClientCert(key PrivateKey) (*x509.Certificate, error) { + info := &certTemplateInfo{ + commonName: key.KeyID(), + clientAuth: true, + } + + return generateCert(key.PublicKey(), key, info, info) +} + +// GenerateCACert creates a certificate which can be used as a trusted +// certificate authority. +func GenerateCACert(signer PrivateKey, trustedKey PublicKey) (*x509.Certificate, error) { + subjectInfo := &certTemplateInfo{ + commonName: trustedKey.KeyID(), + isCA: true, + } + issuerInfo := &certTemplateInfo{ + commonName: signer.KeyID(), + } + + return generateCert(trustedKey, signer, subjectInfo, issuerInfo) +} + +// GenerateCACertPool creates a certificate authority pool to be used for a +// TLS configuration. Any self-signed certificates issued by the specified +// trusted keys will be verified during a TLS handshake +func GenerateCACertPool(signer PrivateKey, trustedKeys []PublicKey) (*x509.CertPool, error) { + certPool := x509.NewCertPool() + + for _, trustedKey := range trustedKeys { + cert, err := GenerateCACert(signer, trustedKey) + if err != nil { + return nil, fmt.Errorf("failed to generate CA certificate: %s", err) + } + + certPool.AddCert(cert) + } + + return certPool, nil +} + +// LoadCertificateBundle loads certificates from the given file. The file should be pem encoded +// containing one or more certificates. The expected pem type is "CERTIFICATE". +func LoadCertificateBundle(filename string) ([]*x509.Certificate, error) { + b, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + certificates := []*x509.Certificate{} + var block *pem.Block + block, b = pem.Decode(b) + for ; block != nil; block, b = pem.Decode(b) { + if block.Type == "CERTIFICATE" { + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, err + } + certificates = append(certificates, cert) + } else { + return nil, fmt.Errorf("invalid pem block type: %s", block.Type) + } + } + + return certificates, nil +} + +// LoadCertificatePool loads a CA pool from the given file. The file should be pem encoded +// containing one or more certificates. The expected pem type is "CERTIFICATE". +func LoadCertificatePool(filename string) (*x509.CertPool, error) { + certs, err := LoadCertificateBundle(filename) + if err != nil { + return nil, err + } + pool := x509.NewCertPool() + for _, cert := range certs { + pool.AddCert(cert) + } + return pool, nil +} diff --git a/vendor/github.com/docker/libtrust/doc.go b/vendor/github.com/docker/libtrust/doc.go new file mode 100644 index 000000000..ec5d2159c --- /dev/null +++ b/vendor/github.com/docker/libtrust/doc.go @@ -0,0 +1,9 @@ +/* +Package libtrust provides an interface for managing authentication and +authorization using public key cryptography. Authentication is handled +using the identity attached to the public key and verified through TLS +x509 certificates, a key challenge, or signature. Authorization and +access control is managed through a trust graph distributed between +both remote trust servers and locally cached and managed data. +*/ +package libtrust diff --git a/vendor/github.com/docker/libtrust/ec_key.go b/vendor/github.com/docker/libtrust/ec_key.go new file mode 100644 index 000000000..00bbe4b3c --- /dev/null +++ b/vendor/github.com/docker/libtrust/ec_key.go @@ -0,0 +1,428 @@ +package libtrust + +import ( + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "math/big" +) + +/* + * EC DSA PUBLIC KEY + */ + +// ecPublicKey implements a libtrust.PublicKey using elliptic curve digital +// signature algorithms. +type ecPublicKey struct { + *ecdsa.PublicKey + curveName string + signatureAlgorithm *signatureAlgorithm + extended map[string]interface{} +} + +func fromECPublicKey(cryptoPublicKey *ecdsa.PublicKey) (*ecPublicKey, error) { + curve := cryptoPublicKey.Curve + + switch { + case curve == elliptic.P256(): + return &ecPublicKey{cryptoPublicKey, "P-256", es256, map[string]interface{}{}}, nil + case curve == elliptic.P384(): + return &ecPublicKey{cryptoPublicKey, "P-384", es384, map[string]interface{}{}}, nil + case curve == elliptic.P521(): + return &ecPublicKey{cryptoPublicKey, "P-521", es512, map[string]interface{}{}}, nil + default: + return nil, errors.New("unsupported elliptic curve") + } +} + +// KeyType returns the key type for elliptic curve keys, i.e., "EC". +func (k *ecPublicKey) KeyType() string { + return "EC" +} + +// CurveName returns the elliptic curve identifier. +// Possible values are "P-256", "P-384", and "P-521". +func (k *ecPublicKey) CurveName() string { + return k.curveName +} + +// KeyID returns a distinct identifier which is unique to this Public Key. +func (k *ecPublicKey) KeyID() string { + return keyIDFromCryptoKey(k) +} + +func (k *ecPublicKey) String() string { + return fmt.Sprintf("EC Public Key <%s>", k.KeyID()) +} + +// Verify verifyies the signature of the data in the io.Reader using this +// PublicKey. The alg parameter should identify the digital signature +// algorithm which was used to produce the signature and should be supported +// by this public key. Returns a nil error if the signature is valid. +func (k *ecPublicKey) Verify(data io.Reader, alg string, signature []byte) error { + // For EC keys there is only one supported signature algorithm depending + // on the curve parameters. + if k.signatureAlgorithm.HeaderParam() != alg { + return fmt.Errorf("unable to verify signature: EC Public Key with curve %q does not support signature algorithm %q", k.curveName, alg) + } + + // signature is the concatenation of (r, s), base64Url encoded. + sigLength := len(signature) + expectedOctetLength := 2 * ((k.Params().BitSize + 7) >> 3) + if sigLength != expectedOctetLength { + return fmt.Errorf("signature length is %d octets long, should be %d", sigLength, expectedOctetLength) + } + + rBytes, sBytes := signature[:sigLength/2], signature[sigLength/2:] + r := new(big.Int).SetBytes(rBytes) + s := new(big.Int).SetBytes(sBytes) + + hasher := k.signatureAlgorithm.HashID().New() + _, err := io.Copy(hasher, data) + if err != nil { + return fmt.Errorf("error reading data to sign: %s", err) + } + hash := hasher.Sum(nil) + + if !ecdsa.Verify(k.PublicKey, hash, r, s) { + return errors.New("invalid signature") + } + + return nil +} + +// CryptoPublicKey returns the internal object which can be used as a +// crypto.PublicKey for use with other standard library operations. The type +// is either *rsa.PublicKey or *ecdsa.PublicKey +func (k *ecPublicKey) CryptoPublicKey() crypto.PublicKey { + return k.PublicKey +} + +func (k *ecPublicKey) toMap() map[string]interface{} { + jwk := make(map[string]interface{}) + for k, v := range k.extended { + jwk[k] = v + } + jwk["kty"] = k.KeyType() + jwk["kid"] = k.KeyID() + jwk["crv"] = k.CurveName() + + xBytes := k.X.Bytes() + yBytes := k.Y.Bytes() + octetLength := (k.Params().BitSize + 7) >> 3 + // MUST include leading zeros in the output so that x, y are each + // *octetLength* bytes long. + xBuf := make([]byte, octetLength-len(xBytes), octetLength) + yBuf := make([]byte, octetLength-len(yBytes), octetLength) + xBuf = append(xBuf, xBytes...) + yBuf = append(yBuf, yBytes...) + + jwk["x"] = joseBase64UrlEncode(xBuf) + jwk["y"] = joseBase64UrlEncode(yBuf) + + return jwk +} + +// MarshalJSON serializes this Public Key using the JWK JSON serialization format for +// elliptic curve keys. +func (k *ecPublicKey) MarshalJSON() (data []byte, err error) { + return json.Marshal(k.toMap()) +} + +// PEMBlock serializes this Public Key to DER-encoded PKIX format. +func (k *ecPublicKey) PEMBlock() (*pem.Block, error) { + derBytes, err := x509.MarshalPKIXPublicKey(k.PublicKey) + if err != nil { + return nil, fmt.Errorf("unable to serialize EC PublicKey to DER-encoded PKIX format: %s", err) + } + k.extended["kid"] = k.KeyID() // For display purposes. + return createPemBlock("PUBLIC KEY", derBytes, k.extended) +} + +func (k *ecPublicKey) AddExtendedField(field string, value interface{}) { + k.extended[field] = value +} + +func (k *ecPublicKey) GetExtendedField(field string) interface{} { + v, ok := k.extended[field] + if !ok { + return nil + } + return v +} + +func ecPublicKeyFromMap(jwk map[string]interface{}) (*ecPublicKey, error) { + // JWK key type (kty) has already been determined to be "EC". + // Need to extract 'crv', 'x', 'y', and 'kid' and check for + // consistency. + + // Get the curve identifier value. + crv, err := stringFromMap(jwk, "crv") + if err != nil { + return nil, fmt.Errorf("JWK EC Public Key curve identifier: %s", err) + } + + var ( + curve elliptic.Curve + sigAlg *signatureAlgorithm + ) + + switch { + case crv == "P-256": + curve = elliptic.P256() + sigAlg = es256 + case crv == "P-384": + curve = elliptic.P384() + sigAlg = es384 + case crv == "P-521": + curve = elliptic.P521() + sigAlg = es512 + default: + return nil, fmt.Errorf("JWK EC Public Key curve identifier not supported: %q\n", crv) + } + + // Get the X and Y coordinates for the public key point. + xB64Url, err := stringFromMap(jwk, "x") + if err != nil { + return nil, fmt.Errorf("JWK EC Public Key x-coordinate: %s", err) + } + x, err := parseECCoordinate(xB64Url, curve) + if err != nil { + return nil, fmt.Errorf("JWK EC Public Key x-coordinate: %s", err) + } + + yB64Url, err := stringFromMap(jwk, "y") + if err != nil { + return nil, fmt.Errorf("JWK EC Public Key y-coordinate: %s", err) + } + y, err := parseECCoordinate(yB64Url, curve) + if err != nil { + return nil, fmt.Errorf("JWK EC Public Key y-coordinate: %s", err) + } + + key := &ecPublicKey{ + PublicKey: &ecdsa.PublicKey{Curve: curve, X: x, Y: y}, + curveName: crv, signatureAlgorithm: sigAlg, + } + + // Key ID is optional too, but if it exists, it should match the key. + _, ok := jwk["kid"] + if ok { + kid, err := stringFromMap(jwk, "kid") + if err != nil { + return nil, fmt.Errorf("JWK EC Public Key ID: %s", err) + } + if kid != key.KeyID() { + return nil, fmt.Errorf("JWK EC Public Key ID does not match: %s", kid) + } + } + + key.extended = jwk + + return key, nil +} + +/* + * EC DSA PRIVATE KEY + */ + +// ecPrivateKey implements a JWK Private Key using elliptic curve digital signature +// algorithms. +type ecPrivateKey struct { + ecPublicKey + *ecdsa.PrivateKey +} + +func fromECPrivateKey(cryptoPrivateKey *ecdsa.PrivateKey) (*ecPrivateKey, error) { + publicKey, err := fromECPublicKey(&cryptoPrivateKey.PublicKey) + if err != nil { + return nil, err + } + + return &ecPrivateKey{*publicKey, cryptoPrivateKey}, nil +} + +// PublicKey returns the Public Key data associated with this Private Key. +func (k *ecPrivateKey) PublicKey() PublicKey { + return &k.ecPublicKey +} + +func (k *ecPrivateKey) String() string { + return fmt.Sprintf("EC Private Key <%s>", k.KeyID()) +} + +// Sign signs the data read from the io.Reader using a signature algorithm supported +// by the elliptic curve private key. If the specified hashing algorithm is +// supported by this key, that hash function is used to generate the signature +// otherwise the the default hashing algorithm for this key is used. Returns +// the signature and the name of the JWK signature algorithm used, e.g., +// "ES256", "ES384", "ES512". +func (k *ecPrivateKey) Sign(data io.Reader, hashID crypto.Hash) (signature []byte, alg string, err error) { + // Generate a signature of the data using the internal alg. + // The given hashId is only a suggestion, and since EC keys only support + // on signature/hash algorithm given the curve name, we disregard it for + // the elliptic curve JWK signature implementation. + hasher := k.signatureAlgorithm.HashID().New() + _, err = io.Copy(hasher, data) + if err != nil { + return nil, "", fmt.Errorf("error reading data to sign: %s", err) + } + hash := hasher.Sum(nil) + + r, s, err := ecdsa.Sign(rand.Reader, k.PrivateKey, hash) + if err != nil { + return nil, "", fmt.Errorf("error producing signature: %s", err) + } + rBytes, sBytes := r.Bytes(), s.Bytes() + octetLength := (k.ecPublicKey.Params().BitSize + 7) >> 3 + // MUST include leading zeros in the output + rBuf := make([]byte, octetLength-len(rBytes), octetLength) + sBuf := make([]byte, octetLength-len(sBytes), octetLength) + + rBuf = append(rBuf, rBytes...) + sBuf = append(sBuf, sBytes...) + + signature = append(rBuf, sBuf...) + alg = k.signatureAlgorithm.HeaderParam() + + return +} + +// CryptoPrivateKey returns the internal object which can be used as a +// crypto.PublicKey for use with other standard library operations. The type +// is either *rsa.PublicKey or *ecdsa.PublicKey +func (k *ecPrivateKey) CryptoPrivateKey() crypto.PrivateKey { + return k.PrivateKey +} + +func (k *ecPrivateKey) toMap() map[string]interface{} { + jwk := k.ecPublicKey.toMap() + + dBytes := k.D.Bytes() + // The length of this octet string MUST be ceiling(log-base-2(n)/8) + // octets (where n is the order of the curve). This is because the private + // key d must be in the interval [1, n-1] so the bitlength of d should be + // no larger than the bitlength of n-1. The easiest way to find the octet + // length is to take bitlength(n-1), add 7 to force a carry, and shift this + // bit sequence right by 3, which is essentially dividing by 8 and adding + // 1 if there is any remainder. Thus, the private key value d should be + // output to (bitlength(n-1)+7)>>3 octets. + n := k.ecPublicKey.Params().N + octetLength := (new(big.Int).Sub(n, big.NewInt(1)).BitLen() + 7) >> 3 + // Create a buffer with the necessary zero-padding. + dBuf := make([]byte, octetLength-len(dBytes), octetLength) + dBuf = append(dBuf, dBytes...) + + jwk["d"] = joseBase64UrlEncode(dBuf) + + return jwk +} + +// MarshalJSON serializes this Private Key using the JWK JSON serialization format for +// elliptic curve keys. +func (k *ecPrivateKey) MarshalJSON() (data []byte, err error) { + return json.Marshal(k.toMap()) +} + +// PEMBlock serializes this Private Key to DER-encoded PKIX format. +func (k *ecPrivateKey) PEMBlock() (*pem.Block, error) { + derBytes, err := x509.MarshalECPrivateKey(k.PrivateKey) + if err != nil { + return nil, fmt.Errorf("unable to serialize EC PrivateKey to DER-encoded PKIX format: %s", err) + } + k.extended["keyID"] = k.KeyID() // For display purposes. + return createPemBlock("EC PRIVATE KEY", derBytes, k.extended) +} + +func ecPrivateKeyFromMap(jwk map[string]interface{}) (*ecPrivateKey, error) { + dB64Url, err := stringFromMap(jwk, "d") + if err != nil { + return nil, fmt.Errorf("JWK EC Private Key: %s", err) + } + + // JWK key type (kty) has already been determined to be "EC". + // Need to extract the public key information, then extract the private + // key value 'd'. + publicKey, err := ecPublicKeyFromMap(jwk) + if err != nil { + return nil, err + } + + d, err := parseECPrivateParam(dB64Url, publicKey.Curve) + if err != nil { + return nil, fmt.Errorf("JWK EC Private Key d-param: %s", err) + } + + key := &ecPrivateKey{ + ecPublicKey: *publicKey, + PrivateKey: &ecdsa.PrivateKey{ + PublicKey: *publicKey.PublicKey, + D: d, + }, + } + + return key, nil +} + +/* + * Key Generation Functions. + */ + +func generateECPrivateKey(curve elliptic.Curve) (k *ecPrivateKey, err error) { + k = new(ecPrivateKey) + k.PrivateKey, err = ecdsa.GenerateKey(curve, rand.Reader) + if err != nil { + return nil, err + } + + k.ecPublicKey.PublicKey = &k.PrivateKey.PublicKey + k.extended = make(map[string]interface{}) + + return +} + +// GenerateECP256PrivateKey generates a key pair using elliptic curve P-256. +func GenerateECP256PrivateKey() (PrivateKey, error) { + k, err := generateECPrivateKey(elliptic.P256()) + if err != nil { + return nil, fmt.Errorf("error generating EC P-256 key: %s", err) + } + + k.curveName = "P-256" + k.signatureAlgorithm = es256 + + return k, nil +} + +// GenerateECP384PrivateKey generates a key pair using elliptic curve P-384. +func GenerateECP384PrivateKey() (PrivateKey, error) { + k, err := generateECPrivateKey(elliptic.P384()) + if err != nil { + return nil, fmt.Errorf("error generating EC P-384 key: %s", err) + } + + k.curveName = "P-384" + k.signatureAlgorithm = es384 + + return k, nil +} + +// GenerateECP521PrivateKey generates aß key pair using elliptic curve P-521. +func GenerateECP521PrivateKey() (PrivateKey, error) { + k, err := generateECPrivateKey(elliptic.P521()) + if err != nil { + return nil, fmt.Errorf("error generating EC P-521 key: %s", err) + } + + k.curveName = "P-521" + k.signatureAlgorithm = es512 + + return k, nil +} diff --git a/vendor/github.com/docker/libtrust/filter.go b/vendor/github.com/docker/libtrust/filter.go new file mode 100644 index 000000000..5b2b4fca6 --- /dev/null +++ b/vendor/github.com/docker/libtrust/filter.go @@ -0,0 +1,50 @@ +package libtrust + +import ( + "path/filepath" +) + +// FilterByHosts filters the list of PublicKeys to only those which contain a +// 'hosts' pattern which matches the given host. If *includeEmpty* is true, +// then keys which do not specify any hosts are also returned. +func FilterByHosts(keys []PublicKey, host string, includeEmpty bool) ([]PublicKey, error) { + filtered := make([]PublicKey, 0, len(keys)) + + for _, pubKey := range keys { + var hosts []string + switch v := pubKey.GetExtendedField("hosts").(type) { + case []string: + hosts = v + case []interface{}: + for _, value := range v { + h, ok := value.(string) + if !ok { + continue + } + hosts = append(hosts, h) + } + } + + if len(hosts) == 0 { + if includeEmpty { + filtered = append(filtered, pubKey) + } + continue + } + + // Check if any hosts match pattern + for _, hostPattern := range hosts { + match, err := filepath.Match(hostPattern, host) + if err != nil { + return nil, err + } + + if match { + filtered = append(filtered, pubKey) + continue + } + } + } + + return filtered, nil +} diff --git a/vendor/github.com/docker/libtrust/hash.go b/vendor/github.com/docker/libtrust/hash.go new file mode 100644 index 000000000..a2df787dd --- /dev/null +++ b/vendor/github.com/docker/libtrust/hash.go @@ -0,0 +1,56 @@ +package libtrust + +import ( + "crypto" + _ "crypto/sha256" // Registrer SHA224 and SHA256 + _ "crypto/sha512" // Registrer SHA384 and SHA512 + "fmt" +) + +type signatureAlgorithm struct { + algHeaderParam string + hashID crypto.Hash +} + +func (h *signatureAlgorithm) HeaderParam() string { + return h.algHeaderParam +} + +func (h *signatureAlgorithm) HashID() crypto.Hash { + return h.hashID +} + +var ( + rs256 = &signatureAlgorithm{"RS256", crypto.SHA256} + rs384 = &signatureAlgorithm{"RS384", crypto.SHA384} + rs512 = &signatureAlgorithm{"RS512", crypto.SHA512} + es256 = &signatureAlgorithm{"ES256", crypto.SHA256} + es384 = &signatureAlgorithm{"ES384", crypto.SHA384} + es512 = &signatureAlgorithm{"ES512", crypto.SHA512} +) + +func rsaSignatureAlgorithmByName(alg string) (*signatureAlgorithm, error) { + switch { + case alg == "RS256": + return rs256, nil + case alg == "RS384": + return rs384, nil + case alg == "RS512": + return rs512, nil + default: + return nil, fmt.Errorf("RSA Digital Signature Algorithm %q not supported", alg) + } +} + +func rsaPKCS1v15SignatureAlgorithmForHashID(hashID crypto.Hash) *signatureAlgorithm { + switch { + case hashID == crypto.SHA512: + return rs512 + case hashID == crypto.SHA384: + return rs384 + case hashID == crypto.SHA256: + fallthrough + default: + return rs256 + } +} diff --git a/vendor/github.com/docker/libtrust/jsonsign.go b/vendor/github.com/docker/libtrust/jsonsign.go new file mode 100644 index 000000000..cb2ca9a76 --- /dev/null +++ b/vendor/github.com/docker/libtrust/jsonsign.go @@ -0,0 +1,657 @@ +package libtrust + +import ( + "bytes" + "crypto" + "crypto/x509" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "sort" + "time" + "unicode" +) + +var ( + // ErrInvalidSignContent is used when the content to be signed is invalid. + ErrInvalidSignContent = errors.New("invalid sign content") + + // ErrInvalidJSONContent is used when invalid json is encountered. + ErrInvalidJSONContent = errors.New("invalid json content") + + // ErrMissingSignatureKey is used when the specified signature key + // does not exist in the JSON content. + ErrMissingSignatureKey = errors.New("missing signature key") +) + +type jsHeader struct { + JWK PublicKey `json:"jwk,omitempty"` + Algorithm string `json:"alg"` + Chain []string `json:"x5c,omitempty"` +} + +type jsSignature struct { + Header jsHeader `json:"header"` + Signature string `json:"signature"` + Protected string `json:"protected,omitempty"` +} + +type jsSignaturesSorted []jsSignature + +func (jsbkid jsSignaturesSorted) Swap(i, j int) { jsbkid[i], jsbkid[j] = jsbkid[j], jsbkid[i] } +func (jsbkid jsSignaturesSorted) Len() int { return len(jsbkid) } + +func (jsbkid jsSignaturesSorted) Less(i, j int) bool { + ki, kj := jsbkid[i].Header.JWK.KeyID(), jsbkid[j].Header.JWK.KeyID() + si, sj := jsbkid[i].Signature, jsbkid[j].Signature + + if ki == kj { + return si < sj + } + + return ki < kj +} + +type signKey struct { + PrivateKey + Chain []*x509.Certificate +} + +// JSONSignature represents a signature of a json object. +type JSONSignature struct { + payload string + signatures []jsSignature + indent string + formatLength int + formatTail []byte +} + +func newJSONSignature() *JSONSignature { + return &JSONSignature{ + signatures: make([]jsSignature, 0, 1), + } +} + +// Payload returns the encoded payload of the signature. This +// payload should not be signed directly +func (js *JSONSignature) Payload() ([]byte, error) { + return joseBase64UrlDecode(js.payload) +} + +func (js *JSONSignature) protectedHeader() (string, error) { + protected := map[string]interface{}{ + "formatLength": js.formatLength, + "formatTail": joseBase64UrlEncode(js.formatTail), + "time": time.Now().UTC().Format(time.RFC3339), + } + protectedBytes, err := json.Marshal(protected) + if err != nil { + return "", err + } + + return joseBase64UrlEncode(protectedBytes), nil +} + +func (js *JSONSignature) signBytes(protectedHeader string) ([]byte, error) { + buf := make([]byte, len(js.payload)+len(protectedHeader)+1) + copy(buf, protectedHeader) + buf[len(protectedHeader)] = '.' + copy(buf[len(protectedHeader)+1:], js.payload) + return buf, nil +} + +// Sign adds a signature using the given private key. +func (js *JSONSignature) Sign(key PrivateKey) error { + protected, err := js.protectedHeader() + if err != nil { + return err + } + signBytes, err := js.signBytes(protected) + if err != nil { + return err + } + sigBytes, algorithm, err := key.Sign(bytes.NewReader(signBytes), crypto.SHA256) + if err != nil { + return err + } + + js.signatures = append(js.signatures, jsSignature{ + Header: jsHeader{ + JWK: key.PublicKey(), + Algorithm: algorithm, + }, + Signature: joseBase64UrlEncode(sigBytes), + Protected: protected, + }) + + return nil +} + +// SignWithChain adds a signature using the given private key +// and setting the x509 chain. The public key of the first element +// in the chain must be the public key corresponding with the sign key. +func (js *JSONSignature) SignWithChain(key PrivateKey, chain []*x509.Certificate) error { + // Ensure key.Chain[0] is public key for key + //key.Chain.PublicKey + //key.PublicKey().CryptoPublicKey() + + // Verify chain + protected, err := js.protectedHeader() + if err != nil { + return err + } + signBytes, err := js.signBytes(protected) + if err != nil { + return err + } + sigBytes, algorithm, err := key.Sign(bytes.NewReader(signBytes), crypto.SHA256) + if err != nil { + return err + } + + header := jsHeader{ + Chain: make([]string, len(chain)), + Algorithm: algorithm, + } + + for i, cert := range chain { + header.Chain[i] = base64.StdEncoding.EncodeToString(cert.Raw) + } + + js.signatures = append(js.signatures, jsSignature{ + Header: header, + Signature: joseBase64UrlEncode(sigBytes), + Protected: protected, + }) + + return nil +} + +// Verify verifies all the signatures and returns the list of +// public keys used to sign. Any x509 chains are not checked. +func (js *JSONSignature) Verify() ([]PublicKey, error) { + keys := make([]PublicKey, len(js.signatures)) + for i, signature := range js.signatures { + signBytes, err := js.signBytes(signature.Protected) + if err != nil { + return nil, err + } + var publicKey PublicKey + if len(signature.Header.Chain) > 0 { + certBytes, err := base64.StdEncoding.DecodeString(signature.Header.Chain[0]) + if err != nil { + return nil, err + } + cert, err := x509.ParseCertificate(certBytes) + if err != nil { + return nil, err + } + publicKey, err = FromCryptoPublicKey(cert.PublicKey) + if err != nil { + return nil, err + } + } else if signature.Header.JWK != nil { + publicKey = signature.Header.JWK + } else { + return nil, errors.New("missing public key") + } + + sigBytes, err := joseBase64UrlDecode(signature.Signature) + if err != nil { + return nil, err + } + + err = publicKey.Verify(bytes.NewReader(signBytes), signature.Header.Algorithm, sigBytes) + if err != nil { + return nil, err + } + + keys[i] = publicKey + } + return keys, nil +} + +// VerifyChains verifies all the signatures and the chains associated +// with each signature and returns the list of verified chains. +// Signatures without an x509 chain are not checked. +func (js *JSONSignature) VerifyChains(ca *x509.CertPool) ([][]*x509.Certificate, error) { + chains := make([][]*x509.Certificate, 0, len(js.signatures)) + for _, signature := range js.signatures { + signBytes, err := js.signBytes(signature.Protected) + if err != nil { + return nil, err + } + var publicKey PublicKey + if len(signature.Header.Chain) > 0 { + certBytes, err := base64.StdEncoding.DecodeString(signature.Header.Chain[0]) + if err != nil { + return nil, err + } + cert, err := x509.ParseCertificate(certBytes) + if err != nil { + return nil, err + } + publicKey, err = FromCryptoPublicKey(cert.PublicKey) + if err != nil { + return nil, err + } + intermediates := x509.NewCertPool() + if len(signature.Header.Chain) > 1 { + intermediateChain := signature.Header.Chain[1:] + for i := range intermediateChain { + certBytes, err := base64.StdEncoding.DecodeString(intermediateChain[i]) + if err != nil { + return nil, err + } + intermediate, err := x509.ParseCertificate(certBytes) + if err != nil { + return nil, err + } + intermediates.AddCert(intermediate) + } + } + + verifyOptions := x509.VerifyOptions{ + Intermediates: intermediates, + Roots: ca, + } + + verifiedChains, err := cert.Verify(verifyOptions) + if err != nil { + return nil, err + } + chains = append(chains, verifiedChains...) + + sigBytes, err := joseBase64UrlDecode(signature.Signature) + if err != nil { + return nil, err + } + + err = publicKey.Verify(bytes.NewReader(signBytes), signature.Header.Algorithm, sigBytes) + if err != nil { + return nil, err + } + } + + } + return chains, nil +} + +// JWS returns JSON serialized JWS according to +// http://tools.ietf.org/html/draft-ietf-jose-json-web-signature-31#section-7.2 +func (js *JSONSignature) JWS() ([]byte, error) { + if len(js.signatures) == 0 { + return nil, errors.New("missing signature") + } + + sort.Sort(jsSignaturesSorted(js.signatures)) + + jsonMap := map[string]interface{}{ + "payload": js.payload, + "signatures": js.signatures, + } + + return json.MarshalIndent(jsonMap, "", " ") +} + +func notSpace(r rune) bool { + return !unicode.IsSpace(r) +} + +func detectJSONIndent(jsonContent []byte) (indent string) { + if len(jsonContent) > 2 && jsonContent[0] == '{' && jsonContent[1] == '\n' { + quoteIndex := bytes.IndexRune(jsonContent[1:], '"') + if quoteIndex > 0 { + indent = string(jsonContent[2 : quoteIndex+1]) + } + } + return +} + +type jsParsedHeader struct { + JWK json.RawMessage `json:"jwk"` + Algorithm string `json:"alg"` + Chain []string `json:"x5c"` +} + +type jsParsedSignature struct { + Header jsParsedHeader `json:"header"` + Signature string `json:"signature"` + Protected string `json:"protected"` +} + +// ParseJWS parses a JWS serialized JSON object into a Json Signature. +func ParseJWS(content []byte) (*JSONSignature, error) { + type jsParsed struct { + Payload string `json:"payload"` + Signatures []jsParsedSignature `json:"signatures"` + } + parsed := &jsParsed{} + err := json.Unmarshal(content, parsed) + if err != nil { + return nil, err + } + if len(parsed.Signatures) == 0 { + return nil, errors.New("missing signatures") + } + payload, err := joseBase64UrlDecode(parsed.Payload) + if err != nil { + return nil, err + } + + js, err := NewJSONSignature(payload) + if err != nil { + return nil, err + } + js.signatures = make([]jsSignature, len(parsed.Signatures)) + for i, signature := range parsed.Signatures { + header := jsHeader{ + Algorithm: signature.Header.Algorithm, + } + if signature.Header.Chain != nil { + header.Chain = signature.Header.Chain + } + if signature.Header.JWK != nil { + publicKey, err := UnmarshalPublicKeyJWK([]byte(signature.Header.JWK)) + if err != nil { + return nil, err + } + header.JWK = publicKey + } + js.signatures[i] = jsSignature{ + Header: header, + Signature: signature.Signature, + Protected: signature.Protected, + } + } + + return js, nil +} + +// NewJSONSignature returns a new unsigned JWS from a json byte array. +// JSONSignature will need to be signed before serializing or storing. +// Optionally, one or more signatures can be provided as byte buffers, +// containing serialized JWS signatures, to assemble a fully signed JWS +// package. It is the callers responsibility to ensure uniqueness of the +// provided signatures. +func NewJSONSignature(content []byte, signatures ...[]byte) (*JSONSignature, error) { + var dataMap map[string]interface{} + err := json.Unmarshal(content, &dataMap) + if err != nil { + return nil, err + } + + js := newJSONSignature() + js.indent = detectJSONIndent(content) + + js.payload = joseBase64UrlEncode(content) + + // Find trailing } and whitespace, put in protected header + closeIndex := bytes.LastIndexFunc(content, notSpace) + if content[closeIndex] != '}' { + return nil, ErrInvalidJSONContent + } + lastRuneIndex := bytes.LastIndexFunc(content[:closeIndex], notSpace) + if content[lastRuneIndex] == ',' { + return nil, ErrInvalidJSONContent + } + js.formatLength = lastRuneIndex + 1 + js.formatTail = content[js.formatLength:] + + if len(signatures) > 0 { + for _, signature := range signatures { + var parsedJSig jsParsedSignature + + if err := json.Unmarshal(signature, &parsedJSig); err != nil { + return nil, err + } + + // TODO(stevvooe): A lot of the code below is repeated in + // ParseJWS. It will require more refactoring to fix that. + jsig := jsSignature{ + Header: jsHeader{ + Algorithm: parsedJSig.Header.Algorithm, + }, + Signature: parsedJSig.Signature, + Protected: parsedJSig.Protected, + } + + if parsedJSig.Header.Chain != nil { + jsig.Header.Chain = parsedJSig.Header.Chain + } + + if parsedJSig.Header.JWK != nil { + publicKey, err := UnmarshalPublicKeyJWK([]byte(parsedJSig.Header.JWK)) + if err != nil { + return nil, err + } + jsig.Header.JWK = publicKey + } + + js.signatures = append(js.signatures, jsig) + } + } + + return js, nil +} + +// NewJSONSignatureFromMap returns a new unsigned JSONSignature from a map or +// struct. JWS will need to be signed before serializing or storing. +func NewJSONSignatureFromMap(content interface{}) (*JSONSignature, error) { + switch content.(type) { + case map[string]interface{}: + case struct{}: + default: + return nil, errors.New("invalid data type") + } + + js := newJSONSignature() + js.indent = " " + + payload, err := json.MarshalIndent(content, "", js.indent) + if err != nil { + return nil, err + } + js.payload = joseBase64UrlEncode(payload) + + // Remove '\n}' from formatted section, put in protected header + js.formatLength = len(payload) - 2 + js.formatTail = payload[js.formatLength:] + + return js, nil +} + +func readIntFromMap(key string, m map[string]interface{}) (int, bool) { + value, ok := m[key] + if !ok { + return 0, false + } + switch v := value.(type) { + case int: + return v, true + case float64: + return int(v), true + default: + return 0, false + } +} + +func readStringFromMap(key string, m map[string]interface{}) (v string, ok bool) { + value, ok := m[key] + if !ok { + return "", false + } + v, ok = value.(string) + return +} + +// ParsePrettySignature parses a formatted signature into a +// JSON signature. If the signatures are missing the format information +// an error is thrown. The formatted signature must be created by +// the same method as format signature. +func ParsePrettySignature(content []byte, signatureKey string) (*JSONSignature, error) { + var contentMap map[string]json.RawMessage + err := json.Unmarshal(content, &contentMap) + if err != nil { + return nil, fmt.Errorf("error unmarshalling content: %s", err) + } + sigMessage, ok := contentMap[signatureKey] + if !ok { + return nil, ErrMissingSignatureKey + } + + var signatureBlocks []jsParsedSignature + err = json.Unmarshal([]byte(sigMessage), &signatureBlocks) + if err != nil { + return nil, fmt.Errorf("error unmarshalling signatures: %s", err) + } + + js := newJSONSignature() + js.signatures = make([]jsSignature, len(signatureBlocks)) + + for i, signatureBlock := range signatureBlocks { + protectedBytes, err := joseBase64UrlDecode(signatureBlock.Protected) + if err != nil { + return nil, fmt.Errorf("base64 decode error: %s", err) + } + var protectedHeader map[string]interface{} + err = json.Unmarshal(protectedBytes, &protectedHeader) + if err != nil { + return nil, fmt.Errorf("error unmarshalling protected header: %s", err) + } + + formatLength, ok := readIntFromMap("formatLength", protectedHeader) + if !ok { + return nil, errors.New("missing formatted length") + } + encodedTail, ok := readStringFromMap("formatTail", protectedHeader) + if !ok { + return nil, errors.New("missing formatted tail") + } + formatTail, err := joseBase64UrlDecode(encodedTail) + if err != nil { + return nil, fmt.Errorf("base64 decode error on tail: %s", err) + } + if js.formatLength == 0 { + js.formatLength = formatLength + } else if js.formatLength != formatLength { + return nil, errors.New("conflicting format length") + } + if len(js.formatTail) == 0 { + js.formatTail = formatTail + } else if bytes.Compare(js.formatTail, formatTail) != 0 { + return nil, errors.New("conflicting format tail") + } + + header := jsHeader{ + Algorithm: signatureBlock.Header.Algorithm, + Chain: signatureBlock.Header.Chain, + } + if signatureBlock.Header.JWK != nil { + publicKey, err := UnmarshalPublicKeyJWK([]byte(signatureBlock.Header.JWK)) + if err != nil { + return nil, fmt.Errorf("error unmarshalling public key: %s", err) + } + header.JWK = publicKey + } + js.signatures[i] = jsSignature{ + Header: header, + Signature: signatureBlock.Signature, + Protected: signatureBlock.Protected, + } + } + if js.formatLength > len(content) { + return nil, errors.New("invalid format length") + } + formatted := make([]byte, js.formatLength+len(js.formatTail)) + copy(formatted, content[:js.formatLength]) + copy(formatted[js.formatLength:], js.formatTail) + js.indent = detectJSONIndent(formatted) + js.payload = joseBase64UrlEncode(formatted) + + return js, nil +} + +// PrettySignature formats a json signature into an easy to read +// single json serialized object. +func (js *JSONSignature) PrettySignature(signatureKey string) ([]byte, error) { + if len(js.signatures) == 0 { + return nil, errors.New("no signatures") + } + payload, err := joseBase64UrlDecode(js.payload) + if err != nil { + return nil, err + } + payload = payload[:js.formatLength] + + sort.Sort(jsSignaturesSorted(js.signatures)) + + var marshalled []byte + var marshallErr error + if js.indent != "" { + marshalled, marshallErr = json.MarshalIndent(js.signatures, js.indent, js.indent) + } else { + marshalled, marshallErr = json.Marshal(js.signatures) + } + if marshallErr != nil { + return nil, marshallErr + } + + buf := bytes.NewBuffer(make([]byte, 0, len(payload)+len(marshalled)+34)) + buf.Write(payload) + buf.WriteByte(',') + if js.indent != "" { + buf.WriteByte('\n') + buf.WriteString(js.indent) + buf.WriteByte('"') + buf.WriteString(signatureKey) + buf.WriteString("\": ") + buf.Write(marshalled) + buf.WriteByte('\n') + } else { + buf.WriteByte('"') + buf.WriteString(signatureKey) + buf.WriteString("\":") + buf.Write(marshalled) + } + buf.WriteByte('}') + + return buf.Bytes(), nil +} + +// Signatures provides the signatures on this JWS as opaque blobs, sorted by +// keyID. These blobs can be stored and reassembled with payloads. Internally, +// they are simply marshaled json web signatures but implementations should +// not rely on this. +func (js *JSONSignature) Signatures() ([][]byte, error) { + sort.Sort(jsSignaturesSorted(js.signatures)) + + var sb [][]byte + for _, jsig := range js.signatures { + p, err := json.Marshal(jsig) + if err != nil { + return nil, err + } + + sb = append(sb, p) + } + + return sb, nil +} + +// Merge combines the signatures from one or more other signatures into the +// method receiver. If the payloads differ for any argument, an error will be +// returned and the receiver will not be modified. +func (js *JSONSignature) Merge(others ...*JSONSignature) error { + merged := js.signatures + for _, other := range others { + if js.payload != other.payload { + return fmt.Errorf("payloads differ from merge target") + } + merged = append(merged, other.signatures...) + } + + js.signatures = merged + return nil +} diff --git a/vendor/github.com/docker/libtrust/key.go b/vendor/github.com/docker/libtrust/key.go new file mode 100644 index 000000000..73642db2a --- /dev/null +++ b/vendor/github.com/docker/libtrust/key.go @@ -0,0 +1,253 @@ +package libtrust + +import ( + "crypto" + "crypto/ecdsa" + "crypto/rsa" + "crypto/x509" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" +) + +// PublicKey is a generic interface for a Public Key. +type PublicKey interface { + // KeyType returns the key type for this key. For elliptic curve keys, + // this value should be "EC". For RSA keys, this value should be "RSA". + KeyType() string + // KeyID returns a distinct identifier which is unique to this Public Key. + // The format generated by this library is a base32 encoding of a 240 bit + // hash of the public key data divided into 12 groups like so: + // ABCD:EFGH:IJKL:MNOP:QRST:UVWX:YZ23:4567:ABCD:EFGH:IJKL:MNOP + KeyID() string + // Verify verifyies the signature of the data in the io.Reader using this + // Public Key. The alg parameter should identify the digital signature + // algorithm which was used to produce the signature and should be + // supported by this public key. Returns a nil error if the signature + // is valid. + Verify(data io.Reader, alg string, signature []byte) error + // CryptoPublicKey returns the internal object which can be used as a + // crypto.PublicKey for use with other standard library operations. The type + // is either *rsa.PublicKey or *ecdsa.PublicKey + CryptoPublicKey() crypto.PublicKey + // These public keys can be serialized to the standard JSON encoding for + // JSON Web Keys. See section 6 of the IETF draft RFC for JOSE JSON Web + // Algorithms. + MarshalJSON() ([]byte, error) + // These keys can also be serialized to the standard PEM encoding. + PEMBlock() (*pem.Block, error) + // The string representation of a key is its key type and ID. + String() string + AddExtendedField(string, interface{}) + GetExtendedField(string) interface{} +} + +// PrivateKey is a generic interface for a Private Key. +type PrivateKey interface { + // A PrivateKey contains all fields and methods of a PublicKey of the + // same type. The MarshalJSON method also outputs the private key as a + // JSON Web Key, and the PEMBlock method outputs the private key as a + // PEM block. + PublicKey + // PublicKey returns the PublicKey associated with this PrivateKey. + PublicKey() PublicKey + // Sign signs the data read from the io.Reader using a signature algorithm + // supported by the private key. If the specified hashing algorithm is + // supported by this key, that hash function is used to generate the + // signature otherwise the the default hashing algorithm for this key is + // used. Returns the signature and identifier of the algorithm used. + Sign(data io.Reader, hashID crypto.Hash) (signature []byte, alg string, err error) + // CryptoPrivateKey returns the internal object which can be used as a + // crypto.PublicKey for use with other standard library operations. The + // type is either *rsa.PublicKey or *ecdsa.PublicKey + CryptoPrivateKey() crypto.PrivateKey +} + +// FromCryptoPublicKey returns a libtrust PublicKey representation of the given +// *ecdsa.PublicKey or *rsa.PublicKey. Returns a non-nil error when the given +// key is of an unsupported type. +func FromCryptoPublicKey(cryptoPublicKey crypto.PublicKey) (PublicKey, error) { + switch cryptoPublicKey := cryptoPublicKey.(type) { + case *ecdsa.PublicKey: + return fromECPublicKey(cryptoPublicKey) + case *rsa.PublicKey: + return fromRSAPublicKey(cryptoPublicKey), nil + default: + return nil, fmt.Errorf("public key type %T is not supported", cryptoPublicKey) + } +} + +// FromCryptoPrivateKey returns a libtrust PrivateKey representation of the given +// *ecdsa.PrivateKey or *rsa.PrivateKey. Returns a non-nil error when the given +// key is of an unsupported type. +func FromCryptoPrivateKey(cryptoPrivateKey crypto.PrivateKey) (PrivateKey, error) { + switch cryptoPrivateKey := cryptoPrivateKey.(type) { + case *ecdsa.PrivateKey: + return fromECPrivateKey(cryptoPrivateKey) + case *rsa.PrivateKey: + return fromRSAPrivateKey(cryptoPrivateKey), nil + default: + return nil, fmt.Errorf("private key type %T is not supported", cryptoPrivateKey) + } +} + +// UnmarshalPublicKeyPEM parses the PEM encoded data and returns a libtrust +// PublicKey or an error if there is a problem with the encoding. +func UnmarshalPublicKeyPEM(data []byte) (PublicKey, error) { + pemBlock, _ := pem.Decode(data) + if pemBlock == nil { + return nil, errors.New("unable to find PEM encoded data") + } else if pemBlock.Type != "PUBLIC KEY" { + return nil, fmt.Errorf("unable to get PublicKey from PEM type: %s", pemBlock.Type) + } + + return pubKeyFromPEMBlock(pemBlock) +} + +// UnmarshalPublicKeyPEMBundle parses the PEM encoded data as a bundle of +// PEM blocks appended one after the other and returns a slice of PublicKey +// objects that it finds. +func UnmarshalPublicKeyPEMBundle(data []byte) ([]PublicKey, error) { + pubKeys := []PublicKey{} + + for { + var pemBlock *pem.Block + pemBlock, data = pem.Decode(data) + if pemBlock == nil { + break + } else if pemBlock.Type != "PUBLIC KEY" { + return nil, fmt.Errorf("unable to get PublicKey from PEM type: %s", pemBlock.Type) + } + + pubKey, err := pubKeyFromPEMBlock(pemBlock) + if err != nil { + return nil, err + } + + pubKeys = append(pubKeys, pubKey) + } + + return pubKeys, nil +} + +// UnmarshalPrivateKeyPEM parses the PEM encoded data and returns a libtrust +// PrivateKey or an error if there is a problem with the encoding. +func UnmarshalPrivateKeyPEM(data []byte) (PrivateKey, error) { + pemBlock, _ := pem.Decode(data) + if pemBlock == nil { + return nil, errors.New("unable to find PEM encoded data") + } + + var key PrivateKey + + switch { + case pemBlock.Type == "RSA PRIVATE KEY": + rsaPrivateKey, err := x509.ParsePKCS1PrivateKey(pemBlock.Bytes) + if err != nil { + return nil, fmt.Errorf("unable to decode RSA Private Key PEM data: %s", err) + } + key = fromRSAPrivateKey(rsaPrivateKey) + case pemBlock.Type == "EC PRIVATE KEY": + ecPrivateKey, err := x509.ParseECPrivateKey(pemBlock.Bytes) + if err != nil { + return nil, fmt.Errorf("unable to decode EC Private Key PEM data: %s", err) + } + key, err = fromECPrivateKey(ecPrivateKey) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unable to get PrivateKey from PEM type: %s", pemBlock.Type) + } + + addPEMHeadersToKey(pemBlock, key.PublicKey()) + + return key, nil +} + +// UnmarshalPublicKeyJWK unmarshals the given JSON Web Key into a generic +// Public Key to be used with libtrust. +func UnmarshalPublicKeyJWK(data []byte) (PublicKey, error) { + jwk := make(map[string]interface{}) + + err := json.Unmarshal(data, &jwk) + if err != nil { + return nil, fmt.Errorf( + "decoding JWK Public Key JSON data: %s\n", err, + ) + } + + // Get the Key Type value. + kty, err := stringFromMap(jwk, "kty") + if err != nil { + return nil, fmt.Errorf("JWK Public Key type: %s", err) + } + + switch { + case kty == "EC": + // Call out to unmarshal EC public key. + return ecPublicKeyFromMap(jwk) + case kty == "RSA": + // Call out to unmarshal RSA public key. + return rsaPublicKeyFromMap(jwk) + default: + return nil, fmt.Errorf( + "JWK Public Key type not supported: %q\n", kty, + ) + } +} + +// UnmarshalPublicKeyJWKSet parses the JSON encoded data as a JSON Web Key Set +// and returns a slice of Public Key objects. +func UnmarshalPublicKeyJWKSet(data []byte) ([]PublicKey, error) { + rawKeys, err := loadJSONKeySetRaw(data) + if err != nil { + return nil, err + } + + pubKeys := make([]PublicKey, 0, len(rawKeys)) + + for _, rawKey := range rawKeys { + pubKey, err := UnmarshalPublicKeyJWK(rawKey) + if err != nil { + return nil, err + } + pubKeys = append(pubKeys, pubKey) + } + + return pubKeys, nil +} + +// UnmarshalPrivateKeyJWK unmarshals the given JSON Web Key into a generic +// Private Key to be used with libtrust. +func UnmarshalPrivateKeyJWK(data []byte) (PrivateKey, error) { + jwk := make(map[string]interface{}) + + err := json.Unmarshal(data, &jwk) + if err != nil { + return nil, fmt.Errorf( + "decoding JWK Private Key JSON data: %s\n", err, + ) + } + + // Get the Key Type value. + kty, err := stringFromMap(jwk, "kty") + if err != nil { + return nil, fmt.Errorf("JWK Private Key type: %s", err) + } + + switch { + case kty == "EC": + // Call out to unmarshal EC private key. + return ecPrivateKeyFromMap(jwk) + case kty == "RSA": + // Call out to unmarshal RSA private key. + return rsaPrivateKeyFromMap(jwk) + default: + return nil, fmt.Errorf( + "JWK Private Key type not supported: %q\n", kty, + ) + } +} diff --git a/vendor/github.com/docker/libtrust/key_files.go b/vendor/github.com/docker/libtrust/key_files.go new file mode 100644 index 000000000..c526de545 --- /dev/null +++ b/vendor/github.com/docker/libtrust/key_files.go @@ -0,0 +1,255 @@ +package libtrust + +import ( + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io/ioutil" + "os" + "strings" +) + +var ( + // ErrKeyFileDoesNotExist indicates that the private key file does not exist. + ErrKeyFileDoesNotExist = errors.New("key file does not exist") +) + +func readKeyFileBytes(filename string) ([]byte, error) { + data, err := ioutil.ReadFile(filename) + if err != nil { + if os.IsNotExist(err) { + err = ErrKeyFileDoesNotExist + } else { + err = fmt.Errorf("unable to read key file %s: %s", filename, err) + } + + return nil, err + } + + return data, nil +} + +/* + Loading and Saving of Public and Private Keys in either PEM or JWK format. +*/ + +// LoadKeyFile opens the given filename and attempts to read a Private Key +// encoded in either PEM or JWK format (if .json or .jwk file extension). +func LoadKeyFile(filename string) (PrivateKey, error) { + contents, err := readKeyFileBytes(filename) + if err != nil { + return nil, err + } + + var key PrivateKey + + if strings.HasSuffix(filename, ".json") || strings.HasSuffix(filename, ".jwk") { + key, err = UnmarshalPrivateKeyJWK(contents) + if err != nil { + return nil, fmt.Errorf("unable to decode private key JWK: %s", err) + } + } else { + key, err = UnmarshalPrivateKeyPEM(contents) + if err != nil { + return nil, fmt.Errorf("unable to decode private key PEM: %s", err) + } + } + + return key, nil +} + +// LoadPublicKeyFile opens the given filename and attempts to read a Public Key +// encoded in either PEM or JWK format (if .json or .jwk file extension). +func LoadPublicKeyFile(filename string) (PublicKey, error) { + contents, err := readKeyFileBytes(filename) + if err != nil { + return nil, err + } + + var key PublicKey + + if strings.HasSuffix(filename, ".json") || strings.HasSuffix(filename, ".jwk") { + key, err = UnmarshalPublicKeyJWK(contents) + if err != nil { + return nil, fmt.Errorf("unable to decode public key JWK: %s", err) + } + } else { + key, err = UnmarshalPublicKeyPEM(contents) + if err != nil { + return nil, fmt.Errorf("unable to decode public key PEM: %s", err) + } + } + + return key, nil +} + +// SaveKey saves the given key to a file using the provided filename. +// This process will overwrite any existing file at the provided location. +func SaveKey(filename string, key PrivateKey) error { + var encodedKey []byte + var err error + + if strings.HasSuffix(filename, ".json") || strings.HasSuffix(filename, ".jwk") { + // Encode in JSON Web Key format. + encodedKey, err = json.MarshalIndent(key, "", " ") + if err != nil { + return fmt.Errorf("unable to encode private key JWK: %s", err) + } + } else { + // Encode in PEM format. + pemBlock, err := key.PEMBlock() + if err != nil { + return fmt.Errorf("unable to encode private key PEM: %s", err) + } + encodedKey = pem.EncodeToMemory(pemBlock) + } + + err = ioutil.WriteFile(filename, encodedKey, os.FileMode(0600)) + if err != nil { + return fmt.Errorf("unable to write private key file %s: %s", filename, err) + } + + return nil +} + +// SavePublicKey saves the given public key to the file. +func SavePublicKey(filename string, key PublicKey) error { + var encodedKey []byte + var err error + + if strings.HasSuffix(filename, ".json") || strings.HasSuffix(filename, ".jwk") { + // Encode in JSON Web Key format. + encodedKey, err = json.MarshalIndent(key, "", " ") + if err != nil { + return fmt.Errorf("unable to encode public key JWK: %s", err) + } + } else { + // Encode in PEM format. + pemBlock, err := key.PEMBlock() + if err != nil { + return fmt.Errorf("unable to encode public key PEM: %s", err) + } + encodedKey = pem.EncodeToMemory(pemBlock) + } + + err = ioutil.WriteFile(filename, encodedKey, os.FileMode(0644)) + if err != nil { + return fmt.Errorf("unable to write public key file %s: %s", filename, err) + } + + return nil +} + +// Public Key Set files + +type jwkSet struct { + Keys []json.RawMessage `json:"keys"` +} + +// LoadKeySetFile loads a key set +func LoadKeySetFile(filename string) ([]PublicKey, error) { + if strings.HasSuffix(filename, ".json") || strings.HasSuffix(filename, ".jwk") { + return loadJSONKeySetFile(filename) + } + + // Must be a PEM format file + return loadPEMKeySetFile(filename) +} + +func loadJSONKeySetRaw(data []byte) ([]json.RawMessage, error) { + if len(data) == 0 { + // This is okay, just return an empty slice. + return []json.RawMessage{}, nil + } + + keySet := jwkSet{} + + err := json.Unmarshal(data, &keySet) + if err != nil { + return nil, fmt.Errorf("unable to decode JSON Web Key Set: %s", err) + } + + return keySet.Keys, nil +} + +func loadJSONKeySetFile(filename string) ([]PublicKey, error) { + contents, err := readKeyFileBytes(filename) + if err != nil && err != ErrKeyFileDoesNotExist { + return nil, err + } + + return UnmarshalPublicKeyJWKSet(contents) +} + +func loadPEMKeySetFile(filename string) ([]PublicKey, error) { + data, err := readKeyFileBytes(filename) + if err != nil && err != ErrKeyFileDoesNotExist { + return nil, err + } + + return UnmarshalPublicKeyPEMBundle(data) +} + +// AddKeySetFile adds a key to a key set +func AddKeySetFile(filename string, key PublicKey) error { + if strings.HasSuffix(filename, ".json") || strings.HasSuffix(filename, ".jwk") { + return addKeySetJSONFile(filename, key) + } + + // Must be a PEM format file + return addKeySetPEMFile(filename, key) +} + +func addKeySetJSONFile(filename string, key PublicKey) error { + encodedKey, err := json.Marshal(key) + if err != nil { + return fmt.Errorf("unable to encode trusted client key: %s", err) + } + + contents, err := readKeyFileBytes(filename) + if err != nil && err != ErrKeyFileDoesNotExist { + return err + } + + rawEntries, err := loadJSONKeySetRaw(contents) + if err != nil { + return err + } + + rawEntries = append(rawEntries, json.RawMessage(encodedKey)) + entriesWrapper := jwkSet{Keys: rawEntries} + + encodedEntries, err := json.MarshalIndent(entriesWrapper, "", " ") + if err != nil { + return fmt.Errorf("unable to encode trusted client keys: %s", err) + } + + err = ioutil.WriteFile(filename, encodedEntries, os.FileMode(0644)) + if err != nil { + return fmt.Errorf("unable to write trusted client keys file %s: %s", filename, err) + } + + return nil +} + +func addKeySetPEMFile(filename string, key PublicKey) error { + // Encode to PEM, open file for appending, write PEM. + file, err := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_RDWR, os.FileMode(0644)) + if err != nil { + return fmt.Errorf("unable to open trusted client keys file %s: %s", filename, err) + } + defer file.Close() + + pemBlock, err := key.PEMBlock() + if err != nil { + return fmt.Errorf("unable to encoded trusted key: %s", err) + } + + _, err = file.Write(pem.EncodeToMemory(pemBlock)) + if err != nil { + return fmt.Errorf("unable to write trusted keys file: %s", err) + } + + return nil +} diff --git a/vendor/github.com/docker/libtrust/key_manager.go b/vendor/github.com/docker/libtrust/key_manager.go new file mode 100644 index 000000000..9a98ae357 --- /dev/null +++ b/vendor/github.com/docker/libtrust/key_manager.go @@ -0,0 +1,175 @@ +package libtrust + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" + "net" + "os" + "path" + "sync" +) + +// ClientKeyManager manages client keys on the filesystem +type ClientKeyManager struct { + key PrivateKey + clientFile string + clientDir string + + clientLock sync.RWMutex + clients []PublicKey + + configLock sync.Mutex + configs []*tls.Config +} + +// NewClientKeyManager loads a new manager from a set of key files +// and managed by the given private key. +func NewClientKeyManager(trustKey PrivateKey, clientFile, clientDir string) (*ClientKeyManager, error) { + m := &ClientKeyManager{ + key: trustKey, + clientFile: clientFile, + clientDir: clientDir, + } + if err := m.loadKeys(); err != nil { + return nil, err + } + // TODO Start watching file and directory + + return m, nil +} + +func (c *ClientKeyManager) loadKeys() (err error) { + // Load authorized keys file + var clients []PublicKey + if c.clientFile != "" { + clients, err = LoadKeySetFile(c.clientFile) + if err != nil { + return fmt.Errorf("unable to load authorized keys: %s", err) + } + } + + // Add clients from authorized keys directory + files, err := ioutil.ReadDir(c.clientDir) + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("unable to open authorized keys directory: %s", err) + } + for _, f := range files { + if !f.IsDir() { + publicKey, err := LoadPublicKeyFile(path.Join(c.clientDir, f.Name())) + if err != nil { + return fmt.Errorf("unable to load authorized key file: %s", err) + } + clients = append(clients, publicKey) + } + } + + c.clientLock.Lock() + c.clients = clients + c.clientLock.Unlock() + + return nil +} + +// RegisterTLSConfig registers a tls configuration to manager +// such that any changes to the keys may be reflected in +// the tls client CA pool +func (c *ClientKeyManager) RegisterTLSConfig(tlsConfig *tls.Config) error { + c.clientLock.RLock() + certPool, err := GenerateCACertPool(c.key, c.clients) + if err != nil { + return fmt.Errorf("CA pool generation error: %s", err) + } + c.clientLock.RUnlock() + + tlsConfig.ClientCAs = certPool + + c.configLock.Lock() + c.configs = append(c.configs, tlsConfig) + c.configLock.Unlock() + + return nil +} + +// NewIdentityAuthTLSConfig creates a tls.Config for the server to use for +// libtrust identity authentication for the domain specified +func NewIdentityAuthTLSConfig(trustKey PrivateKey, clients *ClientKeyManager, addr string, domain string) (*tls.Config, error) { + tlsConfig := newTLSConfig() + + tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert + if err := clients.RegisterTLSConfig(tlsConfig); err != nil { + return nil, err + } + + // Generate cert + ips, domains, err := parseAddr(addr) + if err != nil { + return nil, err + } + // add domain that it expects clients to use + domains = append(domains, domain) + x509Cert, err := GenerateSelfSignedServerCert(trustKey, domains, ips) + if err != nil { + return nil, fmt.Errorf("certificate generation error: %s", err) + } + tlsConfig.Certificates = []tls.Certificate{{ + Certificate: [][]byte{x509Cert.Raw}, + PrivateKey: trustKey.CryptoPrivateKey(), + Leaf: x509Cert, + }} + + return tlsConfig, nil +} + +// NewCertAuthTLSConfig creates a tls.Config for the server to use for +// certificate authentication +func NewCertAuthTLSConfig(caPath, certPath, keyPath string) (*tls.Config, error) { + tlsConfig := newTLSConfig() + + cert, err := tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + return nil, fmt.Errorf("Couldn't load X509 key pair (%s, %s): %s. Key encrypted?", certPath, keyPath, err) + } + tlsConfig.Certificates = []tls.Certificate{cert} + + // Verify client certificates against a CA? + if caPath != "" { + certPool := x509.NewCertPool() + file, err := ioutil.ReadFile(caPath) + if err != nil { + return nil, fmt.Errorf("Couldn't read CA certificate: %s", err) + } + certPool.AppendCertsFromPEM(file) + + tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert + tlsConfig.ClientCAs = certPool + } + + return tlsConfig, nil +} + +func newTLSConfig() *tls.Config { + return &tls.Config{ + NextProtos: []string{"http/1.1"}, + // Avoid fallback on insecure SSL protocols + MinVersion: tls.VersionTLS10, + } +} + +// parseAddr parses an address into an array of IPs and domains +func parseAddr(addr string) ([]net.IP, []string, error) { + host, _, err := net.SplitHostPort(addr) + if err != nil { + return nil, nil, err + } + var domains []string + var ips []net.IP + ip := net.ParseIP(host) + if ip != nil { + ips = []net.IP{ip} + } else { + domains = []string{host} + } + return ips, domains, nil +} diff --git a/vendor/github.com/docker/libtrust/rsa_key.go b/vendor/github.com/docker/libtrust/rsa_key.go new file mode 100644 index 000000000..dac4cacf2 --- /dev/null +++ b/vendor/github.com/docker/libtrust/rsa_key.go @@ -0,0 +1,427 @@ +package libtrust + +import ( + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "math/big" +) + +/* + * RSA DSA PUBLIC KEY + */ + +// rsaPublicKey implements a JWK Public Key using RSA digital signature algorithms. +type rsaPublicKey struct { + *rsa.PublicKey + extended map[string]interface{} +} + +func fromRSAPublicKey(cryptoPublicKey *rsa.PublicKey) *rsaPublicKey { + return &rsaPublicKey{cryptoPublicKey, map[string]interface{}{}} +} + +// KeyType returns the JWK key type for RSA keys, i.e., "RSA". +func (k *rsaPublicKey) KeyType() string { + return "RSA" +} + +// KeyID returns a distinct identifier which is unique to this Public Key. +func (k *rsaPublicKey) KeyID() string { + return keyIDFromCryptoKey(k) +} + +func (k *rsaPublicKey) String() string { + return fmt.Sprintf("RSA Public Key <%s>", k.KeyID()) +} + +// Verify verifyies the signature of the data in the io.Reader using this Public Key. +// The alg parameter should be the name of the JWA digital signature algorithm +// which was used to produce the signature and should be supported by this +// public key. Returns a nil error if the signature is valid. +func (k *rsaPublicKey) Verify(data io.Reader, alg string, signature []byte) error { + // Verify the signature of the given date, return non-nil error if valid. + sigAlg, err := rsaSignatureAlgorithmByName(alg) + if err != nil { + return fmt.Errorf("unable to verify Signature: %s", err) + } + + hasher := sigAlg.HashID().New() + _, err = io.Copy(hasher, data) + if err != nil { + return fmt.Errorf("error reading data to sign: %s", err) + } + hash := hasher.Sum(nil) + + err = rsa.VerifyPKCS1v15(k.PublicKey, sigAlg.HashID(), hash, signature) + if err != nil { + return fmt.Errorf("invalid %s signature: %s", sigAlg.HeaderParam(), err) + } + + return nil +} + +// CryptoPublicKey returns the internal object which can be used as a +// crypto.PublicKey for use with other standard library operations. The type +// is either *rsa.PublicKey or *ecdsa.PublicKey +func (k *rsaPublicKey) CryptoPublicKey() crypto.PublicKey { + return k.PublicKey +} + +func (k *rsaPublicKey) toMap() map[string]interface{} { + jwk := make(map[string]interface{}) + for k, v := range k.extended { + jwk[k] = v + } + jwk["kty"] = k.KeyType() + jwk["kid"] = k.KeyID() + jwk["n"] = joseBase64UrlEncode(k.N.Bytes()) + jwk["e"] = joseBase64UrlEncode(serializeRSAPublicExponentParam(k.E)) + + return jwk +} + +// MarshalJSON serializes this Public Key using the JWK JSON serialization format for +// RSA keys. +func (k *rsaPublicKey) MarshalJSON() (data []byte, err error) { + return json.Marshal(k.toMap()) +} + +// PEMBlock serializes this Public Key to DER-encoded PKIX format. +func (k *rsaPublicKey) PEMBlock() (*pem.Block, error) { + derBytes, err := x509.MarshalPKIXPublicKey(k.PublicKey) + if err != nil { + return nil, fmt.Errorf("unable to serialize RSA PublicKey to DER-encoded PKIX format: %s", err) + } + k.extended["kid"] = k.KeyID() // For display purposes. + return createPemBlock("PUBLIC KEY", derBytes, k.extended) +} + +func (k *rsaPublicKey) AddExtendedField(field string, value interface{}) { + k.extended[field] = value +} + +func (k *rsaPublicKey) GetExtendedField(field string) interface{} { + v, ok := k.extended[field] + if !ok { + return nil + } + return v +} + +func rsaPublicKeyFromMap(jwk map[string]interface{}) (*rsaPublicKey, error) { + // JWK key type (kty) has already been determined to be "RSA". + // Need to extract 'n', 'e', and 'kid' and check for + // consistency. + + // Get the modulus parameter N. + nB64Url, err := stringFromMap(jwk, "n") + if err != nil { + return nil, fmt.Errorf("JWK RSA Public Key modulus: %s", err) + } + + n, err := parseRSAModulusParam(nB64Url) + if err != nil { + return nil, fmt.Errorf("JWK RSA Public Key modulus: %s", err) + } + + // Get the public exponent E. + eB64Url, err := stringFromMap(jwk, "e") + if err != nil { + return nil, fmt.Errorf("JWK RSA Public Key exponent: %s", err) + } + + e, err := parseRSAPublicExponentParam(eB64Url) + if err != nil { + return nil, fmt.Errorf("JWK RSA Public Key exponent: %s", err) + } + + key := &rsaPublicKey{ + PublicKey: &rsa.PublicKey{N: n, E: e}, + } + + // Key ID is optional, but if it exists, it should match the key. + _, ok := jwk["kid"] + if ok { + kid, err := stringFromMap(jwk, "kid") + if err != nil { + return nil, fmt.Errorf("JWK RSA Public Key ID: %s", err) + } + if kid != key.KeyID() { + return nil, fmt.Errorf("JWK RSA Public Key ID does not match: %s", kid) + } + } + + if _, ok := jwk["d"]; ok { + return nil, fmt.Errorf("JWK RSA Public Key cannot contain private exponent") + } + + key.extended = jwk + + return key, nil +} + +/* + * RSA DSA PRIVATE KEY + */ + +// rsaPrivateKey implements a JWK Private Key using RSA digital signature algorithms. +type rsaPrivateKey struct { + rsaPublicKey + *rsa.PrivateKey +} + +func fromRSAPrivateKey(cryptoPrivateKey *rsa.PrivateKey) *rsaPrivateKey { + return &rsaPrivateKey{ + *fromRSAPublicKey(&cryptoPrivateKey.PublicKey), + cryptoPrivateKey, + } +} + +// PublicKey returns the Public Key data associated with this Private Key. +func (k *rsaPrivateKey) PublicKey() PublicKey { + return &k.rsaPublicKey +} + +func (k *rsaPrivateKey) String() string { + return fmt.Sprintf("RSA Private Key <%s>", k.KeyID()) +} + +// Sign signs the data read from the io.Reader using a signature algorithm supported +// by the RSA private key. If the specified hashing algorithm is supported by +// this key, that hash function is used to generate the signature otherwise the +// the default hashing algorithm for this key is used. Returns the signature +// and the name of the JWK signature algorithm used, e.g., "RS256", "RS384", +// "RS512". +func (k *rsaPrivateKey) Sign(data io.Reader, hashID crypto.Hash) (signature []byte, alg string, err error) { + // Generate a signature of the data using the internal alg. + sigAlg := rsaPKCS1v15SignatureAlgorithmForHashID(hashID) + hasher := sigAlg.HashID().New() + + _, err = io.Copy(hasher, data) + if err != nil { + return nil, "", fmt.Errorf("error reading data to sign: %s", err) + } + hash := hasher.Sum(nil) + + signature, err = rsa.SignPKCS1v15(rand.Reader, k.PrivateKey, sigAlg.HashID(), hash) + if err != nil { + return nil, "", fmt.Errorf("error producing signature: %s", err) + } + + alg = sigAlg.HeaderParam() + + return +} + +// CryptoPrivateKey returns the internal object which can be used as a +// crypto.PublicKey for use with other standard library operations. The type +// is either *rsa.PublicKey or *ecdsa.PublicKey +func (k *rsaPrivateKey) CryptoPrivateKey() crypto.PrivateKey { + return k.PrivateKey +} + +func (k *rsaPrivateKey) toMap() map[string]interface{} { + k.Precompute() // Make sure the precomputed values are stored. + jwk := k.rsaPublicKey.toMap() + + jwk["d"] = joseBase64UrlEncode(k.D.Bytes()) + jwk["p"] = joseBase64UrlEncode(k.Primes[0].Bytes()) + jwk["q"] = joseBase64UrlEncode(k.Primes[1].Bytes()) + jwk["dp"] = joseBase64UrlEncode(k.Precomputed.Dp.Bytes()) + jwk["dq"] = joseBase64UrlEncode(k.Precomputed.Dq.Bytes()) + jwk["qi"] = joseBase64UrlEncode(k.Precomputed.Qinv.Bytes()) + + otherPrimes := k.Primes[2:] + + if len(otherPrimes) > 0 { + otherPrimesInfo := make([]interface{}, len(otherPrimes)) + for i, r := range otherPrimes { + otherPrimeInfo := make(map[string]string, 3) + otherPrimeInfo["r"] = joseBase64UrlEncode(r.Bytes()) + crtVal := k.Precomputed.CRTValues[i] + otherPrimeInfo["d"] = joseBase64UrlEncode(crtVal.Exp.Bytes()) + otherPrimeInfo["t"] = joseBase64UrlEncode(crtVal.Coeff.Bytes()) + otherPrimesInfo[i] = otherPrimeInfo + } + jwk["oth"] = otherPrimesInfo + } + + return jwk +} + +// MarshalJSON serializes this Private Key using the JWK JSON serialization format for +// RSA keys. +func (k *rsaPrivateKey) MarshalJSON() (data []byte, err error) { + return json.Marshal(k.toMap()) +} + +// PEMBlock serializes this Private Key to DER-encoded PKIX format. +func (k *rsaPrivateKey) PEMBlock() (*pem.Block, error) { + derBytes := x509.MarshalPKCS1PrivateKey(k.PrivateKey) + k.extended["keyID"] = k.KeyID() // For display purposes. + return createPemBlock("RSA PRIVATE KEY", derBytes, k.extended) +} + +func rsaPrivateKeyFromMap(jwk map[string]interface{}) (*rsaPrivateKey, error) { + // The JWA spec for RSA Private Keys (draft rfc section 5.3.2) states that + // only the private key exponent 'd' is REQUIRED, the others are just for + // signature/decryption optimizations and SHOULD be included when the JWK + // is produced. We MAY choose to accept a JWK which only includes 'd', but + // we're going to go ahead and not choose to accept it without the extra + // fields. Only the 'oth' field will be optional (for multi-prime keys). + privateExponent, err := parseRSAPrivateKeyParamFromMap(jwk, "d") + if err != nil { + return nil, fmt.Errorf("JWK RSA Private Key exponent: %s", err) + } + firstPrimeFactor, err := parseRSAPrivateKeyParamFromMap(jwk, "p") + if err != nil { + return nil, fmt.Errorf("JWK RSA Private Key prime factor: %s", err) + } + secondPrimeFactor, err := parseRSAPrivateKeyParamFromMap(jwk, "q") + if err != nil { + return nil, fmt.Errorf("JWK RSA Private Key prime factor: %s", err) + } + firstFactorCRT, err := parseRSAPrivateKeyParamFromMap(jwk, "dp") + if err != nil { + return nil, fmt.Errorf("JWK RSA Private Key CRT exponent: %s", err) + } + secondFactorCRT, err := parseRSAPrivateKeyParamFromMap(jwk, "dq") + if err != nil { + return nil, fmt.Errorf("JWK RSA Private Key CRT exponent: %s", err) + } + crtCoeff, err := parseRSAPrivateKeyParamFromMap(jwk, "qi") + if err != nil { + return nil, fmt.Errorf("JWK RSA Private Key CRT coefficient: %s", err) + } + + var oth interface{} + if _, ok := jwk["oth"]; ok { + oth = jwk["oth"] + delete(jwk, "oth") + } + + // JWK key type (kty) has already been determined to be "RSA". + // Need to extract the public key information, then extract the private + // key values. + publicKey, err := rsaPublicKeyFromMap(jwk) + if err != nil { + return nil, err + } + + privateKey := &rsa.PrivateKey{ + PublicKey: *publicKey.PublicKey, + D: privateExponent, + Primes: []*big.Int{firstPrimeFactor, secondPrimeFactor}, + Precomputed: rsa.PrecomputedValues{ + Dp: firstFactorCRT, + Dq: secondFactorCRT, + Qinv: crtCoeff, + }, + } + + if oth != nil { + // Should be an array of more JSON objects. + otherPrimesInfo, ok := oth.([]interface{}) + if !ok { + return nil, errors.New("JWK RSA Private Key: Invalid other primes info: must be an array") + } + numOtherPrimeFactors := len(otherPrimesInfo) + if numOtherPrimeFactors == 0 { + return nil, errors.New("JWK RSA Privake Key: Invalid other primes info: must be absent or non-empty") + } + otherPrimeFactors := make([]*big.Int, numOtherPrimeFactors) + productOfPrimes := new(big.Int).Mul(firstPrimeFactor, secondPrimeFactor) + crtValues := make([]rsa.CRTValue, numOtherPrimeFactors) + + for i, val := range otherPrimesInfo { + otherPrimeinfo, ok := val.(map[string]interface{}) + if !ok { + return nil, errors.New("JWK RSA Private Key: Invalid other prime info: must be a JSON object") + } + + otherPrimeFactor, err := parseRSAPrivateKeyParamFromMap(otherPrimeinfo, "r") + if err != nil { + return nil, fmt.Errorf("JWK RSA Private Key prime factor: %s", err) + } + otherFactorCRT, err := parseRSAPrivateKeyParamFromMap(otherPrimeinfo, "d") + if err != nil { + return nil, fmt.Errorf("JWK RSA Private Key CRT exponent: %s", err) + } + otherCrtCoeff, err := parseRSAPrivateKeyParamFromMap(otherPrimeinfo, "t") + if err != nil { + return nil, fmt.Errorf("JWK RSA Private Key CRT coefficient: %s", err) + } + + crtValue := crtValues[i] + crtValue.Exp = otherFactorCRT + crtValue.Coeff = otherCrtCoeff + crtValue.R = productOfPrimes + otherPrimeFactors[i] = otherPrimeFactor + productOfPrimes = new(big.Int).Mul(productOfPrimes, otherPrimeFactor) + } + + privateKey.Primes = append(privateKey.Primes, otherPrimeFactors...) + privateKey.Precomputed.CRTValues = crtValues + } + + key := &rsaPrivateKey{ + rsaPublicKey: *publicKey, + PrivateKey: privateKey, + } + + return key, nil +} + +/* + * Key Generation Functions. + */ + +func generateRSAPrivateKey(bits int) (k *rsaPrivateKey, err error) { + k = new(rsaPrivateKey) + k.PrivateKey, err = rsa.GenerateKey(rand.Reader, bits) + if err != nil { + return nil, err + } + + k.rsaPublicKey.PublicKey = &k.PrivateKey.PublicKey + k.extended = make(map[string]interface{}) + + return +} + +// GenerateRSA2048PrivateKey generates a key pair using 2048-bit RSA. +func GenerateRSA2048PrivateKey() (PrivateKey, error) { + k, err := generateRSAPrivateKey(2048) + if err != nil { + return nil, fmt.Errorf("error generating RSA 2048-bit key: %s", err) + } + + return k, nil +} + +// GenerateRSA3072PrivateKey generates a key pair using 3072-bit RSA. +func GenerateRSA3072PrivateKey() (PrivateKey, error) { + k, err := generateRSAPrivateKey(3072) + if err != nil { + return nil, fmt.Errorf("error generating RSA 3072-bit key: %s", err) + } + + return k, nil +} + +// GenerateRSA4096PrivateKey generates a key pair using 4096-bit RSA. +func GenerateRSA4096PrivateKey() (PrivateKey, error) { + k, err := generateRSAPrivateKey(4096) + if err != nil { + return nil, fmt.Errorf("error generating RSA 4096-bit key: %s", err) + } + + return k, nil +} diff --git a/vendor/github.com/docker/libtrust/util.go b/vendor/github.com/docker/libtrust/util.go new file mode 100644 index 000000000..a5a101d3f --- /dev/null +++ b/vendor/github.com/docker/libtrust/util.go @@ -0,0 +1,363 @@ +package libtrust + +import ( + "bytes" + "crypto" + "crypto/elliptic" + "crypto/tls" + "crypto/x509" + "encoding/base32" + "encoding/base64" + "encoding/binary" + "encoding/pem" + "errors" + "fmt" + "math/big" + "net/url" + "os" + "path/filepath" + "strings" + "time" +) + +// LoadOrCreateTrustKey will load a PrivateKey from the specified path +func LoadOrCreateTrustKey(trustKeyPath string) (PrivateKey, error) { + if err := os.MkdirAll(filepath.Dir(trustKeyPath), 0700); err != nil { + return nil, err + } + + trustKey, err := LoadKeyFile(trustKeyPath) + if err == ErrKeyFileDoesNotExist { + trustKey, err = GenerateECP256PrivateKey() + if err != nil { + return nil, fmt.Errorf("error generating key: %s", err) + } + + if err := SaveKey(trustKeyPath, trustKey); err != nil { + return nil, fmt.Errorf("error saving key file: %s", err) + } + + dir, file := filepath.Split(trustKeyPath) + if err := SavePublicKey(filepath.Join(dir, "public-"+file), trustKey.PublicKey()); err != nil { + return nil, fmt.Errorf("error saving public key file: %s", err) + } + } else if err != nil { + return nil, fmt.Errorf("error loading key file: %s", err) + } + return trustKey, nil +} + +// NewIdentityAuthTLSClientConfig returns a tls.Config configured to use identity +// based authentication from the specified dockerUrl, the rootConfigPath and +// the server name to which it is connecting. +// If trustUnknownHosts is true it will automatically add the host to the +// known-hosts.json in rootConfigPath. +func NewIdentityAuthTLSClientConfig(dockerUrl string, trustUnknownHosts bool, rootConfigPath string, serverName string) (*tls.Config, error) { + tlsConfig := newTLSConfig() + + trustKeyPath := filepath.Join(rootConfigPath, "key.json") + knownHostsPath := filepath.Join(rootConfigPath, "known-hosts.json") + + u, err := url.Parse(dockerUrl) + if err != nil { + return nil, fmt.Errorf("unable to parse machine url") + } + + if u.Scheme == "unix" { + return nil, nil + } + + addr := u.Host + proto := "tcp" + + trustKey, err := LoadOrCreateTrustKey(trustKeyPath) + if err != nil { + return nil, fmt.Errorf("unable to load trust key: %s", err) + } + + knownHosts, err := LoadKeySetFile(knownHostsPath) + if err != nil { + return nil, fmt.Errorf("could not load trusted hosts file: %s", err) + } + + allowedHosts, err := FilterByHosts(knownHosts, addr, false) + if err != nil { + return nil, fmt.Errorf("error filtering hosts: %s", err) + } + + certPool, err := GenerateCACertPool(trustKey, allowedHosts) + if err != nil { + return nil, fmt.Errorf("Could not create CA pool: %s", err) + } + + tlsConfig.ServerName = serverName + tlsConfig.RootCAs = certPool + + x509Cert, err := GenerateSelfSignedClientCert(trustKey) + if err != nil { + return nil, fmt.Errorf("certificate generation error: %s", err) + } + + tlsConfig.Certificates = []tls.Certificate{{ + Certificate: [][]byte{x509Cert.Raw}, + PrivateKey: trustKey.CryptoPrivateKey(), + Leaf: x509Cert, + }} + + tlsConfig.InsecureSkipVerify = true + + testConn, err := tls.Dial(proto, addr, tlsConfig) + if err != nil { + return nil, fmt.Errorf("tls Handshake error: %s", err) + } + + opts := x509.VerifyOptions{ + Roots: tlsConfig.RootCAs, + CurrentTime: time.Now(), + DNSName: tlsConfig.ServerName, + Intermediates: x509.NewCertPool(), + } + + certs := testConn.ConnectionState().PeerCertificates + for i, cert := range certs { + if i == 0 { + continue + } + opts.Intermediates.AddCert(cert) + } + + if _, err := certs[0].Verify(opts); err != nil { + if _, ok := err.(x509.UnknownAuthorityError); ok { + if trustUnknownHosts { + pubKey, err := FromCryptoPublicKey(certs[0].PublicKey) + if err != nil { + return nil, fmt.Errorf("error extracting public key from cert: %s", err) + } + + pubKey.AddExtendedField("hosts", []string{addr}) + + if err := AddKeySetFile(knownHostsPath, pubKey); err != nil { + return nil, fmt.Errorf("error adding machine to known hosts: %s", err) + } + } else { + return nil, fmt.Errorf("unable to connect. unknown host: %s", addr) + } + } + } + + testConn.Close() + tlsConfig.InsecureSkipVerify = false + + return tlsConfig, nil +} + +// joseBase64UrlEncode encodes the given data using the standard base64 url +// encoding format but with all trailing '=' characters omitted in accordance +// with the jose specification. +// http://tools.ietf.org/html/draft-ietf-jose-json-web-signature-31#section-2 +func joseBase64UrlEncode(b []byte) string { + return strings.TrimRight(base64.URLEncoding.EncodeToString(b), "=") +} + +// joseBase64UrlDecode decodes the given string using the standard base64 url +// decoder but first adds the appropriate number of trailing '=' characters in +// accordance with the jose specification. +// http://tools.ietf.org/html/draft-ietf-jose-json-web-signature-31#section-2 +func joseBase64UrlDecode(s string) ([]byte, error) { + s = strings.Replace(s, "\n", "", -1) + s = strings.Replace(s, " ", "", -1) + switch len(s) % 4 { + case 0: + case 2: + s += "==" + case 3: + s += "=" + default: + return nil, errors.New("illegal base64url string") + } + return base64.URLEncoding.DecodeString(s) +} + +func keyIDEncode(b []byte) string { + s := strings.TrimRight(base32.StdEncoding.EncodeToString(b), "=") + var buf bytes.Buffer + var i int + for i = 0; i < len(s)/4-1; i++ { + start := i * 4 + end := start + 4 + buf.WriteString(s[start:end] + ":") + } + buf.WriteString(s[i*4:]) + return buf.String() +} + +func keyIDFromCryptoKey(pubKey PublicKey) string { + // Generate and return a 'libtrust' fingerprint of the public key. + // For an RSA key this should be: + // SHA256(DER encoded ASN1) + // Then truncated to 240 bits and encoded into 12 base32 groups like so: + // ABCD:EFGH:IJKL:MNOP:QRST:UVWX:YZ23:4567:ABCD:EFGH:IJKL:MNOP + derBytes, err := x509.MarshalPKIXPublicKey(pubKey.CryptoPublicKey()) + if err != nil { + return "" + } + hasher := crypto.SHA256.New() + hasher.Write(derBytes) + return keyIDEncode(hasher.Sum(nil)[:30]) +} + +func stringFromMap(m map[string]interface{}, key string) (string, error) { + val, ok := m[key] + if !ok { + return "", fmt.Errorf("%q value not specified", key) + } + + str, ok := val.(string) + if !ok { + return "", fmt.Errorf("%q value must be a string", key) + } + delete(m, key) + + return str, nil +} + +func parseECCoordinate(cB64Url string, curve elliptic.Curve) (*big.Int, error) { + curveByteLen := (curve.Params().BitSize + 7) >> 3 + + cBytes, err := joseBase64UrlDecode(cB64Url) + if err != nil { + return nil, fmt.Errorf("invalid base64 URL encoding: %s", err) + } + cByteLength := len(cBytes) + if cByteLength != curveByteLen { + return nil, fmt.Errorf("invalid number of octets: got %d, should be %d", cByteLength, curveByteLen) + } + return new(big.Int).SetBytes(cBytes), nil +} + +func parseECPrivateParam(dB64Url string, curve elliptic.Curve) (*big.Int, error) { + dBytes, err := joseBase64UrlDecode(dB64Url) + if err != nil { + return nil, fmt.Errorf("invalid base64 URL encoding: %s", err) + } + + // The length of this octet string MUST be ceiling(log-base-2(n)/8) + // octets (where n is the order of the curve). This is because the private + // key d must be in the interval [1, n-1] so the bitlength of d should be + // no larger than the bitlength of n-1. The easiest way to find the octet + // length is to take bitlength(n-1), add 7 to force a carry, and shift this + // bit sequence right by 3, which is essentially dividing by 8 and adding + // 1 if there is any remainder. Thus, the private key value d should be + // output to (bitlength(n-1)+7)>>3 octets. + n := curve.Params().N + octetLength := (new(big.Int).Sub(n, big.NewInt(1)).BitLen() + 7) >> 3 + dByteLength := len(dBytes) + + if dByteLength != octetLength { + return nil, fmt.Errorf("invalid number of octets: got %d, should be %d", dByteLength, octetLength) + } + + return new(big.Int).SetBytes(dBytes), nil +} + +func parseRSAModulusParam(nB64Url string) (*big.Int, error) { + nBytes, err := joseBase64UrlDecode(nB64Url) + if err != nil { + return nil, fmt.Errorf("invalid base64 URL encoding: %s", err) + } + + return new(big.Int).SetBytes(nBytes), nil +} + +func serializeRSAPublicExponentParam(e int) []byte { + // We MUST use the minimum number of octets to represent E. + // E is supposed to be 65537 for performance and security reasons + // and is what golang's rsa package generates, but it might be + // different if imported from some other generator. + buf := make([]byte, 4) + binary.BigEndian.PutUint32(buf, uint32(e)) + var i int + for i = 0; i < 8; i++ { + if buf[i] != 0 { + break + } + } + return buf[i:] +} + +func parseRSAPublicExponentParam(eB64Url string) (int, error) { + eBytes, err := joseBase64UrlDecode(eB64Url) + if err != nil { + return 0, fmt.Errorf("invalid base64 URL encoding: %s", err) + } + // Only the minimum number of bytes were used to represent E, but + // binary.BigEndian.Uint32 expects at least 4 bytes, so we need + // to add zero padding if necassary. + byteLen := len(eBytes) + buf := make([]byte, 4-byteLen, 4) + eBytes = append(buf, eBytes...) + + return int(binary.BigEndian.Uint32(eBytes)), nil +} + +func parseRSAPrivateKeyParamFromMap(m map[string]interface{}, key string) (*big.Int, error) { + b64Url, err := stringFromMap(m, key) + if err != nil { + return nil, err + } + + paramBytes, err := joseBase64UrlDecode(b64Url) + if err != nil { + return nil, fmt.Errorf("invaled base64 URL encoding: %s", err) + } + + return new(big.Int).SetBytes(paramBytes), nil +} + +func createPemBlock(name string, derBytes []byte, headers map[string]interface{}) (*pem.Block, error) { + pemBlock := &pem.Block{Type: name, Bytes: derBytes, Headers: map[string]string{}} + for k, v := range headers { + switch val := v.(type) { + case string: + pemBlock.Headers[k] = val + case []string: + if k == "hosts" { + pemBlock.Headers[k] = strings.Join(val, ",") + } else { + // Return error, non-encodable type + } + default: + // Return error, non-encodable type + } + } + + return pemBlock, nil +} + +func pubKeyFromPEMBlock(pemBlock *pem.Block) (PublicKey, error) { + cryptoPublicKey, err := x509.ParsePKIXPublicKey(pemBlock.Bytes) + if err != nil { + return nil, fmt.Errorf("unable to decode Public Key PEM data: %s", err) + } + + pubKey, err := FromCryptoPublicKey(cryptoPublicKey) + if err != nil { + return nil, err + } + + addPEMHeadersToKey(pemBlock, pubKey) + + return pubKey, nil +} + +func addPEMHeadersToKey(pemBlock *pem.Block, pubKey PublicKey) { + for key, value := range pemBlock.Headers { + var safeVal interface{} + if key == "hosts" { + safeVal = strings.Split(value, ",") + } else { + safeVal = value + } + pubKey.AddExtendedField(key, safeVal) + } +} diff --git a/vendor/github.com/garyburd/redigo/LICENSE b/vendor/github.com/garyburd/redigo/LICENSE new file mode 100644 index 000000000..67db85882 --- /dev/null +++ b/vendor/github.com/garyburd/redigo/LICENSE @@ -0,0 +1,175 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. diff --git a/vendor/github.com/garyburd/redigo/internal/commandinfo.go b/vendor/github.com/garyburd/redigo/internal/commandinfo.go new file mode 100644 index 000000000..11e584257 --- /dev/null +++ b/vendor/github.com/garyburd/redigo/internal/commandinfo.go @@ -0,0 +1,54 @@ +// Copyright 2014 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package internal // import "github.com/garyburd/redigo/internal" + +import ( + "strings" +) + +const ( + WatchState = 1 << iota + MultiState + SubscribeState + MonitorState +) + +type CommandInfo struct { + Set, Clear int +} + +var commandInfos = map[string]CommandInfo{ + "WATCH": {Set: WatchState}, + "UNWATCH": {Clear: WatchState}, + "MULTI": {Set: MultiState}, + "EXEC": {Clear: WatchState | MultiState}, + "DISCARD": {Clear: WatchState | MultiState}, + "PSUBSCRIBE": {Set: SubscribeState}, + "SUBSCRIBE": {Set: SubscribeState}, + "MONITOR": {Set: MonitorState}, +} + +func init() { + for n, ci := range commandInfos { + commandInfos[strings.ToLower(n)] = ci + } +} + +func LookupCommandInfo(commandName string) CommandInfo { + if ci, ok := commandInfos[commandName]; ok { + return ci + } + return commandInfos[strings.ToUpper(commandName)] +} diff --git a/vendor/github.com/garyburd/redigo/redis/conn.go b/vendor/github.com/garyburd/redigo/redis/conn.go new file mode 100644 index 000000000..fad7d6c95 --- /dev/null +++ b/vendor/github.com/garyburd/redigo/redis/conn.go @@ -0,0 +1,679 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis + +import ( + "bufio" + "bytes" + "crypto/tls" + "errors" + "fmt" + "io" + "net" + "net/url" + "regexp" + "strconv" + "sync" + "time" +) + +var ( + _ ConnWithTimeout = (*conn)(nil) +) + +// conn is the low-level implementation of Conn +type conn struct { + // Shared + mu sync.Mutex + pending int + err error + conn net.Conn + + // Read + readTimeout time.Duration + br *bufio.Reader + + // Write + writeTimeout time.Duration + bw *bufio.Writer + + // Scratch space for formatting argument length. + // '*' or '$', length, "\r\n" + lenScratch [32]byte + + // Scratch space for formatting integers and floats. + numScratch [40]byte +} + +// DialTimeout acts like Dial but takes timeouts for establishing the +// connection to the server, writing a command and reading a reply. +// +// Deprecated: This package is replaced by github.com/gomodule/redigo/redis. +func DialTimeout(network, address string, connectTimeout, readTimeout, writeTimeout time.Duration) (Conn, error) { + return Dial(network, address, + DialConnectTimeout(connectTimeout), + DialReadTimeout(readTimeout), + DialWriteTimeout(writeTimeout)) +} + +// DialOption specifies an option for dialing a Redis server. +type DialOption struct { + f func(*dialOptions) +} + +type dialOptions struct { + readTimeout time.Duration + writeTimeout time.Duration + dialer *net.Dialer + dial func(network, addr string) (net.Conn, error) + db int + password string + useTLS bool + skipVerify bool + tlsConfig *tls.Config +} + +// DialReadTimeout specifies the timeout for reading a single command reply. +func DialReadTimeout(d time.Duration) DialOption { + return DialOption{func(do *dialOptions) { + do.readTimeout = d + }} +} + +// DialWriteTimeout specifies the timeout for writing a single command. +func DialWriteTimeout(d time.Duration) DialOption { + return DialOption{func(do *dialOptions) { + do.writeTimeout = d + }} +} + +// DialConnectTimeout specifies the timeout for connecting to the Redis server when +// no DialNetDial option is specified. +func DialConnectTimeout(d time.Duration) DialOption { + return DialOption{func(do *dialOptions) { + do.dialer.Timeout = d + }} +} + +// DialKeepAlive specifies the keep-alive period for TCP connections to the Redis server +// when no DialNetDial option is specified. +// If zero, keep-alives are not enabled. If no DialKeepAlive option is specified then +// the default of 5 minutes is used to ensure that half-closed TCP sessions are detected. +func DialKeepAlive(d time.Duration) DialOption { + return DialOption{func(do *dialOptions) { + do.dialer.KeepAlive = d + }} +} + +// DialNetDial specifies a custom dial function for creating TCP +// connections, otherwise a net.Dialer customized via the other options is used. +// DialNetDial overrides DialConnectTimeout and DialKeepAlive. +func DialNetDial(dial func(network, addr string) (net.Conn, error)) DialOption { + return DialOption{func(do *dialOptions) { + do.dial = dial + }} +} + +// DialDatabase specifies the database to select when dialing a connection. +func DialDatabase(db int) DialOption { + return DialOption{func(do *dialOptions) { + do.db = db + }} +} + +// DialPassword specifies the password to use when connecting to +// the Redis server. +func DialPassword(password string) DialOption { + return DialOption{func(do *dialOptions) { + do.password = password + }} +} + +// DialTLSConfig specifies the config to use when a TLS connection is dialed. +// Has no effect when not dialing a TLS connection. +func DialTLSConfig(c *tls.Config) DialOption { + return DialOption{func(do *dialOptions) { + do.tlsConfig = c + }} +} + +// DialTLSSkipVerify disables server name verification when connecting over +// TLS. Has no effect when not dialing a TLS connection. +func DialTLSSkipVerify(skip bool) DialOption { + return DialOption{func(do *dialOptions) { + do.skipVerify = skip + }} +} + +// DialUseTLS specifies whether TLS should be used when connecting to the +// server. This option is ignore by DialURL. +func DialUseTLS(useTLS bool) DialOption { + return DialOption{func(do *dialOptions) { + do.useTLS = useTLS + }} +} + +// Dial connects to the Redis server at the given network and +// address using the specified options. +// +// Deprecated: This package is replaced by github.com/gomodule/redigo/redis. +func Dial(network, address string, options ...DialOption) (Conn, error) { + do := dialOptions{ + dialer: &net.Dialer{ + KeepAlive: time.Minute * 5, + }, + } + for _, option := range options { + option.f(&do) + } + if do.dial == nil { + do.dial = do.dialer.Dial + } + + netConn, err := do.dial(network, address) + if err != nil { + return nil, err + } + + if do.useTLS { + var tlsConfig *tls.Config + if do.tlsConfig == nil { + tlsConfig = &tls.Config{InsecureSkipVerify: do.skipVerify} + } else { + tlsConfig = cloneTLSConfig(do.tlsConfig) + } + if tlsConfig.ServerName == "" { + host, _, err := net.SplitHostPort(address) + if err != nil { + netConn.Close() + return nil, err + } + tlsConfig.ServerName = host + } + + tlsConn := tls.Client(netConn, tlsConfig) + if err := tlsConn.Handshake(); err != nil { + netConn.Close() + return nil, err + } + netConn = tlsConn + } + + c := &conn{ + conn: netConn, + bw: bufio.NewWriter(netConn), + br: bufio.NewReader(netConn), + readTimeout: do.readTimeout, + writeTimeout: do.writeTimeout, + } + + if do.password != "" { + if _, err := c.Do("AUTH", do.password); err != nil { + netConn.Close() + return nil, err + } + } + + if do.db != 0 { + if _, err := c.Do("SELECT", do.db); err != nil { + netConn.Close() + return nil, err + } + } + + return c, nil +} + +var pathDBRegexp = regexp.MustCompile(`/(\d*)\z`) + +// DialURL connects to a Redis server at the given URL using the Redis +// URI scheme. URLs should follow the draft IANA specification for the +// scheme (https://www.iana.org/assignments/uri-schemes/prov/redis). +// +// Deprecated: This package is replaced by github.com/gomodule/redigo/redis. +func DialURL(rawurl string, options ...DialOption) (Conn, error) { + u, err := url.Parse(rawurl) + if err != nil { + return nil, err + } + + if u.Scheme != "redis" && u.Scheme != "rediss" { + return nil, fmt.Errorf("invalid redis URL scheme: %s", u.Scheme) + } + + // As per the IANA draft spec, the host defaults to localhost and + // the port defaults to 6379. + host, port, err := net.SplitHostPort(u.Host) + if err != nil { + // assume port is missing + host = u.Host + port = "6379" + } + if host == "" { + host = "localhost" + } + address := net.JoinHostPort(host, port) + + if u.User != nil { + password, isSet := u.User.Password() + if isSet { + options = append(options, DialPassword(password)) + } + } + + match := pathDBRegexp.FindStringSubmatch(u.Path) + if len(match) == 2 { + db := 0 + if len(match[1]) > 0 { + db, err = strconv.Atoi(match[1]) + if err != nil { + return nil, fmt.Errorf("invalid database: %s", u.Path[1:]) + } + } + if db != 0 { + options = append(options, DialDatabase(db)) + } + } else if u.Path != "" { + return nil, fmt.Errorf("invalid database: %s", u.Path[1:]) + } + + options = append(options, DialUseTLS(u.Scheme == "rediss")) + + return Dial("tcp", address, options...) +} + +// NewConn returns a new Redigo connection for the given net connection. +// +// Deprecated: This package is replaced by github.com/gomodule/redigo/redis. +func NewConn(netConn net.Conn, readTimeout, writeTimeout time.Duration) Conn { + return &conn{ + conn: netConn, + bw: bufio.NewWriter(netConn), + br: bufio.NewReader(netConn), + readTimeout: readTimeout, + writeTimeout: writeTimeout, + } +} + +func (c *conn) Close() error { + c.mu.Lock() + err := c.err + if c.err == nil { + c.err = errors.New("redigo: closed") + err = c.conn.Close() + } + c.mu.Unlock() + return err +} + +func (c *conn) fatal(err error) error { + c.mu.Lock() + if c.err == nil { + c.err = err + // Close connection to force errors on subsequent calls and to unblock + // other reader or writer. + c.conn.Close() + } + c.mu.Unlock() + return err +} + +func (c *conn) Err() error { + c.mu.Lock() + err := c.err + c.mu.Unlock() + return err +} + +func (c *conn) writeLen(prefix byte, n int) error { + c.lenScratch[len(c.lenScratch)-1] = '\n' + c.lenScratch[len(c.lenScratch)-2] = '\r' + i := len(c.lenScratch) - 3 + for { + c.lenScratch[i] = byte('0' + n%10) + i -= 1 + n = n / 10 + if n == 0 { + break + } + } + c.lenScratch[i] = prefix + _, err := c.bw.Write(c.lenScratch[i:]) + return err +} + +func (c *conn) writeString(s string) error { + c.writeLen('$', len(s)) + c.bw.WriteString(s) + _, err := c.bw.WriteString("\r\n") + return err +} + +func (c *conn) writeBytes(p []byte) error { + c.writeLen('$', len(p)) + c.bw.Write(p) + _, err := c.bw.WriteString("\r\n") + return err +} + +func (c *conn) writeInt64(n int64) error { + return c.writeBytes(strconv.AppendInt(c.numScratch[:0], n, 10)) +} + +func (c *conn) writeFloat64(n float64) error { + return c.writeBytes(strconv.AppendFloat(c.numScratch[:0], n, 'g', -1, 64)) +} + +func (c *conn) writeCommand(cmd string, args []interface{}) error { + c.writeLen('*', 1+len(args)) + if err := c.writeString(cmd); err != nil { + return err + } + for _, arg := range args { + if err := c.writeArg(arg, true); err != nil { + return err + } + } + return nil +} + +func (c *conn) writeArg(arg interface{}, argumentTypeOK bool) (err error) { + switch arg := arg.(type) { + case string: + return c.writeString(arg) + case []byte: + return c.writeBytes(arg) + case int: + return c.writeInt64(int64(arg)) + case int64: + return c.writeInt64(arg) + case float64: + return c.writeFloat64(arg) + case bool: + if arg { + return c.writeString("1") + } else { + return c.writeString("0") + } + case nil: + return c.writeString("") + case Argument: + if argumentTypeOK { + return c.writeArg(arg.RedisArg(), false) + } + // See comment in default clause below. + var buf bytes.Buffer + fmt.Fprint(&buf, arg) + return c.writeBytes(buf.Bytes()) + default: + // This default clause is intended to handle builtin numeric types. + // The function should return an error for other types, but this is not + // done for compatibility with previous versions of the package. + var buf bytes.Buffer + fmt.Fprint(&buf, arg) + return c.writeBytes(buf.Bytes()) + } +} + +type protocolError string + +func (pe protocolError) Error() string { + return fmt.Sprintf("redigo: %s (possible server error or unsupported concurrent read by application)", string(pe)) +} + +func (c *conn) readLine() ([]byte, error) { + p, err := c.br.ReadSlice('\n') + if err == bufio.ErrBufferFull { + return nil, protocolError("long response line") + } + if err != nil { + return nil, err + } + i := len(p) - 2 + if i < 0 || p[i] != '\r' { + return nil, protocolError("bad response line terminator") + } + return p[:i], nil +} + +// parseLen parses bulk string and array lengths. +func parseLen(p []byte) (int, error) { + if len(p) == 0 { + return -1, protocolError("malformed length") + } + + if p[0] == '-' && len(p) == 2 && p[1] == '1' { + // handle $-1 and $-1 null replies. + return -1, nil + } + + var n int + for _, b := range p { + n *= 10 + if b < '0' || b > '9' { + return -1, protocolError("illegal bytes in length") + } + n += int(b - '0') + } + + return n, nil +} + +// parseInt parses an integer reply. +func parseInt(p []byte) (interface{}, error) { + if len(p) == 0 { + return 0, protocolError("malformed integer") + } + + var negate bool + if p[0] == '-' { + negate = true + p = p[1:] + if len(p) == 0 { + return 0, protocolError("malformed integer") + } + } + + var n int64 + for _, b := range p { + n *= 10 + if b < '0' || b > '9' { + return 0, protocolError("illegal bytes in length") + } + n += int64(b - '0') + } + + if negate { + n = -n + } + return n, nil +} + +var ( + okReply interface{} = "OK" + pongReply interface{} = "PONG" +) + +func (c *conn) readReply() (interface{}, error) { + line, err := c.readLine() + if err != nil { + return nil, err + } + if len(line) == 0 { + return nil, protocolError("short response line") + } + switch line[0] { + case '+': + switch { + case len(line) == 3 && line[1] == 'O' && line[2] == 'K': + // Avoid allocation for frequent "+OK" response. + return okReply, nil + case len(line) == 5 && line[1] == 'P' && line[2] == 'O' && line[3] == 'N' && line[4] == 'G': + // Avoid allocation in PING command benchmarks :) + return pongReply, nil + default: + return string(line[1:]), nil + } + case '-': + return Error(string(line[1:])), nil + case ':': + return parseInt(line[1:]) + case '$': + n, err := parseLen(line[1:]) + if n < 0 || err != nil { + return nil, err + } + p := make([]byte, n) + _, err = io.ReadFull(c.br, p) + if err != nil { + return nil, err + } + if line, err := c.readLine(); err != nil { + return nil, err + } else if len(line) != 0 { + return nil, protocolError("bad bulk string format") + } + return p, nil + case '*': + n, err := parseLen(line[1:]) + if n < 0 || err != nil { + return nil, err + } + r := make([]interface{}, n) + for i := range r { + r[i], err = c.readReply() + if err != nil { + return nil, err + } + } + return r, nil + } + return nil, protocolError("unexpected response line") +} + +func (c *conn) Send(cmd string, args ...interface{}) error { + c.mu.Lock() + c.pending += 1 + c.mu.Unlock() + if c.writeTimeout != 0 { + c.conn.SetWriteDeadline(time.Now().Add(c.writeTimeout)) + } + if err := c.writeCommand(cmd, args); err != nil { + return c.fatal(err) + } + return nil +} + +func (c *conn) Flush() error { + if c.writeTimeout != 0 { + c.conn.SetWriteDeadline(time.Now().Add(c.writeTimeout)) + } + if err := c.bw.Flush(); err != nil { + return c.fatal(err) + } + return nil +} + +func (c *conn) Receive() (interface{}, error) { + return c.ReceiveWithTimeout(c.readTimeout) +} + +func (c *conn) ReceiveWithTimeout(timeout time.Duration) (reply interface{}, err error) { + var deadline time.Time + if timeout != 0 { + deadline = time.Now().Add(timeout) + } + c.conn.SetReadDeadline(deadline) + + if reply, err = c.readReply(); err != nil { + return nil, c.fatal(err) + } + // When using pub/sub, the number of receives can be greater than the + // number of sends. To enable normal use of the connection after + // unsubscribing from all channels, we do not decrement pending to a + // negative value. + // + // The pending field is decremented after the reply is read to handle the + // case where Receive is called before Send. + c.mu.Lock() + if c.pending > 0 { + c.pending -= 1 + } + c.mu.Unlock() + if err, ok := reply.(Error); ok { + return nil, err + } + return +} + +func (c *conn) Do(cmd string, args ...interface{}) (interface{}, error) { + return c.DoWithTimeout(c.readTimeout, cmd, args...) +} + +func (c *conn) DoWithTimeout(readTimeout time.Duration, cmd string, args ...interface{}) (interface{}, error) { + c.mu.Lock() + pending := c.pending + c.pending = 0 + c.mu.Unlock() + + if cmd == "" && pending == 0 { + return nil, nil + } + + if c.writeTimeout != 0 { + c.conn.SetWriteDeadline(time.Now().Add(c.writeTimeout)) + } + + if cmd != "" { + if err := c.writeCommand(cmd, args); err != nil { + return nil, c.fatal(err) + } + } + + if err := c.bw.Flush(); err != nil { + return nil, c.fatal(err) + } + + var deadline time.Time + if readTimeout != 0 { + deadline = time.Now().Add(readTimeout) + } + c.conn.SetReadDeadline(deadline) + + if cmd == "" { + reply := make([]interface{}, pending) + for i := range reply { + r, e := c.readReply() + if e != nil { + return nil, c.fatal(e) + } + reply[i] = r + } + return reply, nil + } + + var err error + var reply interface{} + for i := 0; i <= pending; i++ { + var e error + if reply, e = c.readReply(); e != nil { + return nil, c.fatal(e) + } + if e, ok := reply.(Error); ok && err == nil { + err = e + } + } + return reply, err +} diff --git a/vendor/github.com/garyburd/redigo/redis/doc.go b/vendor/github.com/garyburd/redigo/redis/doc.go new file mode 100644 index 000000000..60a0dfd07 --- /dev/null +++ b/vendor/github.com/garyburd/redigo/redis/doc.go @@ -0,0 +1,177 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +// Package redis is a client for the Redis database. +// +// The Redigo FAQ (https://github.com/garyburd/redigo/wiki/FAQ) contains more +// documentation about this package. +// +// # Connections +// +// The Conn interface is the primary interface for working with Redis. +// Applications create connections by calling the Dial, DialWithTimeout or +// NewConn functions. In the future, functions will be added for creating +// sharded and other types of connections. +// +// The application must call the connection Close method when the application +// is done with the connection. +// +// # Executing Commands +// +// The Conn interface has a generic method for executing Redis commands: +// +// Do(commandName string, args ...interface{}) (reply interface{}, err error) +// +// The Redis command reference (http://redis.io/commands) lists the available +// commands. An example of using the Redis APPEND command is: +// +// n, err := conn.Do("APPEND", "key", "value") +// +// The Do method converts command arguments to bulk strings for transmission +// to the server as follows: +// +// Go Type Conversion +// []byte Sent as is +// string Sent as is +// int, int64 strconv.FormatInt(v) +// float64 strconv.FormatFloat(v, 'g', -1, 64) +// bool true -> "1", false -> "0" +// nil "" +// all other types fmt.Fprint(w, v) +// +// Redis command reply types are represented using the following Go types: +// +// Redis type Go type +// error redis.Error +// integer int64 +// simple string string +// bulk string []byte or nil if value not present. +// array []interface{} or nil if value not present. +// +// Use type assertions or the reply helper functions to convert from +// interface{} to the specific Go type for the command result. +// +// # Pipelining +// +// Connections support pipelining using the Send, Flush and Receive methods. +// +// Send(commandName string, args ...interface{}) error +// Flush() error +// Receive() (reply interface{}, err error) +// +// Send writes the command to the connection's output buffer. Flush flushes the +// connection's output buffer to the server. Receive reads a single reply from +// the server. The following example shows a simple pipeline. +// +// c.Send("SET", "foo", "bar") +// c.Send("GET", "foo") +// c.Flush() +// c.Receive() // reply from SET +// v, err = c.Receive() // reply from GET +// +// The Do method combines the functionality of the Send, Flush and Receive +// methods. The Do method starts by writing the command and flushing the output +// buffer. Next, the Do method receives all pending replies including the reply +// for the command just sent by Do. If any of the received replies is an error, +// then Do returns the error. If there are no errors, then Do returns the last +// reply. If the command argument to the Do method is "", then the Do method +// will flush the output buffer and receive pending replies without sending a +// command. +// +// Use the Send and Do methods to implement pipelined transactions. +// +// c.Send("MULTI") +// c.Send("INCR", "foo") +// c.Send("INCR", "bar") +// r, err := c.Do("EXEC") +// fmt.Println(r) // prints [1, 1] +// +// # Concurrency +// +// Connections support one concurrent caller to the Receive method and one +// concurrent caller to the Send and Flush methods. No other concurrency is +// supported including concurrent calls to the Do method. +// +// For full concurrent access to Redis, use the thread-safe Pool to get, use +// and release a connection from within a goroutine. Connections returned from +// a Pool have the concurrency restrictions described in the previous +// paragraph. +// +// # Publish and Subscribe +// +// Use the Send, Flush and Receive methods to implement Pub/Sub subscribers. +// +// c.Send("SUBSCRIBE", "example") +// c.Flush() +// for { +// reply, err := c.Receive() +// if err != nil { +// return err +// } +// // process pushed message +// } +// +// The PubSubConn type wraps a Conn with convenience methods for implementing +// subscribers. The Subscribe, PSubscribe, Unsubscribe and PUnsubscribe methods +// send and flush a subscription management command. The receive method +// converts a pushed message to convenient types for use in a type switch. +// +// psc := redis.PubSubConn{Conn: c} +// psc.Subscribe("example") +// for { +// switch v := psc.Receive().(type) { +// case redis.Message: +// fmt.Printf("%s: message: %s\n", v.Channel, v.Data) +// case redis.Subscription: +// fmt.Printf("%s: %s %d\n", v.Channel, v.Kind, v.Count) +// case error: +// return v +// } +// } +// +// # Reply Helpers +// +// The Bool, Int, Bytes, String, Strings and Values functions convert a reply +// to a value of a specific type. To allow convenient wrapping of calls to the +// connection Do and Receive methods, the functions take a second argument of +// type error. If the error is non-nil, then the helper function returns the +// error. If the error is nil, the function converts the reply to the specified +// type: +// +// exists, err := redis.Bool(c.Do("EXISTS", "foo")) +// if err != nil { +// // handle error return from c.Do or type conversion error. +// } +// +// The Scan function converts elements of a array reply to Go types: +// +// var value1 int +// var value2 string +// reply, err := redis.Values(c.Do("MGET", "key1", "key2")) +// if err != nil { +// // handle error +// } +// if _, err := redis.Scan(reply, &value1, &value2); err != nil { +// // handle error +// } +// +// # Errors +// +// Connection methods return error replies from the server as type redis.Error. +// +// Call the connection Err() method to determine if the connection encountered +// non-recoverable error such as a network error or protocol parsing error. If +// Err() returns a non-nil value, then the connection is not usable and should +// be closed. +package redis // import "github.com/garyburd/redigo/redis" diff --git a/vendor/github.com/garyburd/redigo/redis/go16.go b/vendor/github.com/garyburd/redigo/redis/go16.go new file mode 100644 index 000000000..d83f147eb --- /dev/null +++ b/vendor/github.com/garyburd/redigo/redis/go16.go @@ -0,0 +1,28 @@ +//go:build !go1.7 +// +build !go1.7 + +package redis + +import "crypto/tls" + +func cloneTLSConfig(cfg *tls.Config) *tls.Config { + return &tls.Config{ + Rand: cfg.Rand, + Time: cfg.Time, + Certificates: cfg.Certificates, + NameToCertificate: cfg.NameToCertificate, + GetCertificate: cfg.GetCertificate, + RootCAs: cfg.RootCAs, + NextProtos: cfg.NextProtos, + ServerName: cfg.ServerName, + ClientAuth: cfg.ClientAuth, + ClientCAs: cfg.ClientCAs, + InsecureSkipVerify: cfg.InsecureSkipVerify, + CipherSuites: cfg.CipherSuites, + PreferServerCipherSuites: cfg.PreferServerCipherSuites, + ClientSessionCache: cfg.ClientSessionCache, + MinVersion: cfg.MinVersion, + MaxVersion: cfg.MaxVersion, + CurvePreferences: cfg.CurvePreferences, + } +} diff --git a/vendor/github.com/garyburd/redigo/redis/go17.go b/vendor/github.com/garyburd/redigo/redis/go17.go new file mode 100644 index 000000000..1d51ce5fe --- /dev/null +++ b/vendor/github.com/garyburd/redigo/redis/go17.go @@ -0,0 +1,30 @@ +//go:build go1.7 && !go1.8 +// +build go1.7,!go1.8 + +package redis + +import "crypto/tls" + +func cloneTLSConfig(cfg *tls.Config) *tls.Config { + return &tls.Config{ + Rand: cfg.Rand, + Time: cfg.Time, + Certificates: cfg.Certificates, + NameToCertificate: cfg.NameToCertificate, + GetCertificate: cfg.GetCertificate, + RootCAs: cfg.RootCAs, + NextProtos: cfg.NextProtos, + ServerName: cfg.ServerName, + ClientAuth: cfg.ClientAuth, + ClientCAs: cfg.ClientCAs, + InsecureSkipVerify: cfg.InsecureSkipVerify, + CipherSuites: cfg.CipherSuites, + PreferServerCipherSuites: cfg.PreferServerCipherSuites, + ClientSessionCache: cfg.ClientSessionCache, + MinVersion: cfg.MinVersion, + MaxVersion: cfg.MaxVersion, + CurvePreferences: cfg.CurvePreferences, + DynamicRecordSizingDisabled: cfg.DynamicRecordSizingDisabled, + Renegotiation: cfg.Renegotiation, + } +} diff --git a/vendor/github.com/garyburd/redigo/redis/go18.go b/vendor/github.com/garyburd/redigo/redis/go18.go new file mode 100644 index 000000000..3fa682374 --- /dev/null +++ b/vendor/github.com/garyburd/redigo/redis/go18.go @@ -0,0 +1,10 @@ +//go:build go1.8 +// +build go1.8 + +package redis + +import "crypto/tls" + +func cloneTLSConfig(cfg *tls.Config) *tls.Config { + return cfg.Clone() +} diff --git a/vendor/github.com/garyburd/redigo/redis/log.go b/vendor/github.com/garyburd/redigo/redis/log.go new file mode 100644 index 000000000..b2996611c --- /dev/null +++ b/vendor/github.com/garyburd/redigo/redis/log.go @@ -0,0 +1,134 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis + +import ( + "bytes" + "fmt" + "log" + "time" +) + +var ( + _ ConnWithTimeout = (*loggingConn)(nil) +) + +// NewLoggingConn returns a logging wrapper around a connection. +func NewLoggingConn(conn Conn, logger *log.Logger, prefix string) Conn { + if prefix != "" { + prefix = prefix + "." + } + return &loggingConn{conn, logger, prefix} +} + +type loggingConn struct { + Conn + logger *log.Logger + prefix string +} + +func (c *loggingConn) Close() error { + err := c.Conn.Close() + var buf bytes.Buffer + fmt.Fprintf(&buf, "%sClose() -> (%v)", c.prefix, err) + c.logger.Output(2, buf.String()) + return err +} + +func (c *loggingConn) printValue(buf *bytes.Buffer, v interface{}) { + const chop = 32 + switch v := v.(type) { + case []byte: + if len(v) > chop { + fmt.Fprintf(buf, "%q...", v[:chop]) + } else { + fmt.Fprintf(buf, "%q", v) + } + case string: + if len(v) > chop { + fmt.Fprintf(buf, "%q...", v[:chop]) + } else { + fmt.Fprintf(buf, "%q", v) + } + case []interface{}: + if len(v) == 0 { + buf.WriteString("[]") + } else { + sep := "[" + fin := "]" + if len(v) > chop { + v = v[:chop] + fin = "...]" + } + for _, vv := range v { + buf.WriteString(sep) + c.printValue(buf, vv) + sep = ", " + } + buf.WriteString(fin) + } + default: + fmt.Fprint(buf, v) + } +} + +func (c *loggingConn) print(method, commandName string, args []interface{}, reply interface{}, err error) { + var buf bytes.Buffer + fmt.Fprintf(&buf, "%s%s(", c.prefix, method) + if method != "Receive" { + buf.WriteString(commandName) + for _, arg := range args { + buf.WriteString(", ") + c.printValue(&buf, arg) + } + } + buf.WriteString(") -> (") + if method != "Send" { + c.printValue(&buf, reply) + buf.WriteString(", ") + } + fmt.Fprintf(&buf, "%v)", err) + c.logger.Output(3, buf.String()) +} + +func (c *loggingConn) Do(commandName string, args ...interface{}) (interface{}, error) { + reply, err := c.Conn.Do(commandName, args...) + c.print("Do", commandName, args, reply, err) + return reply, err +} + +func (c *loggingConn) DoWithTimeout(timeout time.Duration, commandName string, args ...interface{}) (interface{}, error) { + reply, err := DoWithTimeout(c.Conn, timeout, commandName, args...) + c.print("DoWithTimeout", commandName, args, reply, err) + return reply, err +} + +func (c *loggingConn) Send(commandName string, args ...interface{}) error { + err := c.Conn.Send(commandName, args...) + c.print("Send", commandName, args, nil, err) + return err +} + +func (c *loggingConn) Receive() (interface{}, error) { + reply, err := c.Conn.Receive() + c.print("Receive", "", nil, reply, err) + return reply, err +} + +func (c *loggingConn) ReceiveWithTimeout(timeout time.Duration) (interface{}, error) { + reply, err := ReceiveWithTimeout(c.Conn, timeout) + c.print("ReceiveWithTimeout", "", nil, reply, err) + return reply, err +} diff --git a/vendor/github.com/garyburd/redigo/redis/pool.go b/vendor/github.com/garyburd/redigo/redis/pool.go new file mode 100644 index 000000000..92afee18c --- /dev/null +++ b/vendor/github.com/garyburd/redigo/redis/pool.go @@ -0,0 +1,561 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis + +import ( + "bytes" + "crypto/rand" + "crypto/sha1" + "errors" + "io" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/garyburd/redigo/internal" +) + +var ( + _ ConnWithTimeout = (*activeConn)(nil) + _ ConnWithTimeout = (*errorConn)(nil) +) + +var nowFunc = time.Now // for testing + +// ErrPoolExhausted is returned from a pool connection method (Do, Send, +// Receive, Flush, Err) when the maximum number of database connections in the +// pool has been reached. +var ErrPoolExhausted = errors.New("redigo: connection pool exhausted") + +var ( + errPoolClosed = errors.New("redigo: connection pool closed") + errConnClosed = errors.New("redigo: connection closed") +) + +// Pool maintains a pool of connections. The application calls the Get method +// to get a connection from the pool and the connection's Close method to +// return the connection's resources to the pool. +// +// The following example shows how to use a pool in a web application. The +// application creates a pool at application startup and makes it available to +// request handlers using a package level variable. The pool configuration used +// here is an example, not a recommendation. +// +// func newPool(addr string) *redis.Pool { +// return &redis.Pool{ +// MaxIdle: 3, +// IdleTimeout: 240 * time.Second, +// Dial: func () (redis.Conn, error) { return redis.Dial("tcp", addr) }, +// } +// } +// +// var ( +// pool *redis.Pool +// redisServer = flag.String("redisServer", ":6379", "") +// ) +// +// func main() { +// flag.Parse() +// pool = newPool(*redisServer) +// ... +// } +// +// A request handler gets a connection from the pool and closes the connection +// when the handler is done: +// +// func serveHome(w http.ResponseWriter, r *http.Request) { +// conn := pool.Get() +// defer conn.Close() +// ... +// } +// +// Use the Dial function to authenticate connections with the AUTH command or +// select a database with the SELECT command: +// +// pool := &redis.Pool{ +// // Other pool configuration not shown in this example. +// Dial: func () (redis.Conn, error) { +// c, err := redis.Dial("tcp", server) +// if err != nil { +// return nil, err +// } +// if _, err := c.Do("AUTH", password); err != nil { +// c.Close() +// return nil, err +// } +// if _, err := c.Do("SELECT", db); err != nil { +// c.Close() +// return nil, err +// } +// return c, nil +// }, +// } +// +// Use the TestOnBorrow function to check the health of an idle connection +// before the connection is returned to the application. This example PINGs +// connections that have been idle more than a minute: +// +// pool := &redis.Pool{ +// // Other pool configuration not shown in this example. +// TestOnBorrow: func(c redis.Conn, t time.Time) error { +// if time.Since(t) < time.Minute { +// return nil +// } +// _, err := c.Do("PING") +// return err +// }, +// } +type Pool struct { + // Dial is an application supplied function for creating and configuring a + // connection. + // + // The connection returned from Dial must not be in a special state + // (subscribed to pubsub channel, transaction started, ...). + Dial func() (Conn, error) + + // TestOnBorrow is an optional application supplied function for checking + // the health of an idle connection before the connection is used again by + // the application. Argument t is the time that the connection was returned + // to the pool. If the function returns an error, then the connection is + // closed. + TestOnBorrow func(c Conn, t time.Time) error + + // Maximum number of idle connections in the pool. + MaxIdle int + + // Maximum number of connections allocated by the pool at a given time. + // When zero, there is no limit on the number of connections in the pool. + MaxActive int + + // Close connections after remaining idle for this duration. If the value + // is zero, then idle connections are not closed. Applications should set + // the timeout to a value less than the server's timeout. + IdleTimeout time.Duration + + // If Wait is true and the pool is at the MaxActive limit, then Get() waits + // for a connection to be returned to the pool before returning. + Wait bool + + // Close connections older than this duration. If the value is zero, then + // the pool does not close connections based on age. + MaxConnLifetime time.Duration + + chInitialized uint32 // set to 1 when field ch is initialized + + mu sync.Mutex // mu protects the following fields + closed bool // set to true when the pool is closed. + active int // the number of open connections in the pool + ch chan struct{} // limits open connections when p.Wait is true + idle idleList // idle connections +} + +// NewPool creates a new pool. +// +// Deprecated: Initialize the Pool directory as shown in the example. +func NewPool(newFn func() (Conn, error), maxIdle int) *Pool { + return &Pool{Dial: newFn, MaxIdle: maxIdle} +} + +// Get gets a connection. The application must close the returned connection. +// This method always returns a valid connection so that applications can defer +// error handling to the first use of the connection. If there is an error +// getting an underlying connection, then the connection Err, Do, Send, Flush +// and Receive methods return that error. +func (p *Pool) Get() Conn { + pc, err := p.get(nil) + if err != nil { + return errorConn{err} + } + return &activeConn{p: p, pc: pc} +} + +// PoolStats contains pool statistics. +type PoolStats struct { + // ActiveCount is the number of connections in the pool. The count includes + // idle connections and connections in use. + ActiveCount int + // IdleCount is the number of idle connections in the pool. + IdleCount int +} + +// Stats returns pool's statistics. +func (p *Pool) Stats() PoolStats { + p.mu.Lock() + stats := PoolStats{ + ActiveCount: p.active, + IdleCount: p.idle.count, + } + p.mu.Unlock() + + return stats +} + +// ActiveCount returns the number of connections in the pool. The count +// includes idle connections and connections in use. +func (p *Pool) ActiveCount() int { + p.mu.Lock() + active := p.active + p.mu.Unlock() + return active +} + +// IdleCount returns the number of idle connections in the pool. +func (p *Pool) IdleCount() int { + p.mu.Lock() + idle := p.idle.count + p.mu.Unlock() + return idle +} + +// Close releases the resources used by the pool. +func (p *Pool) Close() error { + p.mu.Lock() + if p.closed { + p.mu.Unlock() + return nil + } + p.closed = true + p.active -= p.idle.count + pc := p.idle.front + p.idle.count = 0 + p.idle.front, p.idle.back = nil, nil + if p.ch != nil { + close(p.ch) + } + p.mu.Unlock() + for ; pc != nil; pc = pc.next { + pc.c.Close() + } + return nil +} + +func (p *Pool) lazyInit() { + // Fast path. + if atomic.LoadUint32(&p.chInitialized) == 1 { + return + } + // Slow path. + p.mu.Lock() + if p.chInitialized == 0 { + p.ch = make(chan struct{}, p.MaxActive) + if p.closed { + close(p.ch) + } else { + for i := 0; i < p.MaxActive; i++ { + p.ch <- struct{}{} + } + } + atomic.StoreUint32(&p.chInitialized, 1) + } + p.mu.Unlock() +} + +// get prunes stale connections and returns a connection from the idle list or +// creates a new connection. +func (p *Pool) get(ctx interface { + Done() <-chan struct{} + Err() error +}) (*poolConn, error) { + + // Handle limit for p.Wait == true. + if p.Wait && p.MaxActive > 0 { + p.lazyInit() + if ctx == nil { + <-p.ch + } else { + select { + case <-p.ch: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + } + + p.mu.Lock() + + // Prune stale connections at the back of the idle list. + if p.IdleTimeout > 0 { + n := p.idle.count + for i := 0; i < n && p.idle.back != nil && p.idle.back.t.Add(p.IdleTimeout).Before(nowFunc()); i++ { + pc := p.idle.back + p.idle.popBack() + p.mu.Unlock() + pc.c.Close() + p.mu.Lock() + p.active-- + } + } + + // Get idle connection from the front of idle list. + for p.idle.front != nil { + pc := p.idle.front + p.idle.popFront() + p.mu.Unlock() + if (p.TestOnBorrow == nil || p.TestOnBorrow(pc.c, pc.t) == nil) && + (p.MaxConnLifetime == 0 || nowFunc().Sub(pc.created) < p.MaxConnLifetime) { + return pc, nil + } + pc.c.Close() + p.mu.Lock() + p.active-- + } + + // Check for pool closed before dialing a new connection. + if p.closed { + p.mu.Unlock() + return nil, errors.New("redigo: get on closed pool") + } + + // Handle limit for p.Wait == false. + if !p.Wait && p.MaxActive > 0 && p.active >= p.MaxActive { + p.mu.Unlock() + return nil, ErrPoolExhausted + } + + p.active++ + p.mu.Unlock() + c, err := p.Dial() + if err != nil { + c = nil + p.mu.Lock() + p.active-- + if p.ch != nil && !p.closed { + p.ch <- struct{}{} + } + p.mu.Unlock() + } + return &poolConn{c: c, created: nowFunc()}, err +} + +func (p *Pool) put(pc *poolConn, forceClose bool) error { + p.mu.Lock() + if !p.closed && !forceClose { + pc.t = nowFunc() + p.idle.pushFront(pc) + if p.idle.count > p.MaxIdle { + pc = p.idle.back + p.idle.popBack() + } else { + pc = nil + } + } + + if pc != nil { + p.mu.Unlock() + pc.c.Close() + p.mu.Lock() + p.active-- + } + + if p.ch != nil && !p.closed { + p.ch <- struct{}{} + } + p.mu.Unlock() + return nil +} + +type activeConn struct { + p *Pool + pc *poolConn + state int +} + +var ( + sentinel []byte + sentinelOnce sync.Once +) + +func initSentinel() { + p := make([]byte, 64) + if _, err := rand.Read(p); err == nil { + sentinel = p + } else { + h := sha1.New() + io.WriteString(h, "Oops, rand failed. Use time instead.") + io.WriteString(h, strconv.FormatInt(time.Now().UnixNano(), 10)) + sentinel = h.Sum(nil) + } +} + +func (ac *activeConn) Close() error { + pc := ac.pc + if pc == nil { + return nil + } + ac.pc = nil + + if ac.state&internal.MultiState != 0 { + pc.c.Send("DISCARD") + ac.state &^= (internal.MultiState | internal.WatchState) + } else if ac.state&internal.WatchState != 0 { + pc.c.Send("UNWATCH") + ac.state &^= internal.WatchState + } + if ac.state&internal.SubscribeState != 0 { + pc.c.Send("UNSUBSCRIBE") + pc.c.Send("PUNSUBSCRIBE") + // To detect the end of the message stream, ask the server to echo + // a sentinel value and read until we see that value. + sentinelOnce.Do(initSentinel) + pc.c.Send("ECHO", sentinel) + pc.c.Flush() + for { + p, err := pc.c.Receive() + if err != nil { + break + } + if p, ok := p.([]byte); ok && bytes.Equal(p, sentinel) { + ac.state &^= internal.SubscribeState + break + } + } + } + pc.c.Do("") + ac.p.put(pc, ac.state != 0 || pc.c.Err() != nil) + return nil +} + +func (ac *activeConn) Err() error { + pc := ac.pc + if pc == nil { + return errConnClosed + } + return pc.c.Err() +} + +func (ac *activeConn) Do(commandName string, args ...interface{}) (reply interface{}, err error) { + pc := ac.pc + if pc == nil { + return nil, errConnClosed + } + ci := internal.LookupCommandInfo(commandName) + ac.state = (ac.state | ci.Set) &^ ci.Clear + return pc.c.Do(commandName, args...) +} + +func (ac *activeConn) DoWithTimeout(timeout time.Duration, commandName string, args ...interface{}) (reply interface{}, err error) { + pc := ac.pc + if pc == nil { + return nil, errConnClosed + } + cwt, ok := pc.c.(ConnWithTimeout) + if !ok { + return nil, errTimeoutNotSupported + } + ci := internal.LookupCommandInfo(commandName) + ac.state = (ac.state | ci.Set) &^ ci.Clear + return cwt.DoWithTimeout(timeout, commandName, args...) +} + +func (ac *activeConn) Send(commandName string, args ...interface{}) error { + pc := ac.pc + if pc == nil { + return errConnClosed + } + ci := internal.LookupCommandInfo(commandName) + ac.state = (ac.state | ci.Set) &^ ci.Clear + return pc.c.Send(commandName, args...) +} + +func (ac *activeConn) Flush() error { + pc := ac.pc + if pc == nil { + return errConnClosed + } + return pc.c.Flush() +} + +func (ac *activeConn) Receive() (reply interface{}, err error) { + pc := ac.pc + if pc == nil { + return nil, errConnClosed + } + return pc.c.Receive() +} + +func (ac *activeConn) ReceiveWithTimeout(timeout time.Duration) (reply interface{}, err error) { + pc := ac.pc + if pc == nil { + return nil, errConnClosed + } + cwt, ok := pc.c.(ConnWithTimeout) + if !ok { + return nil, errTimeoutNotSupported + } + return cwt.ReceiveWithTimeout(timeout) +} + +type errorConn struct{ err error } + +func (ec errorConn) Do(string, ...interface{}) (interface{}, error) { return nil, ec.err } +func (ec errorConn) DoWithTimeout(time.Duration, string, ...interface{}) (interface{}, error) { + return nil, ec.err +} +func (ec errorConn) Send(string, ...interface{}) error { return ec.err } +func (ec errorConn) Err() error { return ec.err } +func (ec errorConn) Close() error { return nil } +func (ec errorConn) Flush() error { return ec.err } +func (ec errorConn) Receive() (interface{}, error) { return nil, ec.err } +func (ec errorConn) ReceiveWithTimeout(time.Duration) (interface{}, error) { return nil, ec.err } + +type idleList struct { + count int + front, back *poolConn +} + +type poolConn struct { + c Conn + t time.Time + created time.Time + next, prev *poolConn +} + +func (l *idleList) pushFront(pc *poolConn) { + pc.next = l.front + pc.prev = nil + if l.count == 0 { + l.back = pc + } else { + l.front.prev = pc + } + l.front = pc + l.count++ + return +} + +func (l *idleList) popFront() { + pc := l.front + l.count-- + if l.count == 0 { + l.front, l.back = nil, nil + } else { + pc.next.prev = nil + l.front = pc.next + } + pc.next, pc.prev = nil, nil +} + +func (l *idleList) popBack() { + pc := l.back + l.count-- + if l.count == 0 { + l.front, l.back = nil, nil + } else { + pc.prev.next = nil + l.back = pc.prev + } + pc.next, pc.prev = nil, nil +} diff --git a/vendor/github.com/garyburd/redigo/redis/pool17.go b/vendor/github.com/garyburd/redigo/redis/pool17.go new file mode 100644 index 000000000..830ad92ad --- /dev/null +++ b/vendor/github.com/garyburd/redigo/redis/pool17.go @@ -0,0 +1,36 @@ +// Copyright 2018 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +//go:build go1.7 +// +build go1.7 + +package redis + +import "context" + +// GetContext gets a connection using the provided context. +// +// The provided Context must be non-nil. If the context expires before the +// connection is complete, an error is returned. Any expiration on the context +// will not affect the returned connection. +// +// If the function completes without error, then the application must close the +// returned connection. +func (p *Pool) GetContext(ctx context.Context) (Conn, error) { + pc, err := p.get(ctx) + if err != nil { + return errorConn{err}, err + } + return &activeConn{p: p, pc: pc}, nil +} diff --git a/vendor/github.com/garyburd/redigo/redis/pubsub.go b/vendor/github.com/garyburd/redigo/redis/pubsub.go new file mode 100644 index 000000000..f0ac82532 --- /dev/null +++ b/vendor/github.com/garyburd/redigo/redis/pubsub.go @@ -0,0 +1,157 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis + +import ( + "errors" + "time" +) + +// Subscription represents a subscribe or unsubscribe notification. +type Subscription struct { + // Kind is "subscribe", "unsubscribe", "psubscribe" or "punsubscribe" + Kind string + + // The channel that was changed. + Channel string + + // The current number of subscriptions for connection. + Count int +} + +// Message represents a message notification. +type Message struct { + // The originating channel. + Channel string + + // The message data. + Data []byte +} + +// PMessage represents a pmessage notification. +type PMessage struct { + // The matched pattern. + Pattern string + + // The originating channel. + Channel string + + // The message data. + Data []byte +} + +// Pong represents a pubsub pong notification. +type Pong struct { + Data string +} + +// PubSubConn wraps a Conn with convenience methods for subscribers. +type PubSubConn struct { + Conn Conn +} + +// Close closes the connection. +func (c PubSubConn) Close() error { + return c.Conn.Close() +} + +// Subscribe subscribes the connection to the specified channels. +func (c PubSubConn) Subscribe(channel ...interface{}) error { + c.Conn.Send("SUBSCRIBE", channel...) + return c.Conn.Flush() +} + +// PSubscribe subscribes the connection to the given patterns. +func (c PubSubConn) PSubscribe(channel ...interface{}) error { + c.Conn.Send("PSUBSCRIBE", channel...) + return c.Conn.Flush() +} + +// Unsubscribe unsubscribes the connection from the given channels, or from all +// of them if none is given. +func (c PubSubConn) Unsubscribe(channel ...interface{}) error { + c.Conn.Send("UNSUBSCRIBE", channel...) + return c.Conn.Flush() +} + +// PUnsubscribe unsubscribes the connection from the given patterns, or from all +// of them if none is given. +func (c PubSubConn) PUnsubscribe(channel ...interface{}) error { + c.Conn.Send("PUNSUBSCRIBE", channel...) + return c.Conn.Flush() +} + +// Ping sends a PING to the server with the specified data. +// +// The connection must be subscribed to at least one channel or pattern when +// calling this method. +func (c PubSubConn) Ping(data string) error { + c.Conn.Send("PING", data) + return c.Conn.Flush() +} + +// Receive returns a pushed message as a Subscription, Message, PMessage, Pong +// or error. The return value is intended to be used directly in a type switch +// as illustrated in the PubSubConn example. +func (c PubSubConn) Receive() interface{} { + return c.receiveInternal(c.Conn.Receive()) +} + +// ReceiveWithTimeout is like Receive, but it allows the application to +// override the connection's default timeout. +func (c PubSubConn) ReceiveWithTimeout(timeout time.Duration) interface{} { + return c.receiveInternal(ReceiveWithTimeout(c.Conn, timeout)) +} + +func (c PubSubConn) receiveInternal(replyArg interface{}, errArg error) interface{} { + reply, err := Values(replyArg, errArg) + if err != nil { + return err + } + + var kind string + reply, err = Scan(reply, &kind) + if err != nil { + return err + } + + switch kind { + case "message": + var m Message + if _, err := Scan(reply, &m.Channel, &m.Data); err != nil { + return err + } + return m + case "pmessage": + var pm PMessage + if _, err := Scan(reply, &pm.Pattern, &pm.Channel, &pm.Data); err != nil { + return err + } + return pm + case "subscribe", "psubscribe", "unsubscribe", "punsubscribe": + s := Subscription{Kind: kind} + if _, err := Scan(reply, &s.Channel, &s.Count); err != nil { + return err + } + return s + case "pong": + var p Pong + if _, err := Scan(reply, &p.Data); err != nil { + return err + } + return p + } + return errors.New("redigo: unknown pubsub notification") +} diff --git a/vendor/github.com/garyburd/redigo/redis/redis.go b/vendor/github.com/garyburd/redigo/redis/redis.go new file mode 100644 index 000000000..68ad44df7 --- /dev/null +++ b/vendor/github.com/garyburd/redigo/redis/redis.go @@ -0,0 +1,119 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis + +import ( + "errors" + "time" +) + +// Error represents an error returned in a command reply. +type Error string + +func (err Error) Error() string { return string(err) } + +// Conn represents a connection to a Redis server. +// +// Deprecated: This package is replaced by github.com/gomodule/redigo/redis. +type Conn interface { + // Close closes the connection. + Close() error + + // Err returns a non-nil value when the connection is not usable. + Err() error + + // Do sends a command to the server and returns the received reply. + Do(commandName string, args ...interface{}) (reply interface{}, err error) + + // Send writes the command to the client's output buffer. + Send(commandName string, args ...interface{}) error + + // Flush flushes the output buffer to the Redis server. + Flush() error + + // Receive receives a single reply from the Redis server + Receive() (reply interface{}, err error) +} + +// Argument is the interface implemented by an object which wants to control how +// the object is converted to Redis bulk strings. +type Argument interface { + // RedisArg returns a value to be encoded as a bulk string per the + // conversions listed in the section 'Executing Commands'. + // Implementations should typically return a []byte or string. + RedisArg() interface{} +} + +// Scanner is implemented by an object which wants to control its value is +// interpreted when read from Redis. +type Scanner interface { + // RedisScan assigns a value from a Redis value. The argument src is one of + // the reply types listed in the section `Executing Commands`. + // + // An error should be returned if the value cannot be stored without + // loss of information. + RedisScan(src interface{}) error +} + +// ConnWithTimeout is an optional interface that allows the caller to override +// a connection's default read timeout. This interface is useful for executing +// the BLPOP, BRPOP, BRPOPLPUSH, XREAD and other commands that block at the +// server. +// +// A connection's default read timeout is set with the DialReadTimeout dial +// option. Applications should rely on the default timeout for commands that do +// not block at the server. +// +// All of the Conn implementations in this package satisfy the ConnWithTimeout +// interface. +// +// Use the DoWithTimeout and ReceiveWithTimeout helper functions to simplify +// use of this interface. +type ConnWithTimeout interface { + Conn + + // Do sends a command to the server and returns the received reply. + // The timeout overrides the read timeout set when dialing the + // connection. + DoWithTimeout(timeout time.Duration, commandName string, args ...interface{}) (reply interface{}, err error) + + // Receive receives a single reply from the Redis server. The timeout + // overrides the read timeout set when dialing the connection. + ReceiveWithTimeout(timeout time.Duration) (reply interface{}, err error) +} + +var errTimeoutNotSupported = errors.New("redis: connection does not support ConnWithTimeout") + +// DoWithTimeout executes a Redis command with the specified read timeout. If +// the connection does not satisfy the ConnWithTimeout interface, then an error +// is returned. +func DoWithTimeout(c Conn, timeout time.Duration, cmd string, args ...interface{}) (interface{}, error) { + cwt, ok := c.(ConnWithTimeout) + if !ok { + return nil, errTimeoutNotSupported + } + return cwt.DoWithTimeout(timeout, cmd, args...) +} + +// ReceiveWithTimeout receives a reply with the specified read timeout. If the +// connection does not satisfy the ConnWithTimeout interface, then an error is +// returned. +func ReceiveWithTimeout(c Conn, timeout time.Duration) (interface{}, error) { + cwt, ok := c.(ConnWithTimeout) + if !ok { + return nil, errTimeoutNotSupported + } + return cwt.ReceiveWithTimeout(timeout) +} diff --git a/vendor/github.com/garyburd/redigo/redis/reply.go b/vendor/github.com/garyburd/redigo/redis/reply.go new file mode 100644 index 000000000..0874d31df --- /dev/null +++ b/vendor/github.com/garyburd/redigo/redis/reply.go @@ -0,0 +1,479 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis + +import ( + "errors" + "fmt" + "strconv" +) + +// ErrNil indicates that a reply value is nil. +var ErrNil = errors.New("redigo: nil returned") + +// Int is a helper that converts a command reply to an integer. If err is not +// equal to nil, then Int returns 0, err. Otherwise, Int converts the +// reply to an int as follows: +// +// Reply type Result +// integer int(reply), nil +// bulk string parsed reply, nil +// nil 0, ErrNil +// other 0, error +func Int(reply interface{}, err error) (int, error) { + if err != nil { + return 0, err + } + switch reply := reply.(type) { + case int64: + x := int(reply) + if int64(x) != reply { + return 0, strconv.ErrRange + } + return x, nil + case []byte: + n, err := strconv.ParseInt(string(reply), 10, 0) + return int(n), err + case nil: + return 0, ErrNil + case Error: + return 0, reply + } + return 0, fmt.Errorf("redigo: unexpected type for Int, got type %T", reply) +} + +// Int64 is a helper that converts a command reply to 64 bit integer. If err is +// not equal to nil, then Int returns 0, err. Otherwise, Int64 converts the +// reply to an int64 as follows: +// +// Reply type Result +// integer reply, nil +// bulk string parsed reply, nil +// nil 0, ErrNil +// other 0, error +func Int64(reply interface{}, err error) (int64, error) { + if err != nil { + return 0, err + } + switch reply := reply.(type) { + case int64: + return reply, nil + case []byte: + n, err := strconv.ParseInt(string(reply), 10, 64) + return n, err + case nil: + return 0, ErrNil + case Error: + return 0, reply + } + return 0, fmt.Errorf("redigo: unexpected type for Int64, got type %T", reply) +} + +var errNegativeInt = errors.New("redigo: unexpected value for Uint64") + +// Uint64 is a helper that converts a command reply to 64 bit integer. If err is +// not equal to nil, then Int returns 0, err. Otherwise, Int64 converts the +// reply to an int64 as follows: +// +// Reply type Result +// integer reply, nil +// bulk string parsed reply, nil +// nil 0, ErrNil +// other 0, error +func Uint64(reply interface{}, err error) (uint64, error) { + if err != nil { + return 0, err + } + switch reply := reply.(type) { + case int64: + if reply < 0 { + return 0, errNegativeInt + } + return uint64(reply), nil + case []byte: + n, err := strconv.ParseUint(string(reply), 10, 64) + return n, err + case nil: + return 0, ErrNil + case Error: + return 0, reply + } + return 0, fmt.Errorf("redigo: unexpected type for Uint64, got type %T", reply) +} + +// Float64 is a helper that converts a command reply to 64 bit float. If err is +// not equal to nil, then Float64 returns 0, err. Otherwise, Float64 converts +// the reply to an int as follows: +// +// Reply type Result +// bulk string parsed reply, nil +// nil 0, ErrNil +// other 0, error +func Float64(reply interface{}, err error) (float64, error) { + if err != nil { + return 0, err + } + switch reply := reply.(type) { + case []byte: + n, err := strconv.ParseFloat(string(reply), 64) + return n, err + case nil: + return 0, ErrNil + case Error: + return 0, reply + } + return 0, fmt.Errorf("redigo: unexpected type for Float64, got type %T", reply) +} + +// String is a helper that converts a command reply to a string. If err is not +// equal to nil, then String returns "", err. Otherwise String converts the +// reply to a string as follows: +// +// Reply type Result +// bulk string string(reply), nil +// simple string reply, nil +// nil "", ErrNil +// other "", error +func String(reply interface{}, err error) (string, error) { + if err != nil { + return "", err + } + switch reply := reply.(type) { + case []byte: + return string(reply), nil + case string: + return reply, nil + case nil: + return "", ErrNil + case Error: + return "", reply + } + return "", fmt.Errorf("redigo: unexpected type for String, got type %T", reply) +} + +// Bytes is a helper that converts a command reply to a slice of bytes. If err +// is not equal to nil, then Bytes returns nil, err. Otherwise Bytes converts +// the reply to a slice of bytes as follows: +// +// Reply type Result +// bulk string reply, nil +// simple string []byte(reply), nil +// nil nil, ErrNil +// other nil, error +func Bytes(reply interface{}, err error) ([]byte, error) { + if err != nil { + return nil, err + } + switch reply := reply.(type) { + case []byte: + return reply, nil + case string: + return []byte(reply), nil + case nil: + return nil, ErrNil + case Error: + return nil, reply + } + return nil, fmt.Errorf("redigo: unexpected type for Bytes, got type %T", reply) +} + +// Bool is a helper that converts a command reply to a boolean. If err is not +// equal to nil, then Bool returns false, err. Otherwise Bool converts the +// reply to boolean as follows: +// +// Reply type Result +// integer value != 0, nil +// bulk string strconv.ParseBool(reply) +// nil false, ErrNil +// other false, error +func Bool(reply interface{}, err error) (bool, error) { + if err != nil { + return false, err + } + switch reply := reply.(type) { + case int64: + return reply != 0, nil + case []byte: + return strconv.ParseBool(string(reply)) + case nil: + return false, ErrNil + case Error: + return false, reply + } + return false, fmt.Errorf("redigo: unexpected type for Bool, got type %T", reply) +} + +// MultiBulk is a helper that converts an array command reply to a []interface{}. +// +// Deprecated: Use Values instead. +func MultiBulk(reply interface{}, err error) ([]interface{}, error) { return Values(reply, err) } + +// Values is a helper that converts an array command reply to a []interface{}. +// If err is not equal to nil, then Values returns nil, err. Otherwise, Values +// converts the reply as follows: +// +// Reply type Result +// array reply, nil +// nil nil, ErrNil +// other nil, error +func Values(reply interface{}, err error) ([]interface{}, error) { + if err != nil { + return nil, err + } + switch reply := reply.(type) { + case []interface{}: + return reply, nil + case nil: + return nil, ErrNil + case Error: + return nil, reply + } + return nil, fmt.Errorf("redigo: unexpected type for Values, got type %T", reply) +} + +func sliceHelper(reply interface{}, err error, name string, makeSlice func(int), assign func(int, interface{}) error) error { + if err != nil { + return err + } + switch reply := reply.(type) { + case []interface{}: + makeSlice(len(reply)) + for i := range reply { + if reply[i] == nil { + continue + } + if err := assign(i, reply[i]); err != nil { + return err + } + } + return nil + case nil: + return ErrNil + case Error: + return reply + } + return fmt.Errorf("redigo: unexpected type for %s, got type %T", name, reply) +} + +// Float64s is a helper that converts an array command reply to a []float64. If +// err is not equal to nil, then Float64s returns nil, err. Nil array items are +// converted to 0 in the output slice. Floats64 returns an error if an array +// item is not a bulk string or nil. +func Float64s(reply interface{}, err error) ([]float64, error) { + var result []float64 + err = sliceHelper(reply, err, "Float64s", func(n int) { result = make([]float64, n) }, func(i int, v interface{}) error { + p, ok := v.([]byte) + if !ok { + return fmt.Errorf("redigo: unexpected element type for Floats64, got type %T", v) + } + f, err := strconv.ParseFloat(string(p), 64) + result[i] = f + return err + }) + return result, err +} + +// Strings is a helper that converts an array command reply to a []string. If +// err is not equal to nil, then Strings returns nil, err. Nil array items are +// converted to "" in the output slice. Strings returns an error if an array +// item is not a bulk string or nil. +func Strings(reply interface{}, err error) ([]string, error) { + var result []string + err = sliceHelper(reply, err, "Strings", func(n int) { result = make([]string, n) }, func(i int, v interface{}) error { + switch v := v.(type) { + case string: + result[i] = v + return nil + case []byte: + result[i] = string(v) + return nil + default: + return fmt.Errorf("redigo: unexpected element type for Strings, got type %T", v) + } + }) + return result, err +} + +// ByteSlices is a helper that converts an array command reply to a [][]byte. +// If err is not equal to nil, then ByteSlices returns nil, err. Nil array +// items are stay nil. ByteSlices returns an error if an array item is not a +// bulk string or nil. +func ByteSlices(reply interface{}, err error) ([][]byte, error) { + var result [][]byte + err = sliceHelper(reply, err, "ByteSlices", func(n int) { result = make([][]byte, n) }, func(i int, v interface{}) error { + p, ok := v.([]byte) + if !ok { + return fmt.Errorf("redigo: unexpected element type for ByteSlices, got type %T", v) + } + result[i] = p + return nil + }) + return result, err +} + +// Int64s is a helper that converts an array command reply to a []int64. +// If err is not equal to nil, then Int64s returns nil, err. Nil array +// items are stay nil. Int64s returns an error if an array item is not a +// bulk string or nil. +func Int64s(reply interface{}, err error) ([]int64, error) { + var result []int64 + err = sliceHelper(reply, err, "Int64s", func(n int) { result = make([]int64, n) }, func(i int, v interface{}) error { + switch v := v.(type) { + case int64: + result[i] = v + return nil + case []byte: + n, err := strconv.ParseInt(string(v), 10, 64) + result[i] = n + return err + default: + return fmt.Errorf("redigo: unexpected element type for Int64s, got type %T", v) + } + }) + return result, err +} + +// Ints is a helper that converts an array command reply to a []in. +// If err is not equal to nil, then Ints returns nil, err. Nil array +// items are stay nil. Ints returns an error if an array item is not a +// bulk string or nil. +func Ints(reply interface{}, err error) ([]int, error) { + var result []int + err = sliceHelper(reply, err, "Ints", func(n int) { result = make([]int, n) }, func(i int, v interface{}) error { + switch v := v.(type) { + case int64: + n := int(v) + if int64(n) != v { + return strconv.ErrRange + } + result[i] = n + return nil + case []byte: + n, err := strconv.Atoi(string(v)) + result[i] = n + return err + default: + return fmt.Errorf("redigo: unexpected element type for Ints, got type %T", v) + } + }) + return result, err +} + +// StringMap is a helper that converts an array of strings (alternating key, value) +// into a map[string]string. The HGETALL and CONFIG GET commands return replies in this format. +// Requires an even number of values in result. +func StringMap(result interface{}, err error) (map[string]string, error) { + values, err := Values(result, err) + if err != nil { + return nil, err + } + if len(values)%2 != 0 { + return nil, errors.New("redigo: StringMap expects even number of values result") + } + m := make(map[string]string, len(values)/2) + for i := 0; i < len(values); i += 2 { + key, okKey := values[i].([]byte) + value, okValue := values[i+1].([]byte) + if !okKey || !okValue { + return nil, errors.New("redigo: StringMap key not a bulk string value") + } + m[string(key)] = string(value) + } + return m, nil +} + +// IntMap is a helper that converts an array of strings (alternating key, value) +// into a map[string]int. The HGETALL commands return replies in this format. +// Requires an even number of values in result. +func IntMap(result interface{}, err error) (map[string]int, error) { + values, err := Values(result, err) + if err != nil { + return nil, err + } + if len(values)%2 != 0 { + return nil, errors.New("redigo: IntMap expects even number of values result") + } + m := make(map[string]int, len(values)/2) + for i := 0; i < len(values); i += 2 { + key, ok := values[i].([]byte) + if !ok { + return nil, errors.New("redigo: IntMap key not a bulk string value") + } + value, err := Int(values[i+1], nil) + if err != nil { + return nil, err + } + m[string(key)] = value + } + return m, nil +} + +// Int64Map is a helper that converts an array of strings (alternating key, value) +// into a map[string]int64. The HGETALL commands return replies in this format. +// Requires an even number of values in result. +func Int64Map(result interface{}, err error) (map[string]int64, error) { + values, err := Values(result, err) + if err != nil { + return nil, err + } + if len(values)%2 != 0 { + return nil, errors.New("redigo: Int64Map expects even number of values result") + } + m := make(map[string]int64, len(values)/2) + for i := 0; i < len(values); i += 2 { + key, ok := values[i].([]byte) + if !ok { + return nil, errors.New("redigo: Int64Map key not a bulk string value") + } + value, err := Int64(values[i+1], nil) + if err != nil { + return nil, err + } + m[string(key)] = value + } + return m, nil +} + +// Positions is a helper that converts an array of positions (lat, long) +// into a [][2]float64. The GEOPOS command returns replies in this format. +func Positions(result interface{}, err error) ([]*[2]float64, error) { + values, err := Values(result, err) + if err != nil { + return nil, err + } + positions := make([]*[2]float64, len(values)) + for i := range values { + if values[i] == nil { + continue + } + p, ok := values[i].([]interface{}) + if !ok { + return nil, fmt.Errorf("redigo: unexpected element type for interface slice, got type %T", values[i]) + } + if len(p) != 2 { + return nil, fmt.Errorf("redigo: unexpected number of values for a member position, got %d", len(p)) + } + lat, err := Float64(p[0], nil) + if err != nil { + return nil, err + } + long, err := Float64(p[1], nil) + if err != nil { + return nil, err + } + positions[i] = &[2]float64{lat, long} + } + return positions, nil +} diff --git a/vendor/github.com/garyburd/redigo/redis/scan.go b/vendor/github.com/garyburd/redigo/redis/scan.go new file mode 100644 index 000000000..9f6e558b8 --- /dev/null +++ b/vendor/github.com/garyburd/redigo/redis/scan.go @@ -0,0 +1,585 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis + +import ( + "errors" + "fmt" + "reflect" + "strconv" + "strings" + "sync" +) + +func ensureLen(d reflect.Value, n int) { + if n > d.Cap() { + d.Set(reflect.MakeSlice(d.Type(), n, n)) + } else { + d.SetLen(n) + } +} + +func cannotConvert(d reflect.Value, s interface{}) error { + var sname string + switch s.(type) { + case string: + sname = "Redis simple string" + case Error: + sname = "Redis error" + case int64: + sname = "Redis integer" + case []byte: + sname = "Redis bulk string" + case []interface{}: + sname = "Redis array" + default: + sname = reflect.TypeOf(s).String() + } + return fmt.Errorf("cannot convert from %s to %s", sname, d.Type()) +} + +func convertAssignBulkString(d reflect.Value, s []byte) (err error) { + switch d.Type().Kind() { + case reflect.Float32, reflect.Float64: + var x float64 + x, err = strconv.ParseFloat(string(s), d.Type().Bits()) + d.SetFloat(x) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + var x int64 + x, err = strconv.ParseInt(string(s), 10, d.Type().Bits()) + d.SetInt(x) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + var x uint64 + x, err = strconv.ParseUint(string(s), 10, d.Type().Bits()) + d.SetUint(x) + case reflect.Bool: + var x bool + x, err = strconv.ParseBool(string(s)) + d.SetBool(x) + case reflect.String: + d.SetString(string(s)) + case reflect.Slice: + if d.Type().Elem().Kind() != reflect.Uint8 { + err = cannotConvert(d, s) + } else { + d.SetBytes(s) + } + default: + err = cannotConvert(d, s) + } + return +} + +func convertAssignInt(d reflect.Value, s int64) (err error) { + switch d.Type().Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + d.SetInt(s) + if d.Int() != s { + err = strconv.ErrRange + d.SetInt(0) + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + if s < 0 { + err = strconv.ErrRange + } else { + x := uint64(s) + d.SetUint(x) + if d.Uint() != x { + err = strconv.ErrRange + d.SetUint(0) + } + } + case reflect.Bool: + d.SetBool(s != 0) + default: + err = cannotConvert(d, s) + } + return +} + +func convertAssignValue(d reflect.Value, s interface{}) (err error) { + if d.Kind() != reflect.Ptr { + if d.CanAddr() { + d2 := d.Addr() + if d2.CanInterface() { + if scanner, ok := d2.Interface().(Scanner); ok { + return scanner.RedisScan(s) + } + } + } + } else if d.CanInterface() { + // Already a reflect.Ptr + if d.IsNil() { + d.Set(reflect.New(d.Type().Elem())) + } + if scanner, ok := d.Interface().(Scanner); ok { + return scanner.RedisScan(s) + } + } + + switch s := s.(type) { + case []byte: + err = convertAssignBulkString(d, s) + case int64: + err = convertAssignInt(d, s) + default: + err = cannotConvert(d, s) + } + return err +} + +func convertAssignArray(d reflect.Value, s []interface{}) error { + if d.Type().Kind() != reflect.Slice { + return cannotConvert(d, s) + } + ensureLen(d, len(s)) + for i := 0; i < len(s); i++ { + if err := convertAssignValue(d.Index(i), s[i]); err != nil { + return err + } + } + return nil +} + +func convertAssign(d interface{}, s interface{}) (err error) { + if scanner, ok := d.(Scanner); ok { + return scanner.RedisScan(s) + } + + // Handle the most common destination types using type switches and + // fall back to reflection for all other types. + switch s := s.(type) { + case nil: + // ignore + case []byte: + switch d := d.(type) { + case *string: + *d = string(s) + case *int: + *d, err = strconv.Atoi(string(s)) + case *bool: + *d, err = strconv.ParseBool(string(s)) + case *[]byte: + *d = s + case *interface{}: + *d = s + case nil: + // skip value + default: + if d := reflect.ValueOf(d); d.Type().Kind() != reflect.Ptr { + err = cannotConvert(d, s) + } else { + err = convertAssignBulkString(d.Elem(), s) + } + } + case int64: + switch d := d.(type) { + case *int: + x := int(s) + if int64(x) != s { + err = strconv.ErrRange + x = 0 + } + *d = x + case *bool: + *d = s != 0 + case *interface{}: + *d = s + case nil: + // skip value + default: + if d := reflect.ValueOf(d); d.Type().Kind() != reflect.Ptr { + err = cannotConvert(d, s) + } else { + err = convertAssignInt(d.Elem(), s) + } + } + case string: + switch d := d.(type) { + case *string: + *d = s + case *interface{}: + *d = s + case nil: + // skip value + default: + err = cannotConvert(reflect.ValueOf(d), s) + } + case []interface{}: + switch d := d.(type) { + case *[]interface{}: + *d = s + case *interface{}: + *d = s + case nil: + // skip value + default: + if d := reflect.ValueOf(d); d.Type().Kind() != reflect.Ptr { + err = cannotConvert(d, s) + } else { + err = convertAssignArray(d.Elem(), s) + } + } + case Error: + err = s + default: + err = cannotConvert(reflect.ValueOf(d), s) + } + return +} + +// Scan copies from src to the values pointed at by dest. +// +// Scan uses RedisScan if available otherwise: +// +// The values pointed at by dest must be an integer, float, boolean, string, +// []byte, interface{} or slices of these types. Scan uses the standard strconv +// package to convert bulk strings to numeric and boolean types. +// +// If a dest value is nil, then the corresponding src value is skipped. +// +// If a src element is nil, then the corresponding dest value is not modified. +// +// To enable easy use of Scan in a loop, Scan returns the slice of src +// following the copied values. +func Scan(src []interface{}, dest ...interface{}) ([]interface{}, error) { + if len(src) < len(dest) { + return nil, errors.New("redigo.Scan: array short") + } + var err error + for i, d := range dest { + err = convertAssign(d, src[i]) + if err != nil { + err = fmt.Errorf("redigo.Scan: cannot assign to dest %d: %v", i, err) + break + } + } + return src[len(dest):], err +} + +type fieldSpec struct { + name string + index []int + omitEmpty bool +} + +type structSpec struct { + m map[string]*fieldSpec + l []*fieldSpec +} + +func (ss *structSpec) fieldSpec(name []byte) *fieldSpec { + return ss.m[string(name)] +} + +func compileStructSpec(t reflect.Type, depth map[string]int, index []int, ss *structSpec) { + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + switch { + case f.PkgPath != "" && !f.Anonymous: + // Ignore unexported fields. + case f.Anonymous: + // TODO: Handle pointers. Requires change to decoder and + // protection against infinite recursion. + if f.Type.Kind() == reflect.Struct { + compileStructSpec(f.Type, depth, append(index, i), ss) + } + default: + fs := &fieldSpec{name: f.Name} + tag := f.Tag.Get("redis") + p := strings.Split(tag, ",") + if len(p) > 0 { + if p[0] == "-" { + continue + } + if len(p[0]) > 0 { + fs.name = p[0] + } + for _, s := range p[1:] { + switch s { + case "omitempty": + fs.omitEmpty = true + default: + panic(fmt.Errorf("redigo: unknown field tag %s for type %s", s, t.Name())) + } + } + } + d, found := depth[fs.name] + if !found { + d = 1 << 30 + } + switch { + case len(index) == d: + // At same depth, remove from result. + delete(ss.m, fs.name) + j := 0 + for i := 0; i < len(ss.l); i++ { + if fs.name != ss.l[i].name { + ss.l[j] = ss.l[i] + j += 1 + } + } + ss.l = ss.l[:j] + case len(index) < d: + fs.index = make([]int, len(index)+1) + copy(fs.index, index) + fs.index[len(index)] = i + depth[fs.name] = len(index) + ss.m[fs.name] = fs + ss.l = append(ss.l, fs) + } + } + } +} + +var ( + structSpecMutex sync.RWMutex + structSpecCache = make(map[reflect.Type]*structSpec) + defaultFieldSpec = &fieldSpec{} +) + +func structSpecForType(t reflect.Type) *structSpec { + + structSpecMutex.RLock() + ss, found := structSpecCache[t] + structSpecMutex.RUnlock() + if found { + return ss + } + + structSpecMutex.Lock() + defer structSpecMutex.Unlock() + ss, found = structSpecCache[t] + if found { + return ss + } + + ss = &structSpec{m: make(map[string]*fieldSpec)} + compileStructSpec(t, make(map[string]int), nil, ss) + structSpecCache[t] = ss + return ss +} + +var errScanStructValue = errors.New("redigo.ScanStruct: value must be non-nil pointer to a struct") + +// ScanStruct scans alternating names and values from src to a struct. The +// HGETALL and CONFIG GET commands return replies in this format. +// +// ScanStruct uses exported field names to match values in the response. Use +// 'redis' field tag to override the name: +// +// Field int `redis:"myName"` +// +// Fields with the tag redis:"-" are ignored. +// +// Each field uses RedisScan if available otherwise: +// Integer, float, boolean, string and []byte fields are supported. Scan uses the +// standard strconv package to convert bulk string values to numeric and +// boolean types. +// +// If a src element is nil, then the corresponding field is not modified. +func ScanStruct(src []interface{}, dest interface{}) error { + d := reflect.ValueOf(dest) + if d.Kind() != reflect.Ptr || d.IsNil() { + return errScanStructValue + } + d = d.Elem() + if d.Kind() != reflect.Struct { + return errScanStructValue + } + ss := structSpecForType(d.Type()) + + if len(src)%2 != 0 { + return errors.New("redigo.ScanStruct: number of values not a multiple of 2") + } + + for i := 0; i < len(src); i += 2 { + s := src[i+1] + if s == nil { + continue + } + name, ok := src[i].([]byte) + if !ok { + return fmt.Errorf("redigo.ScanStruct: key %d not a bulk string value", i) + } + fs := ss.fieldSpec(name) + if fs == nil { + continue + } + if err := convertAssignValue(d.FieldByIndex(fs.index), s); err != nil { + return fmt.Errorf("redigo.ScanStruct: cannot assign field %s: %v", fs.name, err) + } + } + return nil +} + +var ( + errScanSliceValue = errors.New("redigo.ScanSlice: dest must be non-nil pointer to a struct") +) + +// ScanSlice scans src to the slice pointed to by dest. The elements the dest +// slice must be integer, float, boolean, string, struct or pointer to struct +// values. +// +// Struct fields must be integer, float, boolean or string values. All struct +// fields are used unless a subset is specified using fieldNames. +func ScanSlice(src []interface{}, dest interface{}, fieldNames ...string) error { + d := reflect.ValueOf(dest) + if d.Kind() != reflect.Ptr || d.IsNil() { + return errScanSliceValue + } + d = d.Elem() + if d.Kind() != reflect.Slice { + return errScanSliceValue + } + + isPtr := false + t := d.Type().Elem() + if t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct { + isPtr = true + t = t.Elem() + } + + if t.Kind() != reflect.Struct { + ensureLen(d, len(src)) + for i, s := range src { + if s == nil { + continue + } + if err := convertAssignValue(d.Index(i), s); err != nil { + return fmt.Errorf("redigo.ScanSlice: cannot assign element %d: %v", i, err) + } + } + return nil + } + + ss := structSpecForType(t) + fss := ss.l + if len(fieldNames) > 0 { + fss = make([]*fieldSpec, len(fieldNames)) + for i, name := range fieldNames { + fss[i] = ss.m[name] + if fss[i] == nil { + return fmt.Errorf("redigo.ScanSlice: ScanSlice bad field name %s", name) + } + } + } + + if len(fss) == 0 { + return errors.New("redigo.ScanSlice: no struct fields") + } + + n := len(src) / len(fss) + if n*len(fss) != len(src) { + return errors.New("redigo.ScanSlice: length not a multiple of struct field count") + } + + ensureLen(d, n) + for i := 0; i < n; i++ { + d := d.Index(i) + if isPtr { + if d.IsNil() { + d.Set(reflect.New(t)) + } + d = d.Elem() + } + for j, fs := range fss { + s := src[i*len(fss)+j] + if s == nil { + continue + } + if err := convertAssignValue(d.FieldByIndex(fs.index), s); err != nil { + return fmt.Errorf("redigo.ScanSlice: cannot assign element %d to field %s: %v", i*len(fss)+j, fs.name, err) + } + } + } + return nil +} + +// Args is a helper for constructing command arguments from structured values. +type Args []interface{} + +// Add returns the result of appending value to args. +func (args Args) Add(value ...interface{}) Args { + return append(args, value...) +} + +// AddFlat returns the result of appending the flattened value of v to args. +// +// Maps are flattened by appending the alternating keys and map values to args. +// +// Slices are flattened by appending the slice elements to args. +// +// Structs are flattened by appending the alternating names and values of +// exported fields to args. If v is a nil struct pointer, then nothing is +// appended. The 'redis' field tag overrides struct field names. See ScanStruct +// for more information on the use of the 'redis' field tag. +// +// Other types are appended to args as is. +func (args Args) AddFlat(v interface{}) Args { + rv := reflect.ValueOf(v) + switch rv.Kind() { + case reflect.Struct: + args = flattenStruct(args, rv) + case reflect.Slice: + for i := 0; i < rv.Len(); i++ { + args = append(args, rv.Index(i).Interface()) + } + case reflect.Map: + for _, k := range rv.MapKeys() { + args = append(args, k.Interface(), rv.MapIndex(k).Interface()) + } + case reflect.Ptr: + if rv.Type().Elem().Kind() == reflect.Struct { + if !rv.IsNil() { + args = flattenStruct(args, rv.Elem()) + } + } else { + args = append(args, v) + } + default: + args = append(args, v) + } + return args +} + +func flattenStruct(args Args, v reflect.Value) Args { + ss := structSpecForType(v.Type()) + for _, fs := range ss.l { + fv := v.FieldByIndex(fs.index) + if fs.omitEmpty { + var empty = false + switch fv.Kind() { + case reflect.Array, reflect.Map, reflect.Slice, reflect.String: + empty = fv.Len() == 0 + case reflect.Bool: + empty = !fv.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + empty = fv.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + empty = fv.Uint() == 0 + case reflect.Float32, reflect.Float64: + empty = fv.Float() == 0 + case reflect.Interface, reflect.Ptr: + empty = fv.IsNil() + } + if empty { + continue + } + } + args = append(args, fs.name, fv.Interface()) + } + return args +} diff --git a/vendor/github.com/garyburd/redigo/redis/script.go b/vendor/github.com/garyburd/redigo/redis/script.go new file mode 100644 index 000000000..0ef1c821f --- /dev/null +++ b/vendor/github.com/garyburd/redigo/redis/script.go @@ -0,0 +1,91 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis + +import ( + "crypto/sha1" + "encoding/hex" + "io" + "strings" +) + +// Script encapsulates the source, hash and key count for a Lua script. See +// http://redis.io/commands/eval for information on scripts in Redis. +type Script struct { + keyCount int + src string + hash string +} + +// NewScript returns a new script object. If keyCount is greater than or equal +// to zero, then the count is automatically inserted in the EVAL command +// argument list. If keyCount is less than zero, then the application supplies +// the count as the first value in the keysAndArgs argument to the Do, Send and +// SendHash methods. +func NewScript(keyCount int, src string) *Script { + h := sha1.New() + io.WriteString(h, src) + return &Script{keyCount, src, hex.EncodeToString(h.Sum(nil))} +} + +func (s *Script) args(spec string, keysAndArgs []interface{}) []interface{} { + var args []interface{} + if s.keyCount < 0 { + args = make([]interface{}, 1+len(keysAndArgs)) + args[0] = spec + copy(args[1:], keysAndArgs) + } else { + args = make([]interface{}, 2+len(keysAndArgs)) + args[0] = spec + args[1] = s.keyCount + copy(args[2:], keysAndArgs) + } + return args +} + +// Hash returns the script hash. +func (s *Script) Hash() string { + return s.hash +} + +// Do evaluates the script. Under the covers, Do optimistically evaluates the +// script using the EVALSHA command. If the command fails because the script is +// not loaded, then Do evaluates the script using the EVAL command (thus +// causing the script to load). +func (s *Script) Do(c Conn, keysAndArgs ...interface{}) (interface{}, error) { + v, err := c.Do("EVALSHA", s.args(s.hash, keysAndArgs)...) + if e, ok := err.(Error); ok && strings.HasPrefix(string(e), "NOSCRIPT ") { + v, err = c.Do("EVAL", s.args(s.src, keysAndArgs)...) + } + return v, err +} + +// SendHash evaluates the script without waiting for the reply. The script is +// evaluated with the EVALSHA command. The application must ensure that the +// script is loaded by a previous call to Send, Do or Load methods. +func (s *Script) SendHash(c Conn, keysAndArgs ...interface{}) error { + return c.Send("EVALSHA", s.args(s.hash, keysAndArgs)...) +} + +// Send evaluates the script without waiting for the reply. +func (s *Script) Send(c Conn, keysAndArgs ...interface{}) error { + return c.Send("EVAL", s.args(s.src, keysAndArgs)...) +} + +// Load loads the script without evaluating it. +func (s *Script) Load(c Conn) error { + _, err := c.Do("SCRIPT", "LOAD", s.src) + return err +} diff --git a/vendor/github.com/gorilla/handlers/.editorconfig b/vendor/github.com/gorilla/handlers/.editorconfig new file mode 100644 index 000000000..c6b74c3e0 --- /dev/null +++ b/vendor/github.com/gorilla/handlers/.editorconfig @@ -0,0 +1,20 @@ +; https://editorconfig.org/ + +root = true + +[*] +insert_final_newline = true +charset = utf-8 +trim_trailing_whitespace = true +indent_style = space +indent_size = 2 + +[{Makefile,go.mod,go.sum,*.go,.gitmodules}] +indent_style = tab +indent_size = 4 + +[*.md] +indent_size = 4 +trim_trailing_whitespace = false + +eclint_indent_style = unset \ No newline at end of file diff --git a/vendor/github.com/gorilla/handlers/.gitignore b/vendor/github.com/gorilla/handlers/.gitignore new file mode 100644 index 000000000..577a89e81 --- /dev/null +++ b/vendor/github.com/gorilla/handlers/.gitignore @@ -0,0 +1,2 @@ +# Output of the go test coverage tool +coverage.coverprofile diff --git a/vendor/github.com/gorilla/handlers/LICENSE b/vendor/github.com/gorilla/handlers/LICENSE new file mode 100644 index 000000000..bb9d80bc9 --- /dev/null +++ b/vendor/github.com/gorilla/handlers/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2023 The Gorilla Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/gorilla/handlers/Makefile b/vendor/github.com/gorilla/handlers/Makefile new file mode 100644 index 000000000..003b784f7 --- /dev/null +++ b/vendor/github.com/gorilla/handlers/Makefile @@ -0,0 +1,34 @@ +GO_LINT=$(shell which golangci-lint 2> /dev/null || echo '') +GO_LINT_URI=github.com/golangci/golangci-lint/cmd/golangci-lint@latest + +GO_SEC=$(shell which gosec 2> /dev/null || echo '') +GO_SEC_URI=github.com/securego/gosec/v2/cmd/gosec@latest + +GO_VULNCHECK=$(shell which govulncheck 2> /dev/null || echo '') +GO_VULNCHECK_URI=golang.org/x/vuln/cmd/govulncheck@latest + +.PHONY: verify +verify: sec govulncheck lint test + +.PHONY: lint +lint: + $(if $(GO_LINT), ,go install $(GO_LINT_URI)) + @echo "##### Running golangci-lint #####" + golangci-lint run -v + +.PHONY: sec +sec: + $(if $(GO_SEC), ,go install $(GO_SEC_URI)) + @echo "##### Running gosec #####" + gosec ./... + +.PHONY: govulncheck +govulncheck: + $(if $(GO_VULNCHECK), ,go install $(GO_VULNCHECK_URI)) + @echo "##### Running govulncheck #####" + govulncheck ./... + +.PHONY: test +test: + @echo "##### Running tests #####" + go test -race -cover -coverprofile=coverage.coverprofile -covermode=atomic -v ./... diff --git a/vendor/github.com/gorilla/handlers/README.md b/vendor/github.com/gorilla/handlers/README.md new file mode 100644 index 000000000..02555b264 --- /dev/null +++ b/vendor/github.com/gorilla/handlers/README.md @@ -0,0 +1,56 @@ +# gorilla/handlers + +![Testing](https://github.com/gorilla/handlers/actions/workflows/test.yml/badge.svg) +[![Codecov](https://codecov.io/github/gorilla/handlers/branch/main/graph/badge.svg)](https://codecov.io/github/gorilla/handlers) +[![GoDoc](https://godoc.org/github.com/gorilla/handlers?status.svg)](https://godoc.org/github.com/gorilla/handlers) +[![Sourcegraph](https://sourcegraph.com/github.com/gorilla/handlers/-/badge.svg)](https://sourcegraph.com/github.com/gorilla/handlers?badge) + +Package handlers is a collection of handlers (aka "HTTP middleware") for use +with Go's `net/http` package (or any framework supporting `http.Handler`), including: + +* [**LoggingHandler**](https://godoc.org/github.com/gorilla/handlers#LoggingHandler) for logging HTTP requests in the Apache [Common Log + Format](http://httpd.apache.org/docs/2.2/logs.html#common). +* [**CombinedLoggingHandler**](https://godoc.org/github.com/gorilla/handlers#CombinedLoggingHandler) for logging HTTP requests in the Apache [Combined Log + Format](http://httpd.apache.org/docs/2.2/logs.html#combined) commonly used by + both Apache and nginx. +* [**CompressHandler**](https://godoc.org/github.com/gorilla/handlers#CompressHandler) for gzipping responses. +* [**ContentTypeHandler**](https://godoc.org/github.com/gorilla/handlers#ContentTypeHandler) for validating requests against a list of accepted + content types. +* [**MethodHandler**](https://godoc.org/github.com/gorilla/handlers#MethodHandler) for matching HTTP methods against handlers in a + `map[string]http.Handler` +* [**ProxyHeaders**](https://godoc.org/github.com/gorilla/handlers#ProxyHeaders) for populating `r.RemoteAddr` and `r.URL.Scheme` based on the + `X-Forwarded-For`, `X-Real-IP`, `X-Forwarded-Proto` and RFC7239 `Forwarded` + headers when running a Go server behind a HTTP reverse proxy. +* [**CanonicalHost**](https://godoc.org/github.com/gorilla/handlers#CanonicalHost) for re-directing to the preferred host when handling multiple + domains (i.e. multiple CNAME aliases). +* [**RecoveryHandler**](https://godoc.org/github.com/gorilla/handlers#RecoveryHandler) for recovering from unexpected panics. + +Other handlers are documented [on the Gorilla +website](https://www.gorillatoolkit.org/pkg/handlers). + +## Example + +A simple example using `handlers.LoggingHandler` and `handlers.CompressHandler`: + +```go +import ( + "net/http" + "github.com/gorilla/handlers" +) + +func main() { + r := http.NewServeMux() + + // Only log requests to our admin dashboard to stdout + r.Handle("/admin", handlers.LoggingHandler(os.Stdout, http.HandlerFunc(ShowAdminDashboard))) + r.HandleFunc("/", ShowIndex) + + // Wrap our server with our gzip handler to gzip compress all responses. + http.ListenAndServe(":8000", handlers.CompressHandler(r)) +} +``` + +## License + +BSD licensed. See the included LICENSE file for details. + diff --git a/vendor/github.com/gorilla/handlers/canonical.go b/vendor/github.com/gorilla/handlers/canonical.go new file mode 100644 index 000000000..7121f5307 --- /dev/null +++ b/vendor/github.com/gorilla/handlers/canonical.go @@ -0,0 +1,73 @@ +package handlers + +import ( + "net/http" + "net/url" + "strings" +) + +type canonical struct { + h http.Handler + domain string + code int +} + +// CanonicalHost is HTTP middleware that re-directs requests to the canonical +// domain. It accepts a domain and a status code (e.g. 301 or 302) and +// re-directs clients to this domain. The existing request path is maintained. +// +// Note: If the provided domain is considered invalid by url.Parse or otherwise +// returns an empty scheme or host, clients are not re-directed. +// +// Example: +// +// r := mux.NewRouter() +// canonical := handlers.CanonicalHost("http://www.gorillatoolkit.org", 302) +// r.HandleFunc("/route", YourHandler) +// +// log.Fatal(http.ListenAndServe(":7000", canonical(r))) +func CanonicalHost(domain string, code int) func(h http.Handler) http.Handler { + fn := func(h http.Handler) http.Handler { + return canonical{h, domain, code} + } + + return fn +} + +func (c canonical) ServeHTTP(w http.ResponseWriter, r *http.Request) { + dest, err := url.Parse(c.domain) + if err != nil { + // Call the next handler if the provided domain fails to parse. + c.h.ServeHTTP(w, r) + return + } + + if dest.Scheme == "" || dest.Host == "" { + // Call the next handler if the scheme or host are empty. + // Note that url.Parse won't fail on in this case. + c.h.ServeHTTP(w, r) + return + } + + if !strings.EqualFold(cleanHost(r.Host), dest.Host) { + // Re-build the destination URL + dest := dest.Scheme + "://" + dest.Host + r.URL.Path + if r.URL.RawQuery != "" { + dest += "?" + r.URL.RawQuery + } + http.Redirect(w, r, dest, c.code) + return + } + + c.h.ServeHTTP(w, r) +} + +// cleanHost cleans invalid Host headers by stripping anything after '/' or ' '. +// This is backported from Go 1.5 (in response to issue #11206) and attempts to +// mitigate malformed Host headers that do not match the format in RFC7230. +func cleanHost(in string) string { + if i := strings.IndexAny(in, " /"); i != -1 { + return in[:i] + } + return in +} diff --git a/vendor/github.com/gorilla/handlers/compress.go b/vendor/github.com/gorilla/handlers/compress.go new file mode 100644 index 000000000..d6f589503 --- /dev/null +++ b/vendor/github.com/gorilla/handlers/compress.go @@ -0,0 +1,143 @@ +// Copyright 2013 The Gorilla Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package handlers + +import ( + "compress/flate" + "compress/gzip" + "io" + "net/http" + "strings" + + "github.com/felixge/httpsnoop" +) + +const acceptEncoding string = "Accept-Encoding" + +type compressResponseWriter struct { + compressor io.Writer + w http.ResponseWriter +} + +func (cw *compressResponseWriter) WriteHeader(c int) { + cw.w.Header().Del("Content-Length") + cw.w.WriteHeader(c) +} + +func (cw *compressResponseWriter) Write(b []byte) (int, error) { + h := cw.w.Header() + if h.Get("Content-Type") == "" { + h.Set("Content-Type", http.DetectContentType(b)) + } + h.Del("Content-Length") + + return cw.compressor.Write(b) +} + +func (cw *compressResponseWriter) ReadFrom(r io.Reader) (int64, error) { + return io.Copy(cw.compressor, r) +} + +type flusher interface { + Flush() error +} + +func (cw *compressResponseWriter) Flush() { + // Flush compressed data if compressor supports it. + if f, ok := cw.compressor.(flusher); ok { + _ = f.Flush() + } + // Flush HTTP response. + if f, ok := cw.w.(http.Flusher); ok { + f.Flush() + } +} + +// CompressHandler gzip compresses HTTP responses for clients that support it +// via the 'Accept-Encoding' header. +// +// Compressing TLS traffic may leak the page contents to an attacker if the +// page contains user input: http://security.stackexchange.com/a/102015/12208 +func CompressHandler(h http.Handler) http.Handler { + return CompressHandlerLevel(h, gzip.DefaultCompression) +} + +// CompressHandlerLevel gzip compresses HTTP responses with specified compression level +// for clients that support it via the 'Accept-Encoding' header. +// +// The compression level should be gzip.DefaultCompression, gzip.NoCompression, +// or any integer value between gzip.BestSpeed and gzip.BestCompression inclusive. +// gzip.DefaultCompression is used in case of invalid compression level. +func CompressHandlerLevel(h http.Handler, level int) http.Handler { + if level < gzip.DefaultCompression || level > gzip.BestCompression { + level = gzip.DefaultCompression + } + + const ( + gzipEncoding = "gzip" + flateEncoding = "deflate" + ) + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // detect what encoding to use + var encoding string + for _, curEnc := range strings.Split(r.Header.Get(acceptEncoding), ",") { + curEnc = strings.TrimSpace(curEnc) + if curEnc == gzipEncoding || curEnc == flateEncoding { + encoding = curEnc + break + } + } + + // always add Accept-Encoding to Vary to prevent intermediate caches corruption + w.Header().Add("Vary", acceptEncoding) + + // if we weren't able to identify an encoding we're familiar with, pass on the + // request to the handler and return + if encoding == "" { + h.ServeHTTP(w, r) + return + } + + if r.Header.Get("Upgrade") != "" { + h.ServeHTTP(w, r) + return + } + + // wrap the ResponseWriter with the writer for the chosen encoding + var encWriter io.WriteCloser + if encoding == gzipEncoding { + encWriter, _ = gzip.NewWriterLevel(w, level) + } else if encoding == flateEncoding { + encWriter, _ = flate.NewWriter(w, level) + } + defer encWriter.Close() + + w.Header().Set("Content-Encoding", encoding) + r.Header.Del(acceptEncoding) + + cw := &compressResponseWriter{ + w: w, + compressor: encWriter, + } + + w = httpsnoop.Wrap(w, httpsnoop.Hooks{ + Write: func(httpsnoop.WriteFunc) httpsnoop.WriteFunc { + return cw.Write + }, + WriteHeader: func(httpsnoop.WriteHeaderFunc) httpsnoop.WriteHeaderFunc { + return cw.WriteHeader + }, + Flush: func(httpsnoop.FlushFunc) httpsnoop.FlushFunc { + return cw.Flush + }, + ReadFrom: func(rff httpsnoop.ReadFromFunc) httpsnoop.ReadFromFunc { + return cw.ReadFrom + }, + }) + + h.ServeHTTP(w, r) + }) +} diff --git a/vendor/github.com/gorilla/handlers/cors.go b/vendor/github.com/gorilla/handlers/cors.go new file mode 100644 index 000000000..8af9c096e --- /dev/null +++ b/vendor/github.com/gorilla/handlers/cors.go @@ -0,0 +1,352 @@ +package handlers + +import ( + "net/http" + "strconv" + "strings" +) + +// CORSOption represents a functional option for configuring the CORS middleware. +type CORSOption func(*cors) error + +type cors struct { + h http.Handler + allowedHeaders []string + allowedMethods []string + allowedOrigins []string + allowedOriginValidator OriginValidator + exposedHeaders []string + maxAge int + ignoreOptions bool + allowCredentials bool + optionStatusCode int +} + +// OriginValidator takes an origin string and returns whether or not that origin is allowed. +type OriginValidator func(string) bool + +var ( + defaultCorsOptionStatusCode = http.StatusOK + defaultCorsMethods = []string{http.MethodGet, http.MethodHead, http.MethodPost} + defaultCorsHeaders = []string{"Accept", "Accept-Language", "Content-Language", "Origin"} + // (WebKit/Safari v9 sends the Origin header by default in AJAX requests). +) + +const ( + corsOptionMethod string = http.MethodOptions + corsAllowOriginHeader string = "Access-Control-Allow-Origin" + corsExposeHeadersHeader string = "Access-Control-Expose-Headers" + corsMaxAgeHeader string = "Access-Control-Max-Age" + corsAllowMethodsHeader string = "Access-Control-Allow-Methods" + corsAllowHeadersHeader string = "Access-Control-Allow-Headers" + corsAllowCredentialsHeader string = "Access-Control-Allow-Credentials" + corsRequestMethodHeader string = "Access-Control-Request-Method" + corsRequestHeadersHeader string = "Access-Control-Request-Headers" + corsOriginHeader string = "Origin" + corsVaryHeader string = "Vary" + corsOriginMatchAll string = "*" +) + +func (ch *cors) ServeHTTP(w http.ResponseWriter, r *http.Request) { + origin := r.Header.Get(corsOriginHeader) + if !ch.isOriginAllowed(origin) { + if r.Method != corsOptionMethod || ch.ignoreOptions { + ch.h.ServeHTTP(w, r) + } + + return + } + + if r.Method == corsOptionMethod { + if ch.ignoreOptions { + ch.h.ServeHTTP(w, r) + return + } + + if _, ok := r.Header[corsRequestMethodHeader]; !ok { + w.WriteHeader(http.StatusBadRequest) + return + } + + method := r.Header.Get(corsRequestMethodHeader) + if !ch.isMatch(method, ch.allowedMethods) { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + + requestHeaders := strings.Split(r.Header.Get(corsRequestHeadersHeader), ",") + allowedHeaders := []string{} + for _, v := range requestHeaders { + canonicalHeader := http.CanonicalHeaderKey(strings.TrimSpace(v)) + if canonicalHeader == "" || ch.isMatch(canonicalHeader, defaultCorsHeaders) { + continue + } + + if !ch.isMatch(canonicalHeader, ch.allowedHeaders) { + w.WriteHeader(http.StatusForbidden) + return + } + + allowedHeaders = append(allowedHeaders, canonicalHeader) + } + + if len(allowedHeaders) > 0 { + w.Header().Set(corsAllowHeadersHeader, strings.Join(allowedHeaders, ",")) + } + + if ch.maxAge > 0 { + w.Header().Set(corsMaxAgeHeader, strconv.Itoa(ch.maxAge)) + } + + if !ch.isMatch(method, defaultCorsMethods) { + w.Header().Set(corsAllowMethodsHeader, method) + } + } else if len(ch.exposedHeaders) > 0 { + w.Header().Set(corsExposeHeadersHeader, strings.Join(ch.exposedHeaders, ",")) + } + + if ch.allowCredentials { + w.Header().Set(corsAllowCredentialsHeader, "true") + } + + if len(ch.allowedOrigins) > 1 { + w.Header().Set(corsVaryHeader, corsOriginHeader) + } + + returnOrigin := origin + if ch.allowedOriginValidator == nil && len(ch.allowedOrigins) == 0 { + returnOrigin = "*" + } else { + for _, o := range ch.allowedOrigins { + // A configuration of * is different than explicitly setting an allowed + // origin. Returning arbitrary origin headers in an access control allow + // origin header is unsafe and is not required by any use case. + if o == corsOriginMatchAll { + returnOrigin = "*" + break + } + } + } + w.Header().Set(corsAllowOriginHeader, returnOrigin) + + if r.Method == corsOptionMethod { + w.WriteHeader(ch.optionStatusCode) + return + } + ch.h.ServeHTTP(w, r) +} + +// CORS provides Cross-Origin Resource Sharing middleware. +// Example: +// +// import ( +// "net/http" +// +// "github.com/gorilla/handlers" +// "github.com/gorilla/mux" +// ) +// +// func main() { +// r := mux.NewRouter() +// r.HandleFunc("/users", UserEndpoint) +// r.HandleFunc("/projects", ProjectEndpoint) +// +// // Apply the CORS middleware to our top-level router, with the defaults. +// http.ListenAndServe(":8000", handlers.CORS()(r)) +// } +func CORS(opts ...CORSOption) func(http.Handler) http.Handler { + return func(h http.Handler) http.Handler { + ch := parseCORSOptions(opts...) + ch.h = h + return ch + } +} + +func parseCORSOptions(opts ...CORSOption) *cors { + ch := &cors{ + allowedMethods: defaultCorsMethods, + allowedHeaders: defaultCorsHeaders, + allowedOrigins: []string{}, + optionStatusCode: defaultCorsOptionStatusCode, + } + + for _, option := range opts { + _ = option(ch) //TODO: @bharat-rajani, return error to caller if not nil? + } + + return ch +} + +// +// Functional options for configuring CORS. +// + +// AllowedHeaders adds the provided headers to the list of allowed headers in a +// CORS request. +// This is an append operation so the headers Accept, Accept-Language, +// and Content-Language are always allowed. +// Content-Type must be explicitly declared if accepting Content-Types other than +// application/x-www-form-urlencoded, multipart/form-data, or text/plain. +func AllowedHeaders(headers []string) CORSOption { + return func(ch *cors) error { + for _, v := range headers { + normalizedHeader := http.CanonicalHeaderKey(strings.TrimSpace(v)) + if normalizedHeader == "" { + continue + } + + if !ch.isMatch(normalizedHeader, ch.allowedHeaders) { + ch.allowedHeaders = append(ch.allowedHeaders, normalizedHeader) + } + } + + return nil + } +} + +// AllowedMethods can be used to explicitly allow methods in the +// Access-Control-Allow-Methods header. +// This is a replacement operation so you must also +// pass GET, HEAD, and POST if you wish to support those methods. +func AllowedMethods(methods []string) CORSOption { + return func(ch *cors) error { + ch.allowedMethods = []string{} + for _, v := range methods { + normalizedMethod := strings.ToUpper(strings.TrimSpace(v)) + if normalizedMethod == "" { + continue + } + + if !ch.isMatch(normalizedMethod, ch.allowedMethods) { + ch.allowedMethods = append(ch.allowedMethods, normalizedMethod) + } + } + + return nil + } +} + +// AllowedOrigins sets the allowed origins for CORS requests, as used in the +// 'Allow-Access-Control-Origin' HTTP header. +// Note: Passing in a []string{"*"} will allow any domain. +func AllowedOrigins(origins []string) CORSOption { + return func(ch *cors) error { + for _, v := range origins { + if v == corsOriginMatchAll { + ch.allowedOrigins = []string{corsOriginMatchAll} + return nil + } + } + + ch.allowedOrigins = origins + return nil + } +} + +// AllowedOriginValidator sets a function for evaluating allowed origins in CORS requests, represented by the +// 'Allow-Access-Control-Origin' HTTP header. +func AllowedOriginValidator(fn OriginValidator) CORSOption { + return func(ch *cors) error { + ch.allowedOriginValidator = fn + return nil + } +} + +// OptionStatusCode sets a custom status code on the OPTIONS requests. +// Default behaviour sets it to 200 to reflect best practices. This is option is not mandatory +// and can be used if you need a custom status code (i.e 204). +// +// More informations on the spec: +// https://fetch.spec.whatwg.org/#cors-preflight-fetch +func OptionStatusCode(code int) CORSOption { + return func(ch *cors) error { + ch.optionStatusCode = code + return nil + } +} + +// ExposedHeaders can be used to specify headers that are available +// and will not be stripped out by the user-agent. +func ExposedHeaders(headers []string) CORSOption { + return func(ch *cors) error { + ch.exposedHeaders = []string{} + for _, v := range headers { + normalizedHeader := http.CanonicalHeaderKey(strings.TrimSpace(v)) + if normalizedHeader == "" { + continue + } + + if !ch.isMatch(normalizedHeader, ch.exposedHeaders) { + ch.exposedHeaders = append(ch.exposedHeaders, normalizedHeader) + } + } + + return nil + } +} + +// MaxAge determines the maximum age (in seconds) between preflight requests. A +// maximum of 10 minutes is allowed. An age above this value will default to 10 +// minutes. +func MaxAge(age int) CORSOption { + return func(ch *cors) error { + // Maximum of 10 minutes. + if age > 600 { + age = 600 + } + + ch.maxAge = age + return nil + } +} + +// IgnoreOptions causes the CORS middleware to ignore OPTIONS requests, instead +// passing them through to the next handler. This is useful when your application +// or framework has a pre-existing mechanism for responding to OPTIONS requests. +func IgnoreOptions() CORSOption { + return func(ch *cors) error { + ch.ignoreOptions = true + return nil + } +} + +// AllowCredentials can be used to specify that the user agent may pass +// authentication details along with the request. +func AllowCredentials() CORSOption { + return func(ch *cors) error { + ch.allowCredentials = true + return nil + } +} + +func (ch *cors) isOriginAllowed(origin string) bool { + if origin == "" { + return false + } + + if ch.allowedOriginValidator != nil { + return ch.allowedOriginValidator(origin) + } + + if len(ch.allowedOrigins) == 0 { + return true + } + + for _, allowedOrigin := range ch.allowedOrigins { + if allowedOrigin == origin || allowedOrigin == corsOriginMatchAll { + return true + } + } + + return false +} + +func (ch *cors) isMatch(needle string, haystack []string) bool { + for _, v := range haystack { + if v == needle { + return true + } + } + + return false +} diff --git a/vendor/github.com/gorilla/handlers/doc.go b/vendor/github.com/gorilla/handlers/doc.go new file mode 100644 index 000000000..944e5a8ae --- /dev/null +++ b/vendor/github.com/gorilla/handlers/doc.go @@ -0,0 +1,9 @@ +/* +Package handlers is a collection of handlers (aka "HTTP middleware") for use +with Go's net/http package (or any framework supporting http.Handler). + +The package includes handlers for logging in standardised formats, compressing +HTTP responses, validating content types and other useful tools for manipulating +requests and responses. +*/ +package handlers diff --git a/vendor/github.com/gorilla/handlers/handlers.go b/vendor/github.com/gorilla/handlers/handlers.go new file mode 100644 index 000000000..9b92fce33 --- /dev/null +++ b/vendor/github.com/gorilla/handlers/handlers.go @@ -0,0 +1,150 @@ +// Copyright 2013 The Gorilla Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package handlers + +import ( + "bufio" + "fmt" + "net" + "net/http" + "sort" + "strings" +) + +// MethodHandler is an http.Handler that dispatches to a handler whose key in the +// MethodHandler's map matches the name of the HTTP request's method, eg: GET +// +// If the request's method is OPTIONS and OPTIONS is not a key in the map then +// the handler responds with a status of 200 and sets the Allow header to a +// comma-separated list of available methods. +// +// If the request's method doesn't match any of its keys the handler responds +// with a status of HTTP 405 "Method Not Allowed" and sets the Allow header to a +// comma-separated list of available methods. +type MethodHandler map[string]http.Handler + +func (h MethodHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + if handler, ok := h[req.Method]; ok { + handler.ServeHTTP(w, req) + } else { + allow := []string{} + for k := range h { + allow = append(allow, k) + } + sort.Strings(allow) + w.Header().Set("Allow", strings.Join(allow, ", ")) + if req.Method == http.MethodOptions { + w.WriteHeader(http.StatusOK) + } else { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + } + } +} + +// responseLogger is wrapper of http.ResponseWriter that keeps track of its HTTP +// status code and body size. +type responseLogger struct { + w http.ResponseWriter + status int + size int +} + +func (l *responseLogger) Write(b []byte) (int, error) { + size, err := l.w.Write(b) + l.size += size + return size, err +} + +func (l *responseLogger) WriteHeader(s int) { + l.w.WriteHeader(s) + l.status = s +} + +func (l *responseLogger) Status() int { + return l.status +} + +func (l *responseLogger) Size() int { + return l.size +} + +func (l *responseLogger) Hijack() (net.Conn, *bufio.ReadWriter, error) { + conn, rw, err := l.w.(http.Hijacker).Hijack() + if err == nil && l.status == 0 { + // The status will be StatusSwitchingProtocols if there was no error and + // WriteHeader has not been called yet + l.status = http.StatusSwitchingProtocols + } + return conn, rw, err +} + +// isContentType validates the Content-Type header matches the supplied +// contentType. That is, its type and subtype match. +func isContentType(h http.Header, contentType string) bool { + ct := h.Get("Content-Type") + if i := strings.IndexRune(ct, ';'); i != -1 { + ct = ct[0:i] + } + return ct == contentType +} + +// ContentTypeHandler wraps and returns a http.Handler, validating the request +// content type is compatible with the contentTypes list. It writes a HTTP 415 +// error if that fails. +// +// Only PUT, POST, and PATCH requests are considered. +func ContentTypeHandler(h http.Handler, contentTypes ...string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !(r.Method == http.MethodPut || r.Method == http.MethodPost || r.Method == http.MethodPatch) { + h.ServeHTTP(w, r) + return + } + + for _, ct := range contentTypes { + if isContentType(r.Header, ct) { + h.ServeHTTP(w, r) + return + } + } + http.Error(w, fmt.Sprintf("Unsupported content type %q; expected one of %q", + r.Header.Get("Content-Type"), + contentTypes), + http.StatusUnsupportedMediaType) + }) +} + +const ( + // HTTPMethodOverrideHeader is a commonly used + // http header to override a request method. + HTTPMethodOverrideHeader = "X-HTTP-Method-Override" + // HTTPMethodOverrideFormKey is a commonly used + // HTML form key to override a request method. + HTTPMethodOverrideFormKey = "_method" +) + +// HTTPMethodOverrideHandler wraps and returns a http.Handler which checks for +// the X-HTTP-Method-Override header or the _method form key, and overrides (if +// valid) request.Method with its value. +// +// This is especially useful for HTTP clients that don't support many http verbs. +// It isn't secure to override e.g a GET to a POST, so only POST requests are +// considered. Likewise, the override method can only be a "write" method: PUT, +// PATCH or DELETE. +// +// Form method takes precedence over header method. +func HTTPMethodOverrideHandler(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + om := r.FormValue(HTTPMethodOverrideFormKey) + if om == "" { + om = r.Header.Get(HTTPMethodOverrideHeader) + } + if om == http.MethodPut || om == http.MethodPatch || om == http.MethodDelete { + r.Method = om + } + } + h.ServeHTTP(w, r) + }) +} diff --git a/vendor/github.com/gorilla/handlers/logging.go b/vendor/github.com/gorilla/handlers/logging.go new file mode 100644 index 000000000..2badb6fbf --- /dev/null +++ b/vendor/github.com/gorilla/handlers/logging.go @@ -0,0 +1,246 @@ +// Copyright 2013 The Gorilla Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package handlers + +import ( + "io" + "net" + "net/http" + "net/url" + "strconv" + "time" + "unicode/utf8" + + "github.com/felixge/httpsnoop" +) + +// Logging + +// LogFormatterParams is the structure any formatter will be handed when time to log comes. +type LogFormatterParams struct { + Request *http.Request + URL url.URL + TimeStamp time.Time + StatusCode int + Size int +} + +// LogFormatter gives the signature of the formatter function passed to CustomLoggingHandler. +type LogFormatter func(writer io.Writer, params LogFormatterParams) + +// loggingHandler is the http.Handler implementation for LoggingHandlerTo and its +// friends + +type loggingHandler struct { + writer io.Writer + handler http.Handler + formatter LogFormatter +} + +func (h loggingHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + t := time.Now() + logger, w := makeLogger(w) + url := *req.URL + + h.handler.ServeHTTP(w, req) + if req.MultipartForm != nil { + err := req.MultipartForm.RemoveAll() + if err != nil { + return + } + } + + params := LogFormatterParams{ + Request: req, + URL: url, + TimeStamp: t, + StatusCode: logger.Status(), + Size: logger.Size(), + } + + h.formatter(h.writer, params) +} + +func makeLogger(w http.ResponseWriter) (*responseLogger, http.ResponseWriter) { + logger := &responseLogger{w: w, status: http.StatusOK} + return logger, httpsnoop.Wrap(w, httpsnoop.Hooks{ + Write: func(httpsnoop.WriteFunc) httpsnoop.WriteFunc { + return logger.Write + }, + WriteHeader: func(httpsnoop.WriteHeaderFunc) httpsnoop.WriteHeaderFunc { + return logger.WriteHeader + }, + }) +} + +const lowerhex = "0123456789abcdef" + +func appendQuoted(buf []byte, s string) []byte { + var runeTmp [utf8.UTFMax]byte + for width := 0; len(s) > 0; s = s[width:] { //nolint: wastedassign //TODO: why width starts from 0and reassigned as 1 + r := rune(s[0]) + width = 1 + if r >= utf8.RuneSelf { + r, width = utf8.DecodeRuneInString(s) + } + if width == 1 && r == utf8.RuneError { + buf = append(buf, `\x`...) + buf = append(buf, lowerhex[s[0]>>4]) + buf = append(buf, lowerhex[s[0]&0xF]) + continue + } + if r == rune('"') || r == '\\' { // always backslashed + buf = append(buf, '\\') + buf = append(buf, byte(r)) + continue + } + if strconv.IsPrint(r) { + n := utf8.EncodeRune(runeTmp[:], r) + buf = append(buf, runeTmp[:n]...) + continue + } + switch r { + case '\a': + buf = append(buf, `\a`...) + case '\b': + buf = append(buf, `\b`...) + case '\f': + buf = append(buf, `\f`...) + case '\n': + buf = append(buf, `\n`...) + case '\r': + buf = append(buf, `\r`...) + case '\t': + buf = append(buf, `\t`...) + case '\v': + buf = append(buf, `\v`...) + default: + switch { + case r < ' ': + buf = append(buf, `\x`...) + buf = append(buf, lowerhex[s[0]>>4]) + buf = append(buf, lowerhex[s[0]&0xF]) + case r > utf8.MaxRune: + r = 0xFFFD + fallthrough + case r < 0x10000: + buf = append(buf, `\u`...) + for s := 12; s >= 0; s -= 4 { + buf = append(buf, lowerhex[r>>uint(s)&0xF]) + } + default: + buf = append(buf, `\U`...) + for s := 28; s >= 0; s -= 4 { + buf = append(buf, lowerhex[r>>uint(s)&0xF]) + } + } + } + } + return buf +} + +// buildCommonLogLine builds a log entry for req in Apache Common Log Format. +// ts is the timestamp with which the entry should be logged. +// status and size are used to provide the response HTTP status and size. +func buildCommonLogLine(req *http.Request, url url.URL, ts time.Time, status int, size int) []byte { + username := "-" + if url.User != nil { + if name := url.User.Username(); name != "" { + username = name + } + } + + host, _, err := net.SplitHostPort(req.RemoteAddr) + if err != nil { + host = req.RemoteAddr + } + + uri := req.RequestURI + + // Requests using the CONNECT method over HTTP/2.0 must use + // the authority field (aka r.Host) to identify the target. + // Refer: https://httpwg.github.io/specs/rfc7540.html#CONNECT + if req.ProtoMajor == 2 && req.Method == "CONNECT" { + uri = req.Host + } + if uri == "" { + uri = url.RequestURI() + } + + buf := make([]byte, 0, 3*(len(host)+len(username)+len(req.Method)+len(uri)+len(req.Proto)+50)/2) + buf = append(buf, host...) + buf = append(buf, " - "...) + buf = append(buf, username...) + buf = append(buf, " ["...) + buf = append(buf, ts.Format("02/Jan/2006:15:04:05 -0700")...) + buf = append(buf, `] "`...) + buf = append(buf, req.Method...) + buf = append(buf, " "...) + buf = appendQuoted(buf, uri) + buf = append(buf, " "...) + buf = append(buf, req.Proto...) + buf = append(buf, `" `...) + buf = append(buf, strconv.Itoa(status)...) + buf = append(buf, " "...) + buf = append(buf, strconv.Itoa(size)...) + return buf +} + +// writeLog writes a log entry for req to w in Apache Common Log Format. +// ts is the timestamp with which the entry should be logged. +// status and size are used to provide the response HTTP status and size. +func writeLog(writer io.Writer, params LogFormatterParams) { + buf := buildCommonLogLine(params.Request, params.URL, params.TimeStamp, params.StatusCode, params.Size) + buf = append(buf, '\n') + _, _ = writer.Write(buf) +} + +// writeCombinedLog writes a log entry for req to w in Apache Combined Log Format. +// ts is the timestamp with which the entry should be logged. +// status and size are used to provide the response HTTP status and size. +func writeCombinedLog(writer io.Writer, params LogFormatterParams) { + buf := buildCommonLogLine(params.Request, params.URL, params.TimeStamp, params.StatusCode, params.Size) + buf = append(buf, ` "`...) + buf = appendQuoted(buf, params.Request.Referer()) + buf = append(buf, `" "`...) + buf = appendQuoted(buf, params.Request.UserAgent()) + buf = append(buf, '"', '\n') + _, _ = writer.Write(buf) +} + +// CombinedLoggingHandler return a http.Handler that wraps h and logs requests to out in +// Apache Combined Log Format. +// +// See http://httpd.apache.org/docs/2.2/logs.html#combined for a description of this format. +// +// LoggingHandler always sets the ident field of the log to -. +func CombinedLoggingHandler(out io.Writer, h http.Handler) http.Handler { + return loggingHandler{out, h, writeCombinedLog} +} + +// LoggingHandler return a http.Handler that wraps h and logs requests to out in +// Apache Common Log Format (CLF). +// +// See http://httpd.apache.org/docs/2.2/logs.html#common for a description of this format. +// +// LoggingHandler always sets the ident field of the log to - +// +// Example: +// +// r := mux.NewRouter() +// r.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { +// w.Write([]byte("This is a catch-all route")) +// }) +// loggedRouter := handlers.LoggingHandler(os.Stdout, r) +// http.ListenAndServe(":1123", loggedRouter) +func LoggingHandler(out io.Writer, h http.Handler) http.Handler { + return loggingHandler{out, h, writeLog} +} + +// CustomLoggingHandler provides a way to supply a custom log formatter +// while taking advantage of the mechanisms in this package. +func CustomLoggingHandler(out io.Writer, h http.Handler, f LogFormatter) http.Handler { + return loggingHandler{out, h, f} +} diff --git a/vendor/github.com/gorilla/handlers/proxy_headers.go b/vendor/github.com/gorilla/handlers/proxy_headers.go new file mode 100644 index 000000000..281d753e9 --- /dev/null +++ b/vendor/github.com/gorilla/handlers/proxy_headers.go @@ -0,0 +1,120 @@ +package handlers + +import ( + "net/http" + "regexp" + "strings" +) + +var ( + // De-facto standard header keys. + xForwardedFor = http.CanonicalHeaderKey("X-Forwarded-For") + xForwardedHost = http.CanonicalHeaderKey("X-Forwarded-Host") + xForwardedProto = http.CanonicalHeaderKey("X-Forwarded-Proto") + xForwardedScheme = http.CanonicalHeaderKey("X-Forwarded-Scheme") + xRealIP = http.CanonicalHeaderKey("X-Real-IP") +) + +var ( + // RFC7239 defines a new "Forwarded: " header designed to replace the + // existing use of X-Forwarded-* headers. + // e.g. Forwarded: for=192.0.2.60;proto=https;by=203.0.113.43. + forwarded = http.CanonicalHeaderKey("Forwarded") + // Allows for a sub-match of the first value after 'for=' to the next + // comma, semi-colon or space. The match is case-insensitive. + forRegex = regexp.MustCompile(`(?i)(?:for=)([^(;|,| )]+)`) + // Allows for a sub-match for the first instance of scheme (http|https) + // prefixed by 'proto='. The match is case-insensitive. + protoRegex = regexp.MustCompile(`(?i)(?:proto=)(https|http)`) +) + +// ProxyHeaders inspects common reverse proxy headers and sets the corresponding +// fields in the HTTP request struct. These are X-Forwarded-For and X-Real-IP +// for the remote (client) IP address, X-Forwarded-Proto or X-Forwarded-Scheme +// for the scheme (http|https), X-Forwarded-Host for the host and the RFC7239 +// Forwarded header, which may include both client IPs and schemes. +// +// NOTE: This middleware should only be used when behind a reverse +// proxy like nginx, HAProxy or Apache. Reverse proxies that don't (or are +// configured not to) strip these headers from client requests, or where these +// headers are accepted "as is" from a remote client (e.g. when Go is not behind +// a proxy), can manifest as a vulnerability if your application uses these +// headers for validating the 'trustworthiness' of a request. +func ProxyHeaders(h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + // Set the remote IP with the value passed from the proxy. + if fwd := getIP(r); fwd != "" { + r.RemoteAddr = fwd + } + + // Set the scheme (proto) with the value passed from the proxy. + if scheme := getScheme(r); scheme != "" { + r.URL.Scheme = scheme + } + // Set the host with the value passed by the proxy + if r.Header.Get(xForwardedHost) != "" { + r.Host = r.Header.Get(xForwardedHost) + } + // Call the next handler in the chain. + h.ServeHTTP(w, r) + } + + return http.HandlerFunc(fn) +} + +// getIP retrieves the IP from the X-Forwarded-For, X-Real-IP and RFC7239 +// Forwarded headers (in that order). +func getIP(r *http.Request) string { + var addr string + + switch { + case r.Header.Get(xForwardedFor) != "": + fwd := r.Header.Get(xForwardedFor) + // Only grab the first (client) address. Note that '192.168.0.1, + // 10.1.1.1' is a valid key for X-Forwarded-For where addresses after + // the first may represent forwarding proxies earlier in the chain. + s := strings.Index(fwd, ", ") + if s == -1 { + s = len(fwd) + } + addr = fwd[:s] + case r.Header.Get(xRealIP) != "": + addr = r.Header.Get(xRealIP) + case r.Header.Get(forwarded) != "": + // match should contain at least two elements if the protocol was + // specified in the Forwarded header. The first element will always be + // the 'for=' capture, which we ignore. In the case of multiple IP + // addresses (for=8.8.8.8, 8.8.4.4,172.16.1.20 is valid) we only + // extract the first, which should be the client IP. + if match := forRegex.FindStringSubmatch(r.Header.Get(forwarded)); len(match) > 1 { + // IPv6 addresses in Forwarded headers are quoted-strings. We strip + // these quotes. + addr = strings.Trim(match[1], `"`) + } + } + + return addr +} + +// getScheme retrieves the scheme from the X-Forwarded-Proto and RFC7239 +// Forwarded headers (in that order). +func getScheme(r *http.Request) string { + var scheme string + + // Retrieve the scheme from X-Forwarded-Proto. + if proto := r.Header.Get(xForwardedProto); proto != "" { + scheme = strings.ToLower(proto) + } else if proto = r.Header.Get(xForwardedScheme); proto != "" { + scheme = strings.ToLower(proto) + } else if proto = r.Header.Get(forwarded); proto != "" { + // match should contain at least two elements if the protocol was + // specified in the Forwarded header. The first element will always be + // the 'proto=' capture, which we ignore. In the case of multiple proto + // parameters (invalid) we only extract the first. + if match := protoRegex.FindStringSubmatch(proto); len(match) > 1 { + scheme = strings.ToLower(match[1]) + } + } + + return scheme +} diff --git a/vendor/github.com/gorilla/handlers/recovery.go b/vendor/github.com/gorilla/handlers/recovery.go new file mode 100644 index 000000000..0d4f955ec --- /dev/null +++ b/vendor/github.com/gorilla/handlers/recovery.go @@ -0,0 +1,98 @@ +package handlers + +import ( + "log" + "net/http" + "runtime/debug" +) + +// RecoveryHandlerLogger is an interface used by the recovering handler to print logs. +type RecoveryHandlerLogger interface { + Println(...interface{}) +} + +type recoveryHandler struct { + handler http.Handler + logger RecoveryHandlerLogger + printStack bool +} + +// RecoveryOption provides a functional approach to define +// configuration for a handler; such as setting the logging +// whether or not to print stack traces on panic. +type RecoveryOption func(http.Handler) + +func parseRecoveryOptions(h http.Handler, opts ...RecoveryOption) http.Handler { + for _, option := range opts { + option(h) + } + + return h +} + +// RecoveryHandler is HTTP middleware that recovers from a panic, +// logs the panic, writes http.StatusInternalServerError, and +// continues to the next handler. +// +// Example: +// +// r := mux.NewRouter() +// r.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { +// panic("Unexpected error!") +// }) +// +// http.ListenAndServe(":1123", handlers.RecoveryHandler()(r)) +func RecoveryHandler(opts ...RecoveryOption) func(h http.Handler) http.Handler { + return func(h http.Handler) http.Handler { + r := &recoveryHandler{handler: h} + return parseRecoveryOptions(r, opts...) + } +} + +// RecoveryLogger is a functional option to override +// the default logger. +func RecoveryLogger(logger RecoveryHandlerLogger) RecoveryOption { + return func(h http.Handler) { + r := h.(*recoveryHandler) //nolint:errcheck //TODO: + // @bharat-rajani should return type-assertion error but would break the API? + r.logger = logger + } +} + +// PrintRecoveryStack is a functional option to enable +// or disable printing stack traces on panic. +func PrintRecoveryStack(shouldPrint bool) RecoveryOption { + return func(h http.Handler) { + r := h.(*recoveryHandler) //nolint:errcheck //TODO: + // @bharat-rajani should return type-assertion error but would break the API? + r.printStack = shouldPrint + } +} + +func (h recoveryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + defer func() { + if err := recover(); err != nil { + w.WriteHeader(http.StatusInternalServerError) + h.log(err) + } + }() + + h.handler.ServeHTTP(w, req) +} + +func (h recoveryHandler) log(v ...interface{}) { + if h.logger != nil { + h.logger.Println(v...) + } else { + log.Println(v...) + } + + if h.printStack { + stack := string(debug.Stack()) + if h.logger != nil { + h.logger.Println(stack) + } else { + log.Println(stack) + } + } +} diff --git a/vendor/github.com/gorilla/mux/.editorconfig b/vendor/github.com/gorilla/mux/.editorconfig new file mode 100644 index 000000000..c6b74c3e0 --- /dev/null +++ b/vendor/github.com/gorilla/mux/.editorconfig @@ -0,0 +1,20 @@ +; https://editorconfig.org/ + +root = true + +[*] +insert_final_newline = true +charset = utf-8 +trim_trailing_whitespace = true +indent_style = space +indent_size = 2 + +[{Makefile,go.mod,go.sum,*.go,.gitmodules}] +indent_style = tab +indent_size = 4 + +[*.md] +indent_size = 4 +trim_trailing_whitespace = false + +eclint_indent_style = unset \ No newline at end of file diff --git a/vendor/github.com/gorilla/mux/.gitignore b/vendor/github.com/gorilla/mux/.gitignore new file mode 100644 index 000000000..84039fec6 --- /dev/null +++ b/vendor/github.com/gorilla/mux/.gitignore @@ -0,0 +1 @@ +coverage.coverprofile diff --git a/vendor/github.com/gorilla/mux/LICENSE b/vendor/github.com/gorilla/mux/LICENSE new file mode 100644 index 000000000..bb9d80bc9 --- /dev/null +++ b/vendor/github.com/gorilla/mux/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2023 The Gorilla Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/gorilla/mux/Makefile b/vendor/github.com/gorilla/mux/Makefile new file mode 100644 index 000000000..98f5ab75f --- /dev/null +++ b/vendor/github.com/gorilla/mux/Makefile @@ -0,0 +1,34 @@ +GO_LINT=$(shell which golangci-lint 2> /dev/null || echo '') +GO_LINT_URI=github.com/golangci/golangci-lint/cmd/golangci-lint@latest + +GO_SEC=$(shell which gosec 2> /dev/null || echo '') +GO_SEC_URI=github.com/securego/gosec/v2/cmd/gosec@latest + +GO_VULNCHECK=$(shell which govulncheck 2> /dev/null || echo '') +GO_VULNCHECK_URI=golang.org/x/vuln/cmd/govulncheck@latest + +.PHONY: golangci-lint +golangci-lint: + $(if $(GO_LINT), ,go install $(GO_LINT_URI)) + @echo "##### Running golangci-lint" + golangci-lint run -v + +.PHONY: gosec +gosec: + $(if $(GO_SEC), ,go install $(GO_SEC_URI)) + @echo "##### Running gosec" + gosec ./... + +.PHONY: govulncheck +govulncheck: + $(if $(GO_VULNCHECK), ,go install $(GO_VULNCHECK_URI)) + @echo "##### Running govulncheck" + govulncheck ./... + +.PHONY: verify +verify: golangci-lint gosec govulncheck + +.PHONY: test +test: + @echo "##### Running tests" + go test -race -cover -coverprofile=coverage.coverprofile -covermode=atomic -v ./... \ No newline at end of file diff --git a/vendor/github.com/gorilla/mux/README.md b/vendor/github.com/gorilla/mux/README.md new file mode 100644 index 000000000..382513d57 --- /dev/null +++ b/vendor/github.com/gorilla/mux/README.md @@ -0,0 +1,812 @@ +# gorilla/mux + +![testing](https://github.com/gorilla/mux/actions/workflows/test.yml/badge.svg) +[![codecov](https://codecov.io/github/gorilla/mux/branch/main/graph/badge.svg)](https://codecov.io/github/gorilla/mux) +[![godoc](https://godoc.org/github.com/gorilla/mux?status.svg)](https://godoc.org/github.com/gorilla/mux) +[![sourcegraph](https://sourcegraph.com/github.com/gorilla/mux/-/badge.svg)](https://sourcegraph.com/github.com/gorilla/mux?badge) + + +![Gorilla Logo](https://github.com/gorilla/.github/assets/53367916/d92caabf-98e0-473e-bfbf-ab554ba435e5) + +Package `gorilla/mux` implements a request router and dispatcher for matching incoming requests to +their respective handler. + +The name mux stands for "HTTP request multiplexer". Like the standard `http.ServeMux`, `mux.Router` matches incoming requests against a list of registered routes and calls a handler for the route that matches the URL or other conditions. The main features are: + +* It implements the `http.Handler` interface so it is compatible with the standard `http.ServeMux`. +* Requests can be matched based on URL host, path, path prefix, schemes, header and query values, HTTP methods or using custom matchers. +* URL hosts, paths and query values can have variables with an optional regular expression. +* Registered URLs can be built, or "reversed", which helps maintaining references to resources. +* Routes can be used as subrouters: nested routes are only tested if the parent route matches. This is useful to define groups of routes that share common conditions like a host, a path prefix or other repeated attributes. As a bonus, this optimizes request matching. + +--- + +* [Install](#install) +* [Examples](#examples) +* [Matching Routes](#matching-routes) +* [Static Files](#static-files) +* [Serving Single Page Applications](#serving-single-page-applications) (e.g. React, Vue, Ember.js, etc.) +* [Registered URLs](#registered-urls) +* [Walking Routes](#walking-routes) +* [Graceful Shutdown](#graceful-shutdown) +* [Middleware](#middleware) +* [Handling CORS Requests](#handling-cors-requests) +* [Testing Handlers](#testing-handlers) +* [Full Example](#full-example) + +--- + +## Install + +With a [correctly configured](https://golang.org/doc/install#testing) Go toolchain: + +```sh +go get -u github.com/gorilla/mux +``` + +## Examples + +Let's start registering a couple of URL paths and handlers: + +```go +func main() { + r := mux.NewRouter() + r.HandleFunc("/", HomeHandler) + r.HandleFunc("/products", ProductsHandler) + r.HandleFunc("/articles", ArticlesHandler) + http.Handle("/", r) +} +``` + +Here we register three routes mapping URL paths to handlers. This is equivalent to how `http.HandleFunc()` works: if an incoming request URL matches one of the paths, the corresponding handler is called passing (`http.ResponseWriter`, `*http.Request`) as parameters. + +Paths can have variables. They are defined using the format `{name}` or `{name:pattern}`. If a regular expression pattern is not defined, the matched variable will be anything until the next slash. For example: + +```go +r := mux.NewRouter() +r.HandleFunc("/products/{key}", ProductHandler) +r.HandleFunc("/articles/{category}/", ArticlesCategoryHandler) +r.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler) +``` + +The names are used to create a map of route variables which can be retrieved calling `mux.Vars()`: + +```go +func ArticlesCategoryHandler(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + w.WriteHeader(http.StatusOK) + fmt.Fprintf(w, "Category: %v\n", vars["category"]) +} +``` + +And this is all you need to know about the basic usage. More advanced options are explained below. + +### Matching Routes + +Routes can also be restricted to a domain or subdomain. Just define a host pattern to be matched. They can also have variables: + +```go +r := mux.NewRouter() +// Only matches if domain is "www.example.com". +r.Host("www.example.com") +// Matches a dynamic subdomain. +r.Host("{subdomain:[a-z]+}.example.com") +``` + +There are several other matchers that can be added. To match path prefixes: + +```go +r.PathPrefix("/products/") +``` + +...or HTTP methods: + +```go +r.Methods("GET", "POST") +``` + +...or URL schemes: + +```go +r.Schemes("https") +``` + +...or header values: + +```go +r.Headers("X-Requested-With", "XMLHttpRequest") +``` + +...or query values: + +```go +r.Queries("key", "value") +``` + +...or to use a custom matcher function: + +```go +r.MatcherFunc(func(r *http.Request, rm *RouteMatch) bool { + return r.ProtoMajor == 0 +}) +``` + +...and finally, it is possible to combine several matchers in a single route: + +```go +r.HandleFunc("/products", ProductsHandler). + Host("www.example.com"). + Methods("GET"). + Schemes("http") +``` + +Routes are tested in the order they were added to the router. If two routes match, the first one wins: + +```go +r := mux.NewRouter() +r.HandleFunc("/specific", specificHandler) +r.PathPrefix("/").Handler(catchAllHandler) +``` + +Setting the same matching conditions again and again can be boring, so we have a way to group several routes that share the same requirements. We call it "subrouting". + +For example, let's say we have several URLs that should only match when the host is `www.example.com`. Create a route for that host and get a "subrouter" from it: + +```go +r := mux.NewRouter() +s := r.Host("www.example.com").Subrouter() +``` + +Then register routes in the subrouter: + +```go +s.HandleFunc("/products/", ProductsHandler) +s.HandleFunc("/products/{key}", ProductHandler) +s.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler) +``` + +The three URL paths we registered above will only be tested if the domain is `www.example.com`, because the subrouter is tested first. This is not only convenient, but also optimizes request matching. You can create subrouters combining any attribute matchers accepted by a route. + +Subrouters can be used to create domain or path "namespaces": you define subrouters in a central place and then parts of the app can register its paths relatively to a given subrouter. + +There's one more thing about subroutes. When a subrouter has a path prefix, the inner routes use it as base for their paths: + +```go +r := mux.NewRouter() +s := r.PathPrefix("/products").Subrouter() +// "/products/" +s.HandleFunc("/", ProductsHandler) +// "/products/{key}/" +s.HandleFunc("/{key}/", ProductHandler) +// "/products/{key}/details" +s.HandleFunc("/{key}/details", ProductDetailsHandler) +``` + + +### Static Files + +Note that the path provided to `PathPrefix()` represents a "wildcard": calling +`PathPrefix("/static/").Handler(...)` means that the handler will be passed any +request that matches "/static/\*". This makes it easy to serve static files with mux: + +```go +func main() { + var dir string + + flag.StringVar(&dir, "dir", ".", "the directory to serve files from. Defaults to the current dir") + flag.Parse() + r := mux.NewRouter() + + // This will serve files under http://localhost:8000/static/ + r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir(dir)))) + + srv := &http.Server{ + Handler: r, + Addr: "127.0.0.1:8000", + // Good practice: enforce timeouts for servers you create! + WriteTimeout: 15 * time.Second, + ReadTimeout: 15 * time.Second, + } + + log.Fatal(srv.ListenAndServe()) +} +``` + +### Serving Single Page Applications + +Most of the time it makes sense to serve your SPA on a separate web server from your API, +but sometimes it's desirable to serve them both from one place. It's possible to write a simple +handler for serving your SPA (for use with React Router's [BrowserRouter](https://reacttraining.com/react-router/web/api/BrowserRouter) for example), and leverage +mux's powerful routing for your API endpoints. + +```go +package main + +import ( + "encoding/json" + "log" + "net/http" + "os" + "path/filepath" + "time" + + "github.com/gorilla/mux" +) + +// spaHandler implements the http.Handler interface, so we can use it +// to respond to HTTP requests. The path to the static directory and +// path to the index file within that static directory are used to +// serve the SPA in the given static directory. +type spaHandler struct { + staticPath string + indexPath string +} + +// ServeHTTP inspects the URL path to locate a file within the static dir +// on the SPA handler. If a file is found, it will be served. If not, the +// file located at the index path on the SPA handler will be served. This +// is suitable behavior for serving an SPA (single page application). +func (h spaHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // Join internally call path.Clean to prevent directory traversal + path := filepath.Join(h.staticPath, r.URL.Path) + + // check whether a file exists or is a directory at the given path + fi, err := os.Stat(path) + if os.IsNotExist(err) || fi.IsDir() { + // file does not exist or path is a directory, serve index.html + http.ServeFile(w, r, filepath.Join(h.staticPath, h.indexPath)) + return + } + + if err != nil { + // if we got an error (that wasn't that the file doesn't exist) stating the + // file, return a 500 internal server error and stop + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // otherwise, use http.FileServer to serve the static file + http.FileServer(http.Dir(h.staticPath)).ServeHTTP(w, r) +} + +func main() { + router := mux.NewRouter() + + router.HandleFunc("/api/health", func(w http.ResponseWriter, r *http.Request) { + // an example API handler + json.NewEncoder(w).Encode(map[string]bool{"ok": true}) + }) + + spa := spaHandler{staticPath: "build", indexPath: "index.html"} + router.PathPrefix("/").Handler(spa) + + srv := &http.Server{ + Handler: router, + Addr: "127.0.0.1:8000", + // Good practice: enforce timeouts for servers you create! + WriteTimeout: 15 * time.Second, + ReadTimeout: 15 * time.Second, + } + + log.Fatal(srv.ListenAndServe()) +} +``` + +### Registered URLs + +Now let's see how to build registered URLs. + +Routes can be named. All routes that define a name can have their URLs built, or "reversed". We define a name calling `Name()` on a route. For example: + +```go +r := mux.NewRouter() +r.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler). + Name("article") +``` + +To build a URL, get the route and call the `URL()` method, passing a sequence of key/value pairs for the route variables. For the previous route, we would do: + +```go +url, err := r.Get("article").URL("category", "technology", "id", "42") +``` + +...and the result will be a `url.URL` with the following path: + +``` +"/articles/technology/42" +``` + +This also works for host and query value variables: + +```go +r := mux.NewRouter() +r.Host("{subdomain}.example.com"). + Path("/articles/{category}/{id:[0-9]+}"). + Queries("filter", "{filter}"). + HandlerFunc(ArticleHandler). + Name("article") + +// url.String() will be "http://news.example.com/articles/technology/42?filter=gorilla" +url, err := r.Get("article").URL("subdomain", "news", + "category", "technology", + "id", "42", + "filter", "gorilla") +``` + +All variables defined in the route are required, and their values must conform to the corresponding patterns. These requirements guarantee that a generated URL will always match a registered route -- the only exception is for explicitly defined "build-only" routes which never match. + +Regex support also exists for matching Headers within a route. For example, we could do: + +```go +r.HeadersRegexp("Content-Type", "application/(text|json)") +``` + +...and the route will match both requests with a Content-Type of `application/json` as well as `application/text` + +There's also a way to build only the URL host or path for a route: use the methods `URLHost()` or `URLPath()` instead. For the previous route, we would do: + +```go +// "http://news.example.com/" +host, err := r.Get("article").URLHost("subdomain", "news") + +// "/articles/technology/42" +path, err := r.Get("article").URLPath("category", "technology", "id", "42") +``` + +And if you use subrouters, host and path defined separately can be built as well: + +```go +r := mux.NewRouter() +s := r.Host("{subdomain}.example.com").Subrouter() +s.Path("/articles/{category}/{id:[0-9]+}"). + HandlerFunc(ArticleHandler). + Name("article") + +// "http://news.example.com/articles/technology/42" +url, err := r.Get("article").URL("subdomain", "news", + "category", "technology", + "id", "42") +``` + +To find all the required variables for a given route when calling `URL()`, the method `GetVarNames()` is available: +```go +r := mux.NewRouter() +r.Host("{domain}"). + Path("/{group}/{item_id}"). + Queries("some_data1", "{some_data1}"). + Queries("some_data2", "{some_data2}"). + Name("article") + +// Will print [domain group item_id some_data1 some_data2] +fmt.Println(r.Get("article").GetVarNames()) + +``` +### Walking Routes + +The `Walk` function on `mux.Router` can be used to visit all of the routes that are registered on a router. For example, +the following prints all of the registered routes: + +```go +package main + +import ( + "fmt" + "net/http" + "strings" + + "github.com/gorilla/mux" +) + +func handler(w http.ResponseWriter, r *http.Request) { + return +} + +func main() { + r := mux.NewRouter() + r.HandleFunc("/", handler) + r.HandleFunc("/products", handler).Methods("POST") + r.HandleFunc("/articles", handler).Methods("GET") + r.HandleFunc("/articles/{id}", handler).Methods("GET", "PUT") + r.HandleFunc("/authors", handler).Queries("surname", "{surname}") + err := r.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error { + pathTemplate, err := route.GetPathTemplate() + if err == nil { + fmt.Println("ROUTE:", pathTemplate) + } + pathRegexp, err := route.GetPathRegexp() + if err == nil { + fmt.Println("Path regexp:", pathRegexp) + } + queriesTemplates, err := route.GetQueriesTemplates() + if err == nil { + fmt.Println("Queries templates:", strings.Join(queriesTemplates, ",")) + } + queriesRegexps, err := route.GetQueriesRegexp() + if err == nil { + fmt.Println("Queries regexps:", strings.Join(queriesRegexps, ",")) + } + methods, err := route.GetMethods() + if err == nil { + fmt.Println("Methods:", strings.Join(methods, ",")) + } + fmt.Println() + return nil + }) + + if err != nil { + fmt.Println(err) + } + + http.Handle("/", r) +} +``` + +### Graceful Shutdown + +Go 1.8 introduced the ability to [gracefully shutdown](https://golang.org/doc/go1.8#http_shutdown) a `*http.Server`. Here's how to do that alongside `mux`: + +```go +package main + +import ( + "context" + "flag" + "log" + "net/http" + "os" + "os/signal" + "time" + + "github.com/gorilla/mux" +) + +func main() { + var wait time.Duration + flag.DurationVar(&wait, "graceful-timeout", time.Second * 15, "the duration for which the server gracefully wait for existing connections to finish - e.g. 15s or 1m") + flag.Parse() + + r := mux.NewRouter() + // Add your routes as needed + + srv := &http.Server{ + Addr: "0.0.0.0:8080", + // Good practice to set timeouts to avoid Slowloris attacks. + WriteTimeout: time.Second * 15, + ReadTimeout: time.Second * 15, + IdleTimeout: time.Second * 60, + Handler: r, // Pass our instance of gorilla/mux in. + } + + // Run our server in a goroutine so that it doesn't block. + go func() { + if err := srv.ListenAndServe(); err != nil { + log.Println(err) + } + }() + + c := make(chan os.Signal, 1) + // We'll accept graceful shutdowns when quit via SIGINT (Ctrl+C) + // SIGKILL, SIGQUIT or SIGTERM (Ctrl+/) will not be caught. + signal.Notify(c, os.Interrupt) + + // Block until we receive our signal. + <-c + + // Create a deadline to wait for. + ctx, cancel := context.WithTimeout(context.Background(), wait) + defer cancel() + // Doesn't block if no connections, but will otherwise wait + // until the timeout deadline. + srv.Shutdown(ctx) + // Optionally, you could run srv.Shutdown in a goroutine and block on + // <-ctx.Done() if your application should wait for other services + // to finalize based on context cancellation. + log.Println("shutting down") + os.Exit(0) +} +``` + +### Middleware + +Mux supports the addition of middlewares to a [Router](https://godoc.org/github.com/gorilla/mux#Router), which are executed in the order they are added if a match is found, including its subrouters. +Middlewares are (typically) small pieces of code which take one request, do something with it, and pass it down to another middleware or the final handler. Some common use cases for middleware are request logging, header manipulation, or `ResponseWriter` hijacking. + +Mux middlewares are defined using the de facto standard type: + +```go +type MiddlewareFunc func(http.Handler) http.Handler +``` + +Typically, the returned handler is a closure which does something with the http.ResponseWriter and http.Request passed to it, and then calls the handler passed as parameter to the MiddlewareFunc. This takes advantage of closures being able access variables from the context where they are created, while retaining the signature enforced by the receivers. + +A very basic middleware which logs the URI of the request being handled could be written as: + +```go +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Do stuff here + log.Println(r.RequestURI) + // Call the next handler, which can be another middleware in the chain, or the final handler. + next.ServeHTTP(w, r) + }) +} +``` + +Middlewares can be added to a router using `Router.Use()`: + +```go +r := mux.NewRouter() +r.HandleFunc("/", handler) +r.Use(loggingMiddleware) +``` + +A more complex authentication middleware, which maps session token to users, could be written as: + +```go +// Define our struct +type authenticationMiddleware struct { + tokenUsers map[string]string +} + +// Initialize it somewhere +func (amw *authenticationMiddleware) Populate() { + amw.tokenUsers["00000000"] = "user0" + amw.tokenUsers["aaaaaaaa"] = "userA" + amw.tokenUsers["05f717e5"] = "randomUser" + amw.tokenUsers["deadbeef"] = "user0" +} + +// Middleware function, which will be called for each request +func (amw *authenticationMiddleware) Middleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + token := r.Header.Get("X-Session-Token") + + if user, found := amw.tokenUsers[token]; found { + // We found the token in our map + log.Printf("Authenticated user %s\n", user) + // Pass down the request to the next middleware (or final handler) + next.ServeHTTP(w, r) + } else { + // Write an error and stop the handler chain + http.Error(w, "Forbidden", http.StatusForbidden) + } + }) +} +``` + +```go +r := mux.NewRouter() +r.HandleFunc("/", handler) + +amw := authenticationMiddleware{tokenUsers: make(map[string]string)} +amw.Populate() + +r.Use(amw.Middleware) +``` + +Note: The handler chain will be stopped if your middleware doesn't call `next.ServeHTTP()` with the corresponding parameters. This can be used to abort a request if the middleware writer wants to. Middlewares _should_ write to `ResponseWriter` if they _are_ going to terminate the request, and they _should not_ write to `ResponseWriter` if they _are not_ going to terminate it. + +### Handling CORS Requests + +[CORSMethodMiddleware](https://godoc.org/github.com/gorilla/mux#CORSMethodMiddleware) intends to make it easier to strictly set the `Access-Control-Allow-Methods` response header. + +* You will still need to use your own CORS handler to set the other CORS headers such as `Access-Control-Allow-Origin` +* The middleware will set the `Access-Control-Allow-Methods` header to all the method matchers (e.g. `r.Methods(http.MethodGet, http.MethodPut, http.MethodOptions)` -> `Access-Control-Allow-Methods: GET,PUT,OPTIONS`) on a route +* If you do not specify any methods, then: +> _Important_: there must be an `OPTIONS` method matcher for the middleware to set the headers. + +Here is an example of using `CORSMethodMiddleware` along with a custom `OPTIONS` handler to set all the required CORS headers: + +```go +package main + +import ( + "net/http" + "github.com/gorilla/mux" +) + +func main() { + r := mux.NewRouter() + + // IMPORTANT: you must specify an OPTIONS method matcher for the middleware to set CORS headers + r.HandleFunc("/foo", fooHandler).Methods(http.MethodGet, http.MethodPut, http.MethodPatch, http.MethodOptions) + r.Use(mux.CORSMethodMiddleware(r)) + + http.ListenAndServe(":8080", r) +} + +func fooHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") + if r.Method == http.MethodOptions { + return + } + + w.Write([]byte("foo")) +} +``` + +And an request to `/foo` using something like: + +```bash +curl localhost:8080/foo -v +``` + +Would look like: + +```bash +* Trying ::1... +* TCP_NODELAY set +* Connected to localhost (::1) port 8080 (#0) +> GET /foo HTTP/1.1 +> Host: localhost:8080 +> User-Agent: curl/7.59.0 +> Accept: */* +> +< HTTP/1.1 200 OK +< Access-Control-Allow-Methods: GET,PUT,PATCH,OPTIONS +< Access-Control-Allow-Origin: * +< Date: Fri, 28 Jun 2019 20:13:30 GMT +< Content-Length: 3 +< Content-Type: text/plain; charset=utf-8 +< +* Connection #0 to host localhost left intact +foo +``` + +### Testing Handlers + +Testing handlers in a Go web application is straightforward, and _mux_ doesn't complicate this any further. Given two files: `endpoints.go` and `endpoints_test.go`, here's how we'd test an application using _mux_. + +First, our simple HTTP handler: + +```go +// endpoints.go +package main + +func HealthCheckHandler(w http.ResponseWriter, r *http.Request) { + // A very simple health check. + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + + // In the future we could report back on the status of our DB, or our cache + // (e.g. Redis) by performing a simple PING, and include them in the response. + io.WriteString(w, `{"alive": true}`) +} + +func main() { + r := mux.NewRouter() + r.HandleFunc("/health", HealthCheckHandler) + + log.Fatal(http.ListenAndServe("localhost:8080", r)) +} +``` + +Our test code: + +```go +// endpoints_test.go +package main + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestHealthCheckHandler(t *testing.T) { + // Create a request to pass to our handler. We don't have any query parameters for now, so we'll + // pass 'nil' as the third parameter. + req, err := http.NewRequest("GET", "/health", nil) + if err != nil { + t.Fatal(err) + } + + // We create a ResponseRecorder (which satisfies http.ResponseWriter) to record the response. + rr := httptest.NewRecorder() + handler := http.HandlerFunc(HealthCheckHandler) + + // Our handlers satisfy http.Handler, so we can call their ServeHTTP method + // directly and pass in our Request and ResponseRecorder. + handler.ServeHTTP(rr, req) + + // Check the status code is what we expect. + if status := rr.Code; status != http.StatusOK { + t.Errorf("handler returned wrong status code: got %v want %v", + status, http.StatusOK) + } + + // Check the response body is what we expect. + expected := `{"alive": true}` + if rr.Body.String() != expected { + t.Errorf("handler returned unexpected body: got %v want %v", + rr.Body.String(), expected) + } +} +``` + +In the case that our routes have [variables](#examples), we can pass those in the request. We could write +[table-driven tests](https://dave.cheney.net/2013/06/09/writing-table-driven-tests-in-go) to test multiple +possible route variables as needed. + +```go +// endpoints.go +func main() { + r := mux.NewRouter() + // A route with a route variable: + r.HandleFunc("/metrics/{type}", MetricsHandler) + + log.Fatal(http.ListenAndServe("localhost:8080", r)) +} +``` + +Our test file, with a table-driven test of `routeVariables`: + +```go +// endpoints_test.go +func TestMetricsHandler(t *testing.T) { + tt := []struct{ + routeVariable string + shouldPass bool + }{ + {"goroutines", true}, + {"heap", true}, + {"counters", true}, + {"queries", true}, + {"adhadaeqm3k", false}, + } + + for _, tc := range tt { + path := fmt.Sprintf("/metrics/%s", tc.routeVariable) + req, err := http.NewRequest("GET", path, nil) + if err != nil { + t.Fatal(err) + } + + rr := httptest.NewRecorder() + + // To add the vars to the context, + // we need to create a router through which we can pass the request. + router := mux.NewRouter() + router.HandleFunc("/metrics/{type}", MetricsHandler) + router.ServeHTTP(rr, req) + + // In this case, our MetricsHandler returns a non-200 response + // for a route variable it doesn't know about. + if rr.Code == http.StatusOK && !tc.shouldPass { + t.Errorf("handler should have failed on routeVariable %s: got %v want %v", + tc.routeVariable, rr.Code, http.StatusOK) + } + } +} +``` + +## Full Example + +Here's a complete, runnable example of a small `mux` based server: + +```go +package main + +import ( + "net/http" + "log" + "github.com/gorilla/mux" +) + +func YourHandler(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("Gorilla!\n")) +} + +func main() { + r := mux.NewRouter() + // Routes consist of a path and a handler function. + r.HandleFunc("/", YourHandler) + + // Bind to a port and pass our router in + log.Fatal(http.ListenAndServe(":8000", r)) +} +``` + +## License + +BSD licensed. See the LICENSE file for details. diff --git a/vendor/github.com/gorilla/mux/doc.go b/vendor/github.com/gorilla/mux/doc.go new file mode 100644 index 000000000..80601351f --- /dev/null +++ b/vendor/github.com/gorilla/mux/doc.go @@ -0,0 +1,305 @@ +// Copyright 2012 The Gorilla Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package mux implements a request router and dispatcher. + +The name mux stands for "HTTP request multiplexer". Like the standard +http.ServeMux, mux.Router matches incoming requests against a list of +registered routes and calls a handler for the route that matches the URL +or other conditions. The main features are: + + - Requests can be matched based on URL host, path, path prefix, schemes, + header and query values, HTTP methods or using custom matchers. + - URL hosts, paths and query values can have variables with an optional + regular expression. + - Registered URLs can be built, or "reversed", which helps maintaining + references to resources. + - Routes can be used as subrouters: nested routes are only tested if the + parent route matches. This is useful to define groups of routes that + share common conditions like a host, a path prefix or other repeated + attributes. As a bonus, this optimizes request matching. + - It implements the http.Handler interface so it is compatible with the + standard http.ServeMux. + +Let's start registering a couple of URL paths and handlers: + + func main() { + r := mux.NewRouter() + r.HandleFunc("/", HomeHandler) + r.HandleFunc("/products", ProductsHandler) + r.HandleFunc("/articles", ArticlesHandler) + http.Handle("/", r) + } + +Here we register three routes mapping URL paths to handlers. This is +equivalent to how http.HandleFunc() works: if an incoming request URL matches +one of the paths, the corresponding handler is called passing +(http.ResponseWriter, *http.Request) as parameters. + +Paths can have variables. They are defined using the format {name} or +{name:pattern}. If a regular expression pattern is not defined, the matched +variable will be anything until the next slash. For example: + + r := mux.NewRouter() + r.HandleFunc("/products/{key}", ProductHandler) + r.HandleFunc("/articles/{category}/", ArticlesCategoryHandler) + r.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler) + +Groups can be used inside patterns, as long as they are non-capturing (?:re). For example: + + r.HandleFunc("/articles/{category}/{sort:(?:asc|desc|new)}", ArticlesCategoryHandler) + +The names are used to create a map of route variables which can be retrieved +calling mux.Vars(): + + vars := mux.Vars(request) + category := vars["category"] + +Note that if any capturing groups are present, mux will panic() during parsing. To prevent +this, convert any capturing groups to non-capturing, e.g. change "/{sort:(asc|desc)}" to +"/{sort:(?:asc|desc)}". This is a change from prior versions which behaved unpredictably +when capturing groups were present. + +And this is all you need to know about the basic usage. More advanced options +are explained below. + +Routes can also be restricted to a domain or subdomain. Just define a host +pattern to be matched. They can also have variables: + + r := mux.NewRouter() + // Only matches if domain is "www.example.com". + r.Host("www.example.com") + // Matches a dynamic subdomain. + r.Host("{subdomain:[a-z]+}.domain.com") + +There are several other matchers that can be added. To match path prefixes: + + r.PathPrefix("/products/") + +...or HTTP methods: + + r.Methods("GET", "POST") + +...or URL schemes: + + r.Schemes("https") + +...or header values: + + r.Headers("X-Requested-With", "XMLHttpRequest") + +...or query values: + + r.Queries("key", "value") + +...or to use a custom matcher function: + + r.MatcherFunc(func(r *http.Request, rm *RouteMatch) bool { + return r.ProtoMajor == 0 + }) + +...and finally, it is possible to combine several matchers in a single route: + + r.HandleFunc("/products", ProductsHandler). + Host("www.example.com"). + Methods("GET"). + Schemes("http") + +Setting the same matching conditions again and again can be boring, so we have +a way to group several routes that share the same requirements. +We call it "subrouting". + +For example, let's say we have several URLs that should only match when the +host is "www.example.com". Create a route for that host and get a "subrouter" +from it: + + r := mux.NewRouter() + s := r.Host("www.example.com").Subrouter() + +Then register routes in the subrouter: + + s.HandleFunc("/products/", ProductsHandler) + s.HandleFunc("/products/{key}", ProductHandler) + s.HandleFunc("/articles/{category}/{id:[0-9]+}"), ArticleHandler) + +The three URL paths we registered above will only be tested if the domain is +"www.example.com", because the subrouter is tested first. This is not +only convenient, but also optimizes request matching. You can create +subrouters combining any attribute matchers accepted by a route. + +Subrouters can be used to create domain or path "namespaces": you define +subrouters in a central place and then parts of the app can register its +paths relatively to a given subrouter. + +There's one more thing about subroutes. When a subrouter has a path prefix, +the inner routes use it as base for their paths: + + r := mux.NewRouter() + s := r.PathPrefix("/products").Subrouter() + // "/products/" + s.HandleFunc("/", ProductsHandler) + // "/products/{key}/" + s.HandleFunc("/{key}/", ProductHandler) + // "/products/{key}/details" + s.HandleFunc("/{key}/details", ProductDetailsHandler) + +Note that the path provided to PathPrefix() represents a "wildcard": calling +PathPrefix("/static/").Handler(...) means that the handler will be passed any +request that matches "/static/*". This makes it easy to serve static files with mux: + + func main() { + var dir string + + flag.StringVar(&dir, "dir", ".", "the directory to serve files from. Defaults to the current dir") + flag.Parse() + r := mux.NewRouter() + + // This will serve files under http://localhost:8000/static/ + r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir(dir)))) + + srv := &http.Server{ + Handler: r, + Addr: "127.0.0.1:8000", + // Good practice: enforce timeouts for servers you create! + WriteTimeout: 15 * time.Second, + ReadTimeout: 15 * time.Second, + } + + log.Fatal(srv.ListenAndServe()) + } + +Now let's see how to build registered URLs. + +Routes can be named. All routes that define a name can have their URLs built, +or "reversed". We define a name calling Name() on a route. For example: + + r := mux.NewRouter() + r.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler). + Name("article") + +To build a URL, get the route and call the URL() method, passing a sequence of +key/value pairs for the route variables. For the previous route, we would do: + + url, err := r.Get("article").URL("category", "technology", "id", "42") + +...and the result will be a url.URL with the following path: + + "/articles/technology/42" + +This also works for host and query value variables: + + r := mux.NewRouter() + r.Host("{subdomain}.domain.com"). + Path("/articles/{category}/{id:[0-9]+}"). + Queries("filter", "{filter}"). + HandlerFunc(ArticleHandler). + Name("article") + + // url.String() will be "http://news.domain.com/articles/technology/42?filter=gorilla" + url, err := r.Get("article").URL("subdomain", "news", + "category", "technology", + "id", "42", + "filter", "gorilla") + +All variables defined in the route are required, and their values must +conform to the corresponding patterns. These requirements guarantee that a +generated URL will always match a registered route -- the only exception is +for explicitly defined "build-only" routes which never match. + +Regex support also exists for matching Headers within a route. For example, we could do: + + r.HeadersRegexp("Content-Type", "application/(text|json)") + +...and the route will match both requests with a Content-Type of `application/json` as well as +`application/text` + +There's also a way to build only the URL host or path for a route: +use the methods URLHost() or URLPath() instead. For the previous route, +we would do: + + // "http://news.domain.com/" + host, err := r.Get("article").URLHost("subdomain", "news") + + // "/articles/technology/42" + path, err := r.Get("article").URLPath("category", "technology", "id", "42") + +And if you use subrouters, host and path defined separately can be built +as well: + + r := mux.NewRouter() + s := r.Host("{subdomain}.domain.com").Subrouter() + s.Path("/articles/{category}/{id:[0-9]+}"). + HandlerFunc(ArticleHandler). + Name("article") + + // "http://news.domain.com/articles/technology/42" + url, err := r.Get("article").URL("subdomain", "news", + "category", "technology", + "id", "42") + +Mux supports the addition of middlewares to a Router, which are executed in the order they are added if a match is found, including its subrouters. Middlewares are (typically) small pieces of code which take one request, do something with it, and pass it down to another middleware or the final handler. Some common use cases for middleware are request logging, header manipulation, or ResponseWriter hijacking. + + type MiddlewareFunc func(http.Handler) http.Handler + +Typically, the returned handler is a closure which does something with the http.ResponseWriter and http.Request passed to it, and then calls the handler passed as parameter to the MiddlewareFunc (closures can access variables from the context where they are created). + +A very basic middleware which logs the URI of the request being handled could be written as: + + func simpleMw(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Do stuff here + log.Println(r.RequestURI) + // Call the next handler, which can be another middleware in the chain, or the final handler. + next.ServeHTTP(w, r) + }) + } + +Middlewares can be added to a router using `Router.Use()`: + + r := mux.NewRouter() + r.HandleFunc("/", handler) + r.Use(simpleMw) + +A more complex authentication middleware, which maps session token to users, could be written as: + + // Define our struct + type authenticationMiddleware struct { + tokenUsers map[string]string + } + + // Initialize it somewhere + func (amw *authenticationMiddleware) Populate() { + amw.tokenUsers["00000000"] = "user0" + amw.tokenUsers["aaaaaaaa"] = "userA" + amw.tokenUsers["05f717e5"] = "randomUser" + amw.tokenUsers["deadbeef"] = "user0" + } + + // Middleware function, which will be called for each request + func (amw *authenticationMiddleware) Middleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + token := r.Header.Get("X-Session-Token") + + if user, found := amw.tokenUsers[token]; found { + // We found the token in our map + log.Printf("Authenticated user %s\n", user) + next.ServeHTTP(w, r) + } else { + http.Error(w, "Forbidden", http.StatusForbidden) + } + }) + } + + r := mux.NewRouter() + r.HandleFunc("/", handler) + + amw := authenticationMiddleware{tokenUsers: make(map[string]string)} + amw.Populate() + + r.Use(amw.Middleware) + +Note: The handler chain will be stopped if your middleware doesn't call `next.ServeHTTP()` with the corresponding parameters. This can be used to abort a request if the middleware writer wants to. +*/ +package mux diff --git a/vendor/github.com/gorilla/mux/middleware.go b/vendor/github.com/gorilla/mux/middleware.go new file mode 100644 index 000000000..cb51c565e --- /dev/null +++ b/vendor/github.com/gorilla/mux/middleware.go @@ -0,0 +1,74 @@ +package mux + +import ( + "net/http" + "strings" +) + +// MiddlewareFunc is a function which receives an http.Handler and returns another http.Handler. +// Typically, the returned handler is a closure which does something with the http.ResponseWriter and http.Request passed +// to it, and then calls the handler passed as parameter to the MiddlewareFunc. +type MiddlewareFunc func(http.Handler) http.Handler + +// middleware interface is anything which implements a MiddlewareFunc named Middleware. +type middleware interface { + Middleware(handler http.Handler) http.Handler +} + +// Middleware allows MiddlewareFunc to implement the middleware interface. +func (mw MiddlewareFunc) Middleware(handler http.Handler) http.Handler { + return mw(handler) +} + +// Use appends a MiddlewareFunc to the chain. Middleware can be used to intercept or otherwise modify requests and/or responses, and are executed in the order that they are applied to the Router. +func (r *Router) Use(mwf ...MiddlewareFunc) { + for _, fn := range mwf { + r.middlewares = append(r.middlewares, fn) + } +} + +// useInterface appends a middleware to the chain. Middleware can be used to intercept or otherwise modify requests and/or responses, and are executed in the order that they are applied to the Router. +func (r *Router) useInterface(mw middleware) { + r.middlewares = append(r.middlewares, mw) +} + +// CORSMethodMiddleware automatically sets the Access-Control-Allow-Methods response header +// on requests for routes that have an OPTIONS method matcher to all the method matchers on +// the route. Routes that do not explicitly handle OPTIONS requests will not be processed +// by the middleware. See examples for usage. +func CORSMethodMiddleware(r *Router) MiddlewareFunc { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + allMethods, err := getAllMethodsForRoute(r, req) + if err == nil { + for _, v := range allMethods { + if v == http.MethodOptions { + w.Header().Set("Access-Control-Allow-Methods", strings.Join(allMethods, ",")) + } + } + } + + next.ServeHTTP(w, req) + }) + } +} + +// getAllMethodsForRoute returns all the methods from method matchers matching a given +// request. +func getAllMethodsForRoute(r *Router, req *http.Request) ([]string, error) { + var allMethods []string + + for _, route := range r.routes { + var match RouteMatch + if route.Match(req, &match) || match.MatchErr == ErrMethodMismatch { + methods, err := route.GetMethods() + if err != nil { + return nil, err + } + + allMethods = append(allMethods, methods...) + } + } + + return allMethods, nil +} diff --git a/vendor/github.com/gorilla/mux/mux.go b/vendor/github.com/gorilla/mux/mux.go new file mode 100644 index 000000000..1e089906f --- /dev/null +++ b/vendor/github.com/gorilla/mux/mux.go @@ -0,0 +1,608 @@ +// Copyright 2012 The Gorilla Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mux + +import ( + "context" + "errors" + "fmt" + "net/http" + "path" + "regexp" +) + +var ( + // ErrMethodMismatch is returned when the method in the request does not match + // the method defined against the route. + ErrMethodMismatch = errors.New("method is not allowed") + // ErrNotFound is returned when no route match is found. + ErrNotFound = errors.New("no matching route was found") +) + +// NewRouter returns a new router instance. +func NewRouter() *Router { + return &Router{namedRoutes: make(map[string]*Route)} +} + +// Router registers routes to be matched and dispatches a handler. +// +// It implements the http.Handler interface, so it can be registered to serve +// requests: +// +// var router = mux.NewRouter() +// +// func main() { +// http.Handle("/", router) +// } +// +// Or, for Google App Engine, register it in a init() function: +// +// func init() { +// http.Handle("/", router) +// } +// +// This will send all incoming requests to the router. +type Router struct { + // Configurable Handler to be used when no route matches. + // This can be used to render your own 404 Not Found errors. + NotFoundHandler http.Handler + + // Configurable Handler to be used when the request method does not match the route. + // This can be used to render your own 405 Method Not Allowed errors. + MethodNotAllowedHandler http.Handler + + // Routes to be matched, in order. + routes []*Route + + // Routes by name for URL building. + namedRoutes map[string]*Route + + // If true, do not clear the request context after handling the request. + // + // Deprecated: No effect, since the context is stored on the request itself. + KeepContext bool + + // Slice of middlewares to be called after a match is found + middlewares []middleware + + // configuration shared with `Route` + routeConf +} + +// common route configuration shared between `Router` and `Route` +type routeConf struct { + // If true, "/path/foo%2Fbar/to" will match the path "/path/{var}/to" + useEncodedPath bool + + // If true, when the path pattern is "/path/", accessing "/path" will + // redirect to the former and vice versa. + strictSlash bool + + // If true, when the path pattern is "/path//to", accessing "/path//to" + // will not redirect + skipClean bool + + // Manager for the variables from host and path. + regexp routeRegexpGroup + + // List of matchers. + matchers []matcher + + // The scheme used when building URLs. + buildScheme string + + buildVarsFunc BuildVarsFunc +} + +// returns an effective deep copy of `routeConf` +func copyRouteConf(r routeConf) routeConf { + c := r + + if r.regexp.path != nil { + c.regexp.path = copyRouteRegexp(r.regexp.path) + } + + if r.regexp.host != nil { + c.regexp.host = copyRouteRegexp(r.regexp.host) + } + + c.regexp.queries = make([]*routeRegexp, 0, len(r.regexp.queries)) + for _, q := range r.regexp.queries { + c.regexp.queries = append(c.regexp.queries, copyRouteRegexp(q)) + } + + c.matchers = make([]matcher, len(r.matchers)) + copy(c.matchers, r.matchers) + + return c +} + +func copyRouteRegexp(r *routeRegexp) *routeRegexp { + c := *r + return &c +} + +// Match attempts to match the given request against the router's registered routes. +// +// If the request matches a route of this router or one of its subrouters the Route, +// Handler, and Vars fields of the the match argument are filled and this function +// returns true. +// +// If the request does not match any of this router's or its subrouters' routes +// then this function returns false. If available, a reason for the match failure +// will be filled in the match argument's MatchErr field. If the match failure type +// (eg: not found) has a registered handler, the handler is assigned to the Handler +// field of the match argument. +func (r *Router) Match(req *http.Request, match *RouteMatch) bool { + for _, route := range r.routes { + if route.Match(req, match) { + // Build middleware chain if no error was found + if match.MatchErr == nil { + for i := len(r.middlewares) - 1; i >= 0; i-- { + match.Handler = r.middlewares[i].Middleware(match.Handler) + } + } + return true + } + } + + if match.MatchErr == ErrMethodMismatch { + if r.MethodNotAllowedHandler != nil { + match.Handler = r.MethodNotAllowedHandler + return true + } + + return false + } + + // Closest match for a router (includes sub-routers) + if r.NotFoundHandler != nil { + match.Handler = r.NotFoundHandler + match.MatchErr = ErrNotFound + return true + } + + match.MatchErr = ErrNotFound + return false +} + +// ServeHTTP dispatches the handler registered in the matched route. +// +// When there is a match, the route variables can be retrieved calling +// mux.Vars(request). +func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) { + if !r.skipClean { + path := req.URL.Path + if r.useEncodedPath { + path = req.URL.EscapedPath() + } + // Clean path to canonical form and redirect. + if p := cleanPath(path); p != path { + + // Added 3 lines (Philip Schlump) - It was dropping the query string and #whatever from query. + // This matches with fix in go 1.2 r.c. 4 for same problem. Go Issue: + // http://code.google.com/p/go/issues/detail?id=5252 + url := *req.URL + url.Path = p + p = url.String() + + w.Header().Set("Location", p) + w.WriteHeader(http.StatusMovedPermanently) + return + } + } + var match RouteMatch + var handler http.Handler + if r.Match(req, &match) { + handler = match.Handler + req = requestWithVars(req, match.Vars) + req = requestWithRoute(req, match.Route) + } + + if handler == nil && match.MatchErr == ErrMethodMismatch { + handler = methodNotAllowedHandler() + } + + if handler == nil { + handler = http.NotFoundHandler() + } + + handler.ServeHTTP(w, req) +} + +// Get returns a route registered with the given name. +func (r *Router) Get(name string) *Route { + return r.namedRoutes[name] +} + +// GetRoute returns a route registered with the given name. This method +// was renamed to Get() and remains here for backwards compatibility. +func (r *Router) GetRoute(name string) *Route { + return r.namedRoutes[name] +} + +// StrictSlash defines the trailing slash behavior for new routes. The initial +// value is false. +// +// When true, if the route path is "/path/", accessing "/path" will perform a redirect +// to the former and vice versa. In other words, your application will always +// see the path as specified in the route. +// +// When false, if the route path is "/path", accessing "/path/" will not match +// this route and vice versa. +// +// The re-direct is a HTTP 301 (Moved Permanently). Note that when this is set for +// routes with a non-idempotent method (e.g. POST, PUT), the subsequent re-directed +// request will be made as a GET by most clients. Use middleware or client settings +// to modify this behaviour as needed. +// +// Special case: when a route sets a path prefix using the PathPrefix() method, +// strict slash is ignored for that route because the redirect behavior can't +// be determined from a prefix alone. However, any subrouters created from that +// route inherit the original StrictSlash setting. +func (r *Router) StrictSlash(value bool) *Router { + r.strictSlash = value + return r +} + +// SkipClean defines the path cleaning behaviour for new routes. The initial +// value is false. Users should be careful about which routes are not cleaned +// +// When true, if the route path is "/path//to", it will remain with the double +// slash. This is helpful if you have a route like: /fetch/http://xkcd.com/534/ +// +// When false, the path will be cleaned, so /fetch/http://xkcd.com/534/ will +// become /fetch/http/xkcd.com/534 +func (r *Router) SkipClean(value bool) *Router { + r.skipClean = value + return r +} + +// UseEncodedPath tells the router to match the encoded original path +// to the routes. +// For eg. "/path/foo%2Fbar/to" will match the path "/path/{var}/to". +// +// If not called, the router will match the unencoded path to the routes. +// For eg. "/path/foo%2Fbar/to" will match the path "/path/foo/bar/to" +func (r *Router) UseEncodedPath() *Router { + r.useEncodedPath = true + return r +} + +// ---------------------------------------------------------------------------- +// Route factories +// ---------------------------------------------------------------------------- + +// NewRoute registers an empty route. +func (r *Router) NewRoute() *Route { + // initialize a route with a copy of the parent router's configuration + route := &Route{routeConf: copyRouteConf(r.routeConf), namedRoutes: r.namedRoutes} + r.routes = append(r.routes, route) + return route +} + +// Name registers a new route with a name. +// See Route.Name(). +func (r *Router) Name(name string) *Route { + return r.NewRoute().Name(name) +} + +// Handle registers a new route with a matcher for the URL path. +// See Route.Path() and Route.Handler(). +func (r *Router) Handle(path string, handler http.Handler) *Route { + return r.NewRoute().Path(path).Handler(handler) +} + +// HandleFunc registers a new route with a matcher for the URL path. +// See Route.Path() and Route.HandlerFunc(). +func (r *Router) HandleFunc(path string, f func(http.ResponseWriter, + *http.Request)) *Route { + return r.NewRoute().Path(path).HandlerFunc(f) +} + +// Headers registers a new route with a matcher for request header values. +// See Route.Headers(). +func (r *Router) Headers(pairs ...string) *Route { + return r.NewRoute().Headers(pairs...) +} + +// Host registers a new route with a matcher for the URL host. +// See Route.Host(). +func (r *Router) Host(tpl string) *Route { + return r.NewRoute().Host(tpl) +} + +// MatcherFunc registers a new route with a custom matcher function. +// See Route.MatcherFunc(). +func (r *Router) MatcherFunc(f MatcherFunc) *Route { + return r.NewRoute().MatcherFunc(f) +} + +// Methods registers a new route with a matcher for HTTP methods. +// See Route.Methods(). +func (r *Router) Methods(methods ...string) *Route { + return r.NewRoute().Methods(methods...) +} + +// Path registers a new route with a matcher for the URL path. +// See Route.Path(). +func (r *Router) Path(tpl string) *Route { + return r.NewRoute().Path(tpl) +} + +// PathPrefix registers a new route with a matcher for the URL path prefix. +// See Route.PathPrefix(). +func (r *Router) PathPrefix(tpl string) *Route { + return r.NewRoute().PathPrefix(tpl) +} + +// Queries registers a new route with a matcher for URL query values. +// See Route.Queries(). +func (r *Router) Queries(pairs ...string) *Route { + return r.NewRoute().Queries(pairs...) +} + +// Schemes registers a new route with a matcher for URL schemes. +// See Route.Schemes(). +func (r *Router) Schemes(schemes ...string) *Route { + return r.NewRoute().Schemes(schemes...) +} + +// BuildVarsFunc registers a new route with a custom function for modifying +// route variables before building a URL. +func (r *Router) BuildVarsFunc(f BuildVarsFunc) *Route { + return r.NewRoute().BuildVarsFunc(f) +} + +// Walk walks the router and all its sub-routers, calling walkFn for each route +// in the tree. The routes are walked in the order they were added. Sub-routers +// are explored depth-first. +func (r *Router) Walk(walkFn WalkFunc) error { + return r.walk(walkFn, []*Route{}) +} + +// SkipRouter is used as a return value from WalkFuncs to indicate that the +// router that walk is about to descend down to should be skipped. +var SkipRouter = errors.New("skip this router") + +// WalkFunc is the type of the function called for each route visited by Walk. +// At every invocation, it is given the current route, and the current router, +// and a list of ancestor routes that lead to the current route. +type WalkFunc func(route *Route, router *Router, ancestors []*Route) error + +func (r *Router) walk(walkFn WalkFunc, ancestors []*Route) error { + for _, t := range r.routes { + err := walkFn(t, r, ancestors) + if err == SkipRouter { + continue + } + if err != nil { + return err + } + for _, sr := range t.matchers { + if h, ok := sr.(*Router); ok { + ancestors = append(ancestors, t) + err := h.walk(walkFn, ancestors) + if err != nil { + return err + } + ancestors = ancestors[:len(ancestors)-1] + } + } + if h, ok := t.handler.(*Router); ok { + ancestors = append(ancestors, t) + err := h.walk(walkFn, ancestors) + if err != nil { + return err + } + ancestors = ancestors[:len(ancestors)-1] + } + } + return nil +} + +// ---------------------------------------------------------------------------- +// Context +// ---------------------------------------------------------------------------- + +// RouteMatch stores information about a matched route. +type RouteMatch struct { + Route *Route + Handler http.Handler + Vars map[string]string + + // MatchErr is set to appropriate matching error + // It is set to ErrMethodMismatch if there is a mismatch in + // the request method and route method + MatchErr error +} + +type contextKey int + +const ( + varsKey contextKey = iota + routeKey +) + +// Vars returns the route variables for the current request, if any. +func Vars(r *http.Request) map[string]string { + if rv := r.Context().Value(varsKey); rv != nil { + return rv.(map[string]string) + } + return nil +} + +// CurrentRoute returns the matched route for the current request, if any. +// This only works when called inside the handler of the matched route +// because the matched route is stored in the request context which is cleared +// after the handler returns. +func CurrentRoute(r *http.Request) *Route { + if rv := r.Context().Value(routeKey); rv != nil { + return rv.(*Route) + } + return nil +} + +func requestWithVars(r *http.Request, vars map[string]string) *http.Request { + ctx := context.WithValue(r.Context(), varsKey, vars) + return r.WithContext(ctx) +} + +func requestWithRoute(r *http.Request, route *Route) *http.Request { + ctx := context.WithValue(r.Context(), routeKey, route) + return r.WithContext(ctx) +} + +// ---------------------------------------------------------------------------- +// Helpers +// ---------------------------------------------------------------------------- + +// cleanPath returns the canonical path for p, eliminating . and .. elements. +// Borrowed from the net/http package. +func cleanPath(p string) string { + if p == "" { + return "/" + } + if p[0] != '/' { + p = "/" + p + } + np := path.Clean(p) + // path.Clean removes trailing slash except for root; + // put the trailing slash back if necessary. + if p[len(p)-1] == '/' && np != "/" { + np += "/" + } + + return np +} + +// uniqueVars returns an error if two slices contain duplicated strings. +func uniqueVars(s1, s2 []string) error { + for _, v1 := range s1 { + for _, v2 := range s2 { + if v1 == v2 { + return fmt.Errorf("mux: duplicated route variable %q", v2) + } + } + } + return nil +} + +// checkPairs returns the count of strings passed in, and an error if +// the count is not an even number. +func checkPairs(pairs ...string) (int, error) { + length := len(pairs) + if length%2 != 0 { + return length, fmt.Errorf( + "mux: number of parameters must be multiple of 2, got %v", pairs) + } + return length, nil +} + +// mapFromPairsToString converts variadic string parameters to a +// string to string map. +func mapFromPairsToString(pairs ...string) (map[string]string, error) { + length, err := checkPairs(pairs...) + if err != nil { + return nil, err + } + m := make(map[string]string, length/2) + for i := 0; i < length; i += 2 { + m[pairs[i]] = pairs[i+1] + } + return m, nil +} + +// mapFromPairsToRegex converts variadic string parameters to a +// string to regex map. +func mapFromPairsToRegex(pairs ...string) (map[string]*regexp.Regexp, error) { + length, err := checkPairs(pairs...) + if err != nil { + return nil, err + } + m := make(map[string]*regexp.Regexp, length/2) + for i := 0; i < length; i += 2 { + regex, err := regexp.Compile(pairs[i+1]) + if err != nil { + return nil, err + } + m[pairs[i]] = regex + } + return m, nil +} + +// matchInArray returns true if the given string value is in the array. +func matchInArray(arr []string, value string) bool { + for _, v := range arr { + if v == value { + return true + } + } + return false +} + +// matchMapWithString returns true if the given key/value pairs exist in a given map. +func matchMapWithString(toCheck map[string]string, toMatch map[string][]string, canonicalKey bool) bool { + for k, v := range toCheck { + // Check if key exists. + if canonicalKey { + k = http.CanonicalHeaderKey(k) + } + if values := toMatch[k]; values == nil { + return false + } else if v != "" { + // If value was defined as an empty string we only check that the + // key exists. Otherwise we also check for equality. + valueExists := false + for _, value := range values { + if v == value { + valueExists = true + break + } + } + if !valueExists { + return false + } + } + } + return true +} + +// matchMapWithRegex returns true if the given key/value pairs exist in a given map compiled against +// the given regex +func matchMapWithRegex(toCheck map[string]*regexp.Regexp, toMatch map[string][]string, canonicalKey bool) bool { + for k, v := range toCheck { + // Check if key exists. + if canonicalKey { + k = http.CanonicalHeaderKey(k) + } + if values := toMatch[k]; values == nil { + return false + } else if v != nil { + // If value was defined as an empty string we only check that the + // key exists. Otherwise we also check for equality. + valueExists := false + for _, value := range values { + if v.MatchString(value) { + valueExists = true + break + } + } + if !valueExists { + return false + } + } + } + return true +} + +// methodNotAllowed replies to the request with an HTTP status code 405. +func methodNotAllowed(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusMethodNotAllowed) +} + +// methodNotAllowedHandler returns a simple request handler +// that replies to each request with a status code 405. +func methodNotAllowedHandler() http.Handler { return http.HandlerFunc(methodNotAllowed) } diff --git a/vendor/github.com/gorilla/mux/regexp.go b/vendor/github.com/gorilla/mux/regexp.go new file mode 100644 index 000000000..5d05cfa0e --- /dev/null +++ b/vendor/github.com/gorilla/mux/regexp.go @@ -0,0 +1,388 @@ +// Copyright 2012 The Gorilla Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mux + +import ( + "bytes" + "fmt" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" +) + +type routeRegexpOptions struct { + strictSlash bool + useEncodedPath bool +} + +type regexpType int + +const ( + regexpTypePath regexpType = iota + regexpTypeHost + regexpTypePrefix + regexpTypeQuery +) + +// newRouteRegexp parses a route template and returns a routeRegexp, +// used to match a host, a path or a query string. +// +// It will extract named variables, assemble a regexp to be matched, create +// a "reverse" template to build URLs and compile regexps to validate variable +// values used in URL building. +// +// Previously we accepted only Python-like identifiers for variable +// names ([a-zA-Z_][a-zA-Z0-9_]*), but currently the only restriction is that +// name and pattern can't be empty, and names can't contain a colon. +func newRouteRegexp(tpl string, typ regexpType, options routeRegexpOptions) (*routeRegexp, error) { + // Check if it is well-formed. + idxs, errBraces := braceIndices(tpl) + if errBraces != nil { + return nil, errBraces + } + // Backup the original. + template := tpl + // Now let's parse it. + defaultPattern := "[^/]+" + if typ == regexpTypeQuery { + defaultPattern = ".*" + } else if typ == regexpTypeHost { + defaultPattern = "[^.]+" + } + // Only match strict slash if not matching + if typ != regexpTypePath { + options.strictSlash = false + } + // Set a flag for strictSlash. + endSlash := false + if options.strictSlash && strings.HasSuffix(tpl, "/") { + tpl = tpl[:len(tpl)-1] + endSlash = true + } + varsN := make([]string, len(idxs)/2) + varsR := make([]*regexp.Regexp, len(idxs)/2) + pattern := bytes.NewBufferString("") + pattern.WriteByte('^') + reverse := bytes.NewBufferString("") + var end int + var err error + for i := 0; i < len(idxs); i += 2 { + // Set all values we are interested in. + raw := tpl[end:idxs[i]] + end = idxs[i+1] + parts := strings.SplitN(tpl[idxs[i]+1:end-1], ":", 2) + name := parts[0] + patt := defaultPattern + if len(parts) == 2 { + patt = parts[1] + } + // Name or pattern can't be empty. + if name == "" || patt == "" { + return nil, fmt.Errorf("mux: missing name or pattern in %q", + tpl[idxs[i]:end]) + } + // Build the regexp pattern. + fmt.Fprintf(pattern, "%s(?P<%s>%s)", regexp.QuoteMeta(raw), varGroupName(i/2), patt) + + // Build the reverse template. + fmt.Fprintf(reverse, "%s%%s", raw) + + // Append variable name and compiled pattern. + varsN[i/2] = name + varsR[i/2], err = regexp.Compile(fmt.Sprintf("^%s$", patt)) + if err != nil { + return nil, err + } + } + // Add the remaining. + raw := tpl[end:] + pattern.WriteString(regexp.QuoteMeta(raw)) + if options.strictSlash { + pattern.WriteString("[/]?") + } + if typ == regexpTypeQuery { + // Add the default pattern if the query value is empty + if queryVal := strings.SplitN(template, "=", 2)[1]; queryVal == "" { + pattern.WriteString(defaultPattern) + } + } + if typ != regexpTypePrefix { + pattern.WriteByte('$') + } + + var wildcardHostPort bool + if typ == regexpTypeHost { + if !strings.Contains(pattern.String(), ":") { + wildcardHostPort = true + } + } + reverse.WriteString(raw) + if endSlash { + reverse.WriteByte('/') + } + // Compile full regexp. + reg, errCompile := regexp.Compile(pattern.String()) + if errCompile != nil { + return nil, errCompile + } + + // Check for capturing groups which used to work in older versions + if reg.NumSubexp() != len(idxs)/2 { + panic(fmt.Sprintf("route %s contains capture groups in its regexp. ", template) + + "Only non-capturing groups are accepted: e.g. (?:pattern) instead of (pattern)") + } + + // Done! + return &routeRegexp{ + template: template, + regexpType: typ, + options: options, + regexp: reg, + reverse: reverse.String(), + varsN: varsN, + varsR: varsR, + wildcardHostPort: wildcardHostPort, + }, nil +} + +// routeRegexp stores a regexp to match a host or path and information to +// collect and validate route variables. +type routeRegexp struct { + // The unmodified template. + template string + // The type of match + regexpType regexpType + // Options for matching + options routeRegexpOptions + // Expanded regexp. + regexp *regexp.Regexp + // Reverse template. + reverse string + // Variable names. + varsN []string + // Variable regexps (validators). + varsR []*regexp.Regexp + // Wildcard host-port (no strict port match in hostname) + wildcardHostPort bool +} + +// Match matches the regexp against the URL host or path. +func (r *routeRegexp) Match(req *http.Request, match *RouteMatch) bool { + if r.regexpType == regexpTypeHost { + host := getHost(req) + if r.wildcardHostPort { + // Don't be strict on the port match + if i := strings.Index(host, ":"); i != -1 { + host = host[:i] + } + } + return r.regexp.MatchString(host) + } + + if r.regexpType == regexpTypeQuery { + return r.matchQueryString(req) + } + path := req.URL.Path + if r.options.useEncodedPath { + path = req.URL.EscapedPath() + } + return r.regexp.MatchString(path) +} + +// url builds a URL part using the given values. +func (r *routeRegexp) url(values map[string]string) (string, error) { + urlValues := make([]interface{}, len(r.varsN)) + for k, v := range r.varsN { + value, ok := values[v] + if !ok { + return "", fmt.Errorf("mux: missing route variable %q", v) + } + if r.regexpType == regexpTypeQuery { + value = url.QueryEscape(value) + } + urlValues[k] = value + } + rv := fmt.Sprintf(r.reverse, urlValues...) + if !r.regexp.MatchString(rv) { + // The URL is checked against the full regexp, instead of checking + // individual variables. This is faster but to provide a good error + // message, we check individual regexps if the URL doesn't match. + for k, v := range r.varsN { + if !r.varsR[k].MatchString(values[v]) { + return "", fmt.Errorf( + "mux: variable %q doesn't match, expected %q", values[v], + r.varsR[k].String()) + } + } + } + return rv, nil +} + +// getURLQuery returns a single query parameter from a request URL. +// For a URL with foo=bar&baz=ding, we return only the relevant key +// value pair for the routeRegexp. +func (r *routeRegexp) getURLQuery(req *http.Request) string { + if r.regexpType != regexpTypeQuery { + return "" + } + templateKey := strings.SplitN(r.template, "=", 2)[0] + val, ok := findFirstQueryKey(req.URL.RawQuery, templateKey) + if ok { + return templateKey + "=" + val + } + return "" +} + +// findFirstQueryKey returns the same result as (*url.URL).Query()[key][0]. +// If key was not found, empty string and false is returned. +func findFirstQueryKey(rawQuery, key string) (value string, ok bool) { + query := []byte(rawQuery) + for len(query) > 0 { + foundKey := query + if i := bytes.IndexAny(foundKey, "&;"); i >= 0 { + foundKey, query = foundKey[:i], foundKey[i+1:] + } else { + query = query[:0] + } + if len(foundKey) == 0 { + continue + } + var value []byte + if i := bytes.IndexByte(foundKey, '='); i >= 0 { + foundKey, value = foundKey[:i], foundKey[i+1:] + } + if len(foundKey) < len(key) { + // Cannot possibly be key. + continue + } + keyString, err := url.QueryUnescape(string(foundKey)) + if err != nil { + continue + } + if keyString != key { + continue + } + valueString, err := url.QueryUnescape(string(value)) + if err != nil { + continue + } + return valueString, true + } + return "", false +} + +func (r *routeRegexp) matchQueryString(req *http.Request) bool { + return r.regexp.MatchString(r.getURLQuery(req)) +} + +// braceIndices returns the first level curly brace indices from a string. +// It returns an error in case of unbalanced braces. +func braceIndices(s string) ([]int, error) { + var level, idx int + var idxs []int + for i := 0; i < len(s); i++ { + switch s[i] { + case '{': + if level++; level == 1 { + idx = i + } + case '}': + if level--; level == 0 { + idxs = append(idxs, idx, i+1) + } else if level < 0 { + return nil, fmt.Errorf("mux: unbalanced braces in %q", s) + } + } + } + if level != 0 { + return nil, fmt.Errorf("mux: unbalanced braces in %q", s) + } + return idxs, nil +} + +// varGroupName builds a capturing group name for the indexed variable. +func varGroupName(idx int) string { + return "v" + strconv.Itoa(idx) +} + +// ---------------------------------------------------------------------------- +// routeRegexpGroup +// ---------------------------------------------------------------------------- + +// routeRegexpGroup groups the route matchers that carry variables. +type routeRegexpGroup struct { + host *routeRegexp + path *routeRegexp + queries []*routeRegexp +} + +// setMatch extracts the variables from the URL once a route matches. +func (v routeRegexpGroup) setMatch(req *http.Request, m *RouteMatch, r *Route) { + // Store host variables. + if v.host != nil { + host := getHost(req) + if v.host.wildcardHostPort { + // Don't be strict on the port match + if i := strings.Index(host, ":"); i != -1 { + host = host[:i] + } + } + matches := v.host.regexp.FindStringSubmatchIndex(host) + if len(matches) > 0 { + extractVars(host, matches, v.host.varsN, m.Vars) + } + } + path := req.URL.Path + if r.useEncodedPath { + path = req.URL.EscapedPath() + } + // Store path variables. + if v.path != nil { + matches := v.path.regexp.FindStringSubmatchIndex(path) + if len(matches) > 0 { + extractVars(path, matches, v.path.varsN, m.Vars) + // Check if we should redirect. + if v.path.options.strictSlash { + p1 := strings.HasSuffix(path, "/") + p2 := strings.HasSuffix(v.path.template, "/") + if p1 != p2 { + u, _ := url.Parse(req.URL.String()) + if p1 { + u.Path = u.Path[:len(u.Path)-1] + } else { + u.Path += "/" + } + m.Handler = http.RedirectHandler(u.String(), http.StatusMovedPermanently) + } + } + } + } + // Store query string variables. + for _, q := range v.queries { + queryURL := q.getURLQuery(req) + matches := q.regexp.FindStringSubmatchIndex(queryURL) + if len(matches) > 0 { + extractVars(queryURL, matches, q.varsN, m.Vars) + } + } +} + +// getHost tries its best to return the request host. +// According to section 14.23 of RFC 2616 the Host header +// can include the port number if the default value of 80 is not used. +func getHost(r *http.Request) string { + if r.URL.IsAbs() { + return r.URL.Host + } + return r.Host +} + +func extractVars(input string, matches []int, names []string, output map[string]string) { + for i, name := range names { + output[name] = input[matches[2*i+2]:matches[2*i+3]] + } +} diff --git a/vendor/github.com/gorilla/mux/route.go b/vendor/github.com/gorilla/mux/route.go new file mode 100644 index 000000000..e8f11df22 --- /dev/null +++ b/vendor/github.com/gorilla/mux/route.go @@ -0,0 +1,765 @@ +// Copyright 2012 The Gorilla Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mux + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "regexp" + "strings" +) + +// Route stores information to match a request and build URLs. +type Route struct { + // Request handler for the route. + handler http.Handler + // If true, this route never matches: it is only used to build URLs. + buildOnly bool + // The name used to build URLs. + name string + // Error resulted from building a route. + err error + + // "global" reference to all named routes + namedRoutes map[string]*Route + + // config possibly passed in from `Router` + routeConf +} + +// SkipClean reports whether path cleaning is enabled for this route via +// Router.SkipClean. +func (r *Route) SkipClean() bool { + return r.skipClean +} + +// Match matches the route against the request. +func (r *Route) Match(req *http.Request, match *RouteMatch) bool { + if r.buildOnly || r.err != nil { + return false + } + + var matchErr error + + // Match everything. + for _, m := range r.matchers { + if matched := m.Match(req, match); !matched { + if _, ok := m.(methodMatcher); ok { + matchErr = ErrMethodMismatch + continue + } + + // Ignore ErrNotFound errors. These errors arise from match call + // to Subrouters. + // + // This prevents subsequent matching subrouters from failing to + // run middleware. If not ignored, the middleware would see a + // non-nil MatchErr and be skipped, even when there was a + // matching route. + if match.MatchErr == ErrNotFound { + match.MatchErr = nil + } + + matchErr = nil // nolint:ineffassign + return false + } else { + // Multiple routes may share the same path but use different HTTP methods. For instance: + // Route 1: POST "/users/{id}". + // Route 2: GET "/users/{id}", parameters: "id": "[0-9]+". + // + // The router must handle these cases correctly. For a GET request to "/users/abc" with "id" as "-2", + // The router should return a "Not Found" error as no route fully matches this request. + if match.MatchErr == ErrMethodMismatch { + match.MatchErr = nil + } + } + } + + if matchErr != nil { + match.MatchErr = matchErr + return false + } + + if match.MatchErr == ErrMethodMismatch && r.handler != nil { + // We found a route which matches request method, clear MatchErr + match.MatchErr = nil + // Then override the mis-matched handler + match.Handler = r.handler + } + + // Yay, we have a match. Let's collect some info about it. + if match.Route == nil { + match.Route = r + } + if match.Handler == nil { + match.Handler = r.handler + } + if match.Vars == nil { + match.Vars = make(map[string]string) + } + + // Set variables. + r.regexp.setMatch(req, match, r) + return true +} + +// ---------------------------------------------------------------------------- +// Route attributes +// ---------------------------------------------------------------------------- + +// GetError returns an error resulted from building the route, if any. +func (r *Route) GetError() error { + return r.err +} + +// BuildOnly sets the route to never match: it is only used to build URLs. +func (r *Route) BuildOnly() *Route { + r.buildOnly = true + return r +} + +// Handler -------------------------------------------------------------------- + +// Handler sets a handler for the route. +func (r *Route) Handler(handler http.Handler) *Route { + if r.err == nil { + r.handler = handler + } + return r +} + +// HandlerFunc sets a handler function for the route. +func (r *Route) HandlerFunc(f func(http.ResponseWriter, *http.Request)) *Route { + return r.Handler(http.HandlerFunc(f)) +} + +// GetHandler returns the handler for the route, if any. +func (r *Route) GetHandler() http.Handler { + return r.handler +} + +// Name ----------------------------------------------------------------------- + +// Name sets the name for the route, used to build URLs. +// It is an error to call Name more than once on a route. +func (r *Route) Name(name string) *Route { + if r.name != "" { + r.err = fmt.Errorf("mux: route already has name %q, can't set %q", + r.name, name) + } + if r.err == nil { + r.name = name + r.namedRoutes[name] = r + } + return r +} + +// GetName returns the name for the route, if any. +func (r *Route) GetName() string { + return r.name +} + +// ---------------------------------------------------------------------------- +// Matchers +// ---------------------------------------------------------------------------- + +// matcher types try to match a request. +type matcher interface { + Match(*http.Request, *RouteMatch) bool +} + +// addMatcher adds a matcher to the route. +func (r *Route) addMatcher(m matcher) *Route { + if r.err == nil { + r.matchers = append(r.matchers, m) + } + return r +} + +// addRegexpMatcher adds a host or path matcher and builder to a route. +func (r *Route) addRegexpMatcher(tpl string, typ regexpType) error { + if r.err != nil { + return r.err + } + if typ == regexpTypePath || typ == regexpTypePrefix { + if len(tpl) > 0 && tpl[0] != '/' { + return fmt.Errorf("mux: path must start with a slash, got %q", tpl) + } + if r.regexp.path != nil { + tpl = strings.TrimRight(r.regexp.path.template, "/") + tpl + } + } + rr, err := newRouteRegexp(tpl, typ, routeRegexpOptions{ + strictSlash: r.strictSlash, + useEncodedPath: r.useEncodedPath, + }) + if err != nil { + return err + } + for _, q := range r.regexp.queries { + if err = uniqueVars(rr.varsN, q.varsN); err != nil { + return err + } + } + if typ == regexpTypeHost { + if r.regexp.path != nil { + if err = uniqueVars(rr.varsN, r.regexp.path.varsN); err != nil { + return err + } + } + r.regexp.host = rr + } else { + if r.regexp.host != nil { + if err = uniqueVars(rr.varsN, r.regexp.host.varsN); err != nil { + return err + } + } + if typ == regexpTypeQuery { + r.regexp.queries = append(r.regexp.queries, rr) + } else { + r.regexp.path = rr + } + } + r.addMatcher(rr) + return nil +} + +// Headers -------------------------------------------------------------------- + +// headerMatcher matches the request against header values. +type headerMatcher map[string]string + +func (m headerMatcher) Match(r *http.Request, match *RouteMatch) bool { + return matchMapWithString(m, r.Header, true) +} + +// Headers adds a matcher for request header values. +// It accepts a sequence of key/value pairs to be matched. For example: +// +// r := mux.NewRouter().NewRoute() +// r.Headers("Content-Type", "application/json", +// "X-Requested-With", "XMLHttpRequest") +// +// The above route will only match if both request header values match. +// If the value is an empty string, it will match any value if the key is set. +func (r *Route) Headers(pairs ...string) *Route { + if r.err == nil { + var headers map[string]string + headers, r.err = mapFromPairsToString(pairs...) + return r.addMatcher(headerMatcher(headers)) + } + return r +} + +// headerRegexMatcher matches the request against the route given a regex for the header +type headerRegexMatcher map[string]*regexp.Regexp + +func (m headerRegexMatcher) Match(r *http.Request, match *RouteMatch) bool { + return matchMapWithRegex(m, r.Header, true) +} + +// HeadersRegexp accepts a sequence of key/value pairs, where the value has regex +// support. For example: +// +// r := mux.NewRouter().NewRoute() +// r.HeadersRegexp("Content-Type", "application/(text|json)", +// "X-Requested-With", "XMLHttpRequest") +// +// The above route will only match if both the request header matches both regular expressions. +// If the value is an empty string, it will match any value if the key is set. +// Use the start and end of string anchors (^ and $) to match an exact value. +func (r *Route) HeadersRegexp(pairs ...string) *Route { + if r.err == nil { + var headers map[string]*regexp.Regexp + headers, r.err = mapFromPairsToRegex(pairs...) + return r.addMatcher(headerRegexMatcher(headers)) + } + return r +} + +// Host ----------------------------------------------------------------------- + +// Host adds a matcher for the URL host. +// It accepts a template with zero or more URL variables enclosed by {}. +// Variables can define an optional regexp pattern to be matched: +// +// - {name} matches anything until the next dot. +// +// - {name:pattern} matches the given regexp pattern. +// +// For example: +// +// r := mux.NewRouter().NewRoute() +// r.Host("www.example.com") +// r.Host("{subdomain}.domain.com") +// r.Host("{subdomain:[a-z]+}.domain.com") +// +// Variable names must be unique in a given route. They can be retrieved +// calling mux.Vars(request). +func (r *Route) Host(tpl string) *Route { + r.err = r.addRegexpMatcher(tpl, regexpTypeHost) + return r +} + +// MatcherFunc ---------------------------------------------------------------- + +// MatcherFunc is the function signature used by custom matchers. +type MatcherFunc func(*http.Request, *RouteMatch) bool + +// Match returns the match for a given request. +func (m MatcherFunc) Match(r *http.Request, match *RouteMatch) bool { + return m(r, match) +} + +// MatcherFunc adds a custom function to be used as request matcher. +func (r *Route) MatcherFunc(f MatcherFunc) *Route { + return r.addMatcher(f) +} + +// Methods -------------------------------------------------------------------- + +// methodMatcher matches the request against HTTP methods. +type methodMatcher []string + +func (m methodMatcher) Match(r *http.Request, match *RouteMatch) bool { + return matchInArray(m, r.Method) +} + +// Methods adds a matcher for HTTP methods. +// It accepts a sequence of one or more methods to be matched, e.g.: +// "GET", "POST", "PUT". +func (r *Route) Methods(methods ...string) *Route { + for k, v := range methods { + methods[k] = strings.ToUpper(v) + } + return r.addMatcher(methodMatcher(methods)) +} + +// Path ----------------------------------------------------------------------- + +// Path adds a matcher for the URL path. +// It accepts a template with zero or more URL variables enclosed by {}. The +// template must start with a "/". +// Variables can define an optional regexp pattern to be matched: +// +// - {name} matches anything until the next slash. +// +// - {name:pattern} matches the given regexp pattern. +// +// For example: +// +// r := mux.NewRouter().NewRoute() +// r.Path("/products/").Handler(ProductsHandler) +// r.Path("/products/{key}").Handler(ProductsHandler) +// r.Path("/articles/{category}/{id:[0-9]+}"). +// Handler(ArticleHandler) +// +// Variable names must be unique in a given route. They can be retrieved +// calling mux.Vars(request). +func (r *Route) Path(tpl string) *Route { + r.err = r.addRegexpMatcher(tpl, regexpTypePath) + return r +} + +// PathPrefix ----------------------------------------------------------------- + +// PathPrefix adds a matcher for the URL path prefix. This matches if the given +// template is a prefix of the full URL path. See Route.Path() for details on +// the tpl argument. +// +// Note that it does not treat slashes specially ("/foobar/" will be matched by +// the prefix "/foo") so you may want to use a trailing slash here. +// +// Also note that the setting of Router.StrictSlash() has no effect on routes +// with a PathPrefix matcher. +func (r *Route) PathPrefix(tpl string) *Route { + r.err = r.addRegexpMatcher(tpl, regexpTypePrefix) + return r +} + +// Query ---------------------------------------------------------------------- + +// Queries adds a matcher for URL query values. +// It accepts a sequence of key/value pairs. Values may define variables. +// For example: +// +// r := mux.NewRouter().NewRoute() +// r.Queries("foo", "bar", "id", "{id:[0-9]+}") +// +// The above route will only match if the URL contains the defined queries +// values, e.g.: ?foo=bar&id=42. +// +// If the value is an empty string, it will match any value if the key is set. +// +// Variables can define an optional regexp pattern to be matched: +// +// - {name} matches anything until the next slash. +// +// - {name:pattern} matches the given regexp pattern. +func (r *Route) Queries(pairs ...string) *Route { + length := len(pairs) + if length%2 != 0 { + r.err = fmt.Errorf( + "mux: number of parameters must be multiple of 2, got %v", pairs) + return nil + } + for i := 0; i < length; i += 2 { + if r.err = r.addRegexpMatcher(pairs[i]+"="+pairs[i+1], regexpTypeQuery); r.err != nil { + return r + } + } + + return r +} + +// Schemes -------------------------------------------------------------------- + +// schemeMatcher matches the request against URL schemes. +type schemeMatcher []string + +func (m schemeMatcher) Match(r *http.Request, match *RouteMatch) bool { + scheme := r.URL.Scheme + // https://golang.org/pkg/net/http/#Request + // "For [most] server requests, fields other than Path and RawQuery will be + // empty." + // Since we're an http muxer, the scheme is either going to be http or https + // though, so we can just set it based on the tls termination state. + if scheme == "" { + if r.TLS == nil { + scheme = "http" + } else { + scheme = "https" + } + } + return matchInArray(m, scheme) +} + +// Schemes adds a matcher for URL schemes. +// It accepts a sequence of schemes to be matched, e.g.: "http", "https". +// If the request's URL has a scheme set, it will be matched against. +// Generally, the URL scheme will only be set if a previous handler set it, +// such as the ProxyHeaders handler from gorilla/handlers. +// If unset, the scheme will be determined based on the request's TLS +// termination state. +// The first argument to Schemes will be used when constructing a route URL. +func (r *Route) Schemes(schemes ...string) *Route { + for k, v := range schemes { + schemes[k] = strings.ToLower(v) + } + if len(schemes) > 0 { + r.buildScheme = schemes[0] + } + return r.addMatcher(schemeMatcher(schemes)) +} + +// BuildVarsFunc -------------------------------------------------------------- + +// BuildVarsFunc is the function signature used by custom build variable +// functions (which can modify route variables before a route's URL is built). +type BuildVarsFunc func(map[string]string) map[string]string + +// BuildVarsFunc adds a custom function to be used to modify build variables +// before a route's URL is built. +func (r *Route) BuildVarsFunc(f BuildVarsFunc) *Route { + if r.buildVarsFunc != nil { + // compose the old and new functions + old := r.buildVarsFunc + r.buildVarsFunc = func(m map[string]string) map[string]string { + return f(old(m)) + } + } else { + r.buildVarsFunc = f + } + return r +} + +// Subrouter ------------------------------------------------------------------ + +// Subrouter creates a subrouter for the route. +// +// It will test the inner routes only if the parent route matched. For example: +// +// r := mux.NewRouter().NewRoute() +// s := r.Host("www.example.com").Subrouter() +// s.HandleFunc("/products/", ProductsHandler) +// s.HandleFunc("/products/{key}", ProductHandler) +// s.HandleFunc("/articles/{category}/{id:[0-9]+}"), ArticleHandler) +// +// Here, the routes registered in the subrouter won't be tested if the host +// doesn't match. +func (r *Route) Subrouter() *Router { + // initialize a subrouter with a copy of the parent route's configuration + router := &Router{routeConf: copyRouteConf(r.routeConf), namedRoutes: r.namedRoutes} + r.addMatcher(router) + return router +} + +// ---------------------------------------------------------------------------- +// URL building +// ---------------------------------------------------------------------------- + +// URL builds a URL for the route. +// +// It accepts a sequence of key/value pairs for the route variables. For +// example, given this route: +// +// r := mux.NewRouter() +// r.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler). +// Name("article") +// +// ...a URL for it can be built using: +// +// url, err := r.Get("article").URL("category", "technology", "id", "42") +// +// ...which will return an url.URL with the following path: +// +// "/articles/technology/42" +// +// This also works for host variables: +// +// r := mux.NewRouter() +// r.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler). +// Host("{subdomain}.domain.com"). +// Name("article") +// +// // url.String() will be "http://news.domain.com/articles/technology/42" +// url, err := r.Get("article").URL("subdomain", "news", +// "category", "technology", +// "id", "42") +// +// The scheme of the resulting url will be the first argument that was passed to Schemes: +// +// // url.String() will be "https://example.com" +// r := mux.NewRouter().NewRoute() +// url, err := r.Host("example.com") +// .Schemes("https", "http").URL() +// +// All variables defined in the route are required, and their values must +// conform to the corresponding patterns. +func (r *Route) URL(pairs ...string) (*url.URL, error) { + if r.err != nil { + return nil, r.err + } + values, err := r.prepareVars(pairs...) + if err != nil { + return nil, err + } + var scheme, host, path string + queries := make([]string, 0, len(r.regexp.queries)) + if r.regexp.host != nil { + if host, err = r.regexp.host.url(values); err != nil { + return nil, err + } + scheme = "http" + if r.buildScheme != "" { + scheme = r.buildScheme + } + } + if r.regexp.path != nil { + if path, err = r.regexp.path.url(values); err != nil { + return nil, err + } + } + for _, q := range r.regexp.queries { + var query string + if query, err = q.url(values); err != nil { + return nil, err + } + queries = append(queries, query) + } + return &url.URL{ + Scheme: scheme, + Host: host, + Path: path, + RawQuery: strings.Join(queries, "&"), + }, nil +} + +// URLHost builds the host part of the URL for a route. See Route.URL(). +// +// The route must have a host defined. +func (r *Route) URLHost(pairs ...string) (*url.URL, error) { + if r.err != nil { + return nil, r.err + } + if r.regexp.host == nil { + return nil, errors.New("mux: route doesn't have a host") + } + values, err := r.prepareVars(pairs...) + if err != nil { + return nil, err + } + host, err := r.regexp.host.url(values) + if err != nil { + return nil, err + } + u := &url.URL{ + Scheme: "http", + Host: host, + } + if r.buildScheme != "" { + u.Scheme = r.buildScheme + } + return u, nil +} + +// URLPath builds the path part of the URL for a route. See Route.URL(). +// +// The route must have a path defined. +func (r *Route) URLPath(pairs ...string) (*url.URL, error) { + if r.err != nil { + return nil, r.err + } + if r.regexp.path == nil { + return nil, errors.New("mux: route doesn't have a path") + } + values, err := r.prepareVars(pairs...) + if err != nil { + return nil, err + } + path, err := r.regexp.path.url(values) + if err != nil { + return nil, err + } + return &url.URL{ + Path: path, + }, nil +} + +// GetPathTemplate returns the template used to build the +// route match. +// This is useful for building simple REST API documentation and for instrumentation +// against third-party services. +// An error will be returned if the route does not define a path. +func (r *Route) GetPathTemplate() (string, error) { + if r.err != nil { + return "", r.err + } + if r.regexp.path == nil { + return "", errors.New("mux: route doesn't have a path") + } + return r.regexp.path.template, nil +} + +// GetPathRegexp returns the expanded regular expression used to match route path. +// This is useful for building simple REST API documentation and for instrumentation +// against third-party services. +// An error will be returned if the route does not define a path. +func (r *Route) GetPathRegexp() (string, error) { + if r.err != nil { + return "", r.err + } + if r.regexp.path == nil { + return "", errors.New("mux: route does not have a path") + } + return r.regexp.path.regexp.String(), nil +} + +// GetQueriesRegexp returns the expanded regular expressions used to match the +// route queries. +// This is useful for building simple REST API documentation and for instrumentation +// against third-party services. +// An error will be returned if the route does not have queries. +func (r *Route) GetQueriesRegexp() ([]string, error) { + if r.err != nil { + return nil, r.err + } + if r.regexp.queries == nil { + return nil, errors.New("mux: route doesn't have queries") + } + queries := make([]string, 0, len(r.regexp.queries)) + for _, query := range r.regexp.queries { + queries = append(queries, query.regexp.String()) + } + return queries, nil +} + +// GetQueriesTemplates returns the templates used to build the +// query matching. +// This is useful for building simple REST API documentation and for instrumentation +// against third-party services. +// An error will be returned if the route does not define queries. +func (r *Route) GetQueriesTemplates() ([]string, error) { + if r.err != nil { + return nil, r.err + } + if r.regexp.queries == nil { + return nil, errors.New("mux: route doesn't have queries") + } + queries := make([]string, 0, len(r.regexp.queries)) + for _, query := range r.regexp.queries { + queries = append(queries, query.template) + } + return queries, nil +} + +// GetMethods returns the methods the route matches against +// This is useful for building simple REST API documentation and for instrumentation +// against third-party services. +// An error will be returned if route does not have methods. +func (r *Route) GetMethods() ([]string, error) { + if r.err != nil { + return nil, r.err + } + for _, m := range r.matchers { + if methods, ok := m.(methodMatcher); ok { + return []string(methods), nil + } + } + return nil, errors.New("mux: route doesn't have methods") +} + +// GetHostTemplate returns the template used to build the +// route match. +// This is useful for building simple REST API documentation and for instrumentation +// against third-party services. +// An error will be returned if the route does not define a host. +func (r *Route) GetHostTemplate() (string, error) { + if r.err != nil { + return "", r.err + } + if r.regexp.host == nil { + return "", errors.New("mux: route doesn't have a host") + } + return r.regexp.host.template, nil +} + +// GetVarNames returns the names of all variables added by regexp matchers +// These can be used to know which route variables should be passed into r.URL() +func (r *Route) GetVarNames() ([]string, error) { + if r.err != nil { + return nil, r.err + } + var varNames []string + if r.regexp.host != nil { + varNames = append(varNames, r.regexp.host.varsN...) + } + if r.regexp.path != nil { + varNames = append(varNames, r.regexp.path.varsN...) + } + for _, regx := range r.regexp.queries { + varNames = append(varNames, regx.varsN...) + } + return varNames, nil +} + +// prepareVars converts the route variable pairs into a map. If the route has a +// BuildVarsFunc, it is invoked. +func (r *Route) prepareVars(pairs ...string) (map[string]string, error) { + m, err := mapFromPairsToString(pairs...) + if err != nil { + return nil, err + } + return r.buildVars(m), nil +} + +func (r *Route) buildVars(m map[string]string) map[string]string { + if r.buildVarsFunc != nil { + m = r.buildVarsFunc(m) + } + return m +} diff --git a/vendor/github.com/gorilla/mux/test_helpers.go b/vendor/github.com/gorilla/mux/test_helpers.go new file mode 100644 index 000000000..5f5c496de --- /dev/null +++ b/vendor/github.com/gorilla/mux/test_helpers.go @@ -0,0 +1,19 @@ +// Copyright 2012 The Gorilla Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mux + +import "net/http" + +// SetURLVars sets the URL variables for the given request, to be accessed via +// mux.Vars for testing route behaviour. Arguments are not modified, a shallow +// copy is returned. +// +// This API should only be used for testing purposes; it provides a way to +// inject variables into the request context. Alternatively, URL variables +// can be set by making a route that captures the required variables, +// starting a server and sending the request to that server. +func SetURLVars(r *http.Request, val map[string]string) *http.Request { + return requestWithVars(r, val) +} diff --git a/vendor/github.com/grpc-ecosystem/go-grpc-middleware/.gitignore b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/.gitignore new file mode 100644 index 000000000..826caa390 --- /dev/null +++ b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/.gitignore @@ -0,0 +1,204 @@ +# Created by .ignore support plugin (hsz.mobi) +### Go template +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof +### Windows template +# Windows image file caches +Thumbs.db +ehthumbs.db + +# Folder config file +Desktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msm +*.msp + +# Windows shortcuts +*.lnk +### Kate template +# Swap Files # +.*.kate-swp +.swp.* +### SublimeText template +# cache files for sublime text +*.tmlanguage.cache +*.tmPreferences.cache +*.stTheme.cache + +# workspace files are user-specific +*.sublime-workspace + +# project files should be checked into the repository, unless a significant +# proportion of contributors will probably not be using SublimeText +# *.sublime-project + +# sftp configuration file +sftp-config.json +### Linux template +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* +### JetBrains template +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff: +.idea +.idea/tasks.xml +.idea/dictionaries +.idea/vcs.xml +.idea/jsLibraryMappings.xml + +# Sensitive or high-churn files: +.idea/dataSources.ids +.idea/dataSources.xml +.idea/dataSources.local.xml +.idea/sqlDataSources.xml +.idea/dynamic.xml +.idea/uiDesigner.xml + +# Gradle: +.idea/gradle.xml +.idea/libraries + +# Mongo Explorer plugin: +.idea/mongoSettings.xml + +## File-based project format: +*.iws + +## Plugin-specific files: + +# IntelliJ +/out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties +### Xcode template +# Xcode +# +# gitignore contributors: remember to update Global/Xcode.gitignore, Objective-C.gitignore & Swift.gitignore + +## Build generated +build/ +DerivedData/ + +## Various settings +*.pbxuser +!default.pbxuser +*.mode1v3 +!default.mode1v3 +*.mode2v3 +!default.mode2v3 +*.perspectivev3 +!default.perspectivev3 +xcuserdata/ + +## Other +*.moved-aside +*.xccheckout +*.xcscmblueprint +### Eclipse template + +.metadata +bin/ +tmp/ +*.tmp +*.bak +*.swp +*~.nib +local.properties +.settings/ +.loadpath +.recommenders + +# Eclipse Core +.project + +# External tool builders +.externalToolBuilders/ + +# Locally stored "Eclipse launch configurations" +*.launch + +# PyDev specific (Python IDE for Eclipse) +*.pydevproject + +# CDT-specific (C/C++ Development Tooling) +.cproject + +# JDT-specific (Eclipse Java Development Tools) +.classpath + +# Java annotation processor (APT) +.factorypath + +# PDT-specific (PHP Development Tools) +.buildpath + +# sbteclipse plugin +.target + +# Tern plugin +.tern-project + +# TeXlipse plugin +.texlipse + +# STS (Spring Tool Suite) +.springBeans + +# Code Recommenders +.recommenders/ + + +coverage.txt + +#vendor +vendor/ + +.envrc \ No newline at end of file diff --git a/vendor/github.com/grpc-ecosystem/go-grpc-middleware/CONTRIBUTING.md b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/CONTRIBUTING.md new file mode 100644 index 000000000..dd52ab893 --- /dev/null +++ b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/CONTRIBUTING.md @@ -0,0 +1,20 @@ +# Contributing + +We would love to have people submit pull requests and help make `grpc-ecosystem/go-grpc-middleware` even better 👍. + +Fork, then clone the repo: + +```bash +git clone git@github.com:your-username/go-grpc-middleware.git +``` + +Before checking in please run the following: + +```bash +make all +``` + +This will `vet`, `fmt`, regenerate documentation and run all tests. + + +Push to your fork and open a pull request. \ No newline at end of file diff --git a/vendor/github.com/grpc-ecosystem/go-grpc-middleware/LICENSE b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/LICENSE new file mode 100644 index 000000000..b2b065037 --- /dev/null +++ b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/vendor/github.com/grpc-ecosystem/go-grpc-middleware/README.md b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/README.md new file mode 100644 index 000000000..a12b40904 --- /dev/null +++ b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/README.md @@ -0,0 +1,93 @@ +# Go gRPC Middleware + +[![Travis Build](https://travis-ci.org/grpc-ecosystem/go-grpc-middleware.svg?branch=master)](https://travis-ci.org/grpc-ecosystem/go-grpc-middleware) +[![Go Report Card](https://goreportcard.com/badge/github.com/grpc-ecosystem/go-grpc-middleware)](https://goreportcard.com/report/github.com/grpc-ecosystem/go-grpc-middleware) +[![GoDoc](http://img.shields.io/badge/GoDoc-Reference-blue.svg)](https://godoc.org/github.com/grpc-ecosystem/go-grpc-middleware) +[![SourceGraph](https://sourcegraph.com/github.com/grpc-ecosystem/go-grpc-middleware/-/badge.svg)](https://sourcegraph.com/github.com/grpc-ecosystem/go-grpc-middleware/?badge) +[![codecov](https://codecov.io/gh/grpc-ecosystem/go-grpc-middleware/branch/master/graph/badge.svg)](https://codecov.io/gh/grpc-ecosystem/go-grpc-middleware) +[![Apache 2.0 License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) +[![quality: production](https://img.shields.io/badge/quality-production-orange.svg)](#status) +[![Slack](https://img.shields.io/badge/slack-%23grpc--middleware-brightgreen)](https://gophers.slack.com/archives/CNJL30P4P) + +[gRPC Go](https://github.com/grpc/grpc-go) Middleware: interceptors, helpers, utilities. + +## ⚠️ Status + +Version [v2](https://github.com/grpc-ecosystem/go-grpc-middleware/tree/v2) is about to be released, with migration guide, which will replace v1. Try v2 and give us feedback! + +Version v1 is currently in deprecation mode, which means only critical and safety bug fixes will be merged. + + +## Middleware + +[gRPC Go](https://github.com/grpc/grpc-go) recently acquired support for +Interceptors, i.e. [middleware](https://medium.com/@matryer/writing-middleware-in-golang-and-how-go-makes-it-so-much-fun-4375c1246e81#.gv7tdlghs) +that is executed either on the gRPC Server before the request is passed onto the user's application logic, or on the gRPC client around the user call. It is a perfect way to implement +common patterns: auth, logging, message, validation, retries, or monitoring. + +These are generic building blocks that make it easy to build multiple microservices easily. +The purpose of this repository is to act as a go-to point for such reusable functionality. It contains +some of them itself, but also will link to useful external repos. + +`grpc_middleware` itself provides support for chaining interceptors, here's an example: + +```go +import "github.com/grpc-ecosystem/go-grpc-middleware" + +myServer := grpc.NewServer( + grpc.StreamInterceptor(grpc_middleware.ChainStreamServer( + grpc_ctxtags.StreamServerInterceptor(), + grpc_opentracing.StreamServerInterceptor(), + grpc_prometheus.StreamServerInterceptor, + grpc_zap.StreamServerInterceptor(zapLogger), + grpc_auth.StreamServerInterceptor(myAuthFunction), + grpc_recovery.StreamServerInterceptor(), + )), + grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer( + grpc_ctxtags.UnaryServerInterceptor(), + grpc_opentracing.UnaryServerInterceptor(), + grpc_prometheus.UnaryServerInterceptor, + grpc_zap.UnaryServerInterceptor(zapLogger), + grpc_auth.UnaryServerInterceptor(myAuthFunction), + grpc_recovery.UnaryServerInterceptor(), + )), +) +``` + +## Interceptors + +_Please send a PR to add new interceptors or middleware to this list_ + +#### Auth + +- [`grpc_auth`](auth) - a customizable (via `AuthFunc`) piece of auth middleware + +#### Logging + +- [`grpc_ctxtags`](tags/) - a library that adds a `Tag` map to context, with data populated from request body +- [`grpc_zap`](logging/zap/) - integration of [zap](https://github.com/uber-go/zap) logging library into gRPC handlers. +- [`grpc_logrus`](logging/logrus/) - integration of [logrus](https://github.com/sirupsen/logrus) logging library into gRPC handlers. +- [`grpc_kit`](logging/kit/) - integration of [go-kit/log](https://github.com/go-kit/log) logging library into gRPC handlers. +- [`grpc_grpc_logsettable`](logging/settable/) - a wrapper around `grpclog.LoggerV2` that allows to replace loggers in runtime (thread-safe). + +#### Monitoring + +- [`grpc_prometheus`⚡](https://github.com/grpc-ecosystem/go-grpc-prometheus) - Prometheus client-side and server-side monitoring middleware +- [`otgrpc`⚡](https://github.com/grpc-ecosystem/grpc-opentracing/tree/master/go/otgrpc) - [OpenTracing](http://opentracing.io/) client-side and server-side interceptors +- [`grpc_opentracing`](tracing/opentracing) - [OpenTracing](http://opentracing.io/) client-side and server-side interceptors with support for streaming and handler-returned tags +- [`otelgrpc`](https://github.com/open-telemetry/opentelemetry-go-contrib/tree/main/instrumentation/google.golang.org/grpc/otelgrpc) - [OpenTelemetry](https://opentelemetry.io/) client-side and server-side interceptors + +#### Client + +- [`grpc_retry`](retry/) - a generic gRPC response code retry mechanism, client-side middleware + +#### Server + +- [`grpc_validator`](validator/) - codegen inbound message validation from `.proto` options +- [`grpc_recovery`](recovery/) - turn panics into gRPC errors +- [`ratelimit`](ratelimit/) - grpc rate limiting by your own limiter + + +## License + +`go-grpc-middleware` is released under the Apache 2.0 license. See the [LICENSE](LICENSE) file for details. diff --git a/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go new file mode 100644 index 000000000..407d9332c --- /dev/null +++ b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go @@ -0,0 +1,166 @@ +// Copyright 2016 Michal Witkowski. All Rights Reserved. +// See LICENSE for licensing terms. + +// gRPC Server Interceptor chaining middleware. + +package grpc_middleware + +import ( + "context" + + "google.golang.org/grpc" +) + +// ChainUnaryServer creates a single interceptor out of a chain of many interceptors. +// +// Execution is done in left-to-right order, including passing of context. +// For example ChainUnaryServer(one, two, three) will execute one before two before three, and three +// will see context changes of one and two. +// +// While this can be useful in some scenarios, it is generally advisable to use google.golang.org/grpc.ChainUnaryInterceptor directly. +func ChainUnaryServer(interceptors ...grpc.UnaryServerInterceptor) grpc.UnaryServerInterceptor { + n := len(interceptors) + + // Dummy interceptor maintained for backward compatibility to avoid returning nil. + if n == 0 { + return func(ctx context.Context, req interface{}, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + return handler(ctx, req) + } + } + + // The degenerate case, just return the single wrapped interceptor directly. + if n == 1 { + return interceptors[0] + } + + // Return a function which satisfies the interceptor interface, and which is + // a closure over the given list of interceptors to be chained. + return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + currHandler := handler + // Iterate backwards through all interceptors except the first (outermost). + // Wrap each one in a function which satisfies the handler interface, but + // is also a closure over the `info` and `handler` parameters. Then pass + // each pseudo-handler to the next outer interceptor as the handler to be called. + for i := n - 1; i > 0; i-- { + // Rebind to loop-local vars so they can be closed over. + innerHandler, i := currHandler, i + currHandler = func(currentCtx context.Context, currentReq interface{}) (interface{}, error) { + return interceptors[i](currentCtx, currentReq, info, innerHandler) + } + } + // Finally return the result of calling the outermost interceptor with the + // outermost pseudo-handler created above as its handler. + return interceptors[0](ctx, req, info, currHandler) + } +} + +// ChainStreamServer creates a single interceptor out of a chain of many interceptors. +// +// Execution is done in left-to-right order, including passing of context. +// For example ChainUnaryServer(one, two, three) will execute one before two before three. +// If you want to pass context between interceptors, use WrapServerStream. +// +// While this can be useful in some scenarios, it is generally advisable to use google.golang.org/grpc.ChainStreamInterceptor directly. +func ChainStreamServer(interceptors ...grpc.StreamServerInterceptor) grpc.StreamServerInterceptor { + n := len(interceptors) + + // Dummy interceptor maintained for backward compatibility to avoid returning nil. + if n == 0 { + return func(srv interface{}, stream grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + return handler(srv, stream) + } + } + + if n == 1 { + return interceptors[0] + } + + return func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + currHandler := handler + for i := n - 1; i > 0; i-- { + innerHandler, i := currHandler, i + currHandler = func(currentSrv interface{}, currentStream grpc.ServerStream) error { + return interceptors[i](currentSrv, currentStream, info, innerHandler) + } + } + return interceptors[0](srv, stream, info, currHandler) + } +} + +// ChainUnaryClient creates a single interceptor out of a chain of many interceptors. +// +// Execution is done in left-to-right order, including passing of context. +// For example ChainUnaryClient(one, two, three) will execute one before two before three. +func ChainUnaryClient(interceptors ...grpc.UnaryClientInterceptor) grpc.UnaryClientInterceptor { + n := len(interceptors) + + // Dummy interceptor maintained for backward compatibility to avoid returning nil. + if n == 0 { + return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + return invoker(ctx, method, req, reply, cc, opts...) + } + } + + if n == 1 { + return interceptors[0] + } + + return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + currInvoker := invoker + for i := n - 1; i > 0; i-- { + innerInvoker, i := currInvoker, i + currInvoker = func(currentCtx context.Context, currentMethod string, currentReq, currentRepl interface{}, currentConn *grpc.ClientConn, currentOpts ...grpc.CallOption) error { + return interceptors[i](currentCtx, currentMethod, currentReq, currentRepl, currentConn, innerInvoker, currentOpts...) + } + } + return interceptors[0](ctx, method, req, reply, cc, currInvoker, opts...) + } +} + +// ChainStreamClient creates a single interceptor out of a chain of many interceptors. +// +// Execution is done in left-to-right order, including passing of context. +// For example ChainStreamClient(one, two, three) will execute one before two before three. +func ChainStreamClient(interceptors ...grpc.StreamClientInterceptor) grpc.StreamClientInterceptor { + n := len(interceptors) + + // Dummy interceptor maintained for backward compatibility to avoid returning nil. + if n == 0 { + return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + return streamer(ctx, desc, cc, method, opts...) + } + } + + if n == 1 { + return interceptors[0] + } + + return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + currStreamer := streamer + for i := n - 1; i > 0; i-- { + innerStreamer, i := currStreamer, i + currStreamer = func(currentCtx context.Context, currentDesc *grpc.StreamDesc, currentConn *grpc.ClientConn, currentMethod string, currentOpts ...grpc.CallOption) (grpc.ClientStream, error) { + return interceptors[i](currentCtx, currentDesc, currentConn, currentMethod, innerStreamer, currentOpts...) + } + } + return interceptors[0](ctx, desc, cc, method, currStreamer, opts...) + } +} + +// Chain creates a single interceptor out of a chain of many interceptors. +// +// WithUnaryServerChain is a grpc.Server config option that accepts multiple unary interceptors. +// Basically syntactic sugar. +// +// Deprecated: use google.golang.org/grpc.ChainUnaryInterceptor instead. +func WithUnaryServerChain(interceptors ...grpc.UnaryServerInterceptor) grpc.ServerOption { + return grpc.ChainUnaryInterceptor(interceptors...) +} + +// WithStreamServerChain is a grpc.Server config option that accepts multiple stream interceptors. +// Basically syntactic sugar. +// +// Deprecated: use google.golang.org/grpc.ChainStreamInterceptor instead. +func WithStreamServerChain(interceptors ...grpc.StreamServerInterceptor) grpc.ServerOption { + return grpc.ChainStreamInterceptor(interceptors...) +} diff --git a/vendor/github.com/grpc-ecosystem/go-grpc-middleware/doc.go b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/doc.go new file mode 100644 index 000000000..718e10046 --- /dev/null +++ b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/doc.go @@ -0,0 +1,69 @@ +// Copyright 2016 Michal Witkowski. All Rights Reserved. +// See LICENSE for licensing terms. + +/* +`grpc_middleware` is a collection of gRPC middleware packages: interceptors, helpers and tools. + +Middleware + +gRPC is a fantastic RPC middleware, which sees a lot of adoption in the Golang world. However, the +upstream gRPC codebase is relatively bare bones. + +This package, and most of its child packages provides commonly needed middleware for gRPC: +client-side interceptors for retires, server-side interceptors for input validation and auth, +functions for chaining said interceptors, metadata convenience methods and more. + +Chaining + +By default, gRPC doesn't allow one to have more than one interceptor either on the client nor on +the server side. `grpc_middleware` provides convenient chaining methods + +Simple way of turning a multiple interceptors into a single interceptor. Here's an example for +server chaining: + + myServer := grpc.NewServer( + grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(loggingStream, monitoringStream, authStream)), + grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(loggingUnary, monitoringUnary, authUnary)), + ) + +These interceptors will be executed from left to right: logging, monitoring and auth. + +Here's an example for client side chaining: + + clientConn, err = grpc.Dial( + address, + grpc.WithUnaryInterceptor(grpc_middleware.ChainUnaryClient(monitoringClientUnary, retryUnary)), + grpc.WithStreamInterceptor(grpc_middleware.ChainStreamClient(monitoringClientStream, retryStream)), + ) + client = pb_testproto.NewTestServiceClient(clientConn) + resp, err := client.PingEmpty(s.ctx, &myservice.Request{Msg: "hello"}) + +These interceptors will be executed from left to right: monitoring and then retry logic. + +The retry interceptor will call every interceptor that follows it whenever when a retry happens. + +Writing Your Own + +Implementing your own interceptor is pretty trivial: there are interfaces for that. But the interesting +bit exposing common data to handlers (and other middleware), similarly to HTTP Middleware design. +For example, you may want to pass the identity of the caller from the auth interceptor all the way +to the handling function. + +For example, a client side interceptor example for auth looks like: + + func FakeAuthUnaryInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + newCtx := context.WithValue(ctx, "user_id", "john@example.com") + return handler(newCtx, req) + } + +Unfortunately, it's not as easy for streaming RPCs. These have the `context.Context` embedded within +the `grpc.ServerStream` object. To pass values through context, a wrapper (`WrappedServerStream`) is +needed. For example: + + func FakeAuthStreamingInterceptor(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + newStream := grpc_middleware.WrapServerStream(stream) + newStream.WrappedContext = context.WithValue(ctx, "user_id", "john@example.com") + return handler(srv, newStream) + } +*/ +package grpc_middleware diff --git a/vendor/github.com/grpc-ecosystem/go-grpc-middleware/makefile b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/makefile new file mode 100644 index 000000000..b18d2d2bb --- /dev/null +++ b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/makefile @@ -0,0 +1,17 @@ +SHELL=/bin/bash + +GOFILES_NOVENDOR = $(shell go list ./... | grep -v /vendor/) + +all: vet fmt test + +fmt: + go fmt $(GOFILES_NOVENDOR) + +vet: + # do not check lostcancel, they are intentional. + go vet -lostcancel=false $(GOFILES_NOVENDOR) + +test: vet + ./scripts/test_all.sh + +.PHONY: all test diff --git a/vendor/github.com/grpc-ecosystem/go-grpc-middleware/retry/backoff.go b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/retry/backoff.go new file mode 100644 index 000000000..ad35f09a8 --- /dev/null +++ b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/retry/backoff.go @@ -0,0 +1,44 @@ +// Copyright 2016 Michal Witkowski. All Rights Reserved. +// See LICENSE for licensing terms. + +package grpc_retry + +import ( + "time" + + "github.com/grpc-ecosystem/go-grpc-middleware/util/backoffutils" +) + +// BackoffLinear is very simple: it waits for a fixed period of time between calls. +func BackoffLinear(waitBetween time.Duration) BackoffFunc { + return func(attempt uint) time.Duration { + return waitBetween + } +} + +// BackoffLinearWithJitter waits a set period of time, allowing for jitter (fractional adjustment). +// +// For example waitBetween=1s and jitter=0.10 can generate waits between 900ms and 1100ms. +func BackoffLinearWithJitter(waitBetween time.Duration, jitterFraction float64) BackoffFunc { + return func(attempt uint) time.Duration { + return backoffutils.JitterUp(waitBetween, jitterFraction) + } +} + +// BackoffExponential produces increasing intervals for each attempt. +// +// The scalar is multiplied times 2 raised to the current attempt. So the first +// retry with a scalar of 100ms is 100ms, while the 5th attempt would be 1.6s. +func BackoffExponential(scalar time.Duration) BackoffFunc { + return func(attempt uint) time.Duration { + return scalar * time.Duration(backoffutils.ExponentBase2(attempt)) + } +} + +// BackoffExponentialWithJitter creates an exponential backoff like +// BackoffExponential does, but adds jitter. +func BackoffExponentialWithJitter(scalar time.Duration, jitterFraction float64) BackoffFunc { + return func(attempt uint) time.Duration { + return backoffutils.JitterUp(scalar*time.Duration(backoffutils.ExponentBase2(attempt)), jitterFraction) + } +} diff --git a/vendor/github.com/grpc-ecosystem/go-grpc-middleware/retry/doc.go b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/retry/doc.go new file mode 100644 index 000000000..f8ba7198a --- /dev/null +++ b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/retry/doc.go @@ -0,0 +1,25 @@ +// Copyright 2016 Michal Witkowski. All Rights Reserved. +// See LICENSE for licensing terms. + +/* +`grpc_retry` provides client-side request retry logic for gRPC. + +Client-Side Request Retry Interceptor + +It allows for automatic retry, inside the generated gRPC code of requests based on the gRPC status +of the reply. It supports unary (1:1), and server stream (1:n) requests. + +By default the interceptors *are disabled*, preventing accidental use of retries. You can easily +override the number of retries (setting them to more than 0) with a `grpc.ClientOption`, e.g.: + + myclient.Ping(ctx, goodPing, grpc_retry.WithMax(5)) + +Other default options are: retry on `ResourceExhausted` and `Unavailable` gRPC codes, use a 50ms +linear backoff with 10% jitter. + +For chained interceptors, the retry interceptor will call every interceptor that follows it +whenever a retry happens. + +Please see examples for more advanced use. +*/ +package grpc_retry diff --git a/vendor/github.com/grpc-ecosystem/go-grpc-middleware/retry/options.go b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/retry/options.go new file mode 100644 index 000000000..7a633e293 --- /dev/null +++ b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/retry/options.go @@ -0,0 +1,142 @@ +// Copyright 2016 Michal Witkowski. All Rights Reserved. +// See LICENSE for licensing terms. + +package grpc_retry + +import ( + "context" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" +) + +var ( + // DefaultRetriableCodes is a set of well known types gRPC codes that should be retri-able. + // + // `ResourceExhausted` means that the user quota, e.g. per-RPC limits, have been reached. + // `Unavailable` means that system is currently unavailable and the client should retry again. + DefaultRetriableCodes = []codes.Code{codes.ResourceExhausted, codes.Unavailable} + + defaultOptions = &options{ + max: 0, // disabled + perCallTimeout: 0, // disabled + includeHeader: true, + codes: DefaultRetriableCodes, + backoffFunc: BackoffFuncContext(func(ctx context.Context, attempt uint) time.Duration { + return BackoffLinearWithJitter(50*time.Millisecond /*jitter*/, 0.10)(attempt) + }), + } +) + +// BackoffFunc denotes a family of functions that control the backoff duration between call retries. +// +// They are called with an identifier of the attempt, and should return a time the system client should +// hold off for. If the time returned is longer than the `context.Context.Deadline` of the request +// the deadline of the request takes precedence and the wait will be interrupted before proceeding +// with the next iteration. +type BackoffFunc func(attempt uint) time.Duration + +// BackoffFuncContext denotes a family of functions that control the backoff duration between call retries. +// +// They are called with an identifier of the attempt, and should return a time the system client should +// hold off for. If the time returned is longer than the `context.Context.Deadline` of the request +// the deadline of the request takes precedence and the wait will be interrupted before proceeding +// with the next iteration. The context can be used to extract request scoped metadata and context values. +type BackoffFuncContext func(ctx context.Context, attempt uint) time.Duration + +// Disable disables the retry behaviour on this call, or this interceptor. +// +// Its semantically the same to `WithMax` +func Disable() CallOption { + return WithMax(0) +} + +// WithMax sets the maximum number of retries on this call, or this interceptor. +func WithMax(maxRetries uint) CallOption { + return CallOption{applyFunc: func(o *options) { + o.max = maxRetries + }} +} + +// WithBackoff sets the `BackoffFunc` used to control time between retries. +func WithBackoff(bf BackoffFunc) CallOption { + return CallOption{applyFunc: func(o *options) { + o.backoffFunc = BackoffFuncContext(func(ctx context.Context, attempt uint) time.Duration { + return bf(attempt) + }) + }} +} + +// WithBackoffContext sets the `BackoffFuncContext` used to control time between retries. +func WithBackoffContext(bf BackoffFuncContext) CallOption { + return CallOption{applyFunc: func(o *options) { + o.backoffFunc = bf + }} +} + +// WithCodes sets which codes should be retried. +// +// Please *use with care*, as you may be retrying non-idempotent calls. +// +// You cannot automatically retry on Cancelled and Deadline, please use `WithPerRetryTimeout` for these. +func WithCodes(retryCodes ...codes.Code) CallOption { + return CallOption{applyFunc: func(o *options) { + o.codes = retryCodes + }} +} + +// WithPerRetryTimeout sets the RPC timeout per call (including initial call) on this call, or this interceptor. +// +// The context.Deadline of the call takes precedence and sets the maximum time the whole invocation +// will take, but WithPerRetryTimeout can be used to limit the RPC time per each call. +// +// For example, with context.Deadline = now + 10s, and WithPerRetryTimeout(3 * time.Seconds), each +// of the retry calls (including the initial one) will have a deadline of now + 3s. +// +// A value of 0 disables the timeout overrides completely and returns to each retry call using the +// parent `context.Deadline`. +// +// Note that when this is enabled, any DeadlineExceeded errors that are propagated up will be retried. +func WithPerRetryTimeout(timeout time.Duration) CallOption { + return CallOption{applyFunc: func(o *options) { + o.perCallTimeout = timeout + }} +} + +type options struct { + max uint + perCallTimeout time.Duration + includeHeader bool + codes []codes.Code + backoffFunc BackoffFuncContext +} + +// CallOption is a grpc.CallOption that is local to grpc_retry. +type CallOption struct { + grpc.EmptyCallOption // make sure we implement private after() and before() fields so we don't panic. + applyFunc func(opt *options) +} + +func reuseOrNewWithCallOptions(opt *options, callOptions []CallOption) *options { + if len(callOptions) == 0 { + return opt + } + optCopy := &options{} + *optCopy = *opt + for _, f := range callOptions { + f.applyFunc(optCopy) + } + return optCopy +} + +func filterCallOptions(callOptions []grpc.CallOption) (grpcOptions []grpc.CallOption, retryOptions []CallOption) { + for _, opt := range callOptions { + if co, ok := opt.(CallOption); ok { + retryOptions = append(retryOptions, co) + } else { + grpcOptions = append(grpcOptions, opt) + } + } + return grpcOptions, retryOptions +} diff --git a/vendor/github.com/grpc-ecosystem/go-grpc-middleware/retry/retry.go b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/retry/retry.go new file mode 100644 index 000000000..003bbd906 --- /dev/null +++ b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/retry/retry.go @@ -0,0 +1,319 @@ +// Copyright 2016 Michal Witkowski. All Rights Reserved. +// See LICENSE for licensing terms. + +package grpc_retry + +import ( + "context" + "io" + "strconv" + "sync" + "time" + + "github.com/grpc-ecosystem/go-grpc-middleware/util/metautils" + "golang.org/x/net/trace" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +const ( + AttemptMetadataKey = "x-retry-attempty" +) + +// UnaryClientInterceptor returns a new retrying unary client interceptor. +// +// The default configuration of the interceptor is to not retry *at all*. This behaviour can be +// changed through options (e.g. WithMax) on creation of the interceptor or on call (through grpc.CallOptions). +func UnaryClientInterceptor(optFuncs ...CallOption) grpc.UnaryClientInterceptor { + intOpts := reuseOrNewWithCallOptions(defaultOptions, optFuncs) + return func(parentCtx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + grpcOpts, retryOpts := filterCallOptions(opts) + callOpts := reuseOrNewWithCallOptions(intOpts, retryOpts) + // short circuit for simplicity, and avoiding allocations. + if callOpts.max == 0 { + return invoker(parentCtx, method, req, reply, cc, grpcOpts...) + } + var lastErr error + for attempt := uint(0); attempt < callOpts.max; attempt++ { + if err := waitRetryBackoff(attempt, parentCtx, callOpts); err != nil { + return err + } + callCtx := perCallContext(parentCtx, callOpts, attempt) + lastErr = invoker(callCtx, method, req, reply, cc, grpcOpts...) + // TODO(mwitkow): Maybe dial and transport errors should be retriable? + if lastErr == nil { + return nil + } + logTrace(parentCtx, "grpc_retry attempt: %d, got err: %v", attempt, lastErr) + if isContextError(lastErr) { + if parentCtx.Err() != nil { + logTrace(parentCtx, "grpc_retry attempt: %d, parent context error: %v", attempt, parentCtx.Err()) + // its the parent context deadline or cancellation. + return lastErr + } else if callOpts.perCallTimeout != 0 { + // We have set a perCallTimeout in the retry middleware, which would result in a context error if + // the deadline was exceeded, in which case try again. + logTrace(parentCtx, "grpc_retry attempt: %d, context error from retry call", attempt) + continue + } + } + if !isRetriable(lastErr, callOpts) { + return lastErr + } + } + return lastErr + } +} + +// StreamClientInterceptor returns a new retrying stream client interceptor for server side streaming calls. +// +// The default configuration of the interceptor is to not retry *at all*. This behaviour can be +// changed through options (e.g. WithMax) on creation of the interceptor or on call (through grpc.CallOptions). +// +// Retry logic is available *only for ServerStreams*, i.e. 1:n streams, as the internal logic needs +// to buffer the messages sent by the client. If retry is enabled on any other streams (ClientStreams, +// BidiStreams), the retry interceptor will fail the call. +func StreamClientInterceptor(optFuncs ...CallOption) grpc.StreamClientInterceptor { + intOpts := reuseOrNewWithCallOptions(defaultOptions, optFuncs) + return func(parentCtx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + grpcOpts, retryOpts := filterCallOptions(opts) + callOpts := reuseOrNewWithCallOptions(intOpts, retryOpts) + // short circuit for simplicity, and avoiding allocations. + if callOpts.max == 0 { + return streamer(parentCtx, desc, cc, method, grpcOpts...) + } + if desc.ClientStreams { + return nil, status.Errorf(codes.Unimplemented, "grpc_retry: cannot retry on ClientStreams, set grpc_retry.Disable()") + } + + var lastErr error + for attempt := uint(0); attempt < callOpts.max; attempt++ { + if err := waitRetryBackoff(attempt, parentCtx, callOpts); err != nil { + return nil, err + } + callCtx := perCallContext(parentCtx, callOpts, 0) + + var newStreamer grpc.ClientStream + newStreamer, lastErr = streamer(callCtx, desc, cc, method, grpcOpts...) + if lastErr == nil { + retryingStreamer := &serverStreamingRetryingStream{ + ClientStream: newStreamer, + callOpts: callOpts, + parentCtx: parentCtx, + streamerCall: func(ctx context.Context) (grpc.ClientStream, error) { + return streamer(ctx, desc, cc, method, grpcOpts...) + }, + } + return retryingStreamer, nil + } + + logTrace(parentCtx, "grpc_retry attempt: %d, got err: %v", attempt, lastErr) + if isContextError(lastErr) { + if parentCtx.Err() != nil { + logTrace(parentCtx, "grpc_retry attempt: %d, parent context error: %v", attempt, parentCtx.Err()) + // its the parent context deadline or cancellation. + return nil, lastErr + } else if callOpts.perCallTimeout != 0 { + // We have set a perCallTimeout in the retry middleware, which would result in a context error if + // the deadline was exceeded, in which case try again. + logTrace(parentCtx, "grpc_retry attempt: %d, context error from retry call", attempt) + continue + } + } + if !isRetriable(lastErr, callOpts) { + return nil, lastErr + } + } + return nil, lastErr + } +} + +// type serverStreamingRetryingStream is the implementation of grpc.ClientStream that acts as a +// proxy to the underlying call. If any of the RecvMsg() calls fail, it will try to reestablish +// a new ClientStream according to the retry policy. +type serverStreamingRetryingStream struct { + grpc.ClientStream + bufferedSends []interface{} // single message that the client can sen + wasClosedSend bool // indicates that CloseSend was closed + parentCtx context.Context + callOpts *options + streamerCall func(ctx context.Context) (grpc.ClientStream, error) + mu sync.RWMutex +} + +func (s *serverStreamingRetryingStream) setStream(clientStream grpc.ClientStream) { + s.mu.Lock() + s.ClientStream = clientStream + s.mu.Unlock() +} + +func (s *serverStreamingRetryingStream) getStream() grpc.ClientStream { + s.mu.RLock() + defer s.mu.RUnlock() + return s.ClientStream +} + +func (s *serverStreamingRetryingStream) SendMsg(m interface{}) error { + s.mu.Lock() + s.bufferedSends = append(s.bufferedSends, m) + s.mu.Unlock() + return s.getStream().SendMsg(m) +} + +func (s *serverStreamingRetryingStream) CloseSend() error { + s.mu.Lock() + s.wasClosedSend = true + s.mu.Unlock() + return s.getStream().CloseSend() +} + +func (s *serverStreamingRetryingStream) Header() (metadata.MD, error) { + return s.getStream().Header() +} + +func (s *serverStreamingRetryingStream) Trailer() metadata.MD { + return s.getStream().Trailer() +} + +func (s *serverStreamingRetryingStream) RecvMsg(m interface{}) error { + attemptRetry, lastErr := s.receiveMsgAndIndicateRetry(m) + if !attemptRetry { + return lastErr // success or hard failure + } + // We start off from attempt 1, because zeroth was already made on normal SendMsg(). + for attempt := uint(1); attempt < s.callOpts.max; attempt++ { + if err := waitRetryBackoff(attempt, s.parentCtx, s.callOpts); err != nil { + return err + } + callCtx := perCallContext(s.parentCtx, s.callOpts, attempt) + newStream, err := s.reestablishStreamAndResendBuffer(callCtx) + if err != nil { + // Retry dial and transport errors of establishing stream as grpc doesn't retry. + if isRetriable(err, s.callOpts) { + continue + } + return err + } + + s.setStream(newStream) + attemptRetry, lastErr = s.receiveMsgAndIndicateRetry(m) + //fmt.Printf("Received message and indicate: %v %v\n", attemptRetry, lastErr) + if !attemptRetry { + return lastErr + } + } + return lastErr +} + +func (s *serverStreamingRetryingStream) receiveMsgAndIndicateRetry(m interface{}) (bool, error) { + err := s.getStream().RecvMsg(m) + if err == nil || err == io.EOF { + return false, err + } + if isContextError(err) { + if s.parentCtx.Err() != nil { + logTrace(s.parentCtx, "grpc_retry parent context error: %v", s.parentCtx.Err()) + return false, err + } else if s.callOpts.perCallTimeout != 0 { + // We have set a perCallTimeout in the retry middleware, which would result in a context error if + // the deadline was exceeded, in which case try again. + logTrace(s.parentCtx, "grpc_retry context error from retry call") + return true, err + } + } + return isRetriable(err, s.callOpts), err +} + +func (s *serverStreamingRetryingStream) reestablishStreamAndResendBuffer( + callCtx context.Context, +) (grpc.ClientStream, error) { + s.mu.RLock() + bufferedSends := s.bufferedSends + s.mu.RUnlock() + newStream, err := s.streamerCall(callCtx) + if err != nil { + logTrace(callCtx, "grpc_retry failed redialing new stream: %v", err) + return nil, err + } + for _, msg := range bufferedSends { + if err := newStream.SendMsg(msg); err != nil { + logTrace(callCtx, "grpc_retry failed resending message: %v", err) + return nil, err + } + } + if err := newStream.CloseSend(); err != nil { + logTrace(callCtx, "grpc_retry failed CloseSend on new stream %v", err) + return nil, err + } + return newStream, nil +} + +func waitRetryBackoff(attempt uint, parentCtx context.Context, callOpts *options) error { + var waitTime time.Duration = 0 + if attempt > 0 { + waitTime = callOpts.backoffFunc(parentCtx, attempt) + } + if waitTime > 0 { + logTrace(parentCtx, "grpc_retry attempt: %d, backoff for %v", attempt, waitTime) + timer := time.NewTimer(waitTime) + select { + case <-parentCtx.Done(): + timer.Stop() + return contextErrToGrpcErr(parentCtx.Err()) + case <-timer.C: + } + } + return nil +} + +func isRetriable(err error, callOpts *options) bool { + errCode := status.Code(err) + if isContextError(err) { + // context errors are not retriable based on user settings. + return false + } + for _, code := range callOpts.codes { + if code == errCode { + return true + } + } + return false +} + +func isContextError(err error) bool { + code := status.Code(err) + return code == codes.DeadlineExceeded || code == codes.Canceled +} + +func perCallContext(parentCtx context.Context, callOpts *options, attempt uint) context.Context { + ctx := parentCtx + if callOpts.perCallTimeout != 0 { + ctx, _ = context.WithTimeout(ctx, callOpts.perCallTimeout) + } + if attempt > 0 && callOpts.includeHeader { + mdClone := metautils.ExtractOutgoing(ctx).Clone().Set(AttemptMetadataKey, strconv.FormatUint(uint64(attempt), 10)) + ctx = mdClone.ToOutgoing(ctx) + } + return ctx +} + +func contextErrToGrpcErr(err error) error { + switch err { + case context.DeadlineExceeded: + return status.Error(codes.DeadlineExceeded, err.Error()) + case context.Canceled: + return status.Error(codes.Canceled, err.Error()) + default: + return status.Error(codes.Unknown, err.Error()) + } +} + +func logTrace(ctx context.Context, format string, a ...interface{}) { + tr, ok := trace.FromContext(ctx) + if !ok { + return + } + tr.LazyPrintf(format, a...) +} diff --git a/vendor/github.com/grpc-ecosystem/go-grpc-middleware/slack.png b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/slack.png new file mode 100644 index 000000000..cc8f9a68a Binary files /dev/null and b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/slack.png differ diff --git a/vendor/github.com/grpc-ecosystem/go-grpc-middleware/util/backoffutils/backoff.go b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/util/backoffutils/backoff.go new file mode 100644 index 000000000..4e69a6305 --- /dev/null +++ b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/util/backoffutils/backoff.go @@ -0,0 +1,28 @@ +// Copyright 2016 Michal Witkowski. All Rights Reserved. +// See LICENSE for licensing terms. + +/* +Backoff Helper Utilities + +Implements common backoff features. +*/ +package backoffutils + +import ( + "math/rand" + "time" +) + +// JitterUp adds random jitter to the duration. +// +// This adds or subtracts time from the duration within a given jitter fraction. +// For example for 10s and jitter 0.1, it will return a time within [9s, 11s]) +func JitterUp(duration time.Duration, jitter float64) time.Duration { + multiplier := jitter * (rand.Float64()*2 - 1) + return time.Duration(float64(duration) * (1 + multiplier)) +} + +// ExponentBase2 computes 2^(a-1) where a >= 1. If a is 0, the result is 0. +func ExponentBase2(a uint) uint { + return (1 << a) >> 1 +} diff --git a/vendor/github.com/grpc-ecosystem/go-grpc-middleware/util/metautils/doc.go b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/util/metautils/doc.go new file mode 100644 index 000000000..1ed9bb499 --- /dev/null +++ b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/util/metautils/doc.go @@ -0,0 +1,19 @@ +// Copyright 2016 Michal Witkowski. All Rights Reserved. +// See LICENSE for licensing terms. + +/* +Package `metautils` provides convenience functions for dealing with gRPC metadata.MD objects inside +Context handlers. + +While the upstream grpc-go package contains decent functionality (see https://github.com/grpc/grpc-go/blob/master/Documentation/grpc-metadata.md) +they are hard to use. + +The majority of functions center around the NiceMD, which is a convenience wrapper around metadata.MD. For example +the following code allows you to easily extract incoming metadata (server handler) and put it into a new client context +metadata. + + nmd := metautils.ExtractIncoming(serverCtx).Clone(":authorization", ":custom") + clientCtx := nmd.Set("x-client-header", "2").Set("x-another", "3").ToOutgoing(ctx) +*/ + +package metautils diff --git a/vendor/github.com/grpc-ecosystem/go-grpc-middleware/util/metautils/nicemd.go b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/util/metautils/nicemd.go new file mode 100644 index 000000000..15225d710 --- /dev/null +++ b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/util/metautils/nicemd.go @@ -0,0 +1,126 @@ +// Copyright 2016 Michal Witkowski. All Rights Reserved. +// See LICENSE for licensing terms. + +package metautils + +import ( + "context" + "strings" + + "google.golang.org/grpc/metadata" +) + +// NiceMD is a convenience wrapper defining extra functions on the metadata. +type NiceMD metadata.MD + +// ExtractIncoming extracts an inbound metadata from the server-side context. +// +// This function always returns a NiceMD wrapper of the metadata.MD, in case the context doesn't have metadata it returns +// a new empty NiceMD. +func ExtractIncoming(ctx context.Context) NiceMD { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return NiceMD(metadata.Pairs()) + } + return NiceMD(md) +} + +// ExtractOutgoing extracts an outbound metadata from the client-side context. +// +// This function always returns a NiceMD wrapper of the metadata.MD, in case the context doesn't have metadata it returns +// a new empty NiceMD. +func ExtractOutgoing(ctx context.Context) NiceMD { + md, ok := metadata.FromOutgoingContext(ctx) + if !ok { + return NiceMD(metadata.Pairs()) + } + return NiceMD(md) +} + +// Clone performs a *deep* copy of the metadata.MD. +// +// You can specify the lower-case copiedKeys to only copy certain allow-listed keys. If no keys are explicitly allow-listed +// all keys get copied. +func (m NiceMD) Clone(copiedKeys ...string) NiceMD { + newMd := NiceMD(metadata.Pairs()) + for k, vv := range m { + found := false + if len(copiedKeys) == 0 { + found = true + } else { + for _, allowedKey := range copiedKeys { + if strings.EqualFold(allowedKey, k) { + found = true + break + } + } + } + if !found { + continue + } + newMd[k] = make([]string, len(vv)) + copy(newMd[k], vv) + } + return newMd +} + +// ToOutgoing sets the given NiceMD as a client-side context for dispatching. +func (m NiceMD) ToOutgoing(ctx context.Context) context.Context { + return metadata.NewOutgoingContext(ctx, metadata.MD(m)) +} + +// ToIncoming sets the given NiceMD as a server-side context for dispatching. +// +// This is mostly useful in ServerInterceptors.. +func (m NiceMD) ToIncoming(ctx context.Context) context.Context { + return metadata.NewIncomingContext(ctx, metadata.MD(m)) +} + +// Get retrieves a single value from the metadata. +// +// It works analogously to http.Header.Get, returning the first value if there are many set. If the value is not set, +// an empty string is returned. +// +// The function is binary-key safe. +func (m NiceMD) Get(key string) string { + k := strings.ToLower(key) + vv, ok := m[k] + if !ok { + return "" + } + return vv[0] +} + +// Del retrieves a single value from the metadata. +// +// It works analogously to http.Header.Del, deleting all values if they exist. +// +// The function is binary-key safe. + +func (m NiceMD) Del(key string) NiceMD { + k := strings.ToLower(key) + delete(m, k) + return m +} + +// Set sets the given value in a metadata. +// +// It works analogously to http.Header.Set, overwriting all previous metadata values. +// +// The function is binary-key safe. +func (m NiceMD) Set(key string, value string) NiceMD { + k := strings.ToLower(key) + m[k] = []string{value} + return m +} + +// Add retrieves a single value from the metadata. +// +// It works analogously to http.Header.Add, as it appends to any existing values associated with key. +// +// The function is binary-key safe. +func (m NiceMD) Add(key string, value string) NiceMD { + k := strings.ToLower(key) + m[k] = append(m[k], value) + return m +} diff --git a/vendor/github.com/grpc-ecosystem/go-grpc-middleware/wrappers.go b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/wrappers.go new file mode 100644 index 000000000..05ccfb3f2 --- /dev/null +++ b/vendor/github.com/grpc-ecosystem/go-grpc-middleware/wrappers.go @@ -0,0 +1,30 @@ +// Copyright 2016 Michal Witkowski. All Rights Reserved. +// See LICENSE for licensing terms. + +package grpc_middleware + +import ( + "context" + + "google.golang.org/grpc" +) + +// WrappedServerStream is a thin wrapper around grpc.ServerStream that allows modifying context. +type WrappedServerStream struct { + grpc.ServerStream + // WrappedContext is the wrapper's own Context. You can assign it. + WrappedContext context.Context +} + +// Context returns the wrapper's WrappedContext, overwriting the nested grpc.ServerStream.Context() +func (w *WrappedServerStream) Context() context.Context { + return w.WrappedContext +} + +// WrapServerStream returns a ServerStream that has the ability to overwrite context. +func WrapServerStream(stream grpc.ServerStream) *WrappedServerStream { + if existing, ok := stream.(*WrappedServerStream); ok { + return existing + } + return &WrappedServerStream{ServerStream: stream, WrappedContext: stream.Context()} +} diff --git a/vendor/gopkg.in/yaml.v2/.travis.yml b/vendor/gopkg.in/yaml.v2/.travis.yml new file mode 100644 index 000000000..7348c50c0 --- /dev/null +++ b/vendor/gopkg.in/yaml.v2/.travis.yml @@ -0,0 +1,17 @@ +language: go + +go: + - "1.4.x" + - "1.5.x" + - "1.6.x" + - "1.7.x" + - "1.8.x" + - "1.9.x" + - "1.10.x" + - "1.11.x" + - "1.12.x" + - "1.13.x" + - "1.14.x" + - "tip" + +go_import_path: gopkg.in/yaml.v2 diff --git a/vendor/gopkg.in/yaml.v2/LICENSE b/vendor/gopkg.in/yaml.v2/LICENSE new file mode 100644 index 000000000..8dada3eda --- /dev/null +++ b/vendor/gopkg.in/yaml.v2/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/gopkg.in/yaml.v2/LICENSE.libyaml b/vendor/gopkg.in/yaml.v2/LICENSE.libyaml new file mode 100644 index 000000000..8da58fbf6 --- /dev/null +++ b/vendor/gopkg.in/yaml.v2/LICENSE.libyaml @@ -0,0 +1,31 @@ +The following files were ported to Go from C files of libyaml, and thus +are still covered by their original copyright and license: + + apic.go + emitterc.go + parserc.go + readerc.go + scannerc.go + writerc.go + yamlh.go + yamlprivateh.go + +Copyright (c) 2006 Kirill Simonov + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/gopkg.in/yaml.v2/NOTICE b/vendor/gopkg.in/yaml.v2/NOTICE new file mode 100644 index 000000000..866d74a7a --- /dev/null +++ b/vendor/gopkg.in/yaml.v2/NOTICE @@ -0,0 +1,13 @@ +Copyright 2011-2016 Canonical Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/vendor/gopkg.in/yaml.v2/README.md b/vendor/gopkg.in/yaml.v2/README.md new file mode 100644 index 000000000..b50c6e877 --- /dev/null +++ b/vendor/gopkg.in/yaml.v2/README.md @@ -0,0 +1,133 @@ +# YAML support for the Go language + +Introduction +------------ + +The yaml package enables Go programs to comfortably encode and decode YAML +values. It was developed within [Canonical](https://www.canonical.com) as +part of the [juju](https://juju.ubuntu.com) project, and is based on a +pure Go port of the well-known [libyaml](http://pyyaml.org/wiki/LibYAML) +C library to parse and generate YAML data quickly and reliably. + +Compatibility +------------- + +The yaml package supports most of YAML 1.1 and 1.2, including support for +anchors, tags, map merging, etc. Multi-document unmarshalling is not yet +implemented, and base-60 floats from YAML 1.1 are purposefully not +supported since they're a poor design and are gone in YAML 1.2. + +Installation and usage +---------------------- + +The import path for the package is *gopkg.in/yaml.v2*. + +To install it, run: + + go get gopkg.in/yaml.v2 + +API documentation +----------------- + +If opened in a browser, the import path itself leads to the API documentation: + + * [https://gopkg.in/yaml.v2](https://gopkg.in/yaml.v2) + +API stability +------------- + +The package API for yaml v2 will remain stable as described in [gopkg.in](https://gopkg.in). + + +License +------- + +The yaml package is licensed under the Apache License 2.0. Please see the LICENSE file for details. + + +Example +------- + +```Go +package main + +import ( + "fmt" + "log" + + "gopkg.in/yaml.v2" +) + +var data = ` +a: Easy! +b: + c: 2 + d: [3, 4] +` + +// Note: struct fields must be public in order for unmarshal to +// correctly populate the data. +type T struct { + A string + B struct { + RenamedC int `yaml:"c"` + D []int `yaml:",flow"` + } +} + +func main() { + t := T{} + + err := yaml.Unmarshal([]byte(data), &t) + if err != nil { + log.Fatalf("error: %v", err) + } + fmt.Printf("--- t:\n%v\n\n", t) + + d, err := yaml.Marshal(&t) + if err != nil { + log.Fatalf("error: %v", err) + } + fmt.Printf("--- t dump:\n%s\n\n", string(d)) + + m := make(map[interface{}]interface{}) + + err = yaml.Unmarshal([]byte(data), &m) + if err != nil { + log.Fatalf("error: %v", err) + } + fmt.Printf("--- m:\n%v\n\n", m) + + d, err = yaml.Marshal(&m) + if err != nil { + log.Fatalf("error: %v", err) + } + fmt.Printf("--- m dump:\n%s\n\n", string(d)) +} +``` + +This example will generate the following output: + +``` +--- t: +{Easy! {2 [3 4]}} + +--- t dump: +a: Easy! +b: + c: 2 + d: [3, 4] + + +--- m: +map[a:Easy! b:map[c:2 d:[3 4]]] + +--- m dump: +a: Easy! +b: + c: 2 + d: + - 3 + - 4 +``` + diff --git a/vendor/gopkg.in/yaml.v2/apic.go b/vendor/gopkg.in/yaml.v2/apic.go new file mode 100644 index 000000000..acf71402c --- /dev/null +++ b/vendor/gopkg.in/yaml.v2/apic.go @@ -0,0 +1,744 @@ +package yaml + +import ( + "io" +) + +func yaml_insert_token(parser *yaml_parser_t, pos int, token *yaml_token_t) { + //fmt.Println("yaml_insert_token", "pos:", pos, "typ:", token.typ, "head:", parser.tokens_head, "len:", len(parser.tokens)) + + // Check if we can move the queue at the beginning of the buffer. + if parser.tokens_head > 0 && len(parser.tokens) == cap(parser.tokens) { + if parser.tokens_head != len(parser.tokens) { + copy(parser.tokens, parser.tokens[parser.tokens_head:]) + } + parser.tokens = parser.tokens[:len(parser.tokens)-parser.tokens_head] + parser.tokens_head = 0 + } + parser.tokens = append(parser.tokens, *token) + if pos < 0 { + return + } + copy(parser.tokens[parser.tokens_head+pos+1:], parser.tokens[parser.tokens_head+pos:]) + parser.tokens[parser.tokens_head+pos] = *token +} + +// Create a new parser object. +func yaml_parser_initialize(parser *yaml_parser_t) bool { + *parser = yaml_parser_t{ + raw_buffer: make([]byte, 0, input_raw_buffer_size), + buffer: make([]byte, 0, input_buffer_size), + } + return true +} + +// Destroy a parser object. +func yaml_parser_delete(parser *yaml_parser_t) { + *parser = yaml_parser_t{} +} + +// String read handler. +func yaml_string_read_handler(parser *yaml_parser_t, buffer []byte) (n int, err error) { + if parser.input_pos == len(parser.input) { + return 0, io.EOF + } + n = copy(buffer, parser.input[parser.input_pos:]) + parser.input_pos += n + return n, nil +} + +// Reader read handler. +func yaml_reader_read_handler(parser *yaml_parser_t, buffer []byte) (n int, err error) { + return parser.input_reader.Read(buffer) +} + +// Set a string input. +func yaml_parser_set_input_string(parser *yaml_parser_t, input []byte) { + if parser.read_handler != nil { + panic("must set the input source only once") + } + parser.read_handler = yaml_string_read_handler + parser.input = input + parser.input_pos = 0 +} + +// Set a file input. +func yaml_parser_set_input_reader(parser *yaml_parser_t, r io.Reader) { + if parser.read_handler != nil { + panic("must set the input source only once") + } + parser.read_handler = yaml_reader_read_handler + parser.input_reader = r +} + +// Set the source encoding. +func yaml_parser_set_encoding(parser *yaml_parser_t, encoding yaml_encoding_t) { + if parser.encoding != yaml_ANY_ENCODING { + panic("must set the encoding only once") + } + parser.encoding = encoding +} + +var disableLineWrapping = false + +// Create a new emitter object. +func yaml_emitter_initialize(emitter *yaml_emitter_t) { + *emitter = yaml_emitter_t{ + buffer: make([]byte, output_buffer_size), + raw_buffer: make([]byte, 0, output_raw_buffer_size), + states: make([]yaml_emitter_state_t, 0, initial_stack_size), + events: make([]yaml_event_t, 0, initial_queue_size), + } + if disableLineWrapping { + emitter.best_width = -1 + } +} + +// Destroy an emitter object. +func yaml_emitter_delete(emitter *yaml_emitter_t) { + *emitter = yaml_emitter_t{} +} + +// String write handler. +func yaml_string_write_handler(emitter *yaml_emitter_t, buffer []byte) error { + *emitter.output_buffer = append(*emitter.output_buffer, buffer...) + return nil +} + +// yaml_writer_write_handler uses emitter.output_writer to write the +// emitted text. +func yaml_writer_write_handler(emitter *yaml_emitter_t, buffer []byte) error { + _, err := emitter.output_writer.Write(buffer) + return err +} + +// Set a string output. +func yaml_emitter_set_output_string(emitter *yaml_emitter_t, output_buffer *[]byte) { + if emitter.write_handler != nil { + panic("must set the output target only once") + } + emitter.write_handler = yaml_string_write_handler + emitter.output_buffer = output_buffer +} + +// Set a file output. +func yaml_emitter_set_output_writer(emitter *yaml_emitter_t, w io.Writer) { + if emitter.write_handler != nil { + panic("must set the output target only once") + } + emitter.write_handler = yaml_writer_write_handler + emitter.output_writer = w +} + +// Set the output encoding. +func yaml_emitter_set_encoding(emitter *yaml_emitter_t, encoding yaml_encoding_t) { + if emitter.encoding != yaml_ANY_ENCODING { + panic("must set the output encoding only once") + } + emitter.encoding = encoding +} + +// Set the canonical output style. +func yaml_emitter_set_canonical(emitter *yaml_emitter_t, canonical bool) { + emitter.canonical = canonical +} + +//// Set the indentation increment. +func yaml_emitter_set_indent(emitter *yaml_emitter_t, indent int) { + if indent < 2 || indent > 9 { + indent = 2 + } + emitter.best_indent = indent +} + +// Set the preferred line width. +func yaml_emitter_set_width(emitter *yaml_emitter_t, width int) { + if width < 0 { + width = -1 + } + emitter.best_width = width +} + +// Set if unescaped non-ASCII characters are allowed. +func yaml_emitter_set_unicode(emitter *yaml_emitter_t, unicode bool) { + emitter.unicode = unicode +} + +// Set the preferred line break character. +func yaml_emitter_set_break(emitter *yaml_emitter_t, line_break yaml_break_t) { + emitter.line_break = line_break +} + +///* +// * Destroy a token object. +// */ +// +//YAML_DECLARE(void) +//yaml_token_delete(yaml_token_t *token) +//{ +// assert(token); // Non-NULL token object expected. +// +// switch (token.type) +// { +// case YAML_TAG_DIRECTIVE_TOKEN: +// yaml_free(token.data.tag_directive.handle); +// yaml_free(token.data.tag_directive.prefix); +// break; +// +// case YAML_ALIAS_TOKEN: +// yaml_free(token.data.alias.value); +// break; +// +// case YAML_ANCHOR_TOKEN: +// yaml_free(token.data.anchor.value); +// break; +// +// case YAML_TAG_TOKEN: +// yaml_free(token.data.tag.handle); +// yaml_free(token.data.tag.suffix); +// break; +// +// case YAML_SCALAR_TOKEN: +// yaml_free(token.data.scalar.value); +// break; +// +// default: +// break; +// } +// +// memset(token, 0, sizeof(yaml_token_t)); +//} +// +///* +// * Check if a string is a valid UTF-8 sequence. +// * +// * Check 'reader.c' for more details on UTF-8 encoding. +// */ +// +//static int +//yaml_check_utf8(yaml_char_t *start, size_t length) +//{ +// yaml_char_t *end = start+length; +// yaml_char_t *pointer = start; +// +// while (pointer < end) { +// unsigned char octet; +// unsigned int width; +// unsigned int value; +// size_t k; +// +// octet = pointer[0]; +// width = (octet & 0x80) == 0x00 ? 1 : +// (octet & 0xE0) == 0xC0 ? 2 : +// (octet & 0xF0) == 0xE0 ? 3 : +// (octet & 0xF8) == 0xF0 ? 4 : 0; +// value = (octet & 0x80) == 0x00 ? octet & 0x7F : +// (octet & 0xE0) == 0xC0 ? octet & 0x1F : +// (octet & 0xF0) == 0xE0 ? octet & 0x0F : +// (octet & 0xF8) == 0xF0 ? octet & 0x07 : 0; +// if (!width) return 0; +// if (pointer+width > end) return 0; +// for (k = 1; k < width; k ++) { +// octet = pointer[k]; +// if ((octet & 0xC0) != 0x80) return 0; +// value = (value << 6) + (octet & 0x3F); +// } +// if (!((width == 1) || +// (width == 2 && value >= 0x80) || +// (width == 3 && value >= 0x800) || +// (width == 4 && value >= 0x10000))) return 0; +// +// pointer += width; +// } +// +// return 1; +//} +// + +// Create STREAM-START. +func yaml_stream_start_event_initialize(event *yaml_event_t, encoding yaml_encoding_t) { + *event = yaml_event_t{ + typ: yaml_STREAM_START_EVENT, + encoding: encoding, + } +} + +// Create STREAM-END. +func yaml_stream_end_event_initialize(event *yaml_event_t) { + *event = yaml_event_t{ + typ: yaml_STREAM_END_EVENT, + } +} + +// Create DOCUMENT-START. +func yaml_document_start_event_initialize( + event *yaml_event_t, + version_directive *yaml_version_directive_t, + tag_directives []yaml_tag_directive_t, + implicit bool, +) { + *event = yaml_event_t{ + typ: yaml_DOCUMENT_START_EVENT, + version_directive: version_directive, + tag_directives: tag_directives, + implicit: implicit, + } +} + +// Create DOCUMENT-END. +func yaml_document_end_event_initialize(event *yaml_event_t, implicit bool) { + *event = yaml_event_t{ + typ: yaml_DOCUMENT_END_EVENT, + implicit: implicit, + } +} + +///* +// * Create ALIAS. +// */ +// +//YAML_DECLARE(int) +//yaml_alias_event_initialize(event *yaml_event_t, anchor *yaml_char_t) +//{ +// mark yaml_mark_t = { 0, 0, 0 } +// anchor_copy *yaml_char_t = NULL +// +// assert(event) // Non-NULL event object is expected. +// assert(anchor) // Non-NULL anchor is expected. +// +// if (!yaml_check_utf8(anchor, strlen((char *)anchor))) return 0 +// +// anchor_copy = yaml_strdup(anchor) +// if (!anchor_copy) +// return 0 +// +// ALIAS_EVENT_INIT(*event, anchor_copy, mark, mark) +// +// return 1 +//} + +// Create SCALAR. +func yaml_scalar_event_initialize(event *yaml_event_t, anchor, tag, value []byte, plain_implicit, quoted_implicit bool, style yaml_scalar_style_t) bool { + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + anchor: anchor, + tag: tag, + value: value, + implicit: plain_implicit, + quoted_implicit: quoted_implicit, + style: yaml_style_t(style), + } + return true +} + +// Create SEQUENCE-START. +func yaml_sequence_start_event_initialize(event *yaml_event_t, anchor, tag []byte, implicit bool, style yaml_sequence_style_t) bool { + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(style), + } + return true +} + +// Create SEQUENCE-END. +func yaml_sequence_end_event_initialize(event *yaml_event_t) bool { + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + } + return true +} + +// Create MAPPING-START. +func yaml_mapping_start_event_initialize(event *yaml_event_t, anchor, tag []byte, implicit bool, style yaml_mapping_style_t) { + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(style), + } +} + +// Create MAPPING-END. +func yaml_mapping_end_event_initialize(event *yaml_event_t) { + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + } +} + +// Destroy an event object. +func yaml_event_delete(event *yaml_event_t) { + *event = yaml_event_t{} +} + +///* +// * Create a document object. +// */ +// +//YAML_DECLARE(int) +//yaml_document_initialize(document *yaml_document_t, +// version_directive *yaml_version_directive_t, +// tag_directives_start *yaml_tag_directive_t, +// tag_directives_end *yaml_tag_directive_t, +// start_implicit int, end_implicit int) +//{ +// struct { +// error yaml_error_type_t +// } context +// struct { +// start *yaml_node_t +// end *yaml_node_t +// top *yaml_node_t +// } nodes = { NULL, NULL, NULL } +// version_directive_copy *yaml_version_directive_t = NULL +// struct { +// start *yaml_tag_directive_t +// end *yaml_tag_directive_t +// top *yaml_tag_directive_t +// } tag_directives_copy = { NULL, NULL, NULL } +// value yaml_tag_directive_t = { NULL, NULL } +// mark yaml_mark_t = { 0, 0, 0 } +// +// assert(document) // Non-NULL document object is expected. +// assert((tag_directives_start && tag_directives_end) || +// (tag_directives_start == tag_directives_end)) +// // Valid tag directives are expected. +// +// if (!STACK_INIT(&context, nodes, INITIAL_STACK_SIZE)) goto error +// +// if (version_directive) { +// version_directive_copy = yaml_malloc(sizeof(yaml_version_directive_t)) +// if (!version_directive_copy) goto error +// version_directive_copy.major = version_directive.major +// version_directive_copy.minor = version_directive.minor +// } +// +// if (tag_directives_start != tag_directives_end) { +// tag_directive *yaml_tag_directive_t +// if (!STACK_INIT(&context, tag_directives_copy, INITIAL_STACK_SIZE)) +// goto error +// for (tag_directive = tag_directives_start +// tag_directive != tag_directives_end; tag_directive ++) { +// assert(tag_directive.handle) +// assert(tag_directive.prefix) +// if (!yaml_check_utf8(tag_directive.handle, +// strlen((char *)tag_directive.handle))) +// goto error +// if (!yaml_check_utf8(tag_directive.prefix, +// strlen((char *)tag_directive.prefix))) +// goto error +// value.handle = yaml_strdup(tag_directive.handle) +// value.prefix = yaml_strdup(tag_directive.prefix) +// if (!value.handle || !value.prefix) goto error +// if (!PUSH(&context, tag_directives_copy, value)) +// goto error +// value.handle = NULL +// value.prefix = NULL +// } +// } +// +// DOCUMENT_INIT(*document, nodes.start, nodes.end, version_directive_copy, +// tag_directives_copy.start, tag_directives_copy.top, +// start_implicit, end_implicit, mark, mark) +// +// return 1 +// +//error: +// STACK_DEL(&context, nodes) +// yaml_free(version_directive_copy) +// while (!STACK_EMPTY(&context, tag_directives_copy)) { +// value yaml_tag_directive_t = POP(&context, tag_directives_copy) +// yaml_free(value.handle) +// yaml_free(value.prefix) +// } +// STACK_DEL(&context, tag_directives_copy) +// yaml_free(value.handle) +// yaml_free(value.prefix) +// +// return 0 +//} +// +///* +// * Destroy a document object. +// */ +// +//YAML_DECLARE(void) +//yaml_document_delete(document *yaml_document_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// tag_directive *yaml_tag_directive_t +// +// context.error = YAML_NO_ERROR // Eliminate a compiler warning. +// +// assert(document) // Non-NULL document object is expected. +// +// while (!STACK_EMPTY(&context, document.nodes)) { +// node yaml_node_t = POP(&context, document.nodes) +// yaml_free(node.tag) +// switch (node.type) { +// case YAML_SCALAR_NODE: +// yaml_free(node.data.scalar.value) +// break +// case YAML_SEQUENCE_NODE: +// STACK_DEL(&context, node.data.sequence.items) +// break +// case YAML_MAPPING_NODE: +// STACK_DEL(&context, node.data.mapping.pairs) +// break +// default: +// assert(0) // Should not happen. +// } +// } +// STACK_DEL(&context, document.nodes) +// +// yaml_free(document.version_directive) +// for (tag_directive = document.tag_directives.start +// tag_directive != document.tag_directives.end +// tag_directive++) { +// yaml_free(tag_directive.handle) +// yaml_free(tag_directive.prefix) +// } +// yaml_free(document.tag_directives.start) +// +// memset(document, 0, sizeof(yaml_document_t)) +//} +// +///** +// * Get a document node. +// */ +// +//YAML_DECLARE(yaml_node_t *) +//yaml_document_get_node(document *yaml_document_t, index int) +//{ +// assert(document) // Non-NULL document object is expected. +// +// if (index > 0 && document.nodes.start + index <= document.nodes.top) { +// return document.nodes.start + index - 1 +// } +// return NULL +//} +// +///** +// * Get the root object. +// */ +// +//YAML_DECLARE(yaml_node_t *) +//yaml_document_get_root_node(document *yaml_document_t) +//{ +// assert(document) // Non-NULL document object is expected. +// +// if (document.nodes.top != document.nodes.start) { +// return document.nodes.start +// } +// return NULL +//} +// +///* +// * Add a scalar node to a document. +// */ +// +//YAML_DECLARE(int) +//yaml_document_add_scalar(document *yaml_document_t, +// tag *yaml_char_t, value *yaml_char_t, length int, +// style yaml_scalar_style_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// mark yaml_mark_t = { 0, 0, 0 } +// tag_copy *yaml_char_t = NULL +// value_copy *yaml_char_t = NULL +// node yaml_node_t +// +// assert(document) // Non-NULL document object is expected. +// assert(value) // Non-NULL value is expected. +// +// if (!tag) { +// tag = (yaml_char_t *)YAML_DEFAULT_SCALAR_TAG +// } +// +// if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error +// tag_copy = yaml_strdup(tag) +// if (!tag_copy) goto error +// +// if (length < 0) { +// length = strlen((char *)value) +// } +// +// if (!yaml_check_utf8(value, length)) goto error +// value_copy = yaml_malloc(length+1) +// if (!value_copy) goto error +// memcpy(value_copy, value, length) +// value_copy[length] = '\0' +// +// SCALAR_NODE_INIT(node, tag_copy, value_copy, length, style, mark, mark) +// if (!PUSH(&context, document.nodes, node)) goto error +// +// return document.nodes.top - document.nodes.start +// +//error: +// yaml_free(tag_copy) +// yaml_free(value_copy) +// +// return 0 +//} +// +///* +// * Add a sequence node to a document. +// */ +// +//YAML_DECLARE(int) +//yaml_document_add_sequence(document *yaml_document_t, +// tag *yaml_char_t, style yaml_sequence_style_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// mark yaml_mark_t = { 0, 0, 0 } +// tag_copy *yaml_char_t = NULL +// struct { +// start *yaml_node_item_t +// end *yaml_node_item_t +// top *yaml_node_item_t +// } items = { NULL, NULL, NULL } +// node yaml_node_t +// +// assert(document) // Non-NULL document object is expected. +// +// if (!tag) { +// tag = (yaml_char_t *)YAML_DEFAULT_SEQUENCE_TAG +// } +// +// if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error +// tag_copy = yaml_strdup(tag) +// if (!tag_copy) goto error +// +// if (!STACK_INIT(&context, items, INITIAL_STACK_SIZE)) goto error +// +// SEQUENCE_NODE_INIT(node, tag_copy, items.start, items.end, +// style, mark, mark) +// if (!PUSH(&context, document.nodes, node)) goto error +// +// return document.nodes.top - document.nodes.start +// +//error: +// STACK_DEL(&context, items) +// yaml_free(tag_copy) +// +// return 0 +//} +// +///* +// * Add a mapping node to a document. +// */ +// +//YAML_DECLARE(int) +//yaml_document_add_mapping(document *yaml_document_t, +// tag *yaml_char_t, style yaml_mapping_style_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// mark yaml_mark_t = { 0, 0, 0 } +// tag_copy *yaml_char_t = NULL +// struct { +// start *yaml_node_pair_t +// end *yaml_node_pair_t +// top *yaml_node_pair_t +// } pairs = { NULL, NULL, NULL } +// node yaml_node_t +// +// assert(document) // Non-NULL document object is expected. +// +// if (!tag) { +// tag = (yaml_char_t *)YAML_DEFAULT_MAPPING_TAG +// } +// +// if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error +// tag_copy = yaml_strdup(tag) +// if (!tag_copy) goto error +// +// if (!STACK_INIT(&context, pairs, INITIAL_STACK_SIZE)) goto error +// +// MAPPING_NODE_INIT(node, tag_copy, pairs.start, pairs.end, +// style, mark, mark) +// if (!PUSH(&context, document.nodes, node)) goto error +// +// return document.nodes.top - document.nodes.start +// +//error: +// STACK_DEL(&context, pairs) +// yaml_free(tag_copy) +// +// return 0 +//} +// +///* +// * Append an item to a sequence node. +// */ +// +//YAML_DECLARE(int) +//yaml_document_append_sequence_item(document *yaml_document_t, +// sequence int, item int) +//{ +// struct { +// error yaml_error_type_t +// } context +// +// assert(document) // Non-NULL document is required. +// assert(sequence > 0 +// && document.nodes.start + sequence <= document.nodes.top) +// // Valid sequence id is required. +// assert(document.nodes.start[sequence-1].type == YAML_SEQUENCE_NODE) +// // A sequence node is required. +// assert(item > 0 && document.nodes.start + item <= document.nodes.top) +// // Valid item id is required. +// +// if (!PUSH(&context, +// document.nodes.start[sequence-1].data.sequence.items, item)) +// return 0 +// +// return 1 +//} +// +///* +// * Append a pair of a key and a value to a mapping node. +// */ +// +//YAML_DECLARE(int) +//yaml_document_append_mapping_pair(document *yaml_document_t, +// mapping int, key int, value int) +//{ +// struct { +// error yaml_error_type_t +// } context +// +// pair yaml_node_pair_t +// +// assert(document) // Non-NULL document is required. +// assert(mapping > 0 +// && document.nodes.start + mapping <= document.nodes.top) +// // Valid mapping id is required. +// assert(document.nodes.start[mapping-1].type == YAML_MAPPING_NODE) +// // A mapping node is required. +// assert(key > 0 && document.nodes.start + key <= document.nodes.top) +// // Valid key id is required. +// assert(value > 0 && document.nodes.start + value <= document.nodes.top) +// // Valid value id is required. +// +// pair.key = key +// pair.value = value +// +// if (!PUSH(&context, +// document.nodes.start[mapping-1].data.mapping.pairs, pair)) +// return 0 +// +// return 1 +//} +// +// diff --git a/vendor/gopkg.in/yaml.v2/decode.go b/vendor/gopkg.in/yaml.v2/decode.go new file mode 100644 index 000000000..129bc2a97 --- /dev/null +++ b/vendor/gopkg.in/yaml.v2/decode.go @@ -0,0 +1,815 @@ +package yaml + +import ( + "encoding" + "encoding/base64" + "fmt" + "io" + "math" + "reflect" + "strconv" + "time" +) + +const ( + documentNode = 1 << iota + mappingNode + sequenceNode + scalarNode + aliasNode +) + +type node struct { + kind int + line, column int + tag string + // For an alias node, alias holds the resolved alias. + alias *node + value string + implicit bool + children []*node + anchors map[string]*node +} + +// ---------------------------------------------------------------------------- +// Parser, produces a node tree out of a libyaml event stream. + +type parser struct { + parser yaml_parser_t + event yaml_event_t + doc *node + doneInit bool +} + +func newParser(b []byte) *parser { + p := parser{} + if !yaml_parser_initialize(&p.parser) { + panic("failed to initialize YAML emitter") + } + if len(b) == 0 { + b = []byte{'\n'} + } + yaml_parser_set_input_string(&p.parser, b) + return &p +} + +func newParserFromReader(r io.Reader) *parser { + p := parser{} + if !yaml_parser_initialize(&p.parser) { + panic("failed to initialize YAML emitter") + } + yaml_parser_set_input_reader(&p.parser, r) + return &p +} + +func (p *parser) init() { + if p.doneInit { + return + } + p.expect(yaml_STREAM_START_EVENT) + p.doneInit = true +} + +func (p *parser) destroy() { + if p.event.typ != yaml_NO_EVENT { + yaml_event_delete(&p.event) + } + yaml_parser_delete(&p.parser) +} + +// expect consumes an event from the event stream and +// checks that it's of the expected type. +func (p *parser) expect(e yaml_event_type_t) { + if p.event.typ == yaml_NO_EVENT { + if !yaml_parser_parse(&p.parser, &p.event) { + p.fail() + } + } + if p.event.typ == yaml_STREAM_END_EVENT { + failf("attempted to go past the end of stream; corrupted value?") + } + if p.event.typ != e { + p.parser.problem = fmt.Sprintf("expected %s event but got %s", e, p.event.typ) + p.fail() + } + yaml_event_delete(&p.event) + p.event.typ = yaml_NO_EVENT +} + +// peek peeks at the next event in the event stream, +// puts the results into p.event and returns the event type. +func (p *parser) peek() yaml_event_type_t { + if p.event.typ != yaml_NO_EVENT { + return p.event.typ + } + if !yaml_parser_parse(&p.parser, &p.event) { + p.fail() + } + return p.event.typ +} + +func (p *parser) fail() { + var where string + var line int + if p.parser.problem_mark.line != 0 { + line = p.parser.problem_mark.line + // Scanner errors don't iterate line before returning error + if p.parser.error == yaml_SCANNER_ERROR { + line++ + } + } else if p.parser.context_mark.line != 0 { + line = p.parser.context_mark.line + } + if line != 0 { + where = "line " + strconv.Itoa(line) + ": " + } + var msg string + if len(p.parser.problem) > 0 { + msg = p.parser.problem + } else { + msg = "unknown problem parsing YAML content" + } + failf("%s%s", where, msg) +} + +func (p *parser) anchor(n *node, anchor []byte) { + if anchor != nil { + p.doc.anchors[string(anchor)] = n + } +} + +func (p *parser) parse() *node { + p.init() + switch p.peek() { + case yaml_SCALAR_EVENT: + return p.scalar() + case yaml_ALIAS_EVENT: + return p.alias() + case yaml_MAPPING_START_EVENT: + return p.mapping() + case yaml_SEQUENCE_START_EVENT: + return p.sequence() + case yaml_DOCUMENT_START_EVENT: + return p.document() + case yaml_STREAM_END_EVENT: + // Happens when attempting to decode an empty buffer. + return nil + default: + panic("attempted to parse unknown event: " + p.event.typ.String()) + } +} + +func (p *parser) node(kind int) *node { + return &node{ + kind: kind, + line: p.event.start_mark.line, + column: p.event.start_mark.column, + } +} + +func (p *parser) document() *node { + n := p.node(documentNode) + n.anchors = make(map[string]*node) + p.doc = n + p.expect(yaml_DOCUMENT_START_EVENT) + n.children = append(n.children, p.parse()) + p.expect(yaml_DOCUMENT_END_EVENT) + return n +} + +func (p *parser) alias() *node { + n := p.node(aliasNode) + n.value = string(p.event.anchor) + n.alias = p.doc.anchors[n.value] + if n.alias == nil { + failf("unknown anchor '%s' referenced", n.value) + } + p.expect(yaml_ALIAS_EVENT) + return n +} + +func (p *parser) scalar() *node { + n := p.node(scalarNode) + n.value = string(p.event.value) + n.tag = string(p.event.tag) + n.implicit = p.event.implicit + p.anchor(n, p.event.anchor) + p.expect(yaml_SCALAR_EVENT) + return n +} + +func (p *parser) sequence() *node { + n := p.node(sequenceNode) + p.anchor(n, p.event.anchor) + p.expect(yaml_SEQUENCE_START_EVENT) + for p.peek() != yaml_SEQUENCE_END_EVENT { + n.children = append(n.children, p.parse()) + } + p.expect(yaml_SEQUENCE_END_EVENT) + return n +} + +func (p *parser) mapping() *node { + n := p.node(mappingNode) + p.anchor(n, p.event.anchor) + p.expect(yaml_MAPPING_START_EVENT) + for p.peek() != yaml_MAPPING_END_EVENT { + n.children = append(n.children, p.parse(), p.parse()) + } + p.expect(yaml_MAPPING_END_EVENT) + return n +} + +// ---------------------------------------------------------------------------- +// Decoder, unmarshals a node into a provided value. + +type decoder struct { + doc *node + aliases map[*node]bool + mapType reflect.Type + terrors []string + strict bool + + decodeCount int + aliasCount int + aliasDepth int +} + +var ( + mapItemType = reflect.TypeOf(MapItem{}) + durationType = reflect.TypeOf(time.Duration(0)) + defaultMapType = reflect.TypeOf(map[interface{}]interface{}{}) + ifaceType = defaultMapType.Elem() + timeType = reflect.TypeOf(time.Time{}) + ptrTimeType = reflect.TypeOf(&time.Time{}) +) + +func newDecoder(strict bool) *decoder { + d := &decoder{mapType: defaultMapType, strict: strict} + d.aliases = make(map[*node]bool) + return d +} + +func (d *decoder) terror(n *node, tag string, out reflect.Value) { + if n.tag != "" { + tag = n.tag + } + value := n.value + if tag != yaml_SEQ_TAG && tag != yaml_MAP_TAG { + if len(value) > 10 { + value = " `" + value[:7] + "...`" + } else { + value = " `" + value + "`" + } + } + d.terrors = append(d.terrors, fmt.Sprintf("line %d: cannot unmarshal %s%s into %s", n.line+1, shortTag(tag), value, out.Type())) +} + +func (d *decoder) callUnmarshaler(n *node, u Unmarshaler) (good bool) { + terrlen := len(d.terrors) + err := u.UnmarshalYAML(func(v interface{}) (err error) { + defer handleErr(&err) + d.unmarshal(n, reflect.ValueOf(v)) + if len(d.terrors) > terrlen { + issues := d.terrors[terrlen:] + d.terrors = d.terrors[:terrlen] + return &TypeError{issues} + } + return nil + }) + if e, ok := err.(*TypeError); ok { + d.terrors = append(d.terrors, e.Errors...) + return false + } + if err != nil { + fail(err) + } + return true +} + +// d.prepare initializes and dereferences pointers and calls UnmarshalYAML +// if a value is found to implement it. +// It returns the initialized and dereferenced out value, whether +// unmarshalling was already done by UnmarshalYAML, and if so whether +// its types unmarshalled appropriately. +// +// If n holds a null value, prepare returns before doing anything. +func (d *decoder) prepare(n *node, out reflect.Value) (newout reflect.Value, unmarshaled, good bool) { + if n.tag == yaml_NULL_TAG || n.kind == scalarNode && n.tag == "" && (n.value == "null" || n.value == "~" || n.value == "" && n.implicit) { + return out, false, false + } + again := true + for again { + again = false + if out.Kind() == reflect.Ptr { + if out.IsNil() { + out.Set(reflect.New(out.Type().Elem())) + } + out = out.Elem() + again = true + } + if out.CanAddr() { + if u, ok := out.Addr().Interface().(Unmarshaler); ok { + good = d.callUnmarshaler(n, u) + return out, true, good + } + } + } + return out, false, false +} + +const ( + // 400,000 decode operations is ~500kb of dense object declarations, or + // ~5kb of dense object declarations with 10000% alias expansion + alias_ratio_range_low = 400000 + + // 4,000,000 decode operations is ~5MB of dense object declarations, or + // ~4.5MB of dense object declarations with 10% alias expansion + alias_ratio_range_high = 4000000 + + // alias_ratio_range is the range over which we scale allowed alias ratios + alias_ratio_range = float64(alias_ratio_range_high - alias_ratio_range_low) +) + +func allowedAliasRatio(decodeCount int) float64 { + switch { + case decodeCount <= alias_ratio_range_low: + // allow 99% to come from alias expansion for small-to-medium documents + return 0.99 + case decodeCount >= alias_ratio_range_high: + // allow 10% to come from alias expansion for very large documents + return 0.10 + default: + // scale smoothly from 99% down to 10% over the range. + // this maps to 396,000 - 400,000 allowed alias-driven decodes over the range. + // 400,000 decode operations is ~100MB of allocations in worst-case scenarios (single-item maps). + return 0.99 - 0.89*(float64(decodeCount-alias_ratio_range_low)/alias_ratio_range) + } +} + +func (d *decoder) unmarshal(n *node, out reflect.Value) (good bool) { + d.decodeCount++ + if d.aliasDepth > 0 { + d.aliasCount++ + } + if d.aliasCount > 100 && d.decodeCount > 1000 && float64(d.aliasCount)/float64(d.decodeCount) > allowedAliasRatio(d.decodeCount) { + failf("document contains excessive aliasing") + } + switch n.kind { + case documentNode: + return d.document(n, out) + case aliasNode: + return d.alias(n, out) + } + out, unmarshaled, good := d.prepare(n, out) + if unmarshaled { + return good + } + switch n.kind { + case scalarNode: + good = d.scalar(n, out) + case mappingNode: + good = d.mapping(n, out) + case sequenceNode: + good = d.sequence(n, out) + default: + panic("internal error: unknown node kind: " + strconv.Itoa(n.kind)) + } + return good +} + +func (d *decoder) document(n *node, out reflect.Value) (good bool) { + if len(n.children) == 1 { + d.doc = n + d.unmarshal(n.children[0], out) + return true + } + return false +} + +func (d *decoder) alias(n *node, out reflect.Value) (good bool) { + if d.aliases[n] { + // TODO this could actually be allowed in some circumstances. + failf("anchor '%s' value contains itself", n.value) + } + d.aliases[n] = true + d.aliasDepth++ + good = d.unmarshal(n.alias, out) + d.aliasDepth-- + delete(d.aliases, n) + return good +} + +var zeroValue reflect.Value + +func resetMap(out reflect.Value) { + for _, k := range out.MapKeys() { + out.SetMapIndex(k, zeroValue) + } +} + +func (d *decoder) scalar(n *node, out reflect.Value) bool { + var tag string + var resolved interface{} + if n.tag == "" && !n.implicit { + tag = yaml_STR_TAG + resolved = n.value + } else { + tag, resolved = resolve(n.tag, n.value) + if tag == yaml_BINARY_TAG { + data, err := base64.StdEncoding.DecodeString(resolved.(string)) + if err != nil { + failf("!!binary value contains invalid base64 data") + } + resolved = string(data) + } + } + if resolved == nil { + if out.Kind() == reflect.Map && !out.CanAddr() { + resetMap(out) + } else { + out.Set(reflect.Zero(out.Type())) + } + return true + } + if resolvedv := reflect.ValueOf(resolved); out.Type() == resolvedv.Type() { + // We've resolved to exactly the type we want, so use that. + out.Set(resolvedv) + return true + } + // Perhaps we can use the value as a TextUnmarshaler to + // set its value. + if out.CanAddr() { + u, ok := out.Addr().Interface().(encoding.TextUnmarshaler) + if ok { + var text []byte + if tag == yaml_BINARY_TAG { + text = []byte(resolved.(string)) + } else { + // We let any value be unmarshaled into TextUnmarshaler. + // That might be more lax than we'd like, but the + // TextUnmarshaler itself should bowl out any dubious values. + text = []byte(n.value) + } + err := u.UnmarshalText(text) + if err != nil { + fail(err) + } + return true + } + } + switch out.Kind() { + case reflect.String: + if tag == yaml_BINARY_TAG { + out.SetString(resolved.(string)) + return true + } + if resolved != nil { + out.SetString(n.value) + return true + } + case reflect.Interface: + if resolved == nil { + out.Set(reflect.Zero(out.Type())) + } else if tag == yaml_TIMESTAMP_TAG { + // It looks like a timestamp but for backward compatibility + // reasons we set it as a string, so that code that unmarshals + // timestamp-like values into interface{} will continue to + // see a string and not a time.Time. + // TODO(v3) Drop this. + out.Set(reflect.ValueOf(n.value)) + } else { + out.Set(reflect.ValueOf(resolved)) + } + return true + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + switch resolved := resolved.(type) { + case int: + if !out.OverflowInt(int64(resolved)) { + out.SetInt(int64(resolved)) + return true + } + case int64: + if !out.OverflowInt(resolved) { + out.SetInt(resolved) + return true + } + case uint64: + if resolved <= math.MaxInt64 && !out.OverflowInt(int64(resolved)) { + out.SetInt(int64(resolved)) + return true + } + case float64: + if resolved <= math.MaxInt64 && !out.OverflowInt(int64(resolved)) { + out.SetInt(int64(resolved)) + return true + } + case string: + if out.Type() == durationType { + d, err := time.ParseDuration(resolved) + if err == nil { + out.SetInt(int64(d)) + return true + } + } + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + switch resolved := resolved.(type) { + case int: + if resolved >= 0 && !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + case int64: + if resolved >= 0 && !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + case uint64: + if !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + case float64: + if resolved <= math.MaxUint64 && !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + } + case reflect.Bool: + switch resolved := resolved.(type) { + case bool: + out.SetBool(resolved) + return true + } + case reflect.Float32, reflect.Float64: + switch resolved := resolved.(type) { + case int: + out.SetFloat(float64(resolved)) + return true + case int64: + out.SetFloat(float64(resolved)) + return true + case uint64: + out.SetFloat(float64(resolved)) + return true + case float64: + out.SetFloat(resolved) + return true + } + case reflect.Struct: + if resolvedv := reflect.ValueOf(resolved); out.Type() == resolvedv.Type() { + out.Set(resolvedv) + return true + } + case reflect.Ptr: + if out.Type().Elem() == reflect.TypeOf(resolved) { + // TODO DOes this make sense? When is out a Ptr except when decoding a nil value? + elem := reflect.New(out.Type().Elem()) + elem.Elem().Set(reflect.ValueOf(resolved)) + out.Set(elem) + return true + } + } + d.terror(n, tag, out) + return false +} + +func settableValueOf(i interface{}) reflect.Value { + v := reflect.ValueOf(i) + sv := reflect.New(v.Type()).Elem() + sv.Set(v) + return sv +} + +func (d *decoder) sequence(n *node, out reflect.Value) (good bool) { + l := len(n.children) + + var iface reflect.Value + switch out.Kind() { + case reflect.Slice: + out.Set(reflect.MakeSlice(out.Type(), l, l)) + case reflect.Array: + if l != out.Len() { + failf("invalid array: want %d elements but got %d", out.Len(), l) + } + case reflect.Interface: + // No type hints. Will have to use a generic sequence. + iface = out + out = settableValueOf(make([]interface{}, l)) + default: + d.terror(n, yaml_SEQ_TAG, out) + return false + } + et := out.Type().Elem() + + j := 0 + for i := 0; i < l; i++ { + e := reflect.New(et).Elem() + if ok := d.unmarshal(n.children[i], e); ok { + out.Index(j).Set(e) + j++ + } + } + if out.Kind() != reflect.Array { + out.Set(out.Slice(0, j)) + } + if iface.IsValid() { + iface.Set(out) + } + return true +} + +func (d *decoder) mapping(n *node, out reflect.Value) (good bool) { + switch out.Kind() { + case reflect.Struct: + return d.mappingStruct(n, out) + case reflect.Slice: + return d.mappingSlice(n, out) + case reflect.Map: + // okay + case reflect.Interface: + if d.mapType.Kind() == reflect.Map { + iface := out + out = reflect.MakeMap(d.mapType) + iface.Set(out) + } else { + slicev := reflect.New(d.mapType).Elem() + if !d.mappingSlice(n, slicev) { + return false + } + out.Set(slicev) + return true + } + default: + d.terror(n, yaml_MAP_TAG, out) + return false + } + outt := out.Type() + kt := outt.Key() + et := outt.Elem() + + mapType := d.mapType + if outt.Key() == ifaceType && outt.Elem() == ifaceType { + d.mapType = outt + } + + if out.IsNil() { + out.Set(reflect.MakeMap(outt)) + } + l := len(n.children) + for i := 0; i < l; i += 2 { + if isMerge(n.children[i]) { + d.merge(n.children[i+1], out) + continue + } + k := reflect.New(kt).Elem() + if d.unmarshal(n.children[i], k) { + kkind := k.Kind() + if kkind == reflect.Interface { + kkind = k.Elem().Kind() + } + if kkind == reflect.Map || kkind == reflect.Slice { + failf("invalid map key: %#v", k.Interface()) + } + e := reflect.New(et).Elem() + if d.unmarshal(n.children[i+1], e) { + d.setMapIndex(n.children[i+1], out, k, e) + } + } + } + d.mapType = mapType + return true +} + +func (d *decoder) setMapIndex(n *node, out, k, v reflect.Value) { + if d.strict && out.MapIndex(k) != zeroValue { + d.terrors = append(d.terrors, fmt.Sprintf("line %d: key %#v already set in map", n.line+1, k.Interface())) + return + } + out.SetMapIndex(k, v) +} + +func (d *decoder) mappingSlice(n *node, out reflect.Value) (good bool) { + outt := out.Type() + if outt.Elem() != mapItemType { + d.terror(n, yaml_MAP_TAG, out) + return false + } + + mapType := d.mapType + d.mapType = outt + + var slice []MapItem + var l = len(n.children) + for i := 0; i < l; i += 2 { + if isMerge(n.children[i]) { + d.merge(n.children[i+1], out) + continue + } + item := MapItem{} + k := reflect.ValueOf(&item.Key).Elem() + if d.unmarshal(n.children[i], k) { + v := reflect.ValueOf(&item.Value).Elem() + if d.unmarshal(n.children[i+1], v) { + slice = append(slice, item) + } + } + } + out.Set(reflect.ValueOf(slice)) + d.mapType = mapType + return true +} + +func (d *decoder) mappingStruct(n *node, out reflect.Value) (good bool) { + sinfo, err := getStructInfo(out.Type()) + if err != nil { + panic(err) + } + name := settableValueOf("") + l := len(n.children) + + var inlineMap reflect.Value + var elemType reflect.Type + if sinfo.InlineMap != -1 { + inlineMap = out.Field(sinfo.InlineMap) + inlineMap.Set(reflect.New(inlineMap.Type()).Elem()) + elemType = inlineMap.Type().Elem() + } + + var doneFields []bool + if d.strict { + doneFields = make([]bool, len(sinfo.FieldsList)) + } + for i := 0; i < l; i += 2 { + ni := n.children[i] + if isMerge(ni) { + d.merge(n.children[i+1], out) + continue + } + if !d.unmarshal(ni, name) { + continue + } + if info, ok := sinfo.FieldsMap[name.String()]; ok { + if d.strict { + if doneFields[info.Id] { + d.terrors = append(d.terrors, fmt.Sprintf("line %d: field %s already set in type %s", ni.line+1, name.String(), out.Type())) + continue + } + doneFields[info.Id] = true + } + var field reflect.Value + if info.Inline == nil { + field = out.Field(info.Num) + } else { + field = out.FieldByIndex(info.Inline) + } + d.unmarshal(n.children[i+1], field) + } else if sinfo.InlineMap != -1 { + if inlineMap.IsNil() { + inlineMap.Set(reflect.MakeMap(inlineMap.Type())) + } + value := reflect.New(elemType).Elem() + d.unmarshal(n.children[i+1], value) + d.setMapIndex(n.children[i+1], inlineMap, name, value) + } else if d.strict { + d.terrors = append(d.terrors, fmt.Sprintf("line %d: field %s not found in type %s", ni.line+1, name.String(), out.Type())) + } + } + return true +} + +func failWantMap() { + failf("map merge requires map or sequence of maps as the value") +} + +func (d *decoder) merge(n *node, out reflect.Value) { + switch n.kind { + case mappingNode: + d.unmarshal(n, out) + case aliasNode: + if n.alias != nil && n.alias.kind != mappingNode { + failWantMap() + } + d.unmarshal(n, out) + case sequenceNode: + // Step backwards as earlier nodes take precedence. + for i := len(n.children) - 1; i >= 0; i-- { + ni := n.children[i] + if ni.kind == aliasNode { + if ni.alias != nil && ni.alias.kind != mappingNode { + failWantMap() + } + } else if ni.kind != mappingNode { + failWantMap() + } + d.unmarshal(ni, out) + } + default: + failWantMap() + } +} + +func isMerge(n *node) bool { + return n.kind == scalarNode && n.value == "<<" && (n.implicit == true || n.tag == yaml_MERGE_TAG) +} diff --git a/vendor/gopkg.in/yaml.v2/emitterc.go b/vendor/gopkg.in/yaml.v2/emitterc.go new file mode 100644 index 000000000..a1c2cc526 --- /dev/null +++ b/vendor/gopkg.in/yaml.v2/emitterc.go @@ -0,0 +1,1685 @@ +package yaml + +import ( + "bytes" + "fmt" +) + +// Flush the buffer if needed. +func flush(emitter *yaml_emitter_t) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) { + return yaml_emitter_flush(emitter) + } + return true +} + +// Put a character to the output buffer. +func put(emitter *yaml_emitter_t, value byte) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) && !yaml_emitter_flush(emitter) { + return false + } + emitter.buffer[emitter.buffer_pos] = value + emitter.buffer_pos++ + emitter.column++ + return true +} + +// Put a line break to the output buffer. +func put_break(emitter *yaml_emitter_t) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) && !yaml_emitter_flush(emitter) { + return false + } + switch emitter.line_break { + case yaml_CR_BREAK: + emitter.buffer[emitter.buffer_pos] = '\r' + emitter.buffer_pos += 1 + case yaml_LN_BREAK: + emitter.buffer[emitter.buffer_pos] = '\n' + emitter.buffer_pos += 1 + case yaml_CRLN_BREAK: + emitter.buffer[emitter.buffer_pos+0] = '\r' + emitter.buffer[emitter.buffer_pos+1] = '\n' + emitter.buffer_pos += 2 + default: + panic("unknown line break setting") + } + emitter.column = 0 + emitter.line++ + return true +} + +// Copy a character from a string into buffer. +func write(emitter *yaml_emitter_t, s []byte, i *int) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) && !yaml_emitter_flush(emitter) { + return false + } + p := emitter.buffer_pos + w := width(s[*i]) + switch w { + case 4: + emitter.buffer[p+3] = s[*i+3] + fallthrough + case 3: + emitter.buffer[p+2] = s[*i+2] + fallthrough + case 2: + emitter.buffer[p+1] = s[*i+1] + fallthrough + case 1: + emitter.buffer[p+0] = s[*i+0] + default: + panic("unknown character width") + } + emitter.column++ + emitter.buffer_pos += w + *i += w + return true +} + +// Write a whole string into buffer. +func write_all(emitter *yaml_emitter_t, s []byte) bool { + for i := 0; i < len(s); { + if !write(emitter, s, &i) { + return false + } + } + return true +} + +// Copy a line break character from a string into buffer. +func write_break(emitter *yaml_emitter_t, s []byte, i *int) bool { + if s[*i] == '\n' { + if !put_break(emitter) { + return false + } + *i++ + } else { + if !write(emitter, s, i) { + return false + } + emitter.column = 0 + emitter.line++ + } + return true +} + +// Set an emitter error and return false. +func yaml_emitter_set_emitter_error(emitter *yaml_emitter_t, problem string) bool { + emitter.error = yaml_EMITTER_ERROR + emitter.problem = problem + return false +} + +// Emit an event. +func yaml_emitter_emit(emitter *yaml_emitter_t, event *yaml_event_t) bool { + emitter.events = append(emitter.events, *event) + for !yaml_emitter_need_more_events(emitter) { + event := &emitter.events[emitter.events_head] + if !yaml_emitter_analyze_event(emitter, event) { + return false + } + if !yaml_emitter_state_machine(emitter, event) { + return false + } + yaml_event_delete(event) + emitter.events_head++ + } + return true +} + +// Check if we need to accumulate more events before emitting. +// +// We accumulate extra +// - 1 event for DOCUMENT-START +// - 2 events for SEQUENCE-START +// - 3 events for MAPPING-START +// +func yaml_emitter_need_more_events(emitter *yaml_emitter_t) bool { + if emitter.events_head == len(emitter.events) { + return true + } + var accumulate int + switch emitter.events[emitter.events_head].typ { + case yaml_DOCUMENT_START_EVENT: + accumulate = 1 + break + case yaml_SEQUENCE_START_EVENT: + accumulate = 2 + break + case yaml_MAPPING_START_EVENT: + accumulate = 3 + break + default: + return false + } + if len(emitter.events)-emitter.events_head > accumulate { + return false + } + var level int + for i := emitter.events_head; i < len(emitter.events); i++ { + switch emitter.events[i].typ { + case yaml_STREAM_START_EVENT, yaml_DOCUMENT_START_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT: + level++ + case yaml_STREAM_END_EVENT, yaml_DOCUMENT_END_EVENT, yaml_SEQUENCE_END_EVENT, yaml_MAPPING_END_EVENT: + level-- + } + if level == 0 { + return false + } + } + return true +} + +// Append a directive to the directives stack. +func yaml_emitter_append_tag_directive(emitter *yaml_emitter_t, value *yaml_tag_directive_t, allow_duplicates bool) bool { + for i := 0; i < len(emitter.tag_directives); i++ { + if bytes.Equal(value.handle, emitter.tag_directives[i].handle) { + if allow_duplicates { + return true + } + return yaml_emitter_set_emitter_error(emitter, "duplicate %TAG directive") + } + } + + // [Go] Do we actually need to copy this given garbage collection + // and the lack of deallocating destructors? + tag_copy := yaml_tag_directive_t{ + handle: make([]byte, len(value.handle)), + prefix: make([]byte, len(value.prefix)), + } + copy(tag_copy.handle, value.handle) + copy(tag_copy.prefix, value.prefix) + emitter.tag_directives = append(emitter.tag_directives, tag_copy) + return true +} + +// Increase the indentation level. +func yaml_emitter_increase_indent(emitter *yaml_emitter_t, flow, indentless bool) bool { + emitter.indents = append(emitter.indents, emitter.indent) + if emitter.indent < 0 { + if flow { + emitter.indent = emitter.best_indent + } else { + emitter.indent = 0 + } + } else if !indentless { + emitter.indent += emitter.best_indent + } + return true +} + +// State dispatcher. +func yaml_emitter_state_machine(emitter *yaml_emitter_t, event *yaml_event_t) bool { + switch emitter.state { + default: + case yaml_EMIT_STREAM_START_STATE: + return yaml_emitter_emit_stream_start(emitter, event) + + case yaml_EMIT_FIRST_DOCUMENT_START_STATE: + return yaml_emitter_emit_document_start(emitter, event, true) + + case yaml_EMIT_DOCUMENT_START_STATE: + return yaml_emitter_emit_document_start(emitter, event, false) + + case yaml_EMIT_DOCUMENT_CONTENT_STATE: + return yaml_emitter_emit_document_content(emitter, event) + + case yaml_EMIT_DOCUMENT_END_STATE: + return yaml_emitter_emit_document_end(emitter, event) + + case yaml_EMIT_FLOW_SEQUENCE_FIRST_ITEM_STATE: + return yaml_emitter_emit_flow_sequence_item(emitter, event, true) + + case yaml_EMIT_FLOW_SEQUENCE_ITEM_STATE: + return yaml_emitter_emit_flow_sequence_item(emitter, event, false) + + case yaml_EMIT_FLOW_MAPPING_FIRST_KEY_STATE: + return yaml_emitter_emit_flow_mapping_key(emitter, event, true) + + case yaml_EMIT_FLOW_MAPPING_KEY_STATE: + return yaml_emitter_emit_flow_mapping_key(emitter, event, false) + + case yaml_EMIT_FLOW_MAPPING_SIMPLE_VALUE_STATE: + return yaml_emitter_emit_flow_mapping_value(emitter, event, true) + + case yaml_EMIT_FLOW_MAPPING_VALUE_STATE: + return yaml_emitter_emit_flow_mapping_value(emitter, event, false) + + case yaml_EMIT_BLOCK_SEQUENCE_FIRST_ITEM_STATE: + return yaml_emitter_emit_block_sequence_item(emitter, event, true) + + case yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE: + return yaml_emitter_emit_block_sequence_item(emitter, event, false) + + case yaml_EMIT_BLOCK_MAPPING_FIRST_KEY_STATE: + return yaml_emitter_emit_block_mapping_key(emitter, event, true) + + case yaml_EMIT_BLOCK_MAPPING_KEY_STATE: + return yaml_emitter_emit_block_mapping_key(emitter, event, false) + + case yaml_EMIT_BLOCK_MAPPING_SIMPLE_VALUE_STATE: + return yaml_emitter_emit_block_mapping_value(emitter, event, true) + + case yaml_EMIT_BLOCK_MAPPING_VALUE_STATE: + return yaml_emitter_emit_block_mapping_value(emitter, event, false) + + case yaml_EMIT_END_STATE: + return yaml_emitter_set_emitter_error(emitter, "expected nothing after STREAM-END") + } + panic("invalid emitter state") +} + +// Expect STREAM-START. +func yaml_emitter_emit_stream_start(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if event.typ != yaml_STREAM_START_EVENT { + return yaml_emitter_set_emitter_error(emitter, "expected STREAM-START") + } + if emitter.encoding == yaml_ANY_ENCODING { + emitter.encoding = event.encoding + if emitter.encoding == yaml_ANY_ENCODING { + emitter.encoding = yaml_UTF8_ENCODING + } + } + if emitter.best_indent < 2 || emitter.best_indent > 9 { + emitter.best_indent = 2 + } + if emitter.best_width >= 0 && emitter.best_width <= emitter.best_indent*2 { + emitter.best_width = 80 + } + if emitter.best_width < 0 { + emitter.best_width = 1<<31 - 1 + } + if emitter.line_break == yaml_ANY_BREAK { + emitter.line_break = yaml_LN_BREAK + } + + emitter.indent = -1 + emitter.line = 0 + emitter.column = 0 + emitter.whitespace = true + emitter.indention = true + + if emitter.encoding != yaml_UTF8_ENCODING { + if !yaml_emitter_write_bom(emitter) { + return false + } + } + emitter.state = yaml_EMIT_FIRST_DOCUMENT_START_STATE + return true +} + +// Expect DOCUMENT-START or STREAM-END. +func yaml_emitter_emit_document_start(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + + if event.typ == yaml_DOCUMENT_START_EVENT { + + if event.version_directive != nil { + if !yaml_emitter_analyze_version_directive(emitter, event.version_directive) { + return false + } + } + + for i := 0; i < len(event.tag_directives); i++ { + tag_directive := &event.tag_directives[i] + if !yaml_emitter_analyze_tag_directive(emitter, tag_directive) { + return false + } + if !yaml_emitter_append_tag_directive(emitter, tag_directive, false) { + return false + } + } + + for i := 0; i < len(default_tag_directives); i++ { + tag_directive := &default_tag_directives[i] + if !yaml_emitter_append_tag_directive(emitter, tag_directive, true) { + return false + } + } + + implicit := event.implicit + if !first || emitter.canonical { + implicit = false + } + + if emitter.open_ended && (event.version_directive != nil || len(event.tag_directives) > 0) { + if !yaml_emitter_write_indicator(emitter, []byte("..."), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + + if event.version_directive != nil { + implicit = false + if !yaml_emitter_write_indicator(emitter, []byte("%YAML"), true, false, false) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte("1.1"), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + + if len(event.tag_directives) > 0 { + implicit = false + for i := 0; i < len(event.tag_directives); i++ { + tag_directive := &event.tag_directives[i] + if !yaml_emitter_write_indicator(emitter, []byte("%TAG"), true, false, false) { + return false + } + if !yaml_emitter_write_tag_handle(emitter, tag_directive.handle) { + return false + } + if !yaml_emitter_write_tag_content(emitter, tag_directive.prefix, true) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + } + + if yaml_emitter_check_empty_document(emitter) { + implicit = false + } + if !implicit { + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte("---"), true, false, false) { + return false + } + if emitter.canonical { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + } + + emitter.state = yaml_EMIT_DOCUMENT_CONTENT_STATE + return true + } + + if event.typ == yaml_STREAM_END_EVENT { + if emitter.open_ended { + if !yaml_emitter_write_indicator(emitter, []byte("..."), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_flush(emitter) { + return false + } + emitter.state = yaml_EMIT_END_STATE + return true + } + + return yaml_emitter_set_emitter_error(emitter, "expected DOCUMENT-START or STREAM-END") +} + +// Expect the root node. +func yaml_emitter_emit_document_content(emitter *yaml_emitter_t, event *yaml_event_t) bool { + emitter.states = append(emitter.states, yaml_EMIT_DOCUMENT_END_STATE) + return yaml_emitter_emit_node(emitter, event, true, false, false, false) +} + +// Expect DOCUMENT-END. +func yaml_emitter_emit_document_end(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if event.typ != yaml_DOCUMENT_END_EVENT { + return yaml_emitter_set_emitter_error(emitter, "expected DOCUMENT-END") + } + if !yaml_emitter_write_indent(emitter) { + return false + } + if !event.implicit { + // [Go] Allocate the slice elsewhere. + if !yaml_emitter_write_indicator(emitter, []byte("..."), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_flush(emitter) { + return false + } + emitter.state = yaml_EMIT_DOCUMENT_START_STATE + emitter.tag_directives = emitter.tag_directives[:0] + return true +} + +// Expect a flow item node. +func yaml_emitter_emit_flow_sequence_item(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + if first { + if !yaml_emitter_write_indicator(emitter, []byte{'['}, true, true, false) { + return false + } + if !yaml_emitter_increase_indent(emitter, true, false) { + return false + } + emitter.flow_level++ + } + + if event.typ == yaml_SEQUENCE_END_EVENT { + emitter.flow_level-- + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + if emitter.canonical && !first { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{']'}, false, false, false) { + return false + } + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + + return true + } + + if !first { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + } + + if emitter.canonical || emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + emitter.states = append(emitter.states, yaml_EMIT_FLOW_SEQUENCE_ITEM_STATE) + return yaml_emitter_emit_node(emitter, event, false, true, false, false) +} + +// Expect a flow key node. +func yaml_emitter_emit_flow_mapping_key(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + if first { + if !yaml_emitter_write_indicator(emitter, []byte{'{'}, true, true, false) { + return false + } + if !yaml_emitter_increase_indent(emitter, true, false) { + return false + } + emitter.flow_level++ + } + + if event.typ == yaml_MAPPING_END_EVENT { + emitter.flow_level-- + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + if emitter.canonical && !first { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{'}'}, false, false, false) { + return false + } + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true + } + + if !first { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + } + if emitter.canonical || emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + + if !emitter.canonical && yaml_emitter_check_simple_key(emitter) { + emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_SIMPLE_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, true) + } + if !yaml_emitter_write_indicator(emitter, []byte{'?'}, true, false, false) { + return false + } + emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, false) +} + +// Expect a flow value node. +func yaml_emitter_emit_flow_mapping_value(emitter *yaml_emitter_t, event *yaml_event_t, simple bool) bool { + if simple { + if !yaml_emitter_write_indicator(emitter, []byte{':'}, false, false, false) { + return false + } + } else { + if emitter.canonical || emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{':'}, true, false, false) { + return false + } + } + emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_KEY_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, false) +} + +// Expect a block item node. +func yaml_emitter_emit_block_sequence_item(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + if first { + if !yaml_emitter_increase_indent(emitter, false, emitter.mapping_context && !emitter.indention) { + return false + } + } + if event.typ == yaml_SEQUENCE_END_EVENT { + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true + } + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte{'-'}, true, false, true) { + return false + } + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE) + return yaml_emitter_emit_node(emitter, event, false, true, false, false) +} + +// Expect a block key node. +func yaml_emitter_emit_block_mapping_key(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + if first { + if !yaml_emitter_increase_indent(emitter, false, false) { + return false + } + } + if event.typ == yaml_MAPPING_END_EVENT { + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true + } + if !yaml_emitter_write_indent(emitter) { + return false + } + if yaml_emitter_check_simple_key(emitter) { + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_SIMPLE_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, true) + } + if !yaml_emitter_write_indicator(emitter, []byte{'?'}, true, false, true) { + return false + } + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, false) +} + +// Expect a block value node. +func yaml_emitter_emit_block_mapping_value(emitter *yaml_emitter_t, event *yaml_event_t, simple bool) bool { + if simple { + if !yaml_emitter_write_indicator(emitter, []byte{':'}, false, false, false) { + return false + } + } else { + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte{':'}, true, false, true) { + return false + } + } + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_KEY_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, false) +} + +// Expect a node. +func yaml_emitter_emit_node(emitter *yaml_emitter_t, event *yaml_event_t, + root bool, sequence bool, mapping bool, simple_key bool) bool { + + emitter.root_context = root + emitter.sequence_context = sequence + emitter.mapping_context = mapping + emitter.simple_key_context = simple_key + + switch event.typ { + case yaml_ALIAS_EVENT: + return yaml_emitter_emit_alias(emitter, event) + case yaml_SCALAR_EVENT: + return yaml_emitter_emit_scalar(emitter, event) + case yaml_SEQUENCE_START_EVENT: + return yaml_emitter_emit_sequence_start(emitter, event) + case yaml_MAPPING_START_EVENT: + return yaml_emitter_emit_mapping_start(emitter, event) + default: + return yaml_emitter_set_emitter_error(emitter, + fmt.Sprintf("expected SCALAR, SEQUENCE-START, MAPPING-START, or ALIAS, but got %v", event.typ)) + } +} + +// Expect ALIAS. +func yaml_emitter_emit_alias(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_process_anchor(emitter) { + return false + } + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true +} + +// Expect SCALAR. +func yaml_emitter_emit_scalar(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_select_scalar_style(emitter, event) { + return false + } + if !yaml_emitter_process_anchor(emitter) { + return false + } + if !yaml_emitter_process_tag(emitter) { + return false + } + if !yaml_emitter_increase_indent(emitter, true, false) { + return false + } + if !yaml_emitter_process_scalar(emitter) { + return false + } + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true +} + +// Expect SEQUENCE-START. +func yaml_emitter_emit_sequence_start(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_process_anchor(emitter) { + return false + } + if !yaml_emitter_process_tag(emitter) { + return false + } + if emitter.flow_level > 0 || emitter.canonical || event.sequence_style() == yaml_FLOW_SEQUENCE_STYLE || + yaml_emitter_check_empty_sequence(emitter) { + emitter.state = yaml_EMIT_FLOW_SEQUENCE_FIRST_ITEM_STATE + } else { + emitter.state = yaml_EMIT_BLOCK_SEQUENCE_FIRST_ITEM_STATE + } + return true +} + +// Expect MAPPING-START. +func yaml_emitter_emit_mapping_start(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_process_anchor(emitter) { + return false + } + if !yaml_emitter_process_tag(emitter) { + return false + } + if emitter.flow_level > 0 || emitter.canonical || event.mapping_style() == yaml_FLOW_MAPPING_STYLE || + yaml_emitter_check_empty_mapping(emitter) { + emitter.state = yaml_EMIT_FLOW_MAPPING_FIRST_KEY_STATE + } else { + emitter.state = yaml_EMIT_BLOCK_MAPPING_FIRST_KEY_STATE + } + return true +} + +// Check if the document content is an empty scalar. +func yaml_emitter_check_empty_document(emitter *yaml_emitter_t) bool { + return false // [Go] Huh? +} + +// Check if the next events represent an empty sequence. +func yaml_emitter_check_empty_sequence(emitter *yaml_emitter_t) bool { + if len(emitter.events)-emitter.events_head < 2 { + return false + } + return emitter.events[emitter.events_head].typ == yaml_SEQUENCE_START_EVENT && + emitter.events[emitter.events_head+1].typ == yaml_SEQUENCE_END_EVENT +} + +// Check if the next events represent an empty mapping. +func yaml_emitter_check_empty_mapping(emitter *yaml_emitter_t) bool { + if len(emitter.events)-emitter.events_head < 2 { + return false + } + return emitter.events[emitter.events_head].typ == yaml_MAPPING_START_EVENT && + emitter.events[emitter.events_head+1].typ == yaml_MAPPING_END_EVENT +} + +// Check if the next node can be expressed as a simple key. +func yaml_emitter_check_simple_key(emitter *yaml_emitter_t) bool { + length := 0 + switch emitter.events[emitter.events_head].typ { + case yaml_ALIAS_EVENT: + length += len(emitter.anchor_data.anchor) + case yaml_SCALAR_EVENT: + if emitter.scalar_data.multiline { + return false + } + length += len(emitter.anchor_data.anchor) + + len(emitter.tag_data.handle) + + len(emitter.tag_data.suffix) + + len(emitter.scalar_data.value) + case yaml_SEQUENCE_START_EVENT: + if !yaml_emitter_check_empty_sequence(emitter) { + return false + } + length += len(emitter.anchor_data.anchor) + + len(emitter.tag_data.handle) + + len(emitter.tag_data.suffix) + case yaml_MAPPING_START_EVENT: + if !yaml_emitter_check_empty_mapping(emitter) { + return false + } + length += len(emitter.anchor_data.anchor) + + len(emitter.tag_data.handle) + + len(emitter.tag_data.suffix) + default: + return false + } + return length <= 128 +} + +// Determine an acceptable scalar style. +func yaml_emitter_select_scalar_style(emitter *yaml_emitter_t, event *yaml_event_t) bool { + + no_tag := len(emitter.tag_data.handle) == 0 && len(emitter.tag_data.suffix) == 0 + if no_tag && !event.implicit && !event.quoted_implicit { + return yaml_emitter_set_emitter_error(emitter, "neither tag nor implicit flags are specified") + } + + style := event.scalar_style() + if style == yaml_ANY_SCALAR_STYLE { + style = yaml_PLAIN_SCALAR_STYLE + } + if emitter.canonical { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + if emitter.simple_key_context && emitter.scalar_data.multiline { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + + if style == yaml_PLAIN_SCALAR_STYLE { + if emitter.flow_level > 0 && !emitter.scalar_data.flow_plain_allowed || + emitter.flow_level == 0 && !emitter.scalar_data.block_plain_allowed { + style = yaml_SINGLE_QUOTED_SCALAR_STYLE + } + if len(emitter.scalar_data.value) == 0 && (emitter.flow_level > 0 || emitter.simple_key_context) { + style = yaml_SINGLE_QUOTED_SCALAR_STYLE + } + if no_tag && !event.implicit { + style = yaml_SINGLE_QUOTED_SCALAR_STYLE + } + } + if style == yaml_SINGLE_QUOTED_SCALAR_STYLE { + if !emitter.scalar_data.single_quoted_allowed { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + } + if style == yaml_LITERAL_SCALAR_STYLE || style == yaml_FOLDED_SCALAR_STYLE { + if !emitter.scalar_data.block_allowed || emitter.flow_level > 0 || emitter.simple_key_context { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + } + + if no_tag && !event.quoted_implicit && style != yaml_PLAIN_SCALAR_STYLE { + emitter.tag_data.handle = []byte{'!'} + } + emitter.scalar_data.style = style + return true +} + +// Write an anchor. +func yaml_emitter_process_anchor(emitter *yaml_emitter_t) bool { + if emitter.anchor_data.anchor == nil { + return true + } + c := []byte{'&'} + if emitter.anchor_data.alias { + c[0] = '*' + } + if !yaml_emitter_write_indicator(emitter, c, true, false, false) { + return false + } + return yaml_emitter_write_anchor(emitter, emitter.anchor_data.anchor) +} + +// Write a tag. +func yaml_emitter_process_tag(emitter *yaml_emitter_t) bool { + if len(emitter.tag_data.handle) == 0 && len(emitter.tag_data.suffix) == 0 { + return true + } + if len(emitter.tag_data.handle) > 0 { + if !yaml_emitter_write_tag_handle(emitter, emitter.tag_data.handle) { + return false + } + if len(emitter.tag_data.suffix) > 0 { + if !yaml_emitter_write_tag_content(emitter, emitter.tag_data.suffix, false) { + return false + } + } + } else { + // [Go] Allocate these slices elsewhere. + if !yaml_emitter_write_indicator(emitter, []byte("!<"), true, false, false) { + return false + } + if !yaml_emitter_write_tag_content(emitter, emitter.tag_data.suffix, false) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte{'>'}, false, false, false) { + return false + } + } + return true +} + +// Write a scalar. +func yaml_emitter_process_scalar(emitter *yaml_emitter_t) bool { + switch emitter.scalar_data.style { + case yaml_PLAIN_SCALAR_STYLE: + return yaml_emitter_write_plain_scalar(emitter, emitter.scalar_data.value, !emitter.simple_key_context) + + case yaml_SINGLE_QUOTED_SCALAR_STYLE: + return yaml_emitter_write_single_quoted_scalar(emitter, emitter.scalar_data.value, !emitter.simple_key_context) + + case yaml_DOUBLE_QUOTED_SCALAR_STYLE: + return yaml_emitter_write_double_quoted_scalar(emitter, emitter.scalar_data.value, !emitter.simple_key_context) + + case yaml_LITERAL_SCALAR_STYLE: + return yaml_emitter_write_literal_scalar(emitter, emitter.scalar_data.value) + + case yaml_FOLDED_SCALAR_STYLE: + return yaml_emitter_write_folded_scalar(emitter, emitter.scalar_data.value) + } + panic("unknown scalar style") +} + +// Check if a %YAML directive is valid. +func yaml_emitter_analyze_version_directive(emitter *yaml_emitter_t, version_directive *yaml_version_directive_t) bool { + if version_directive.major != 1 || version_directive.minor != 1 { + return yaml_emitter_set_emitter_error(emitter, "incompatible %YAML directive") + } + return true +} + +// Check if a %TAG directive is valid. +func yaml_emitter_analyze_tag_directive(emitter *yaml_emitter_t, tag_directive *yaml_tag_directive_t) bool { + handle := tag_directive.handle + prefix := tag_directive.prefix + if len(handle) == 0 { + return yaml_emitter_set_emitter_error(emitter, "tag handle must not be empty") + } + if handle[0] != '!' { + return yaml_emitter_set_emitter_error(emitter, "tag handle must start with '!'") + } + if handle[len(handle)-1] != '!' { + return yaml_emitter_set_emitter_error(emitter, "tag handle must end with '!'") + } + for i := 1; i < len(handle)-1; i += width(handle[i]) { + if !is_alpha(handle, i) { + return yaml_emitter_set_emitter_error(emitter, "tag handle must contain alphanumerical characters only") + } + } + if len(prefix) == 0 { + return yaml_emitter_set_emitter_error(emitter, "tag prefix must not be empty") + } + return true +} + +// Check if an anchor is valid. +func yaml_emitter_analyze_anchor(emitter *yaml_emitter_t, anchor []byte, alias bool) bool { + if len(anchor) == 0 { + problem := "anchor value must not be empty" + if alias { + problem = "alias value must not be empty" + } + return yaml_emitter_set_emitter_error(emitter, problem) + } + for i := 0; i < len(anchor); i += width(anchor[i]) { + if !is_alpha(anchor, i) { + problem := "anchor value must contain alphanumerical characters only" + if alias { + problem = "alias value must contain alphanumerical characters only" + } + return yaml_emitter_set_emitter_error(emitter, problem) + } + } + emitter.anchor_data.anchor = anchor + emitter.anchor_data.alias = alias + return true +} + +// Check if a tag is valid. +func yaml_emitter_analyze_tag(emitter *yaml_emitter_t, tag []byte) bool { + if len(tag) == 0 { + return yaml_emitter_set_emitter_error(emitter, "tag value must not be empty") + } + for i := 0; i < len(emitter.tag_directives); i++ { + tag_directive := &emitter.tag_directives[i] + if bytes.HasPrefix(tag, tag_directive.prefix) { + emitter.tag_data.handle = tag_directive.handle + emitter.tag_data.suffix = tag[len(tag_directive.prefix):] + return true + } + } + emitter.tag_data.suffix = tag + return true +} + +// Check if a scalar is valid. +func yaml_emitter_analyze_scalar(emitter *yaml_emitter_t, value []byte) bool { + var ( + block_indicators = false + flow_indicators = false + line_breaks = false + special_characters = false + + leading_space = false + leading_break = false + trailing_space = false + trailing_break = false + break_space = false + space_break = false + + preceded_by_whitespace = false + followed_by_whitespace = false + previous_space = false + previous_break = false + ) + + emitter.scalar_data.value = value + + if len(value) == 0 { + emitter.scalar_data.multiline = false + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = true + emitter.scalar_data.single_quoted_allowed = true + emitter.scalar_data.block_allowed = false + return true + } + + if len(value) >= 3 && ((value[0] == '-' && value[1] == '-' && value[2] == '-') || (value[0] == '.' && value[1] == '.' && value[2] == '.')) { + block_indicators = true + flow_indicators = true + } + + preceded_by_whitespace = true + for i, w := 0, 0; i < len(value); i += w { + w = width(value[i]) + followed_by_whitespace = i+w >= len(value) || is_blank(value, i+w) + + if i == 0 { + switch value[i] { + case '#', ',', '[', ']', '{', '}', '&', '*', '!', '|', '>', '\'', '"', '%', '@', '`': + flow_indicators = true + block_indicators = true + case '?', ':': + flow_indicators = true + if followed_by_whitespace { + block_indicators = true + } + case '-': + if followed_by_whitespace { + flow_indicators = true + block_indicators = true + } + } + } else { + switch value[i] { + case ',', '?', '[', ']', '{', '}': + flow_indicators = true + case ':': + flow_indicators = true + if followed_by_whitespace { + block_indicators = true + } + case '#': + if preceded_by_whitespace { + flow_indicators = true + block_indicators = true + } + } + } + + if !is_printable(value, i) || !is_ascii(value, i) && !emitter.unicode { + special_characters = true + } + if is_space(value, i) { + if i == 0 { + leading_space = true + } + if i+width(value[i]) == len(value) { + trailing_space = true + } + if previous_break { + break_space = true + } + previous_space = true + previous_break = false + } else if is_break(value, i) { + line_breaks = true + if i == 0 { + leading_break = true + } + if i+width(value[i]) == len(value) { + trailing_break = true + } + if previous_space { + space_break = true + } + previous_space = false + previous_break = true + } else { + previous_space = false + previous_break = false + } + + // [Go]: Why 'z'? Couldn't be the end of the string as that's the loop condition. + preceded_by_whitespace = is_blankz(value, i) + } + + emitter.scalar_data.multiline = line_breaks + emitter.scalar_data.flow_plain_allowed = true + emitter.scalar_data.block_plain_allowed = true + emitter.scalar_data.single_quoted_allowed = true + emitter.scalar_data.block_allowed = true + + if leading_space || leading_break || trailing_space || trailing_break { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + } + if trailing_space { + emitter.scalar_data.block_allowed = false + } + if break_space { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + emitter.scalar_data.single_quoted_allowed = false + } + if space_break || special_characters { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + emitter.scalar_data.single_quoted_allowed = false + emitter.scalar_data.block_allowed = false + } + if line_breaks { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + } + if flow_indicators { + emitter.scalar_data.flow_plain_allowed = false + } + if block_indicators { + emitter.scalar_data.block_plain_allowed = false + } + return true +} + +// Check if the event data is valid. +func yaml_emitter_analyze_event(emitter *yaml_emitter_t, event *yaml_event_t) bool { + + emitter.anchor_data.anchor = nil + emitter.tag_data.handle = nil + emitter.tag_data.suffix = nil + emitter.scalar_data.value = nil + + switch event.typ { + case yaml_ALIAS_EVENT: + if !yaml_emitter_analyze_anchor(emitter, event.anchor, true) { + return false + } + + case yaml_SCALAR_EVENT: + if len(event.anchor) > 0 { + if !yaml_emitter_analyze_anchor(emitter, event.anchor, false) { + return false + } + } + if len(event.tag) > 0 && (emitter.canonical || (!event.implicit && !event.quoted_implicit)) { + if !yaml_emitter_analyze_tag(emitter, event.tag) { + return false + } + } + if !yaml_emitter_analyze_scalar(emitter, event.value) { + return false + } + + case yaml_SEQUENCE_START_EVENT: + if len(event.anchor) > 0 { + if !yaml_emitter_analyze_anchor(emitter, event.anchor, false) { + return false + } + } + if len(event.tag) > 0 && (emitter.canonical || !event.implicit) { + if !yaml_emitter_analyze_tag(emitter, event.tag) { + return false + } + } + + case yaml_MAPPING_START_EVENT: + if len(event.anchor) > 0 { + if !yaml_emitter_analyze_anchor(emitter, event.anchor, false) { + return false + } + } + if len(event.tag) > 0 && (emitter.canonical || !event.implicit) { + if !yaml_emitter_analyze_tag(emitter, event.tag) { + return false + } + } + } + return true +} + +// Write the BOM character. +func yaml_emitter_write_bom(emitter *yaml_emitter_t) bool { + if !flush(emitter) { + return false + } + pos := emitter.buffer_pos + emitter.buffer[pos+0] = '\xEF' + emitter.buffer[pos+1] = '\xBB' + emitter.buffer[pos+2] = '\xBF' + emitter.buffer_pos += 3 + return true +} + +func yaml_emitter_write_indent(emitter *yaml_emitter_t) bool { + indent := emitter.indent + if indent < 0 { + indent = 0 + } + if !emitter.indention || emitter.column > indent || (emitter.column == indent && !emitter.whitespace) { + if !put_break(emitter) { + return false + } + } + for emitter.column < indent { + if !put(emitter, ' ') { + return false + } + } + emitter.whitespace = true + emitter.indention = true + return true +} + +func yaml_emitter_write_indicator(emitter *yaml_emitter_t, indicator []byte, need_whitespace, is_whitespace, is_indention bool) bool { + if need_whitespace && !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + if !write_all(emitter, indicator) { + return false + } + emitter.whitespace = is_whitespace + emitter.indention = (emitter.indention && is_indention) + emitter.open_ended = false + return true +} + +func yaml_emitter_write_anchor(emitter *yaml_emitter_t, value []byte) bool { + if !write_all(emitter, value) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_tag_handle(emitter *yaml_emitter_t, value []byte) bool { + if !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + if !write_all(emitter, value) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_tag_content(emitter *yaml_emitter_t, value []byte, need_whitespace bool) bool { + if need_whitespace && !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + for i := 0; i < len(value); { + var must_write bool + switch value[i] { + case ';', '/', '?', ':', '@', '&', '=', '+', '$', ',', '_', '.', '~', '*', '\'', '(', ')', '[', ']': + must_write = true + default: + must_write = is_alpha(value, i) + } + if must_write { + if !write(emitter, value, &i) { + return false + } + } else { + w := width(value[i]) + for k := 0; k < w; k++ { + octet := value[i] + i++ + if !put(emitter, '%') { + return false + } + + c := octet >> 4 + if c < 10 { + c += '0' + } else { + c += 'A' - 10 + } + if !put(emitter, c) { + return false + } + + c = octet & 0x0f + if c < 10 { + c += '0' + } else { + c += 'A' - 10 + } + if !put(emitter, c) { + return false + } + } + } + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_plain_scalar(emitter *yaml_emitter_t, value []byte, allow_breaks bool) bool { + if !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + + spaces := false + breaks := false + for i := 0; i < len(value); { + if is_space(value, i) { + if allow_breaks && !spaces && emitter.column > emitter.best_width && !is_space(value, i+1) { + if !yaml_emitter_write_indent(emitter) { + return false + } + i += width(value[i]) + } else { + if !write(emitter, value, &i) { + return false + } + } + spaces = true + } else if is_break(value, i) { + if !breaks && value[i] == '\n' { + if !put_break(emitter) { + return false + } + } + if !write_break(emitter, value, &i) { + return false + } + emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !write(emitter, value, &i) { + return false + } + emitter.indention = false + spaces = false + breaks = false + } + } + + emitter.whitespace = false + emitter.indention = false + if emitter.root_context { + emitter.open_ended = true + } + + return true +} + +func yaml_emitter_write_single_quoted_scalar(emitter *yaml_emitter_t, value []byte, allow_breaks bool) bool { + + if !yaml_emitter_write_indicator(emitter, []byte{'\''}, true, false, false) { + return false + } + + spaces := false + breaks := false + for i := 0; i < len(value); { + if is_space(value, i) { + if allow_breaks && !spaces && emitter.column > emitter.best_width && i > 0 && i < len(value)-1 && !is_space(value, i+1) { + if !yaml_emitter_write_indent(emitter) { + return false + } + i += width(value[i]) + } else { + if !write(emitter, value, &i) { + return false + } + } + spaces = true + } else if is_break(value, i) { + if !breaks && value[i] == '\n' { + if !put_break(emitter) { + return false + } + } + if !write_break(emitter, value, &i) { + return false + } + emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if value[i] == '\'' { + if !put(emitter, '\'') { + return false + } + } + if !write(emitter, value, &i) { + return false + } + emitter.indention = false + spaces = false + breaks = false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{'\''}, false, false, false) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_double_quoted_scalar(emitter *yaml_emitter_t, value []byte, allow_breaks bool) bool { + spaces := false + if !yaml_emitter_write_indicator(emitter, []byte{'"'}, true, false, false) { + return false + } + + for i := 0; i < len(value); { + if !is_printable(value, i) || (!emitter.unicode && !is_ascii(value, i)) || + is_bom(value, i) || is_break(value, i) || + value[i] == '"' || value[i] == '\\' { + + octet := value[i] + + var w int + var v rune + switch { + case octet&0x80 == 0x00: + w, v = 1, rune(octet&0x7F) + case octet&0xE0 == 0xC0: + w, v = 2, rune(octet&0x1F) + case octet&0xF0 == 0xE0: + w, v = 3, rune(octet&0x0F) + case octet&0xF8 == 0xF0: + w, v = 4, rune(octet&0x07) + } + for k := 1; k < w; k++ { + octet = value[i+k] + v = (v << 6) + (rune(octet) & 0x3F) + } + i += w + + if !put(emitter, '\\') { + return false + } + + var ok bool + switch v { + case 0x00: + ok = put(emitter, '0') + case 0x07: + ok = put(emitter, 'a') + case 0x08: + ok = put(emitter, 'b') + case 0x09: + ok = put(emitter, 't') + case 0x0A: + ok = put(emitter, 'n') + case 0x0b: + ok = put(emitter, 'v') + case 0x0c: + ok = put(emitter, 'f') + case 0x0d: + ok = put(emitter, 'r') + case 0x1b: + ok = put(emitter, 'e') + case 0x22: + ok = put(emitter, '"') + case 0x5c: + ok = put(emitter, '\\') + case 0x85: + ok = put(emitter, 'N') + case 0xA0: + ok = put(emitter, '_') + case 0x2028: + ok = put(emitter, 'L') + case 0x2029: + ok = put(emitter, 'P') + default: + if v <= 0xFF { + ok = put(emitter, 'x') + w = 2 + } else if v <= 0xFFFF { + ok = put(emitter, 'u') + w = 4 + } else { + ok = put(emitter, 'U') + w = 8 + } + for k := (w - 1) * 4; ok && k >= 0; k -= 4 { + digit := byte((v >> uint(k)) & 0x0F) + if digit < 10 { + ok = put(emitter, digit+'0') + } else { + ok = put(emitter, digit+'A'-10) + } + } + } + if !ok { + return false + } + spaces = false + } else if is_space(value, i) { + if allow_breaks && !spaces && emitter.column > emitter.best_width && i > 0 && i < len(value)-1 { + if !yaml_emitter_write_indent(emitter) { + return false + } + if is_space(value, i+1) { + if !put(emitter, '\\') { + return false + } + } + i += width(value[i]) + } else if !write(emitter, value, &i) { + return false + } + spaces = true + } else { + if !write(emitter, value, &i) { + return false + } + spaces = false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{'"'}, false, false, false) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_block_scalar_hints(emitter *yaml_emitter_t, value []byte) bool { + if is_space(value, 0) || is_break(value, 0) { + indent_hint := []byte{'0' + byte(emitter.best_indent)} + if !yaml_emitter_write_indicator(emitter, indent_hint, false, false, false) { + return false + } + } + + emitter.open_ended = false + + var chomp_hint [1]byte + if len(value) == 0 { + chomp_hint[0] = '-' + } else { + i := len(value) - 1 + for value[i]&0xC0 == 0x80 { + i-- + } + if !is_break(value, i) { + chomp_hint[0] = '-' + } else if i == 0 { + chomp_hint[0] = '+' + emitter.open_ended = true + } else { + i-- + for value[i]&0xC0 == 0x80 { + i-- + } + if is_break(value, i) { + chomp_hint[0] = '+' + emitter.open_ended = true + } + } + } + if chomp_hint[0] != 0 { + if !yaml_emitter_write_indicator(emitter, chomp_hint[:], false, false, false) { + return false + } + } + return true +} + +func yaml_emitter_write_literal_scalar(emitter *yaml_emitter_t, value []byte) bool { + if !yaml_emitter_write_indicator(emitter, []byte{'|'}, true, false, false) { + return false + } + if !yaml_emitter_write_block_scalar_hints(emitter, value) { + return false + } + if !put_break(emitter) { + return false + } + emitter.indention = true + emitter.whitespace = true + breaks := true + for i := 0; i < len(value); { + if is_break(value, i) { + if !write_break(emitter, value, &i) { + return false + } + emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !write(emitter, value, &i) { + return false + } + emitter.indention = false + breaks = false + } + } + + return true +} + +func yaml_emitter_write_folded_scalar(emitter *yaml_emitter_t, value []byte) bool { + if !yaml_emitter_write_indicator(emitter, []byte{'>'}, true, false, false) { + return false + } + if !yaml_emitter_write_block_scalar_hints(emitter, value) { + return false + } + + if !put_break(emitter) { + return false + } + emitter.indention = true + emitter.whitespace = true + + breaks := true + leading_spaces := true + for i := 0; i < len(value); { + if is_break(value, i) { + if !breaks && !leading_spaces && value[i] == '\n' { + k := 0 + for is_break(value, k) { + k += width(value[k]) + } + if !is_blankz(value, k) { + if !put_break(emitter) { + return false + } + } + } + if !write_break(emitter, value, &i) { + return false + } + emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + leading_spaces = is_blank(value, i) + } + if !breaks && is_space(value, i) && !is_space(value, i+1) && emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + i += width(value[i]) + } else { + if !write(emitter, value, &i) { + return false + } + } + emitter.indention = false + breaks = false + } + } + return true +} diff --git a/vendor/gopkg.in/yaml.v2/encode.go b/vendor/gopkg.in/yaml.v2/encode.go new file mode 100644 index 000000000..0ee738e11 --- /dev/null +++ b/vendor/gopkg.in/yaml.v2/encode.go @@ -0,0 +1,390 @@ +package yaml + +import ( + "encoding" + "fmt" + "io" + "reflect" + "regexp" + "sort" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +// jsonNumber is the interface of the encoding/json.Number datatype. +// Repeating the interface here avoids a dependency on encoding/json, and also +// supports other libraries like jsoniter, which use a similar datatype with +// the same interface. Detecting this interface is useful when dealing with +// structures containing json.Number, which is a string under the hood. The +// encoder should prefer the use of Int64(), Float64() and string(), in that +// order, when encoding this type. +type jsonNumber interface { + Float64() (float64, error) + Int64() (int64, error) + String() string +} + +type encoder struct { + emitter yaml_emitter_t + event yaml_event_t + out []byte + flow bool + // doneInit holds whether the initial stream_start_event has been + // emitted. + doneInit bool +} + +func newEncoder() *encoder { + e := &encoder{} + yaml_emitter_initialize(&e.emitter) + yaml_emitter_set_output_string(&e.emitter, &e.out) + yaml_emitter_set_unicode(&e.emitter, true) + return e +} + +func newEncoderWithWriter(w io.Writer) *encoder { + e := &encoder{} + yaml_emitter_initialize(&e.emitter) + yaml_emitter_set_output_writer(&e.emitter, w) + yaml_emitter_set_unicode(&e.emitter, true) + return e +} + +func (e *encoder) init() { + if e.doneInit { + return + } + yaml_stream_start_event_initialize(&e.event, yaml_UTF8_ENCODING) + e.emit() + e.doneInit = true +} + +func (e *encoder) finish() { + e.emitter.open_ended = false + yaml_stream_end_event_initialize(&e.event) + e.emit() +} + +func (e *encoder) destroy() { + yaml_emitter_delete(&e.emitter) +} + +func (e *encoder) emit() { + // This will internally delete the e.event value. + e.must(yaml_emitter_emit(&e.emitter, &e.event)) +} + +func (e *encoder) must(ok bool) { + if !ok { + msg := e.emitter.problem + if msg == "" { + msg = "unknown problem generating YAML content" + } + failf("%s", msg) + } +} + +func (e *encoder) marshalDoc(tag string, in reflect.Value) { + e.init() + yaml_document_start_event_initialize(&e.event, nil, nil, true) + e.emit() + e.marshal(tag, in) + yaml_document_end_event_initialize(&e.event, true) + e.emit() +} + +func (e *encoder) marshal(tag string, in reflect.Value) { + if !in.IsValid() || in.Kind() == reflect.Ptr && in.IsNil() { + e.nilv() + return + } + iface := in.Interface() + switch m := iface.(type) { + case jsonNumber: + integer, err := m.Int64() + if err == nil { + // In this case the json.Number is a valid int64 + in = reflect.ValueOf(integer) + break + } + float, err := m.Float64() + if err == nil { + // In this case the json.Number is a valid float64 + in = reflect.ValueOf(float) + break + } + // fallback case - no number could be obtained + in = reflect.ValueOf(m.String()) + case time.Time, *time.Time: + // Although time.Time implements TextMarshaler, + // we don't want to treat it as a string for YAML + // purposes because YAML has special support for + // timestamps. + case Marshaler: + v, err := m.MarshalYAML() + if err != nil { + fail(err) + } + if v == nil { + e.nilv() + return + } + in = reflect.ValueOf(v) + case encoding.TextMarshaler: + text, err := m.MarshalText() + if err != nil { + fail(err) + } + in = reflect.ValueOf(string(text)) + case nil: + e.nilv() + return + } + switch in.Kind() { + case reflect.Interface: + e.marshal(tag, in.Elem()) + case reflect.Map: + e.mapv(tag, in) + case reflect.Ptr: + if in.Type() == ptrTimeType { + e.timev(tag, in.Elem()) + } else { + e.marshal(tag, in.Elem()) + } + case reflect.Struct: + if in.Type() == timeType { + e.timev(tag, in) + } else { + e.structv(tag, in) + } + case reflect.Slice, reflect.Array: + if in.Type().Elem() == mapItemType { + e.itemsv(tag, in) + } else { + e.slicev(tag, in) + } + case reflect.String: + e.stringv(tag, in) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + if in.Type() == durationType { + e.stringv(tag, reflect.ValueOf(iface.(time.Duration).String())) + } else { + e.intv(tag, in) + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + e.uintv(tag, in) + case reflect.Float32, reflect.Float64: + e.floatv(tag, in) + case reflect.Bool: + e.boolv(tag, in) + default: + panic("cannot marshal type: " + in.Type().String()) + } +} + +func (e *encoder) mapv(tag string, in reflect.Value) { + e.mappingv(tag, func() { + keys := keyList(in.MapKeys()) + sort.Sort(keys) + for _, k := range keys { + e.marshal("", k) + e.marshal("", in.MapIndex(k)) + } + }) +} + +func (e *encoder) itemsv(tag string, in reflect.Value) { + e.mappingv(tag, func() { + slice := in.Convert(reflect.TypeOf([]MapItem{})).Interface().([]MapItem) + for _, item := range slice { + e.marshal("", reflect.ValueOf(item.Key)) + e.marshal("", reflect.ValueOf(item.Value)) + } + }) +} + +func (e *encoder) structv(tag string, in reflect.Value) { + sinfo, err := getStructInfo(in.Type()) + if err != nil { + panic(err) + } + e.mappingv(tag, func() { + for _, info := range sinfo.FieldsList { + var value reflect.Value + if info.Inline == nil { + value = in.Field(info.Num) + } else { + value = in.FieldByIndex(info.Inline) + } + if info.OmitEmpty && isZero(value) { + continue + } + e.marshal("", reflect.ValueOf(info.Key)) + e.flow = info.Flow + e.marshal("", value) + } + if sinfo.InlineMap >= 0 { + m := in.Field(sinfo.InlineMap) + if m.Len() > 0 { + e.flow = false + keys := keyList(m.MapKeys()) + sort.Sort(keys) + for _, k := range keys { + if _, found := sinfo.FieldsMap[k.String()]; found { + panic(fmt.Sprintf("Can't have key %q in inlined map; conflicts with struct field", k.String())) + } + e.marshal("", k) + e.flow = false + e.marshal("", m.MapIndex(k)) + } + } + } + }) +} + +func (e *encoder) mappingv(tag string, f func()) { + implicit := tag == "" + style := yaml_BLOCK_MAPPING_STYLE + if e.flow { + e.flow = false + style = yaml_FLOW_MAPPING_STYLE + } + yaml_mapping_start_event_initialize(&e.event, nil, []byte(tag), implicit, style) + e.emit() + f() + yaml_mapping_end_event_initialize(&e.event) + e.emit() +} + +func (e *encoder) slicev(tag string, in reflect.Value) { + implicit := tag == "" + style := yaml_BLOCK_SEQUENCE_STYLE + if e.flow { + e.flow = false + style = yaml_FLOW_SEQUENCE_STYLE + } + e.must(yaml_sequence_start_event_initialize(&e.event, nil, []byte(tag), implicit, style)) + e.emit() + n := in.Len() + for i := 0; i < n; i++ { + e.marshal("", in.Index(i)) + } + e.must(yaml_sequence_end_event_initialize(&e.event)) + e.emit() +} + +// isBase60 returns whether s is in base 60 notation as defined in YAML 1.1. +// +// The base 60 float notation in YAML 1.1 is a terrible idea and is unsupported +// in YAML 1.2 and by this package, but these should be marshalled quoted for +// the time being for compatibility with other parsers. +func isBase60Float(s string) (result bool) { + // Fast path. + if s == "" { + return false + } + c := s[0] + if !(c == '+' || c == '-' || c >= '0' && c <= '9') || strings.IndexByte(s, ':') < 0 { + return false + } + // Do the full match. + return base60float.MatchString(s) +} + +// From http://yaml.org/type/float.html, except the regular expression there +// is bogus. In practice parsers do not enforce the "\.[0-9_]*" suffix. +var base60float = regexp.MustCompile(`^[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+(?:\.[0-9_]*)?$`) + +func (e *encoder) stringv(tag string, in reflect.Value) { + var style yaml_scalar_style_t + s := in.String() + canUsePlain := true + switch { + case !utf8.ValidString(s): + if tag == yaml_BINARY_TAG { + failf("explicitly tagged !!binary data must be base64-encoded") + } + if tag != "" { + failf("cannot marshal invalid UTF-8 data as %s", shortTag(tag)) + } + // It can't be encoded directly as YAML so use a binary tag + // and encode it as base64. + tag = yaml_BINARY_TAG + s = encodeBase64(s) + case tag == "": + // Check to see if it would resolve to a specific + // tag when encoded unquoted. If it doesn't, + // there's no need to quote it. + rtag, _ := resolve("", s) + canUsePlain = rtag == yaml_STR_TAG && !isBase60Float(s) + } + // Note: it's possible for user code to emit invalid YAML + // if they explicitly specify a tag and a string containing + // text that's incompatible with that tag. + switch { + case strings.Contains(s, "\n"): + style = yaml_LITERAL_SCALAR_STYLE + case canUsePlain: + style = yaml_PLAIN_SCALAR_STYLE + default: + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + e.emitScalar(s, "", tag, style) +} + +func (e *encoder) boolv(tag string, in reflect.Value) { + var s string + if in.Bool() { + s = "true" + } else { + s = "false" + } + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) intv(tag string, in reflect.Value) { + s := strconv.FormatInt(in.Int(), 10) + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) uintv(tag string, in reflect.Value) { + s := strconv.FormatUint(in.Uint(), 10) + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) timev(tag string, in reflect.Value) { + t := in.Interface().(time.Time) + s := t.Format(time.RFC3339Nano) + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) floatv(tag string, in reflect.Value) { + // Issue #352: When formatting, use the precision of the underlying value + precision := 64 + if in.Kind() == reflect.Float32 { + precision = 32 + } + + s := strconv.FormatFloat(in.Float(), 'g', -1, precision) + switch s { + case "+Inf": + s = ".inf" + case "-Inf": + s = "-.inf" + case "NaN": + s = ".nan" + } + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) nilv() { + e.emitScalar("null", "", "", yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) emitScalar(value, anchor, tag string, style yaml_scalar_style_t) { + implicit := tag == "" + e.must(yaml_scalar_event_initialize(&e.event, []byte(anchor), []byte(tag), []byte(value), implicit, implicit, style)) + e.emit() +} diff --git a/vendor/gopkg.in/yaml.v2/parserc.go b/vendor/gopkg.in/yaml.v2/parserc.go new file mode 100644 index 000000000..81d05dfe5 --- /dev/null +++ b/vendor/gopkg.in/yaml.v2/parserc.go @@ -0,0 +1,1095 @@ +package yaml + +import ( + "bytes" +) + +// The parser implements the following grammar: +// +// stream ::= STREAM-START implicit_document? explicit_document* STREAM-END +// implicit_document ::= block_node DOCUMENT-END* +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// block_node_or_indentless_sequence ::= +// ALIAS +// | properties (block_content | indentless_block_sequence)? +// | block_content +// | indentless_block_sequence +// block_node ::= ALIAS +// | properties block_content? +// | block_content +// flow_node ::= ALIAS +// | properties flow_content? +// | flow_content +// properties ::= TAG ANCHOR? | ANCHOR TAG? +// block_content ::= block_collection | flow_collection | SCALAR +// flow_content ::= flow_collection | SCALAR +// block_collection ::= block_sequence | block_mapping +// flow_collection ::= flow_sequence | flow_mapping +// block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END +// indentless_sequence ::= (BLOCK-ENTRY block_node?)+ +// block_mapping ::= BLOCK-MAPPING_START +// ((KEY block_node_or_indentless_sequence?)? +// (VALUE block_node_or_indentless_sequence?)?)* +// BLOCK-END +// flow_sequence ::= FLOW-SEQUENCE-START +// (flow_sequence_entry FLOW-ENTRY)* +// flow_sequence_entry? +// FLOW-SEQUENCE-END +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// flow_mapping ::= FLOW-MAPPING-START +// (flow_mapping_entry FLOW-ENTRY)* +// flow_mapping_entry? +// FLOW-MAPPING-END +// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? + +// Peek the next token in the token queue. +func peek_token(parser *yaml_parser_t) *yaml_token_t { + if parser.token_available || yaml_parser_fetch_more_tokens(parser) { + return &parser.tokens[parser.tokens_head] + } + return nil +} + +// Remove the next token from the queue (must be called after peek_token). +func skip_token(parser *yaml_parser_t) { + parser.token_available = false + parser.tokens_parsed++ + parser.stream_end_produced = parser.tokens[parser.tokens_head].typ == yaml_STREAM_END_TOKEN + parser.tokens_head++ +} + +// Get the next event. +func yaml_parser_parse(parser *yaml_parser_t, event *yaml_event_t) bool { + // Erase the event object. + *event = yaml_event_t{} + + // No events after the end of the stream or error. + if parser.stream_end_produced || parser.error != yaml_NO_ERROR || parser.state == yaml_PARSE_END_STATE { + return true + } + + // Generate the next event. + return yaml_parser_state_machine(parser, event) +} + +// Set parser error. +func yaml_parser_set_parser_error(parser *yaml_parser_t, problem string, problem_mark yaml_mark_t) bool { + parser.error = yaml_PARSER_ERROR + parser.problem = problem + parser.problem_mark = problem_mark + return false +} + +func yaml_parser_set_parser_error_context(parser *yaml_parser_t, context string, context_mark yaml_mark_t, problem string, problem_mark yaml_mark_t) bool { + parser.error = yaml_PARSER_ERROR + parser.context = context + parser.context_mark = context_mark + parser.problem = problem + parser.problem_mark = problem_mark + return false +} + +// State dispatcher. +func yaml_parser_state_machine(parser *yaml_parser_t, event *yaml_event_t) bool { + //trace("yaml_parser_state_machine", "state:", parser.state.String()) + + switch parser.state { + case yaml_PARSE_STREAM_START_STATE: + return yaml_parser_parse_stream_start(parser, event) + + case yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE: + return yaml_parser_parse_document_start(parser, event, true) + + case yaml_PARSE_DOCUMENT_START_STATE: + return yaml_parser_parse_document_start(parser, event, false) + + case yaml_PARSE_DOCUMENT_CONTENT_STATE: + return yaml_parser_parse_document_content(parser, event) + + case yaml_PARSE_DOCUMENT_END_STATE: + return yaml_parser_parse_document_end(parser, event) + + case yaml_PARSE_BLOCK_NODE_STATE: + return yaml_parser_parse_node(parser, event, true, false) + + case yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE: + return yaml_parser_parse_node(parser, event, true, true) + + case yaml_PARSE_FLOW_NODE_STATE: + return yaml_parser_parse_node(parser, event, false, false) + + case yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE: + return yaml_parser_parse_block_sequence_entry(parser, event, true) + + case yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE: + return yaml_parser_parse_block_sequence_entry(parser, event, false) + + case yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE: + return yaml_parser_parse_indentless_sequence_entry(parser, event) + + case yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE: + return yaml_parser_parse_block_mapping_key(parser, event, true) + + case yaml_PARSE_BLOCK_MAPPING_KEY_STATE: + return yaml_parser_parse_block_mapping_key(parser, event, false) + + case yaml_PARSE_BLOCK_MAPPING_VALUE_STATE: + return yaml_parser_parse_block_mapping_value(parser, event) + + case yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE: + return yaml_parser_parse_flow_sequence_entry(parser, event, true) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE: + return yaml_parser_parse_flow_sequence_entry(parser, event, false) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE: + return yaml_parser_parse_flow_sequence_entry_mapping_key(parser, event) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE: + return yaml_parser_parse_flow_sequence_entry_mapping_value(parser, event) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE: + return yaml_parser_parse_flow_sequence_entry_mapping_end(parser, event) + + case yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE: + return yaml_parser_parse_flow_mapping_key(parser, event, true) + + case yaml_PARSE_FLOW_MAPPING_KEY_STATE: + return yaml_parser_parse_flow_mapping_key(parser, event, false) + + case yaml_PARSE_FLOW_MAPPING_VALUE_STATE: + return yaml_parser_parse_flow_mapping_value(parser, event, false) + + case yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE: + return yaml_parser_parse_flow_mapping_value(parser, event, true) + + default: + panic("invalid parser state") + } +} + +// Parse the production: +// stream ::= STREAM-START implicit_document? explicit_document* STREAM-END +// ************ +func yaml_parser_parse_stream_start(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_STREAM_START_TOKEN { + return yaml_parser_set_parser_error(parser, "did not find expected ", token.start_mark) + } + parser.state = yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE + *event = yaml_event_t{ + typ: yaml_STREAM_START_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + encoding: token.encoding, + } + skip_token(parser) + return true +} + +// Parse the productions: +// implicit_document ::= block_node DOCUMENT-END* +// * +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// ************************* +func yaml_parser_parse_document_start(parser *yaml_parser_t, event *yaml_event_t, implicit bool) bool { + + token := peek_token(parser) + if token == nil { + return false + } + + // Parse extra document end indicators. + if !implicit { + for token.typ == yaml_DOCUMENT_END_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + } + + if implicit && token.typ != yaml_VERSION_DIRECTIVE_TOKEN && + token.typ != yaml_TAG_DIRECTIVE_TOKEN && + token.typ != yaml_DOCUMENT_START_TOKEN && + token.typ != yaml_STREAM_END_TOKEN { + // Parse an implicit document. + if !yaml_parser_process_directives(parser, nil, nil) { + return false + } + parser.states = append(parser.states, yaml_PARSE_DOCUMENT_END_STATE) + parser.state = yaml_PARSE_BLOCK_NODE_STATE + + *event = yaml_event_t{ + typ: yaml_DOCUMENT_START_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + + } else if token.typ != yaml_STREAM_END_TOKEN { + // Parse an explicit document. + var version_directive *yaml_version_directive_t + var tag_directives []yaml_tag_directive_t + start_mark := token.start_mark + if !yaml_parser_process_directives(parser, &version_directive, &tag_directives) { + return false + } + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_DOCUMENT_START_TOKEN { + yaml_parser_set_parser_error(parser, + "did not find expected ", token.start_mark) + return false + } + parser.states = append(parser.states, yaml_PARSE_DOCUMENT_END_STATE) + parser.state = yaml_PARSE_DOCUMENT_CONTENT_STATE + end_mark := token.end_mark + + *event = yaml_event_t{ + typ: yaml_DOCUMENT_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + version_directive: version_directive, + tag_directives: tag_directives, + implicit: false, + } + skip_token(parser) + + } else { + // Parse the stream end. + parser.state = yaml_PARSE_END_STATE + *event = yaml_event_t{ + typ: yaml_STREAM_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + skip_token(parser) + } + + return true +} + +// Parse the productions: +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// *********** +// +func yaml_parser_parse_document_content(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_VERSION_DIRECTIVE_TOKEN || + token.typ == yaml_TAG_DIRECTIVE_TOKEN || + token.typ == yaml_DOCUMENT_START_TOKEN || + token.typ == yaml_DOCUMENT_END_TOKEN || + token.typ == yaml_STREAM_END_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + return yaml_parser_process_empty_scalar(parser, event, + token.start_mark) + } + return yaml_parser_parse_node(parser, event, true, false) +} + +// Parse the productions: +// implicit_document ::= block_node DOCUMENT-END* +// ************* +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// +func yaml_parser_parse_document_end(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + + start_mark := token.start_mark + end_mark := token.start_mark + + implicit := true + if token.typ == yaml_DOCUMENT_END_TOKEN { + end_mark = token.end_mark + skip_token(parser) + implicit = false + } + + parser.tag_directives = parser.tag_directives[:0] + + parser.state = yaml_PARSE_DOCUMENT_START_STATE + *event = yaml_event_t{ + typ: yaml_DOCUMENT_END_EVENT, + start_mark: start_mark, + end_mark: end_mark, + implicit: implicit, + } + return true +} + +// Parse the productions: +// block_node_or_indentless_sequence ::= +// ALIAS +// ***** +// | properties (block_content | indentless_block_sequence)? +// ********** * +// | block_content | indentless_block_sequence +// * +// block_node ::= ALIAS +// ***** +// | properties block_content? +// ********** * +// | block_content +// * +// flow_node ::= ALIAS +// ***** +// | properties flow_content? +// ********** * +// | flow_content +// * +// properties ::= TAG ANCHOR? | ANCHOR TAG? +// ************************* +// block_content ::= block_collection | flow_collection | SCALAR +// ****** +// flow_content ::= flow_collection | SCALAR +// ****** +func yaml_parser_parse_node(parser *yaml_parser_t, event *yaml_event_t, block, indentless_sequence bool) bool { + //defer trace("yaml_parser_parse_node", "block:", block, "indentless_sequence:", indentless_sequence)() + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_ALIAS_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + *event = yaml_event_t{ + typ: yaml_ALIAS_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + anchor: token.value, + } + skip_token(parser) + return true + } + + start_mark := token.start_mark + end_mark := token.start_mark + + var tag_token bool + var tag_handle, tag_suffix, anchor []byte + var tag_mark yaml_mark_t + if token.typ == yaml_ANCHOR_TOKEN { + anchor = token.value + start_mark = token.start_mark + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_TAG_TOKEN { + tag_token = true + tag_handle = token.value + tag_suffix = token.suffix + tag_mark = token.start_mark + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + } else if token.typ == yaml_TAG_TOKEN { + tag_token = true + tag_handle = token.value + tag_suffix = token.suffix + start_mark = token.start_mark + tag_mark = token.start_mark + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_ANCHOR_TOKEN { + anchor = token.value + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + } + + var tag []byte + if tag_token { + if len(tag_handle) == 0 { + tag = tag_suffix + tag_suffix = nil + } else { + for i := range parser.tag_directives { + if bytes.Equal(parser.tag_directives[i].handle, tag_handle) { + tag = append([]byte(nil), parser.tag_directives[i].prefix...) + tag = append(tag, tag_suffix...) + break + } + } + if len(tag) == 0 { + yaml_parser_set_parser_error_context(parser, + "while parsing a node", start_mark, + "found undefined tag handle", tag_mark) + return false + } + } + } + + implicit := len(tag) == 0 + if indentless_sequence && token.typ == yaml_BLOCK_ENTRY_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_BLOCK_SEQUENCE_STYLE), + } + return true + } + if token.typ == yaml_SCALAR_TOKEN { + var plain_implicit, quoted_implicit bool + end_mark = token.end_mark + if (len(tag) == 0 && token.style == yaml_PLAIN_SCALAR_STYLE) || (len(tag) == 1 && tag[0] == '!') { + plain_implicit = true + } else if len(tag) == 0 { + quoted_implicit = true + } + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + value: token.value, + implicit: plain_implicit, + quoted_implicit: quoted_implicit, + style: yaml_style_t(token.style), + } + skip_token(parser) + return true + } + if token.typ == yaml_FLOW_SEQUENCE_START_TOKEN { + // [Go] Some of the events below can be merged as they differ only on style. + end_mark = token.end_mark + parser.state = yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_FLOW_SEQUENCE_STYLE), + } + return true + } + if token.typ == yaml_FLOW_MAPPING_START_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_FLOW_MAPPING_STYLE), + } + return true + } + if block && token.typ == yaml_BLOCK_SEQUENCE_START_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_BLOCK_SEQUENCE_STYLE), + } + return true + } + if block && token.typ == yaml_BLOCK_MAPPING_START_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_BLOCK_MAPPING_STYLE), + } + return true + } + if len(anchor) > 0 || len(tag) > 0 { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + quoted_implicit: false, + style: yaml_style_t(yaml_PLAIN_SCALAR_STYLE), + } + return true + } + + context := "while parsing a flow node" + if block { + context = "while parsing a block node" + } + yaml_parser_set_parser_error_context(parser, context, start_mark, + "did not find expected node content", token.start_mark) + return false +} + +// Parse the productions: +// block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END +// ******************** *********** * ********* +// +func yaml_parser_parse_block_sequence_entry(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_BLOCK_ENTRY_TOKEN { + mark := token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_BLOCK_ENTRY_TOKEN && token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE) + return yaml_parser_parse_node(parser, event, true, false) + } else { + parser.state = yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + } + if token.typ == yaml_BLOCK_END_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + + skip_token(parser) + return true + } + + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a block collection", context_mark, + "did not find expected '-' indicator", token.start_mark) +} + +// Parse the productions: +// indentless_sequence ::= (BLOCK-ENTRY block_node?)+ +// *********** * +func yaml_parser_parse_indentless_sequence_entry(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_BLOCK_ENTRY_TOKEN { + mark := token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_BLOCK_ENTRY_TOKEN && + token.typ != yaml_KEY_TOKEN && + token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE) + return yaml_parser_parse_node(parser, event, true, false) + } + parser.state = yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + start_mark: token.start_mark, + end_mark: token.start_mark, // [Go] Shouldn't this be token.end_mark? + } + return true +} + +// Parse the productions: +// block_mapping ::= BLOCK-MAPPING_START +// ******************* +// ((KEY block_node_or_indentless_sequence?)? +// *** * +// (VALUE block_node_or_indentless_sequence?)?)* +// +// BLOCK-END +// ********* +// +func yaml_parser_parse_block_mapping_key(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_KEY_TOKEN { + mark := token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_KEY_TOKEN && + token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_BLOCK_MAPPING_VALUE_STATE) + return yaml_parser_parse_node(parser, event, true, true) + } else { + parser.state = yaml_PARSE_BLOCK_MAPPING_VALUE_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + } else if token.typ == yaml_BLOCK_END_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + skip_token(parser) + return true + } + + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a block mapping", context_mark, + "did not find expected key", token.start_mark) +} + +// Parse the productions: +// block_mapping ::= BLOCK-MAPPING_START +// +// ((KEY block_node_or_indentless_sequence?)? +// +// (VALUE block_node_or_indentless_sequence?)?)* +// ***** * +// BLOCK-END +// +// +func yaml_parser_parse_block_mapping_value(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_VALUE_TOKEN { + mark := token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_KEY_TOKEN && + token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_BLOCK_MAPPING_KEY_STATE) + return yaml_parser_parse_node(parser, event, true, true) + } + parser.state = yaml_PARSE_BLOCK_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + parser.state = yaml_PARSE_BLOCK_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) +} + +// Parse the productions: +// flow_sequence ::= FLOW-SEQUENCE-START +// ******************* +// (flow_sequence_entry FLOW-ENTRY)* +// * ********** +// flow_sequence_entry? +// * +// FLOW-SEQUENCE-END +// ***************** +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * +// +func yaml_parser_parse_flow_sequence_entry(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + if !first { + if token.typ == yaml_FLOW_ENTRY_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } else { + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a flow sequence", context_mark, + "did not find expected ',' or ']'", token.start_mark) + } + } + + if token.typ == yaml_KEY_TOKEN { + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + implicit: true, + style: yaml_style_t(yaml_FLOW_MAPPING_STYLE), + } + skip_token(parser) + return true + } else if token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + + skip_token(parser) + return true +} + +// +// Parse the productions: +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// *** * +// +func yaml_parser_parse_flow_sequence_entry_mapping_key(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_FLOW_ENTRY_TOKEN && + token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + mark := token.end_mark + skip_token(parser) + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) +} + +// Parse the productions: +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// ***** * +// +func yaml_parser_parse_flow_sequence_entry_mapping_value(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_VALUE_TOKEN { + skip_token(parser) + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_FLOW_ENTRY_TOKEN && token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) +} + +// Parse the productions: +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * +// +func yaml_parser_parse_flow_sequence_entry_mapping_end(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + start_mark: token.start_mark, + end_mark: token.start_mark, // [Go] Shouldn't this be end_mark? + } + return true +} + +// Parse the productions: +// flow_mapping ::= FLOW-MAPPING-START +// ****************** +// (flow_mapping_entry FLOW-ENTRY)* +// * ********** +// flow_mapping_entry? +// ****************** +// FLOW-MAPPING-END +// **************** +// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * *** * +// +func yaml_parser_parse_flow_mapping_key(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ != yaml_FLOW_MAPPING_END_TOKEN { + if !first { + if token.typ == yaml_FLOW_ENTRY_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } else { + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a flow mapping", context_mark, + "did not find expected ',' or '}'", token.start_mark) + } + } + + if token.typ == yaml_KEY_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_FLOW_ENTRY_TOKEN && + token.typ != yaml_FLOW_MAPPING_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_MAPPING_VALUE_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } else { + parser.state = yaml_PARSE_FLOW_MAPPING_VALUE_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) + } + } else if token.typ != yaml_FLOW_MAPPING_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + skip_token(parser) + return true +} + +// Parse the productions: +// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * ***** * +// +func yaml_parser_parse_flow_mapping_value(parser *yaml_parser_t, event *yaml_event_t, empty bool) bool { + token := peek_token(parser) + if token == nil { + return false + } + if empty { + parser.state = yaml_PARSE_FLOW_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) + } + if token.typ == yaml_VALUE_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_FLOW_ENTRY_TOKEN && token.typ != yaml_FLOW_MAPPING_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_MAPPING_KEY_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + parser.state = yaml_PARSE_FLOW_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) +} + +// Generate an empty scalar event. +func yaml_parser_process_empty_scalar(parser *yaml_parser_t, event *yaml_event_t, mark yaml_mark_t) bool { + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + start_mark: mark, + end_mark: mark, + value: nil, // Empty + implicit: true, + style: yaml_style_t(yaml_PLAIN_SCALAR_STYLE), + } + return true +} + +var default_tag_directives = []yaml_tag_directive_t{ + {[]byte("!"), []byte("!")}, + {[]byte("!!"), []byte("tag:yaml.org,2002:")}, +} + +// Parse directives. +func yaml_parser_process_directives(parser *yaml_parser_t, + version_directive_ref **yaml_version_directive_t, + tag_directives_ref *[]yaml_tag_directive_t) bool { + + var version_directive *yaml_version_directive_t + var tag_directives []yaml_tag_directive_t + + token := peek_token(parser) + if token == nil { + return false + } + + for token.typ == yaml_VERSION_DIRECTIVE_TOKEN || token.typ == yaml_TAG_DIRECTIVE_TOKEN { + if token.typ == yaml_VERSION_DIRECTIVE_TOKEN { + if version_directive != nil { + yaml_parser_set_parser_error(parser, + "found duplicate %YAML directive", token.start_mark) + return false + } + if token.major != 1 || token.minor != 1 { + yaml_parser_set_parser_error(parser, + "found incompatible YAML document", token.start_mark) + return false + } + version_directive = &yaml_version_directive_t{ + major: token.major, + minor: token.minor, + } + } else if token.typ == yaml_TAG_DIRECTIVE_TOKEN { + value := yaml_tag_directive_t{ + handle: token.value, + prefix: token.prefix, + } + if !yaml_parser_append_tag_directive(parser, value, false, token.start_mark) { + return false + } + tag_directives = append(tag_directives, value) + } + + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + + for i := range default_tag_directives { + if !yaml_parser_append_tag_directive(parser, default_tag_directives[i], true, token.start_mark) { + return false + } + } + + if version_directive_ref != nil { + *version_directive_ref = version_directive + } + if tag_directives_ref != nil { + *tag_directives_ref = tag_directives + } + return true +} + +// Append a tag directive to the directives stack. +func yaml_parser_append_tag_directive(parser *yaml_parser_t, value yaml_tag_directive_t, allow_duplicates bool, mark yaml_mark_t) bool { + for i := range parser.tag_directives { + if bytes.Equal(value.handle, parser.tag_directives[i].handle) { + if allow_duplicates { + return true + } + return yaml_parser_set_parser_error(parser, "found duplicate %TAG directive", mark) + } + } + + // [Go] I suspect the copy is unnecessary. This was likely done + // because there was no way to track ownership of the data. + value_copy := yaml_tag_directive_t{ + handle: make([]byte, len(value.handle)), + prefix: make([]byte, len(value.prefix)), + } + copy(value_copy.handle, value.handle) + copy(value_copy.prefix, value.prefix) + parser.tag_directives = append(parser.tag_directives, value_copy) + return true +} diff --git a/vendor/gopkg.in/yaml.v2/readerc.go b/vendor/gopkg.in/yaml.v2/readerc.go new file mode 100644 index 000000000..7c1f5fac3 --- /dev/null +++ b/vendor/gopkg.in/yaml.v2/readerc.go @@ -0,0 +1,412 @@ +package yaml + +import ( + "io" +) + +// Set the reader error and return 0. +func yaml_parser_set_reader_error(parser *yaml_parser_t, problem string, offset int, value int) bool { + parser.error = yaml_READER_ERROR + parser.problem = problem + parser.problem_offset = offset + parser.problem_value = value + return false +} + +// Byte order marks. +const ( + bom_UTF8 = "\xef\xbb\xbf" + bom_UTF16LE = "\xff\xfe" + bom_UTF16BE = "\xfe\xff" +) + +// Determine the input stream encoding by checking the BOM symbol. If no BOM is +// found, the UTF-8 encoding is assumed. Return 1 on success, 0 on failure. +func yaml_parser_determine_encoding(parser *yaml_parser_t) bool { + // Ensure that we had enough bytes in the raw buffer. + for !parser.eof && len(parser.raw_buffer)-parser.raw_buffer_pos < 3 { + if !yaml_parser_update_raw_buffer(parser) { + return false + } + } + + // Determine the encoding. + buf := parser.raw_buffer + pos := parser.raw_buffer_pos + avail := len(buf) - pos + if avail >= 2 && buf[pos] == bom_UTF16LE[0] && buf[pos+1] == bom_UTF16LE[1] { + parser.encoding = yaml_UTF16LE_ENCODING + parser.raw_buffer_pos += 2 + parser.offset += 2 + } else if avail >= 2 && buf[pos] == bom_UTF16BE[0] && buf[pos+1] == bom_UTF16BE[1] { + parser.encoding = yaml_UTF16BE_ENCODING + parser.raw_buffer_pos += 2 + parser.offset += 2 + } else if avail >= 3 && buf[pos] == bom_UTF8[0] && buf[pos+1] == bom_UTF8[1] && buf[pos+2] == bom_UTF8[2] { + parser.encoding = yaml_UTF8_ENCODING + parser.raw_buffer_pos += 3 + parser.offset += 3 + } else { + parser.encoding = yaml_UTF8_ENCODING + } + return true +} + +// Update the raw buffer. +func yaml_parser_update_raw_buffer(parser *yaml_parser_t) bool { + size_read := 0 + + // Return if the raw buffer is full. + if parser.raw_buffer_pos == 0 && len(parser.raw_buffer) == cap(parser.raw_buffer) { + return true + } + + // Return on EOF. + if parser.eof { + return true + } + + // Move the remaining bytes in the raw buffer to the beginning. + if parser.raw_buffer_pos > 0 && parser.raw_buffer_pos < len(parser.raw_buffer) { + copy(parser.raw_buffer, parser.raw_buffer[parser.raw_buffer_pos:]) + } + parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)-parser.raw_buffer_pos] + parser.raw_buffer_pos = 0 + + // Call the read handler to fill the buffer. + size_read, err := parser.read_handler(parser, parser.raw_buffer[len(parser.raw_buffer):cap(parser.raw_buffer)]) + parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)+size_read] + if err == io.EOF { + parser.eof = true + } else if err != nil { + return yaml_parser_set_reader_error(parser, "input error: "+err.Error(), parser.offset, -1) + } + return true +} + +// Ensure that the buffer contains at least `length` characters. +// Return true on success, false on failure. +// +// The length is supposed to be significantly less that the buffer size. +func yaml_parser_update_buffer(parser *yaml_parser_t, length int) bool { + if parser.read_handler == nil { + panic("read handler must be set") + } + + // [Go] This function was changed to guarantee the requested length size at EOF. + // The fact we need to do this is pretty awful, but the description above implies + // for that to be the case, and there are tests + + // If the EOF flag is set and the raw buffer is empty, do nothing. + if parser.eof && parser.raw_buffer_pos == len(parser.raw_buffer) { + // [Go] ACTUALLY! Read the documentation of this function above. + // This is just broken. To return true, we need to have the + // given length in the buffer. Not doing that means every single + // check that calls this function to make sure the buffer has a + // given length is Go) panicking; or C) accessing invalid memory. + //return true + } + + // Return if the buffer contains enough characters. + if parser.unread >= length { + return true + } + + // Determine the input encoding if it is not known yet. + if parser.encoding == yaml_ANY_ENCODING { + if !yaml_parser_determine_encoding(parser) { + return false + } + } + + // Move the unread characters to the beginning of the buffer. + buffer_len := len(parser.buffer) + if parser.buffer_pos > 0 && parser.buffer_pos < buffer_len { + copy(parser.buffer, parser.buffer[parser.buffer_pos:]) + buffer_len -= parser.buffer_pos + parser.buffer_pos = 0 + } else if parser.buffer_pos == buffer_len { + buffer_len = 0 + parser.buffer_pos = 0 + } + + // Open the whole buffer for writing, and cut it before returning. + parser.buffer = parser.buffer[:cap(parser.buffer)] + + // Fill the buffer until it has enough characters. + first := true + for parser.unread < length { + + // Fill the raw buffer if necessary. + if !first || parser.raw_buffer_pos == len(parser.raw_buffer) { + if !yaml_parser_update_raw_buffer(parser) { + parser.buffer = parser.buffer[:buffer_len] + return false + } + } + first = false + + // Decode the raw buffer. + inner: + for parser.raw_buffer_pos != len(parser.raw_buffer) { + var value rune + var width int + + raw_unread := len(parser.raw_buffer) - parser.raw_buffer_pos + + // Decode the next character. + switch parser.encoding { + case yaml_UTF8_ENCODING: + // Decode a UTF-8 character. Check RFC 3629 + // (http://www.ietf.org/rfc/rfc3629.txt) for more details. + // + // The following table (taken from the RFC) is used for + // decoding. + // + // Char. number range | UTF-8 octet sequence + // (hexadecimal) | (binary) + // --------------------+------------------------------------ + // 0000 0000-0000 007F | 0xxxxxxx + // 0000 0080-0000 07FF | 110xxxxx 10xxxxxx + // 0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx + // 0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + // + // Additionally, the characters in the range 0xD800-0xDFFF + // are prohibited as they are reserved for use with UTF-16 + // surrogate pairs. + + // Determine the length of the UTF-8 sequence. + octet := parser.raw_buffer[parser.raw_buffer_pos] + switch { + case octet&0x80 == 0x00: + width = 1 + case octet&0xE0 == 0xC0: + width = 2 + case octet&0xF0 == 0xE0: + width = 3 + case octet&0xF8 == 0xF0: + width = 4 + default: + // The leading octet is invalid. + return yaml_parser_set_reader_error(parser, + "invalid leading UTF-8 octet", + parser.offset, int(octet)) + } + + // Check if the raw buffer contains an incomplete character. + if width > raw_unread { + if parser.eof { + return yaml_parser_set_reader_error(parser, + "incomplete UTF-8 octet sequence", + parser.offset, -1) + } + break inner + } + + // Decode the leading octet. + switch { + case octet&0x80 == 0x00: + value = rune(octet & 0x7F) + case octet&0xE0 == 0xC0: + value = rune(octet & 0x1F) + case octet&0xF0 == 0xE0: + value = rune(octet & 0x0F) + case octet&0xF8 == 0xF0: + value = rune(octet & 0x07) + default: + value = 0 + } + + // Check and decode the trailing octets. + for k := 1; k < width; k++ { + octet = parser.raw_buffer[parser.raw_buffer_pos+k] + + // Check if the octet is valid. + if (octet & 0xC0) != 0x80 { + return yaml_parser_set_reader_error(parser, + "invalid trailing UTF-8 octet", + parser.offset+k, int(octet)) + } + + // Decode the octet. + value = (value << 6) + rune(octet&0x3F) + } + + // Check the length of the sequence against the value. + switch { + case width == 1: + case width == 2 && value >= 0x80: + case width == 3 && value >= 0x800: + case width == 4 && value >= 0x10000: + default: + return yaml_parser_set_reader_error(parser, + "invalid length of a UTF-8 sequence", + parser.offset, -1) + } + + // Check the range of the value. + if value >= 0xD800 && value <= 0xDFFF || value > 0x10FFFF { + return yaml_parser_set_reader_error(parser, + "invalid Unicode character", + parser.offset, int(value)) + } + + case yaml_UTF16LE_ENCODING, yaml_UTF16BE_ENCODING: + var low, high int + if parser.encoding == yaml_UTF16LE_ENCODING { + low, high = 0, 1 + } else { + low, high = 1, 0 + } + + // The UTF-16 encoding is not as simple as one might + // naively think. Check RFC 2781 + // (http://www.ietf.org/rfc/rfc2781.txt). + // + // Normally, two subsequent bytes describe a Unicode + // character. However a special technique (called a + // surrogate pair) is used for specifying character + // values larger than 0xFFFF. + // + // A surrogate pair consists of two pseudo-characters: + // high surrogate area (0xD800-0xDBFF) + // low surrogate area (0xDC00-0xDFFF) + // + // The following formulas are used for decoding + // and encoding characters using surrogate pairs: + // + // U = U' + 0x10000 (0x01 00 00 <= U <= 0x10 FF FF) + // U' = yyyyyyyyyyxxxxxxxxxx (0 <= U' <= 0x0F FF FF) + // W1 = 110110yyyyyyyyyy + // W2 = 110111xxxxxxxxxx + // + // where U is the character value, W1 is the high surrogate + // area, W2 is the low surrogate area. + + // Check for incomplete UTF-16 character. + if raw_unread < 2 { + if parser.eof { + return yaml_parser_set_reader_error(parser, + "incomplete UTF-16 character", + parser.offset, -1) + } + break inner + } + + // Get the character. + value = rune(parser.raw_buffer[parser.raw_buffer_pos+low]) + + (rune(parser.raw_buffer[parser.raw_buffer_pos+high]) << 8) + + // Check for unexpected low surrogate area. + if value&0xFC00 == 0xDC00 { + return yaml_parser_set_reader_error(parser, + "unexpected low surrogate area", + parser.offset, int(value)) + } + + // Check for a high surrogate area. + if value&0xFC00 == 0xD800 { + width = 4 + + // Check for incomplete surrogate pair. + if raw_unread < 4 { + if parser.eof { + return yaml_parser_set_reader_error(parser, + "incomplete UTF-16 surrogate pair", + parser.offset, -1) + } + break inner + } + + // Get the next character. + value2 := rune(parser.raw_buffer[parser.raw_buffer_pos+low+2]) + + (rune(parser.raw_buffer[parser.raw_buffer_pos+high+2]) << 8) + + // Check for a low surrogate area. + if value2&0xFC00 != 0xDC00 { + return yaml_parser_set_reader_error(parser, + "expected low surrogate area", + parser.offset+2, int(value2)) + } + + // Generate the value of the surrogate pair. + value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF) + } else { + width = 2 + } + + default: + panic("impossible") + } + + // Check if the character is in the allowed range: + // #x9 | #xA | #xD | [#x20-#x7E] (8 bit) + // | #x85 | [#xA0-#xD7FF] | [#xE000-#xFFFD] (16 bit) + // | [#x10000-#x10FFFF] (32 bit) + switch { + case value == 0x09: + case value == 0x0A: + case value == 0x0D: + case value >= 0x20 && value <= 0x7E: + case value == 0x85: + case value >= 0xA0 && value <= 0xD7FF: + case value >= 0xE000 && value <= 0xFFFD: + case value >= 0x10000 && value <= 0x10FFFF: + default: + return yaml_parser_set_reader_error(parser, + "control characters are not allowed", + parser.offset, int(value)) + } + + // Move the raw pointers. + parser.raw_buffer_pos += width + parser.offset += width + + // Finally put the character into the buffer. + if value <= 0x7F { + // 0000 0000-0000 007F . 0xxxxxxx + parser.buffer[buffer_len+0] = byte(value) + buffer_len += 1 + } else if value <= 0x7FF { + // 0000 0080-0000 07FF . 110xxxxx 10xxxxxx + parser.buffer[buffer_len+0] = byte(0xC0 + (value >> 6)) + parser.buffer[buffer_len+1] = byte(0x80 + (value & 0x3F)) + buffer_len += 2 + } else if value <= 0xFFFF { + // 0000 0800-0000 FFFF . 1110xxxx 10xxxxxx 10xxxxxx + parser.buffer[buffer_len+0] = byte(0xE0 + (value >> 12)) + parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 6) & 0x3F)) + parser.buffer[buffer_len+2] = byte(0x80 + (value & 0x3F)) + buffer_len += 3 + } else { + // 0001 0000-0010 FFFF . 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + parser.buffer[buffer_len+0] = byte(0xF0 + (value >> 18)) + parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 12) & 0x3F)) + parser.buffer[buffer_len+2] = byte(0x80 + ((value >> 6) & 0x3F)) + parser.buffer[buffer_len+3] = byte(0x80 + (value & 0x3F)) + buffer_len += 4 + } + + parser.unread++ + } + + // On EOF, put NUL into the buffer and return. + if parser.eof { + parser.buffer[buffer_len] = 0 + buffer_len++ + parser.unread++ + break + } + } + // [Go] Read the documentation of this function above. To return true, + // we need to have the given length in the buffer. Not doing that means + // every single check that calls this function to make sure the buffer + // has a given length is Go) panicking; or C) accessing invalid memory. + // This happens here due to the EOF above breaking early. + for buffer_len < length { + parser.buffer[buffer_len] = 0 + buffer_len++ + } + parser.buffer = parser.buffer[:buffer_len] + return true +} diff --git a/vendor/gopkg.in/yaml.v2/resolve.go b/vendor/gopkg.in/yaml.v2/resolve.go new file mode 100644 index 000000000..4120e0c91 --- /dev/null +++ b/vendor/gopkg.in/yaml.v2/resolve.go @@ -0,0 +1,258 @@ +package yaml + +import ( + "encoding/base64" + "math" + "regexp" + "strconv" + "strings" + "time" +) + +type resolveMapItem struct { + value interface{} + tag string +} + +var resolveTable = make([]byte, 256) +var resolveMap = make(map[string]resolveMapItem) + +func init() { + t := resolveTable + t[int('+')] = 'S' // Sign + t[int('-')] = 'S' + for _, c := range "0123456789" { + t[int(c)] = 'D' // Digit + } + for _, c := range "yYnNtTfFoO~" { + t[int(c)] = 'M' // In map + } + t[int('.')] = '.' // Float (potentially in map) + + var resolveMapList = []struct { + v interface{} + tag string + l []string + }{ + {true, yaml_BOOL_TAG, []string{"y", "Y", "yes", "Yes", "YES"}}, + {true, yaml_BOOL_TAG, []string{"true", "True", "TRUE"}}, + {true, yaml_BOOL_TAG, []string{"on", "On", "ON"}}, + {false, yaml_BOOL_TAG, []string{"n", "N", "no", "No", "NO"}}, + {false, yaml_BOOL_TAG, []string{"false", "False", "FALSE"}}, + {false, yaml_BOOL_TAG, []string{"off", "Off", "OFF"}}, + {nil, yaml_NULL_TAG, []string{"", "~", "null", "Null", "NULL"}}, + {math.NaN(), yaml_FLOAT_TAG, []string{".nan", ".NaN", ".NAN"}}, + {math.Inf(+1), yaml_FLOAT_TAG, []string{".inf", ".Inf", ".INF"}}, + {math.Inf(+1), yaml_FLOAT_TAG, []string{"+.inf", "+.Inf", "+.INF"}}, + {math.Inf(-1), yaml_FLOAT_TAG, []string{"-.inf", "-.Inf", "-.INF"}}, + {"<<", yaml_MERGE_TAG, []string{"<<"}}, + } + + m := resolveMap + for _, item := range resolveMapList { + for _, s := range item.l { + m[s] = resolveMapItem{item.v, item.tag} + } + } +} + +const longTagPrefix = "tag:yaml.org,2002:" + +func shortTag(tag string) string { + // TODO This can easily be made faster and produce less garbage. + if strings.HasPrefix(tag, longTagPrefix) { + return "!!" + tag[len(longTagPrefix):] + } + return tag +} + +func longTag(tag string) string { + if strings.HasPrefix(tag, "!!") { + return longTagPrefix + tag[2:] + } + return tag +} + +func resolvableTag(tag string) bool { + switch tag { + case "", yaml_STR_TAG, yaml_BOOL_TAG, yaml_INT_TAG, yaml_FLOAT_TAG, yaml_NULL_TAG, yaml_TIMESTAMP_TAG: + return true + } + return false +} + +var yamlStyleFloat = regexp.MustCompile(`^[-+]?(\.[0-9]+|[0-9]+(\.[0-9]*)?)([eE][-+]?[0-9]+)?$`) + +func resolve(tag string, in string) (rtag string, out interface{}) { + if !resolvableTag(tag) { + return tag, in + } + + defer func() { + switch tag { + case "", rtag, yaml_STR_TAG, yaml_BINARY_TAG: + return + case yaml_FLOAT_TAG: + if rtag == yaml_INT_TAG { + switch v := out.(type) { + case int64: + rtag = yaml_FLOAT_TAG + out = float64(v) + return + case int: + rtag = yaml_FLOAT_TAG + out = float64(v) + return + } + } + } + failf("cannot decode %s `%s` as a %s", shortTag(rtag), in, shortTag(tag)) + }() + + // Any data is accepted as a !!str or !!binary. + // Otherwise, the prefix is enough of a hint about what it might be. + hint := byte('N') + if in != "" { + hint = resolveTable[in[0]] + } + if hint != 0 && tag != yaml_STR_TAG && tag != yaml_BINARY_TAG { + // Handle things we can lookup in a map. + if item, ok := resolveMap[in]; ok { + return item.tag, item.value + } + + // Base 60 floats are a bad idea, were dropped in YAML 1.2, and + // are purposefully unsupported here. They're still quoted on + // the way out for compatibility with other parser, though. + + switch hint { + case 'M': + // We've already checked the map above. + + case '.': + // Not in the map, so maybe a normal float. + floatv, err := strconv.ParseFloat(in, 64) + if err == nil { + return yaml_FLOAT_TAG, floatv + } + + case 'D', 'S': + // Int, float, or timestamp. + // Only try values as a timestamp if the value is unquoted or there's an explicit + // !!timestamp tag. + if tag == "" || tag == yaml_TIMESTAMP_TAG { + t, ok := parseTimestamp(in) + if ok { + return yaml_TIMESTAMP_TAG, t + } + } + + plain := strings.Replace(in, "_", "", -1) + intv, err := strconv.ParseInt(plain, 0, 64) + if err == nil { + if intv == int64(int(intv)) { + return yaml_INT_TAG, int(intv) + } else { + return yaml_INT_TAG, intv + } + } + uintv, err := strconv.ParseUint(plain, 0, 64) + if err == nil { + return yaml_INT_TAG, uintv + } + if yamlStyleFloat.MatchString(plain) { + floatv, err := strconv.ParseFloat(plain, 64) + if err == nil { + return yaml_FLOAT_TAG, floatv + } + } + if strings.HasPrefix(plain, "0b") { + intv, err := strconv.ParseInt(plain[2:], 2, 64) + if err == nil { + if intv == int64(int(intv)) { + return yaml_INT_TAG, int(intv) + } else { + return yaml_INT_TAG, intv + } + } + uintv, err := strconv.ParseUint(plain[2:], 2, 64) + if err == nil { + return yaml_INT_TAG, uintv + } + } else if strings.HasPrefix(plain, "-0b") { + intv, err := strconv.ParseInt("-" + plain[3:], 2, 64) + if err == nil { + if true || intv == int64(int(intv)) { + return yaml_INT_TAG, int(intv) + } else { + return yaml_INT_TAG, intv + } + } + } + default: + panic("resolveTable item not yet handled: " + string(rune(hint)) + " (with " + in + ")") + } + } + return yaml_STR_TAG, in +} + +// encodeBase64 encodes s as base64 that is broken up into multiple lines +// as appropriate for the resulting length. +func encodeBase64(s string) string { + const lineLen = 70 + encLen := base64.StdEncoding.EncodedLen(len(s)) + lines := encLen/lineLen + 1 + buf := make([]byte, encLen*2+lines) + in := buf[0:encLen] + out := buf[encLen:] + base64.StdEncoding.Encode(in, []byte(s)) + k := 0 + for i := 0; i < len(in); i += lineLen { + j := i + lineLen + if j > len(in) { + j = len(in) + } + k += copy(out[k:], in[i:j]) + if lines > 1 { + out[k] = '\n' + k++ + } + } + return string(out[:k]) +} + +// This is a subset of the formats allowed by the regular expression +// defined at http://yaml.org/type/timestamp.html. +var allowedTimestampFormats = []string{ + "2006-1-2T15:4:5.999999999Z07:00", // RCF3339Nano with short date fields. + "2006-1-2t15:4:5.999999999Z07:00", // RFC3339Nano with short date fields and lower-case "t". + "2006-1-2 15:4:5.999999999", // space separated with no time zone + "2006-1-2", // date only + // Notable exception: time.Parse cannot handle: "2001-12-14 21:59:43.10 -5" + // from the set of examples. +} + +// parseTimestamp parses s as a timestamp string and +// returns the timestamp and reports whether it succeeded. +// Timestamp formats are defined at http://yaml.org/type/timestamp.html +func parseTimestamp(s string) (time.Time, bool) { + // TODO write code to check all the formats supported by + // http://yaml.org/type/timestamp.html instead of using time.Parse. + + // Quick check: all date formats start with YYYY-. + i := 0 + for ; i < len(s); i++ { + if c := s[i]; c < '0' || c > '9' { + break + } + } + if i != 4 || i == len(s) || s[i] != '-' { + return time.Time{}, false + } + for _, format := range allowedTimestampFormats { + if t, err := time.Parse(format, s); err == nil { + return t, true + } + } + return time.Time{}, false +} diff --git a/vendor/gopkg.in/yaml.v2/scannerc.go b/vendor/gopkg.in/yaml.v2/scannerc.go new file mode 100644 index 000000000..0b9bb6030 --- /dev/null +++ b/vendor/gopkg.in/yaml.v2/scannerc.go @@ -0,0 +1,2711 @@ +package yaml + +import ( + "bytes" + "fmt" +) + +// Introduction +// ************ +// +// The following notes assume that you are familiar with the YAML specification +// (http://yaml.org/spec/1.2/spec.html). We mostly follow it, although in +// some cases we are less restrictive that it requires. +// +// The process of transforming a YAML stream into a sequence of events is +// divided on two steps: Scanning and Parsing. +// +// The Scanner transforms the input stream into a sequence of tokens, while the +// parser transform the sequence of tokens produced by the Scanner into a +// sequence of parsing events. +// +// The Scanner is rather clever and complicated. The Parser, on the contrary, +// is a straightforward implementation of a recursive-descendant parser (or, +// LL(1) parser, as it is usually called). +// +// Actually there are two issues of Scanning that might be called "clever", the +// rest is quite straightforward. The issues are "block collection start" and +// "simple keys". Both issues are explained below in details. +// +// Here the Scanning step is explained and implemented. We start with the list +// of all the tokens produced by the Scanner together with short descriptions. +// +// Now, tokens: +// +// STREAM-START(encoding) # The stream start. +// STREAM-END # The stream end. +// VERSION-DIRECTIVE(major,minor) # The '%YAML' directive. +// TAG-DIRECTIVE(handle,prefix) # The '%TAG' directive. +// DOCUMENT-START # '---' +// DOCUMENT-END # '...' +// BLOCK-SEQUENCE-START # Indentation increase denoting a block +// BLOCK-MAPPING-START # sequence or a block mapping. +// BLOCK-END # Indentation decrease. +// FLOW-SEQUENCE-START # '[' +// FLOW-SEQUENCE-END # ']' +// BLOCK-SEQUENCE-START # '{' +// BLOCK-SEQUENCE-END # '}' +// BLOCK-ENTRY # '-' +// FLOW-ENTRY # ',' +// KEY # '?' or nothing (simple keys). +// VALUE # ':' +// ALIAS(anchor) # '*anchor' +// ANCHOR(anchor) # '&anchor' +// TAG(handle,suffix) # '!handle!suffix' +// SCALAR(value,style) # A scalar. +// +// The following two tokens are "virtual" tokens denoting the beginning and the +// end of the stream: +// +// STREAM-START(encoding) +// STREAM-END +// +// We pass the information about the input stream encoding with the +// STREAM-START token. +// +// The next two tokens are responsible for tags: +// +// VERSION-DIRECTIVE(major,minor) +// TAG-DIRECTIVE(handle,prefix) +// +// Example: +// +// %YAML 1.1 +// %TAG ! !foo +// %TAG !yaml! tag:yaml.org,2002: +// --- +// +// The correspoding sequence of tokens: +// +// STREAM-START(utf-8) +// VERSION-DIRECTIVE(1,1) +// TAG-DIRECTIVE("!","!foo") +// TAG-DIRECTIVE("!yaml","tag:yaml.org,2002:") +// DOCUMENT-START +// STREAM-END +// +// Note that the VERSION-DIRECTIVE and TAG-DIRECTIVE tokens occupy a whole +// line. +// +// The document start and end indicators are represented by: +// +// DOCUMENT-START +// DOCUMENT-END +// +// Note that if a YAML stream contains an implicit document (without '---' +// and '...' indicators), no DOCUMENT-START and DOCUMENT-END tokens will be +// produced. +// +// In the following examples, we present whole documents together with the +// produced tokens. +// +// 1. An implicit document: +// +// 'a scalar' +// +// Tokens: +// +// STREAM-START(utf-8) +// SCALAR("a scalar",single-quoted) +// STREAM-END +// +// 2. An explicit document: +// +// --- +// 'a scalar' +// ... +// +// Tokens: +// +// STREAM-START(utf-8) +// DOCUMENT-START +// SCALAR("a scalar",single-quoted) +// DOCUMENT-END +// STREAM-END +// +// 3. Several documents in a stream: +// +// 'a scalar' +// --- +// 'another scalar' +// --- +// 'yet another scalar' +// +// Tokens: +// +// STREAM-START(utf-8) +// SCALAR("a scalar",single-quoted) +// DOCUMENT-START +// SCALAR("another scalar",single-quoted) +// DOCUMENT-START +// SCALAR("yet another scalar",single-quoted) +// STREAM-END +// +// We have already introduced the SCALAR token above. The following tokens are +// used to describe aliases, anchors, tag, and scalars: +// +// ALIAS(anchor) +// ANCHOR(anchor) +// TAG(handle,suffix) +// SCALAR(value,style) +// +// The following series of examples illustrate the usage of these tokens: +// +// 1. A recursive sequence: +// +// &A [ *A ] +// +// Tokens: +// +// STREAM-START(utf-8) +// ANCHOR("A") +// FLOW-SEQUENCE-START +// ALIAS("A") +// FLOW-SEQUENCE-END +// STREAM-END +// +// 2. A tagged scalar: +// +// !!float "3.14" # A good approximation. +// +// Tokens: +// +// STREAM-START(utf-8) +// TAG("!!","float") +// SCALAR("3.14",double-quoted) +// STREAM-END +// +// 3. Various scalar styles: +// +// --- # Implicit empty plain scalars do not produce tokens. +// --- a plain scalar +// --- 'a single-quoted scalar' +// --- "a double-quoted scalar" +// --- |- +// a literal scalar +// --- >- +// a folded +// scalar +// +// Tokens: +// +// STREAM-START(utf-8) +// DOCUMENT-START +// DOCUMENT-START +// SCALAR("a plain scalar",plain) +// DOCUMENT-START +// SCALAR("a single-quoted scalar",single-quoted) +// DOCUMENT-START +// SCALAR("a double-quoted scalar",double-quoted) +// DOCUMENT-START +// SCALAR("a literal scalar",literal) +// DOCUMENT-START +// SCALAR("a folded scalar",folded) +// STREAM-END +// +// Now it's time to review collection-related tokens. We will start with +// flow collections: +// +// FLOW-SEQUENCE-START +// FLOW-SEQUENCE-END +// FLOW-MAPPING-START +// FLOW-MAPPING-END +// FLOW-ENTRY +// KEY +// VALUE +// +// The tokens FLOW-SEQUENCE-START, FLOW-SEQUENCE-END, FLOW-MAPPING-START, and +// FLOW-MAPPING-END represent the indicators '[', ']', '{', and '}' +// correspondingly. FLOW-ENTRY represent the ',' indicator. Finally the +// indicators '?' and ':', which are used for denoting mapping keys and values, +// are represented by the KEY and VALUE tokens. +// +// The following examples show flow collections: +// +// 1. A flow sequence: +// +// [item 1, item 2, item 3] +// +// Tokens: +// +// STREAM-START(utf-8) +// FLOW-SEQUENCE-START +// SCALAR("item 1",plain) +// FLOW-ENTRY +// SCALAR("item 2",plain) +// FLOW-ENTRY +// SCALAR("item 3",plain) +// FLOW-SEQUENCE-END +// STREAM-END +// +// 2. A flow mapping: +// +// { +// a simple key: a value, # Note that the KEY token is produced. +// ? a complex key: another value, +// } +// +// Tokens: +// +// STREAM-START(utf-8) +// FLOW-MAPPING-START +// KEY +// SCALAR("a simple key",plain) +// VALUE +// SCALAR("a value",plain) +// FLOW-ENTRY +// KEY +// SCALAR("a complex key",plain) +// VALUE +// SCALAR("another value",plain) +// FLOW-ENTRY +// FLOW-MAPPING-END +// STREAM-END +// +// A simple key is a key which is not denoted by the '?' indicator. Note that +// the Scanner still produce the KEY token whenever it encounters a simple key. +// +// For scanning block collections, the following tokens are used (note that we +// repeat KEY and VALUE here): +// +// BLOCK-SEQUENCE-START +// BLOCK-MAPPING-START +// BLOCK-END +// BLOCK-ENTRY +// KEY +// VALUE +// +// The tokens BLOCK-SEQUENCE-START and BLOCK-MAPPING-START denote indentation +// increase that precedes a block collection (cf. the INDENT token in Python). +// The token BLOCK-END denote indentation decrease that ends a block collection +// (cf. the DEDENT token in Python). However YAML has some syntax pecularities +// that makes detections of these tokens more complex. +// +// The tokens BLOCK-ENTRY, KEY, and VALUE are used to represent the indicators +// '-', '?', and ':' correspondingly. +// +// The following examples show how the tokens BLOCK-SEQUENCE-START, +// BLOCK-MAPPING-START, and BLOCK-END are emitted by the Scanner: +// +// 1. Block sequences: +// +// - item 1 +// - item 2 +// - +// - item 3.1 +// - item 3.2 +// - +// key 1: value 1 +// key 2: value 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-ENTRY +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 3.1",plain) +// BLOCK-ENTRY +// SCALAR("item 3.2",plain) +// BLOCK-END +// BLOCK-ENTRY +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// 2. Block mappings: +// +// a simple key: a value # The KEY token is produced here. +// ? a complex key +// : another value +// a mapping: +// key 1: value 1 +// key 2: value 2 +// a sequence: +// - item 1 +// - item 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-MAPPING-START +// KEY +// SCALAR("a simple key",plain) +// VALUE +// SCALAR("a value",plain) +// KEY +// SCALAR("a complex key",plain) +// VALUE +// SCALAR("another value",plain) +// KEY +// SCALAR("a mapping",plain) +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// KEY +// SCALAR("a sequence",plain) +// VALUE +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// YAML does not always require to start a new block collection from a new +// line. If the current line contains only '-', '?', and ':' indicators, a new +// block collection may start at the current line. The following examples +// illustrate this case: +// +// 1. Collections in a sequence: +// +// - - item 1 +// - item 2 +// - key 1: value 1 +// key 2: value 2 +// - ? complex key +// : complex value +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// BLOCK-ENTRY +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// BLOCK-ENTRY +// BLOCK-MAPPING-START +// KEY +// SCALAR("complex key") +// VALUE +// SCALAR("complex value") +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// 2. Collections in a mapping: +// +// ? a sequence +// : - item 1 +// - item 2 +// ? a mapping +// : key 1: value 1 +// key 2: value 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-MAPPING-START +// KEY +// SCALAR("a sequence",plain) +// VALUE +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// KEY +// SCALAR("a mapping",plain) +// VALUE +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// YAML also permits non-indented sequences if they are included into a block +// mapping. In this case, the token BLOCK-SEQUENCE-START is not produced: +// +// key: +// - item 1 # BLOCK-SEQUENCE-START is NOT produced here. +// - item 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-MAPPING-START +// KEY +// SCALAR("key",plain) +// VALUE +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// + +// Ensure that the buffer contains the required number of characters. +// Return true on success, false on failure (reader error or memory error). +func cache(parser *yaml_parser_t, length int) bool { + // [Go] This was inlined: !cache(A, B) -> unread < B && !update(A, B) + return parser.unread >= length || yaml_parser_update_buffer(parser, length) +} + +// Advance the buffer pointer. +func skip(parser *yaml_parser_t) { + parser.mark.index++ + parser.mark.column++ + parser.unread-- + parser.buffer_pos += width(parser.buffer[parser.buffer_pos]) +} + +func skip_line(parser *yaml_parser_t) { + if is_crlf(parser.buffer, parser.buffer_pos) { + parser.mark.index += 2 + parser.mark.column = 0 + parser.mark.line++ + parser.unread -= 2 + parser.buffer_pos += 2 + } else if is_break(parser.buffer, parser.buffer_pos) { + parser.mark.index++ + parser.mark.column = 0 + parser.mark.line++ + parser.unread-- + parser.buffer_pos += width(parser.buffer[parser.buffer_pos]) + } +} + +// Copy a character to a string buffer and advance pointers. +func read(parser *yaml_parser_t, s []byte) []byte { + w := width(parser.buffer[parser.buffer_pos]) + if w == 0 { + panic("invalid character sequence") + } + if len(s) == 0 { + s = make([]byte, 0, 32) + } + if w == 1 && len(s)+w <= cap(s) { + s = s[:len(s)+1] + s[len(s)-1] = parser.buffer[parser.buffer_pos] + parser.buffer_pos++ + } else { + s = append(s, parser.buffer[parser.buffer_pos:parser.buffer_pos+w]...) + parser.buffer_pos += w + } + parser.mark.index++ + parser.mark.column++ + parser.unread-- + return s +} + +// Copy a line break character to a string buffer and advance pointers. +func read_line(parser *yaml_parser_t, s []byte) []byte { + buf := parser.buffer + pos := parser.buffer_pos + switch { + case buf[pos] == '\r' && buf[pos+1] == '\n': + // CR LF . LF + s = append(s, '\n') + parser.buffer_pos += 2 + parser.mark.index++ + parser.unread-- + case buf[pos] == '\r' || buf[pos] == '\n': + // CR|LF . LF + s = append(s, '\n') + parser.buffer_pos += 1 + case buf[pos] == '\xC2' && buf[pos+1] == '\x85': + // NEL . LF + s = append(s, '\n') + parser.buffer_pos += 2 + case buf[pos] == '\xE2' && buf[pos+1] == '\x80' && (buf[pos+2] == '\xA8' || buf[pos+2] == '\xA9'): + // LS|PS . LS|PS + s = append(s, buf[parser.buffer_pos:pos+3]...) + parser.buffer_pos += 3 + default: + return s + } + parser.mark.index++ + parser.mark.column = 0 + parser.mark.line++ + parser.unread-- + return s +} + +// Get the next token. +func yaml_parser_scan(parser *yaml_parser_t, token *yaml_token_t) bool { + // Erase the token object. + *token = yaml_token_t{} // [Go] Is this necessary? + + // No tokens after STREAM-END or error. + if parser.stream_end_produced || parser.error != yaml_NO_ERROR { + return true + } + + // Ensure that the tokens queue contains enough tokens. + if !parser.token_available { + if !yaml_parser_fetch_more_tokens(parser) { + return false + } + } + + // Fetch the next token from the queue. + *token = parser.tokens[parser.tokens_head] + parser.tokens_head++ + parser.tokens_parsed++ + parser.token_available = false + + if token.typ == yaml_STREAM_END_TOKEN { + parser.stream_end_produced = true + } + return true +} + +// Set the scanner error and return false. +func yaml_parser_set_scanner_error(parser *yaml_parser_t, context string, context_mark yaml_mark_t, problem string) bool { + parser.error = yaml_SCANNER_ERROR + parser.context = context + parser.context_mark = context_mark + parser.problem = problem + parser.problem_mark = parser.mark + return false +} + +func yaml_parser_set_scanner_tag_error(parser *yaml_parser_t, directive bool, context_mark yaml_mark_t, problem string) bool { + context := "while parsing a tag" + if directive { + context = "while parsing a %TAG directive" + } + return yaml_parser_set_scanner_error(parser, context, context_mark, problem) +} + +func trace(args ...interface{}) func() { + pargs := append([]interface{}{"+++"}, args...) + fmt.Println(pargs...) + pargs = append([]interface{}{"---"}, args...) + return func() { fmt.Println(pargs...) } +} + +// Ensure that the tokens queue contains at least one token which can be +// returned to the Parser. +func yaml_parser_fetch_more_tokens(parser *yaml_parser_t) bool { + // While we need more tokens to fetch, do it. + for { + if parser.tokens_head != len(parser.tokens) { + // If queue is non-empty, check if any potential simple key may + // occupy the head position. + head_tok_idx, ok := parser.simple_keys_by_tok[parser.tokens_parsed] + if !ok { + break + } else if valid, ok := yaml_simple_key_is_valid(parser, &parser.simple_keys[head_tok_idx]); !ok { + return false + } else if !valid { + break + } + } + // Fetch the next token. + if !yaml_parser_fetch_next_token(parser) { + return false + } + } + + parser.token_available = true + return true +} + +// The dispatcher for token fetchers. +func yaml_parser_fetch_next_token(parser *yaml_parser_t) bool { + // Ensure that the buffer is initialized. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // Check if we just started scanning. Fetch STREAM-START then. + if !parser.stream_start_produced { + return yaml_parser_fetch_stream_start(parser) + } + + // Eat whitespaces and comments until we reach the next token. + if !yaml_parser_scan_to_next_token(parser) { + return false + } + + // Check the indentation level against the current column. + if !yaml_parser_unroll_indent(parser, parser.mark.column) { + return false + } + + // Ensure that the buffer contains at least 4 characters. 4 is the length + // of the longest indicators ('--- ' and '... '). + if parser.unread < 4 && !yaml_parser_update_buffer(parser, 4) { + return false + } + + // Is it the end of the stream? + if is_z(parser.buffer, parser.buffer_pos) { + return yaml_parser_fetch_stream_end(parser) + } + + // Is it a directive? + if parser.mark.column == 0 && parser.buffer[parser.buffer_pos] == '%' { + return yaml_parser_fetch_directive(parser) + } + + buf := parser.buffer + pos := parser.buffer_pos + + // Is it the document start indicator? + if parser.mark.column == 0 && buf[pos] == '-' && buf[pos+1] == '-' && buf[pos+2] == '-' && is_blankz(buf, pos+3) { + return yaml_parser_fetch_document_indicator(parser, yaml_DOCUMENT_START_TOKEN) + } + + // Is it the document end indicator? + if parser.mark.column == 0 && buf[pos] == '.' && buf[pos+1] == '.' && buf[pos+2] == '.' && is_blankz(buf, pos+3) { + return yaml_parser_fetch_document_indicator(parser, yaml_DOCUMENT_END_TOKEN) + } + + // Is it the flow sequence start indicator? + if buf[pos] == '[' { + return yaml_parser_fetch_flow_collection_start(parser, yaml_FLOW_SEQUENCE_START_TOKEN) + } + + // Is it the flow mapping start indicator? + if parser.buffer[parser.buffer_pos] == '{' { + return yaml_parser_fetch_flow_collection_start(parser, yaml_FLOW_MAPPING_START_TOKEN) + } + + // Is it the flow sequence end indicator? + if parser.buffer[parser.buffer_pos] == ']' { + return yaml_parser_fetch_flow_collection_end(parser, + yaml_FLOW_SEQUENCE_END_TOKEN) + } + + // Is it the flow mapping end indicator? + if parser.buffer[parser.buffer_pos] == '}' { + return yaml_parser_fetch_flow_collection_end(parser, + yaml_FLOW_MAPPING_END_TOKEN) + } + + // Is it the flow entry indicator? + if parser.buffer[parser.buffer_pos] == ',' { + return yaml_parser_fetch_flow_entry(parser) + } + + // Is it the block entry indicator? + if parser.buffer[parser.buffer_pos] == '-' && is_blankz(parser.buffer, parser.buffer_pos+1) { + return yaml_parser_fetch_block_entry(parser) + } + + // Is it the key indicator? + if parser.buffer[parser.buffer_pos] == '?' && (parser.flow_level > 0 || is_blankz(parser.buffer, parser.buffer_pos+1)) { + return yaml_parser_fetch_key(parser) + } + + // Is it the value indicator? + if parser.buffer[parser.buffer_pos] == ':' && (parser.flow_level > 0 || is_blankz(parser.buffer, parser.buffer_pos+1)) { + return yaml_parser_fetch_value(parser) + } + + // Is it an alias? + if parser.buffer[parser.buffer_pos] == '*' { + return yaml_parser_fetch_anchor(parser, yaml_ALIAS_TOKEN) + } + + // Is it an anchor? + if parser.buffer[parser.buffer_pos] == '&' { + return yaml_parser_fetch_anchor(parser, yaml_ANCHOR_TOKEN) + } + + // Is it a tag? + if parser.buffer[parser.buffer_pos] == '!' { + return yaml_parser_fetch_tag(parser) + } + + // Is it a literal scalar? + if parser.buffer[parser.buffer_pos] == '|' && parser.flow_level == 0 { + return yaml_parser_fetch_block_scalar(parser, true) + } + + // Is it a folded scalar? + if parser.buffer[parser.buffer_pos] == '>' && parser.flow_level == 0 { + return yaml_parser_fetch_block_scalar(parser, false) + } + + // Is it a single-quoted scalar? + if parser.buffer[parser.buffer_pos] == '\'' { + return yaml_parser_fetch_flow_scalar(parser, true) + } + + // Is it a double-quoted scalar? + if parser.buffer[parser.buffer_pos] == '"' { + return yaml_parser_fetch_flow_scalar(parser, false) + } + + // Is it a plain scalar? + // + // A plain scalar may start with any non-blank characters except + // + // '-', '?', ':', ',', '[', ']', '{', '}', + // '#', '&', '*', '!', '|', '>', '\'', '\"', + // '%', '@', '`'. + // + // In the block context (and, for the '-' indicator, in the flow context + // too), it may also start with the characters + // + // '-', '?', ':' + // + // if it is followed by a non-space character. + // + // The last rule is more restrictive than the specification requires. + // [Go] Make this logic more reasonable. + //switch parser.buffer[parser.buffer_pos] { + //case '-', '?', ':', ',', '?', '-', ',', ':', ']', '[', '}', '{', '&', '#', '!', '*', '>', '|', '"', '\'', '@', '%', '-', '`': + //} + if !(is_blankz(parser.buffer, parser.buffer_pos) || parser.buffer[parser.buffer_pos] == '-' || + parser.buffer[parser.buffer_pos] == '?' || parser.buffer[parser.buffer_pos] == ':' || + parser.buffer[parser.buffer_pos] == ',' || parser.buffer[parser.buffer_pos] == '[' || + parser.buffer[parser.buffer_pos] == ']' || parser.buffer[parser.buffer_pos] == '{' || + parser.buffer[parser.buffer_pos] == '}' || parser.buffer[parser.buffer_pos] == '#' || + parser.buffer[parser.buffer_pos] == '&' || parser.buffer[parser.buffer_pos] == '*' || + parser.buffer[parser.buffer_pos] == '!' || parser.buffer[parser.buffer_pos] == '|' || + parser.buffer[parser.buffer_pos] == '>' || parser.buffer[parser.buffer_pos] == '\'' || + parser.buffer[parser.buffer_pos] == '"' || parser.buffer[parser.buffer_pos] == '%' || + parser.buffer[parser.buffer_pos] == '@' || parser.buffer[parser.buffer_pos] == '`') || + (parser.buffer[parser.buffer_pos] == '-' && !is_blank(parser.buffer, parser.buffer_pos+1)) || + (parser.flow_level == 0 && + (parser.buffer[parser.buffer_pos] == '?' || parser.buffer[parser.buffer_pos] == ':') && + !is_blankz(parser.buffer, parser.buffer_pos+1)) { + return yaml_parser_fetch_plain_scalar(parser) + } + + // If we don't determine the token type so far, it is an error. + return yaml_parser_set_scanner_error(parser, + "while scanning for the next token", parser.mark, + "found character that cannot start any token") +} + +func yaml_simple_key_is_valid(parser *yaml_parser_t, simple_key *yaml_simple_key_t) (valid, ok bool) { + if !simple_key.possible { + return false, true + } + + // The 1.2 specification says: + // + // "If the ? indicator is omitted, parsing needs to see past the + // implicit key to recognize it as such. To limit the amount of + // lookahead required, the “:” indicator must appear at most 1024 + // Unicode characters beyond the start of the key. In addition, the key + // is restricted to a single line." + // + if simple_key.mark.line < parser.mark.line || simple_key.mark.index+1024 < parser.mark.index { + // Check if the potential simple key to be removed is required. + if simple_key.required { + return false, yaml_parser_set_scanner_error(parser, + "while scanning a simple key", simple_key.mark, + "could not find expected ':'") + } + simple_key.possible = false + return false, true + } + return true, true +} + +// Check if a simple key may start at the current position and add it if +// needed. +func yaml_parser_save_simple_key(parser *yaml_parser_t) bool { + // A simple key is required at the current position if the scanner is in + // the block context and the current column coincides with the indentation + // level. + + required := parser.flow_level == 0 && parser.indent == parser.mark.column + + // + // If the current position may start a simple key, save it. + // + if parser.simple_key_allowed { + simple_key := yaml_simple_key_t{ + possible: true, + required: required, + token_number: parser.tokens_parsed + (len(parser.tokens) - parser.tokens_head), + mark: parser.mark, + } + + if !yaml_parser_remove_simple_key(parser) { + return false + } + parser.simple_keys[len(parser.simple_keys)-1] = simple_key + parser.simple_keys_by_tok[simple_key.token_number] = len(parser.simple_keys) - 1 + } + return true +} + +// Remove a potential simple key at the current flow level. +func yaml_parser_remove_simple_key(parser *yaml_parser_t) bool { + i := len(parser.simple_keys) - 1 + if parser.simple_keys[i].possible { + // If the key is required, it is an error. + if parser.simple_keys[i].required { + return yaml_parser_set_scanner_error(parser, + "while scanning a simple key", parser.simple_keys[i].mark, + "could not find expected ':'") + } + // Remove the key from the stack. + parser.simple_keys[i].possible = false + delete(parser.simple_keys_by_tok, parser.simple_keys[i].token_number) + } + return true +} + +// max_flow_level limits the flow_level +const max_flow_level = 10000 + +// Increase the flow level and resize the simple key list if needed. +func yaml_parser_increase_flow_level(parser *yaml_parser_t) bool { + // Reset the simple key on the next level. + parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{ + possible: false, + required: false, + token_number: parser.tokens_parsed + (len(parser.tokens) - parser.tokens_head), + mark: parser.mark, + }) + + // Increase the flow level. + parser.flow_level++ + if parser.flow_level > max_flow_level { + return yaml_parser_set_scanner_error(parser, + "while increasing flow level", parser.simple_keys[len(parser.simple_keys)-1].mark, + fmt.Sprintf("exceeded max depth of %d", max_flow_level)) + } + return true +} + +// Decrease the flow level. +func yaml_parser_decrease_flow_level(parser *yaml_parser_t) bool { + if parser.flow_level > 0 { + parser.flow_level-- + last := len(parser.simple_keys) - 1 + delete(parser.simple_keys_by_tok, parser.simple_keys[last].token_number) + parser.simple_keys = parser.simple_keys[:last] + } + return true +} + +// max_indents limits the indents stack size +const max_indents = 10000 + +// Push the current indentation level to the stack and set the new level +// the current column is greater than the indentation level. In this case, +// append or insert the specified token into the token queue. +func yaml_parser_roll_indent(parser *yaml_parser_t, column, number int, typ yaml_token_type_t, mark yaml_mark_t) bool { + // In the flow context, do nothing. + if parser.flow_level > 0 { + return true + } + + if parser.indent < column { + // Push the current indentation level to the stack and set the new + // indentation level. + parser.indents = append(parser.indents, parser.indent) + parser.indent = column + if len(parser.indents) > max_indents { + return yaml_parser_set_scanner_error(parser, + "while increasing indent level", parser.simple_keys[len(parser.simple_keys)-1].mark, + fmt.Sprintf("exceeded max depth of %d", max_indents)) + } + + // Create a token and insert it into the queue. + token := yaml_token_t{ + typ: typ, + start_mark: mark, + end_mark: mark, + } + if number > -1 { + number -= parser.tokens_parsed + } + yaml_insert_token(parser, number, &token) + } + return true +} + +// Pop indentation levels from the indents stack until the current level +// becomes less or equal to the column. For each indentation level, append +// the BLOCK-END token. +func yaml_parser_unroll_indent(parser *yaml_parser_t, column int) bool { + // In the flow context, do nothing. + if parser.flow_level > 0 { + return true + } + + // Loop through the indentation levels in the stack. + for parser.indent > column { + // Create a token and append it to the queue. + token := yaml_token_t{ + typ: yaml_BLOCK_END_TOKEN, + start_mark: parser.mark, + end_mark: parser.mark, + } + yaml_insert_token(parser, -1, &token) + + // Pop the indentation level. + parser.indent = parser.indents[len(parser.indents)-1] + parser.indents = parser.indents[:len(parser.indents)-1] + } + return true +} + +// Initialize the scanner and produce the STREAM-START token. +func yaml_parser_fetch_stream_start(parser *yaml_parser_t) bool { + + // Set the initial indentation. + parser.indent = -1 + + // Initialize the simple key stack. + parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{}) + + parser.simple_keys_by_tok = make(map[int]int) + + // A simple key is allowed at the beginning of the stream. + parser.simple_key_allowed = true + + // We have started. + parser.stream_start_produced = true + + // Create the STREAM-START token and append it to the queue. + token := yaml_token_t{ + typ: yaml_STREAM_START_TOKEN, + start_mark: parser.mark, + end_mark: parser.mark, + encoding: parser.encoding, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the STREAM-END token and shut down the scanner. +func yaml_parser_fetch_stream_end(parser *yaml_parser_t) bool { + + // Force new line. + if parser.mark.column != 0 { + parser.mark.column = 0 + parser.mark.line++ + } + + // Reset the indentation level. + if !yaml_parser_unroll_indent(parser, -1) { + return false + } + + // Reset simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + parser.simple_key_allowed = false + + // Create the STREAM-END token and append it to the queue. + token := yaml_token_t{ + typ: yaml_STREAM_END_TOKEN, + start_mark: parser.mark, + end_mark: parser.mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce a VERSION-DIRECTIVE or TAG-DIRECTIVE token. +func yaml_parser_fetch_directive(parser *yaml_parser_t) bool { + // Reset the indentation level. + if !yaml_parser_unroll_indent(parser, -1) { + return false + } + + // Reset simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + parser.simple_key_allowed = false + + // Create the YAML-DIRECTIVE or TAG-DIRECTIVE token. + token := yaml_token_t{} + if !yaml_parser_scan_directive(parser, &token) { + return false + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the DOCUMENT-START or DOCUMENT-END token. +func yaml_parser_fetch_document_indicator(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // Reset the indentation level. + if !yaml_parser_unroll_indent(parser, -1) { + return false + } + + // Reset simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + parser.simple_key_allowed = false + + // Consume the token. + start_mark := parser.mark + + skip(parser) + skip(parser) + skip(parser) + + end_mark := parser.mark + + // Create the DOCUMENT-START or DOCUMENT-END token. + token := yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the FLOW-SEQUENCE-START or FLOW-MAPPING-START token. +func yaml_parser_fetch_flow_collection_start(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // The indicators '[' and '{' may start a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // Increase the flow level. + if !yaml_parser_increase_flow_level(parser) { + return false + } + + // A simple key may follow the indicators '[' and '{'. + parser.simple_key_allowed = true + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the FLOW-SEQUENCE-START of FLOW-MAPPING-START token. + token := yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the FLOW-SEQUENCE-END or FLOW-MAPPING-END token. +func yaml_parser_fetch_flow_collection_end(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // Reset any potential simple key on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Decrease the flow level. + if !yaml_parser_decrease_flow_level(parser) { + return false + } + + // No simple keys after the indicators ']' and '}'. + parser.simple_key_allowed = false + + // Consume the token. + + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the FLOW-SEQUENCE-END of FLOW-MAPPING-END token. + token := yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the FLOW-ENTRY token. +func yaml_parser_fetch_flow_entry(parser *yaml_parser_t) bool { + // Reset any potential simple keys on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Simple keys are allowed after ','. + parser.simple_key_allowed = true + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the FLOW-ENTRY token and append it to the queue. + token := yaml_token_t{ + typ: yaml_FLOW_ENTRY_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the BLOCK-ENTRY token. +func yaml_parser_fetch_block_entry(parser *yaml_parser_t) bool { + // Check if the scanner is in the block context. + if parser.flow_level == 0 { + // Check if we are allowed to start a new entry. + if !parser.simple_key_allowed { + return yaml_parser_set_scanner_error(parser, "", parser.mark, + "block sequence entries are not allowed in this context") + } + // Add the BLOCK-SEQUENCE-START token if needed. + if !yaml_parser_roll_indent(parser, parser.mark.column, -1, yaml_BLOCK_SEQUENCE_START_TOKEN, parser.mark) { + return false + } + } else { + // It is an error for the '-' indicator to occur in the flow context, + // but we let the Parser detect and report about it because the Parser + // is able to point to the context. + } + + // Reset any potential simple keys on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Simple keys are allowed after '-'. + parser.simple_key_allowed = true + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the BLOCK-ENTRY token and append it to the queue. + token := yaml_token_t{ + typ: yaml_BLOCK_ENTRY_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the KEY token. +func yaml_parser_fetch_key(parser *yaml_parser_t) bool { + + // In the block context, additional checks are required. + if parser.flow_level == 0 { + // Check if we are allowed to start a new key (not nessesary simple). + if !parser.simple_key_allowed { + return yaml_parser_set_scanner_error(parser, "", parser.mark, + "mapping keys are not allowed in this context") + } + // Add the BLOCK-MAPPING-START token if needed. + if !yaml_parser_roll_indent(parser, parser.mark.column, -1, yaml_BLOCK_MAPPING_START_TOKEN, parser.mark) { + return false + } + } + + // Reset any potential simple keys on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Simple keys are allowed after '?' in the block context. + parser.simple_key_allowed = parser.flow_level == 0 + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the KEY token and append it to the queue. + token := yaml_token_t{ + typ: yaml_KEY_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the VALUE token. +func yaml_parser_fetch_value(parser *yaml_parser_t) bool { + + simple_key := &parser.simple_keys[len(parser.simple_keys)-1] + + // Have we found a simple key? + if valid, ok := yaml_simple_key_is_valid(parser, simple_key); !ok { + return false + + } else if valid { + + // Create the KEY token and insert it into the queue. + token := yaml_token_t{ + typ: yaml_KEY_TOKEN, + start_mark: simple_key.mark, + end_mark: simple_key.mark, + } + yaml_insert_token(parser, simple_key.token_number-parser.tokens_parsed, &token) + + // In the block context, we may need to add the BLOCK-MAPPING-START token. + if !yaml_parser_roll_indent(parser, simple_key.mark.column, + simple_key.token_number, + yaml_BLOCK_MAPPING_START_TOKEN, simple_key.mark) { + return false + } + + // Remove the simple key. + simple_key.possible = false + delete(parser.simple_keys_by_tok, simple_key.token_number) + + // A simple key cannot follow another simple key. + parser.simple_key_allowed = false + + } else { + // The ':' indicator follows a complex key. + + // In the block context, extra checks are required. + if parser.flow_level == 0 { + + // Check if we are allowed to start a complex value. + if !parser.simple_key_allowed { + return yaml_parser_set_scanner_error(parser, "", parser.mark, + "mapping values are not allowed in this context") + } + + // Add the BLOCK-MAPPING-START token if needed. + if !yaml_parser_roll_indent(parser, parser.mark.column, -1, yaml_BLOCK_MAPPING_START_TOKEN, parser.mark) { + return false + } + } + + // Simple keys after ':' are allowed in the block context. + parser.simple_key_allowed = parser.flow_level == 0 + } + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the VALUE token and append it to the queue. + token := yaml_token_t{ + typ: yaml_VALUE_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the ALIAS or ANCHOR token. +func yaml_parser_fetch_anchor(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // An anchor or an alias could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow an anchor or an alias. + parser.simple_key_allowed = false + + // Create the ALIAS or ANCHOR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_anchor(parser, &token, typ) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the TAG token. +func yaml_parser_fetch_tag(parser *yaml_parser_t) bool { + // A tag could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow a tag. + parser.simple_key_allowed = false + + // Create the TAG token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_tag(parser, &token) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the SCALAR(...,literal) or SCALAR(...,folded) tokens. +func yaml_parser_fetch_block_scalar(parser *yaml_parser_t, literal bool) bool { + // Remove any potential simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // A simple key may follow a block scalar. + parser.simple_key_allowed = true + + // Create the SCALAR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_block_scalar(parser, &token, literal) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the SCALAR(...,single-quoted) or SCALAR(...,double-quoted) tokens. +func yaml_parser_fetch_flow_scalar(parser *yaml_parser_t, single bool) bool { + // A plain scalar could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow a flow scalar. + parser.simple_key_allowed = false + + // Create the SCALAR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_flow_scalar(parser, &token, single) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the SCALAR(...,plain) token. +func yaml_parser_fetch_plain_scalar(parser *yaml_parser_t) bool { + // A plain scalar could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow a flow scalar. + parser.simple_key_allowed = false + + // Create the SCALAR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_plain_scalar(parser, &token) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Eat whitespaces and comments until the next token is found. +func yaml_parser_scan_to_next_token(parser *yaml_parser_t) bool { + + // Until the next token is not found. + for { + // Allow the BOM mark to start a line. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if parser.mark.column == 0 && is_bom(parser.buffer, parser.buffer_pos) { + skip(parser) + } + + // Eat whitespaces. + // Tabs are allowed: + // - in the flow context + // - in the block context, but not at the beginning of the line or + // after '-', '?', or ':' (complex value). + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for parser.buffer[parser.buffer_pos] == ' ' || ((parser.flow_level > 0 || !parser.simple_key_allowed) && parser.buffer[parser.buffer_pos] == '\t') { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Eat a comment until a line break. + if parser.buffer[parser.buffer_pos] == '#' { + for !is_breakz(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + } + + // If it is a line break, eat it. + if is_break(parser.buffer, parser.buffer_pos) { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + skip_line(parser) + + // In the block context, a new line may start a simple key. + if parser.flow_level == 0 { + parser.simple_key_allowed = true + } + } else { + break // We have found a token. + } + } + + return true +} + +// Scan a YAML-DIRECTIVE or TAG-DIRECTIVE token. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// %TAG !yaml! tag:yaml.org,2002: \n +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// +func yaml_parser_scan_directive(parser *yaml_parser_t, token *yaml_token_t) bool { + // Eat '%'. + start_mark := parser.mark + skip(parser) + + // Scan the directive name. + var name []byte + if !yaml_parser_scan_directive_name(parser, start_mark, &name) { + return false + } + + // Is it a YAML directive? + if bytes.Equal(name, []byte("YAML")) { + // Scan the VERSION directive value. + var major, minor int8 + if !yaml_parser_scan_version_directive_value(parser, start_mark, &major, &minor) { + return false + } + end_mark := parser.mark + + // Create a VERSION-DIRECTIVE token. + *token = yaml_token_t{ + typ: yaml_VERSION_DIRECTIVE_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + major: major, + minor: minor, + } + + // Is it a TAG directive? + } else if bytes.Equal(name, []byte("TAG")) { + // Scan the TAG directive value. + var handle, prefix []byte + if !yaml_parser_scan_tag_directive_value(parser, start_mark, &handle, &prefix) { + return false + } + end_mark := parser.mark + + // Create a TAG-DIRECTIVE token. + *token = yaml_token_t{ + typ: yaml_TAG_DIRECTIVE_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: handle, + prefix: prefix, + } + + // Unknown directive. + } else { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "found unknown directive name") + return false + } + + // Eat the rest of the line including any comments. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + if parser.buffer[parser.buffer_pos] == '#' { + for !is_breakz(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + } + + // Check if we are at the end of the line. + if !is_breakz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "did not find expected comment or line break") + return false + } + + // Eat a line break. + if is_break(parser.buffer, parser.buffer_pos) { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + skip_line(parser) + } + + return true +} + +// Scan the directive name. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^^^^ +// %TAG !yaml! tag:yaml.org,2002: \n +// ^^^ +// +func yaml_parser_scan_directive_name(parser *yaml_parser_t, start_mark yaml_mark_t, name *[]byte) bool { + // Consume the directive name. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + var s []byte + for is_alpha(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check if the name is empty. + if len(s) == 0 { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "could not find expected directive name") + return false + } + + // Check for an blank character after the name. + if !is_blankz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "found unexpected non-alphabetical character") + return false + } + *name = s + return true +} + +// Scan the value of VERSION-DIRECTIVE. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^^^^^^ +func yaml_parser_scan_version_directive_value(parser *yaml_parser_t, start_mark yaml_mark_t, major, minor *int8) bool { + // Eat whitespaces. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Consume the major version number. + if !yaml_parser_scan_version_directive_number(parser, start_mark, major) { + return false + } + + // Eat '.'. + if parser.buffer[parser.buffer_pos] != '.' { + return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive", + start_mark, "did not find expected digit or '.' character") + } + + skip(parser) + + // Consume the minor version number. + if !yaml_parser_scan_version_directive_number(parser, start_mark, minor) { + return false + } + return true +} + +const max_number_length = 2 + +// Scan the version number of VERSION-DIRECTIVE. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^ +// %YAML 1.1 # a comment \n +// ^ +func yaml_parser_scan_version_directive_number(parser *yaml_parser_t, start_mark yaml_mark_t, number *int8) bool { + + // Repeat while the next character is digit. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + var value, length int8 + for is_digit(parser.buffer, parser.buffer_pos) { + // Check if the number is too long. + length++ + if length > max_number_length { + return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive", + start_mark, "found extremely long version number") + } + value = value*10 + int8(as_digit(parser.buffer, parser.buffer_pos)) + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check if the number was present. + if length == 0 { + return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive", + start_mark, "did not find expected version number") + } + *number = value + return true +} + +// Scan the value of a TAG-DIRECTIVE token. +// +// Scope: +// %TAG !yaml! tag:yaml.org,2002: \n +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// +func yaml_parser_scan_tag_directive_value(parser *yaml_parser_t, start_mark yaml_mark_t, handle, prefix *[]byte) bool { + var handle_value, prefix_value []byte + + // Eat whitespaces. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Scan a handle. + if !yaml_parser_scan_tag_handle(parser, true, start_mark, &handle_value) { + return false + } + + // Expect a whitespace. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if !is_blank(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a %TAG directive", + start_mark, "did not find expected whitespace") + return false + } + + // Eat whitespaces. + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Scan a prefix. + if !yaml_parser_scan_tag_uri(parser, true, nil, start_mark, &prefix_value) { + return false + } + + // Expect a whitespace or line break. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if !is_blankz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a %TAG directive", + start_mark, "did not find expected whitespace or line break") + return false + } + + *handle = handle_value + *prefix = prefix_value + return true +} + +func yaml_parser_scan_anchor(parser *yaml_parser_t, token *yaml_token_t, typ yaml_token_type_t) bool { + var s []byte + + // Eat the indicator character. + start_mark := parser.mark + skip(parser) + + // Consume the value. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_alpha(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + end_mark := parser.mark + + /* + * Check if length of the anchor is greater than 0 and it is followed by + * a whitespace character or one of the indicators: + * + * '?', ':', ',', ']', '}', '%', '@', '`'. + */ + + if len(s) == 0 || + !(is_blankz(parser.buffer, parser.buffer_pos) || parser.buffer[parser.buffer_pos] == '?' || + parser.buffer[parser.buffer_pos] == ':' || parser.buffer[parser.buffer_pos] == ',' || + parser.buffer[parser.buffer_pos] == ']' || parser.buffer[parser.buffer_pos] == '}' || + parser.buffer[parser.buffer_pos] == '%' || parser.buffer[parser.buffer_pos] == '@' || + parser.buffer[parser.buffer_pos] == '`') { + context := "while scanning an alias" + if typ == yaml_ANCHOR_TOKEN { + context = "while scanning an anchor" + } + yaml_parser_set_scanner_error(parser, context, start_mark, + "did not find expected alphabetic or numeric character") + return false + } + + // Create a token. + *token = yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + value: s, + } + + return true +} + +/* + * Scan a TAG token. + */ + +func yaml_parser_scan_tag(parser *yaml_parser_t, token *yaml_token_t) bool { + var handle, suffix []byte + + start_mark := parser.mark + + // Check if the tag is in the canonical form. + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + if parser.buffer[parser.buffer_pos+1] == '<' { + // Keep the handle as '' + + // Eat '!<' + skip(parser) + skip(parser) + + // Consume the tag value. + if !yaml_parser_scan_tag_uri(parser, false, nil, start_mark, &suffix) { + return false + } + + // Check for '>' and eat it. + if parser.buffer[parser.buffer_pos] != '>' { + yaml_parser_set_scanner_error(parser, "while scanning a tag", + start_mark, "did not find the expected '>'") + return false + } + + skip(parser) + } else { + // The tag has either the '!suffix' or the '!handle!suffix' form. + + // First, try to scan a handle. + if !yaml_parser_scan_tag_handle(parser, false, start_mark, &handle) { + return false + } + + // Check if it is, indeed, handle. + if handle[0] == '!' && len(handle) > 1 && handle[len(handle)-1] == '!' { + // Scan the suffix now. + if !yaml_parser_scan_tag_uri(parser, false, nil, start_mark, &suffix) { + return false + } + } else { + // It wasn't a handle after all. Scan the rest of the tag. + if !yaml_parser_scan_tag_uri(parser, false, handle, start_mark, &suffix) { + return false + } + + // Set the handle to '!'. + handle = []byte{'!'} + + // A special case: the '!' tag. Set the handle to '' and the + // suffix to '!'. + if len(suffix) == 0 { + handle, suffix = suffix, handle + } + } + } + + // Check the character which ends the tag. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if !is_blankz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a tag", + start_mark, "did not find expected whitespace or line break") + return false + } + + end_mark := parser.mark + + // Create a token. + *token = yaml_token_t{ + typ: yaml_TAG_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: handle, + suffix: suffix, + } + return true +} + +// Scan a tag handle. +func yaml_parser_scan_tag_handle(parser *yaml_parser_t, directive bool, start_mark yaml_mark_t, handle *[]byte) bool { + // Check the initial '!' character. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if parser.buffer[parser.buffer_pos] != '!' { + yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find expected '!'") + return false + } + + var s []byte + + // Copy the '!' character. + s = read(parser, s) + + // Copy all subsequent alphabetical and numerical characters. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for is_alpha(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check if the trailing character is '!' and copy it. + if parser.buffer[parser.buffer_pos] == '!' { + s = read(parser, s) + } else { + // It's either the '!' tag or not really a tag handle. If it's a %TAG + // directive, it's an error. If it's a tag token, it must be a part of URI. + if directive && string(s) != "!" { + yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find expected '!'") + return false + } + } + + *handle = s + return true +} + +// Scan a tag. +func yaml_parser_scan_tag_uri(parser *yaml_parser_t, directive bool, head []byte, start_mark yaml_mark_t, uri *[]byte) bool { + //size_t length = head ? strlen((char *)head) : 0 + var s []byte + hasTag := len(head) > 0 + + // Copy the head if needed. + // + // Note that we don't copy the leading '!' character. + if len(head) > 1 { + s = append(s, head[1:]...) + } + + // Scan the tag. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // The set of characters that may appear in URI is as follows: + // + // '0'-'9', 'A'-'Z', 'a'-'z', '_', '-', ';', '/', '?', ':', '@', '&', + // '=', '+', '$', ',', '.', '!', '~', '*', '\'', '(', ')', '[', ']', + // '%'. + // [Go] Convert this into more reasonable logic. + for is_alpha(parser.buffer, parser.buffer_pos) || parser.buffer[parser.buffer_pos] == ';' || + parser.buffer[parser.buffer_pos] == '/' || parser.buffer[parser.buffer_pos] == '?' || + parser.buffer[parser.buffer_pos] == ':' || parser.buffer[parser.buffer_pos] == '@' || + parser.buffer[parser.buffer_pos] == '&' || parser.buffer[parser.buffer_pos] == '=' || + parser.buffer[parser.buffer_pos] == '+' || parser.buffer[parser.buffer_pos] == '$' || + parser.buffer[parser.buffer_pos] == ',' || parser.buffer[parser.buffer_pos] == '.' || + parser.buffer[parser.buffer_pos] == '!' || parser.buffer[parser.buffer_pos] == '~' || + parser.buffer[parser.buffer_pos] == '*' || parser.buffer[parser.buffer_pos] == '\'' || + parser.buffer[parser.buffer_pos] == '(' || parser.buffer[parser.buffer_pos] == ')' || + parser.buffer[parser.buffer_pos] == '[' || parser.buffer[parser.buffer_pos] == ']' || + parser.buffer[parser.buffer_pos] == '%' { + // Check if it is a URI-escape sequence. + if parser.buffer[parser.buffer_pos] == '%' { + if !yaml_parser_scan_uri_escapes(parser, directive, start_mark, &s) { + return false + } + } else { + s = read(parser, s) + } + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + hasTag = true + } + + if !hasTag { + yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find expected tag URI") + return false + } + *uri = s + return true +} + +// Decode an URI-escape sequence corresponding to a single UTF-8 character. +func yaml_parser_scan_uri_escapes(parser *yaml_parser_t, directive bool, start_mark yaml_mark_t, s *[]byte) bool { + + // Decode the required number of characters. + w := 1024 + for w > 0 { + // Check for a URI-escaped octet. + if parser.unread < 3 && !yaml_parser_update_buffer(parser, 3) { + return false + } + + if !(parser.buffer[parser.buffer_pos] == '%' && + is_hex(parser.buffer, parser.buffer_pos+1) && + is_hex(parser.buffer, parser.buffer_pos+2)) { + return yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find URI escaped octet") + } + + // Get the octet. + octet := byte((as_hex(parser.buffer, parser.buffer_pos+1) << 4) + as_hex(parser.buffer, parser.buffer_pos+2)) + + // If it is the leading octet, determine the length of the UTF-8 sequence. + if w == 1024 { + w = width(octet) + if w == 0 { + return yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "found an incorrect leading UTF-8 octet") + } + } else { + // Check if the trailing octet is correct. + if octet&0xC0 != 0x80 { + return yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "found an incorrect trailing UTF-8 octet") + } + } + + // Copy the octet and move the pointers. + *s = append(*s, octet) + skip(parser) + skip(parser) + skip(parser) + w-- + } + return true +} + +// Scan a block scalar. +func yaml_parser_scan_block_scalar(parser *yaml_parser_t, token *yaml_token_t, literal bool) bool { + // Eat the indicator '|' or '>'. + start_mark := parser.mark + skip(parser) + + // Scan the additional block scalar indicators. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // Check for a chomping indicator. + var chomping, increment int + if parser.buffer[parser.buffer_pos] == '+' || parser.buffer[parser.buffer_pos] == '-' { + // Set the chomping method and eat the indicator. + if parser.buffer[parser.buffer_pos] == '+' { + chomping = +1 + } else { + chomping = -1 + } + skip(parser) + + // Check for an indentation indicator. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if is_digit(parser.buffer, parser.buffer_pos) { + // Check that the indentation is greater than 0. + if parser.buffer[parser.buffer_pos] == '0' { + yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "found an indentation indicator equal to 0") + return false + } + + // Get the indentation level and eat the indicator. + increment = as_digit(parser.buffer, parser.buffer_pos) + skip(parser) + } + + } else if is_digit(parser.buffer, parser.buffer_pos) { + // Do the same as above, but in the opposite order. + + if parser.buffer[parser.buffer_pos] == '0' { + yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "found an indentation indicator equal to 0") + return false + } + increment = as_digit(parser.buffer, parser.buffer_pos) + skip(parser) + + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if parser.buffer[parser.buffer_pos] == '+' || parser.buffer[parser.buffer_pos] == '-' { + if parser.buffer[parser.buffer_pos] == '+' { + chomping = +1 + } else { + chomping = -1 + } + skip(parser) + } + } + + // Eat whitespaces and comments to the end of the line. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + if parser.buffer[parser.buffer_pos] == '#' { + for !is_breakz(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + } + + // Check if we are at the end of the line. + if !is_breakz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "did not find expected comment or line break") + return false + } + + // Eat a line break. + if is_break(parser.buffer, parser.buffer_pos) { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + skip_line(parser) + } + + end_mark := parser.mark + + // Set the indentation level if it was specified. + var indent int + if increment > 0 { + if parser.indent >= 0 { + indent = parser.indent + increment + } else { + indent = increment + } + } + + // Scan the leading line breaks and determine the indentation level if needed. + var s, leading_break, trailing_breaks []byte + if !yaml_parser_scan_block_scalar_breaks(parser, &indent, &trailing_breaks, start_mark, &end_mark) { + return false + } + + // Scan the block scalar content. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + var leading_blank, trailing_blank bool + for parser.mark.column == indent && !is_z(parser.buffer, parser.buffer_pos) { + // We are at the beginning of a non-empty line. + + // Is it a trailing whitespace? + trailing_blank = is_blank(parser.buffer, parser.buffer_pos) + + // Check if we need to fold the leading line break. + if !literal && !leading_blank && !trailing_blank && len(leading_break) > 0 && leading_break[0] == '\n' { + // Do we need to join the lines by space? + if len(trailing_breaks) == 0 { + s = append(s, ' ') + } + } else { + s = append(s, leading_break...) + } + leading_break = leading_break[:0] + + // Append the remaining line breaks. + s = append(s, trailing_breaks...) + trailing_breaks = trailing_breaks[:0] + + // Is it a leading whitespace? + leading_blank = is_blank(parser.buffer, parser.buffer_pos) + + // Consume the current line. + for !is_breakz(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Consume the line break. + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + leading_break = read_line(parser, leading_break) + + // Eat the following indentation spaces and line breaks. + if !yaml_parser_scan_block_scalar_breaks(parser, &indent, &trailing_breaks, start_mark, &end_mark) { + return false + } + } + + // Chomp the tail. + if chomping != -1 { + s = append(s, leading_break...) + } + if chomping == 1 { + s = append(s, trailing_breaks...) + } + + // Create a token. + *token = yaml_token_t{ + typ: yaml_SCALAR_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: s, + style: yaml_LITERAL_SCALAR_STYLE, + } + if !literal { + token.style = yaml_FOLDED_SCALAR_STYLE + } + return true +} + +// Scan indentation spaces and line breaks for a block scalar. Determine the +// indentation level if needed. +func yaml_parser_scan_block_scalar_breaks(parser *yaml_parser_t, indent *int, breaks *[]byte, start_mark yaml_mark_t, end_mark *yaml_mark_t) bool { + *end_mark = parser.mark + + // Eat the indentation spaces and line breaks. + max_indent := 0 + for { + // Eat the indentation spaces. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for (*indent == 0 || parser.mark.column < *indent) && is_space(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + if parser.mark.column > max_indent { + max_indent = parser.mark.column + } + + // Check for a tab character messing the indentation. + if (*indent == 0 || parser.mark.column < *indent) && is_tab(parser.buffer, parser.buffer_pos) { + return yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "found a tab character where an indentation space is expected") + } + + // Have we found a non-empty line? + if !is_break(parser.buffer, parser.buffer_pos) { + break + } + + // Consume the line break. + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + // [Go] Should really be returning breaks instead. + *breaks = read_line(parser, *breaks) + *end_mark = parser.mark + } + + // Determine the indentation level if needed. + if *indent == 0 { + *indent = max_indent + if *indent < parser.indent+1 { + *indent = parser.indent + 1 + } + if *indent < 1 { + *indent = 1 + } + } + return true +} + +// Scan a quoted scalar. +func yaml_parser_scan_flow_scalar(parser *yaml_parser_t, token *yaml_token_t, single bool) bool { + // Eat the left quote. + start_mark := parser.mark + skip(parser) + + // Consume the content of the quoted scalar. + var s, leading_break, trailing_breaks, whitespaces []byte + for { + // Check that there are no document indicators at the beginning of the line. + if parser.unread < 4 && !yaml_parser_update_buffer(parser, 4) { + return false + } + + if parser.mark.column == 0 && + ((parser.buffer[parser.buffer_pos+0] == '-' && + parser.buffer[parser.buffer_pos+1] == '-' && + parser.buffer[parser.buffer_pos+2] == '-') || + (parser.buffer[parser.buffer_pos+0] == '.' && + parser.buffer[parser.buffer_pos+1] == '.' && + parser.buffer[parser.buffer_pos+2] == '.')) && + is_blankz(parser.buffer, parser.buffer_pos+3) { + yaml_parser_set_scanner_error(parser, "while scanning a quoted scalar", + start_mark, "found unexpected document indicator") + return false + } + + // Check for EOF. + if is_z(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a quoted scalar", + start_mark, "found unexpected end of stream") + return false + } + + // Consume non-blank characters. + leading_blanks := false + for !is_blankz(parser.buffer, parser.buffer_pos) { + if single && parser.buffer[parser.buffer_pos] == '\'' && parser.buffer[parser.buffer_pos+1] == '\'' { + // Is is an escaped single quote. + s = append(s, '\'') + skip(parser) + skip(parser) + + } else if single && parser.buffer[parser.buffer_pos] == '\'' { + // It is a right single quote. + break + } else if !single && parser.buffer[parser.buffer_pos] == '"' { + // It is a right double quote. + break + + } else if !single && parser.buffer[parser.buffer_pos] == '\\' && is_break(parser.buffer, parser.buffer_pos+1) { + // It is an escaped line break. + if parser.unread < 3 && !yaml_parser_update_buffer(parser, 3) { + return false + } + skip(parser) + skip_line(parser) + leading_blanks = true + break + + } else if !single && parser.buffer[parser.buffer_pos] == '\\' { + // It is an escape sequence. + code_length := 0 + + // Check the escape character. + switch parser.buffer[parser.buffer_pos+1] { + case '0': + s = append(s, 0) + case 'a': + s = append(s, '\x07') + case 'b': + s = append(s, '\x08') + case 't', '\t': + s = append(s, '\x09') + case 'n': + s = append(s, '\x0A') + case 'v': + s = append(s, '\x0B') + case 'f': + s = append(s, '\x0C') + case 'r': + s = append(s, '\x0D') + case 'e': + s = append(s, '\x1B') + case ' ': + s = append(s, '\x20') + case '"': + s = append(s, '"') + case '\'': + s = append(s, '\'') + case '\\': + s = append(s, '\\') + case 'N': // NEL (#x85) + s = append(s, '\xC2') + s = append(s, '\x85') + case '_': // #xA0 + s = append(s, '\xC2') + s = append(s, '\xA0') + case 'L': // LS (#x2028) + s = append(s, '\xE2') + s = append(s, '\x80') + s = append(s, '\xA8') + case 'P': // PS (#x2029) + s = append(s, '\xE2') + s = append(s, '\x80') + s = append(s, '\xA9') + case 'x': + code_length = 2 + case 'u': + code_length = 4 + case 'U': + code_length = 8 + default: + yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar", + start_mark, "found unknown escape character") + return false + } + + skip(parser) + skip(parser) + + // Consume an arbitrary escape code. + if code_length > 0 { + var value int + + // Scan the character value. + if parser.unread < code_length && !yaml_parser_update_buffer(parser, code_length) { + return false + } + for k := 0; k < code_length; k++ { + if !is_hex(parser.buffer, parser.buffer_pos+k) { + yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar", + start_mark, "did not find expected hexdecimal number") + return false + } + value = (value << 4) + as_hex(parser.buffer, parser.buffer_pos+k) + } + + // Check the value and write the character. + if (value >= 0xD800 && value <= 0xDFFF) || value > 0x10FFFF { + yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar", + start_mark, "found invalid Unicode character escape code") + return false + } + if value <= 0x7F { + s = append(s, byte(value)) + } else if value <= 0x7FF { + s = append(s, byte(0xC0+(value>>6))) + s = append(s, byte(0x80+(value&0x3F))) + } else if value <= 0xFFFF { + s = append(s, byte(0xE0+(value>>12))) + s = append(s, byte(0x80+((value>>6)&0x3F))) + s = append(s, byte(0x80+(value&0x3F))) + } else { + s = append(s, byte(0xF0+(value>>18))) + s = append(s, byte(0x80+((value>>12)&0x3F))) + s = append(s, byte(0x80+((value>>6)&0x3F))) + s = append(s, byte(0x80+(value&0x3F))) + } + + // Advance the pointer. + for k := 0; k < code_length; k++ { + skip(parser) + } + } + } else { + // It is a non-escaped non-blank character. + s = read(parser, s) + } + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + } + + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // Check if we are at the end of the scalar. + if single { + if parser.buffer[parser.buffer_pos] == '\'' { + break + } + } else { + if parser.buffer[parser.buffer_pos] == '"' { + break + } + } + + // Consume blank characters. + for is_blank(parser.buffer, parser.buffer_pos) || is_break(parser.buffer, parser.buffer_pos) { + if is_blank(parser.buffer, parser.buffer_pos) { + // Consume a space or a tab character. + if !leading_blanks { + whitespaces = read(parser, whitespaces) + } else { + skip(parser) + } + } else { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + // Check if it is a first line break. + if !leading_blanks { + whitespaces = whitespaces[:0] + leading_break = read_line(parser, leading_break) + leading_blanks = true + } else { + trailing_breaks = read_line(parser, trailing_breaks) + } + } + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Join the whitespaces or fold line breaks. + if leading_blanks { + // Do we need to fold line breaks? + if len(leading_break) > 0 && leading_break[0] == '\n' { + if len(trailing_breaks) == 0 { + s = append(s, ' ') + } else { + s = append(s, trailing_breaks...) + } + } else { + s = append(s, leading_break...) + s = append(s, trailing_breaks...) + } + trailing_breaks = trailing_breaks[:0] + leading_break = leading_break[:0] + } else { + s = append(s, whitespaces...) + whitespaces = whitespaces[:0] + } + } + + // Eat the right quote. + skip(parser) + end_mark := parser.mark + + // Create a token. + *token = yaml_token_t{ + typ: yaml_SCALAR_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: s, + style: yaml_SINGLE_QUOTED_SCALAR_STYLE, + } + if !single { + token.style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + return true +} + +// Scan a plain scalar. +func yaml_parser_scan_plain_scalar(parser *yaml_parser_t, token *yaml_token_t) bool { + + var s, leading_break, trailing_breaks, whitespaces []byte + var leading_blanks bool + var indent = parser.indent + 1 + + start_mark := parser.mark + end_mark := parser.mark + + // Consume the content of the plain scalar. + for { + // Check for a document indicator. + if parser.unread < 4 && !yaml_parser_update_buffer(parser, 4) { + return false + } + if parser.mark.column == 0 && + ((parser.buffer[parser.buffer_pos+0] == '-' && + parser.buffer[parser.buffer_pos+1] == '-' && + parser.buffer[parser.buffer_pos+2] == '-') || + (parser.buffer[parser.buffer_pos+0] == '.' && + parser.buffer[parser.buffer_pos+1] == '.' && + parser.buffer[parser.buffer_pos+2] == '.')) && + is_blankz(parser.buffer, parser.buffer_pos+3) { + break + } + + // Check for a comment. + if parser.buffer[parser.buffer_pos] == '#' { + break + } + + // Consume non-blank characters. + for !is_blankz(parser.buffer, parser.buffer_pos) { + + // Check for indicators that may end a plain scalar. + if (parser.buffer[parser.buffer_pos] == ':' && is_blankz(parser.buffer, parser.buffer_pos+1)) || + (parser.flow_level > 0 && + (parser.buffer[parser.buffer_pos] == ',' || + parser.buffer[parser.buffer_pos] == '?' || parser.buffer[parser.buffer_pos] == '[' || + parser.buffer[parser.buffer_pos] == ']' || parser.buffer[parser.buffer_pos] == '{' || + parser.buffer[parser.buffer_pos] == '}')) { + break + } + + // Check if we need to join whitespaces and breaks. + if leading_blanks || len(whitespaces) > 0 { + if leading_blanks { + // Do we need to fold line breaks? + if leading_break[0] == '\n' { + if len(trailing_breaks) == 0 { + s = append(s, ' ') + } else { + s = append(s, trailing_breaks...) + } + } else { + s = append(s, leading_break...) + s = append(s, trailing_breaks...) + } + trailing_breaks = trailing_breaks[:0] + leading_break = leading_break[:0] + leading_blanks = false + } else { + s = append(s, whitespaces...) + whitespaces = whitespaces[:0] + } + } + + // Copy the character. + s = read(parser, s) + + end_mark = parser.mark + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + } + + // Is it the end? + if !(is_blank(parser.buffer, parser.buffer_pos) || is_break(parser.buffer, parser.buffer_pos)) { + break + } + + // Consume blank characters. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_blank(parser.buffer, parser.buffer_pos) || is_break(parser.buffer, parser.buffer_pos) { + if is_blank(parser.buffer, parser.buffer_pos) { + + // Check for tab characters that abuse indentation. + if leading_blanks && parser.mark.column < indent && is_tab(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a plain scalar", + start_mark, "found a tab character that violates indentation") + return false + } + + // Consume a space or a tab character. + if !leading_blanks { + whitespaces = read(parser, whitespaces) + } else { + skip(parser) + } + } else { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + // Check if it is a first line break. + if !leading_blanks { + whitespaces = whitespaces[:0] + leading_break = read_line(parser, leading_break) + leading_blanks = true + } else { + trailing_breaks = read_line(parser, trailing_breaks) + } + } + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check indentation level. + if parser.flow_level == 0 && parser.mark.column < indent { + break + } + } + + // Create a token. + *token = yaml_token_t{ + typ: yaml_SCALAR_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: s, + style: yaml_PLAIN_SCALAR_STYLE, + } + + // Note that we change the 'simple_key_allowed' flag. + if leading_blanks { + parser.simple_key_allowed = true + } + return true +} diff --git a/vendor/gopkg.in/yaml.v2/sorter.go b/vendor/gopkg.in/yaml.v2/sorter.go new file mode 100644 index 000000000..4c45e660a --- /dev/null +++ b/vendor/gopkg.in/yaml.v2/sorter.go @@ -0,0 +1,113 @@ +package yaml + +import ( + "reflect" + "unicode" +) + +type keyList []reflect.Value + +func (l keyList) Len() int { return len(l) } +func (l keyList) Swap(i, j int) { l[i], l[j] = l[j], l[i] } +func (l keyList) Less(i, j int) bool { + a := l[i] + b := l[j] + ak := a.Kind() + bk := b.Kind() + for (ak == reflect.Interface || ak == reflect.Ptr) && !a.IsNil() { + a = a.Elem() + ak = a.Kind() + } + for (bk == reflect.Interface || bk == reflect.Ptr) && !b.IsNil() { + b = b.Elem() + bk = b.Kind() + } + af, aok := keyFloat(a) + bf, bok := keyFloat(b) + if aok && bok { + if af != bf { + return af < bf + } + if ak != bk { + return ak < bk + } + return numLess(a, b) + } + if ak != reflect.String || bk != reflect.String { + return ak < bk + } + ar, br := []rune(a.String()), []rune(b.String()) + for i := 0; i < len(ar) && i < len(br); i++ { + if ar[i] == br[i] { + continue + } + al := unicode.IsLetter(ar[i]) + bl := unicode.IsLetter(br[i]) + if al && bl { + return ar[i] < br[i] + } + if al || bl { + return bl + } + var ai, bi int + var an, bn int64 + if ar[i] == '0' || br[i] == '0' { + for j := i-1; j >= 0 && unicode.IsDigit(ar[j]); j-- { + if ar[j] != '0' { + an = 1 + bn = 1 + break + } + } + } + for ai = i; ai < len(ar) && unicode.IsDigit(ar[ai]); ai++ { + an = an*10 + int64(ar[ai]-'0') + } + for bi = i; bi < len(br) && unicode.IsDigit(br[bi]); bi++ { + bn = bn*10 + int64(br[bi]-'0') + } + if an != bn { + return an < bn + } + if ai != bi { + return ai < bi + } + return ar[i] < br[i] + } + return len(ar) < len(br) +} + +// keyFloat returns a float value for v if it is a number/bool +// and whether it is a number/bool or not. +func keyFloat(v reflect.Value) (f float64, ok bool) { + switch v.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return float64(v.Int()), true + case reflect.Float32, reflect.Float64: + return v.Float(), true + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return float64(v.Uint()), true + case reflect.Bool: + if v.Bool() { + return 1, true + } + return 0, true + } + return 0, false +} + +// numLess returns whether a < b. +// a and b must necessarily have the same kind. +func numLess(a, b reflect.Value) bool { + switch a.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return a.Int() < b.Int() + case reflect.Float32, reflect.Float64: + return a.Float() < b.Float() + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return a.Uint() < b.Uint() + case reflect.Bool: + return !a.Bool() && b.Bool() + } + panic("not a number") +} diff --git a/vendor/gopkg.in/yaml.v2/writerc.go b/vendor/gopkg.in/yaml.v2/writerc.go new file mode 100644 index 000000000..a2dde608c --- /dev/null +++ b/vendor/gopkg.in/yaml.v2/writerc.go @@ -0,0 +1,26 @@ +package yaml + +// Set the writer error and return false. +func yaml_emitter_set_writer_error(emitter *yaml_emitter_t, problem string) bool { + emitter.error = yaml_WRITER_ERROR + emitter.problem = problem + return false +} + +// Flush the output buffer. +func yaml_emitter_flush(emitter *yaml_emitter_t) bool { + if emitter.write_handler == nil { + panic("write handler not set") + } + + // Check if the buffer is empty. + if emitter.buffer_pos == 0 { + return true + } + + if err := emitter.write_handler(emitter, emitter.buffer[:emitter.buffer_pos]); err != nil { + return yaml_emitter_set_writer_error(emitter, "write error: "+err.Error()) + } + emitter.buffer_pos = 0 + return true +} diff --git a/vendor/gopkg.in/yaml.v2/yaml.go b/vendor/gopkg.in/yaml.v2/yaml.go new file mode 100644 index 000000000..30813884c --- /dev/null +++ b/vendor/gopkg.in/yaml.v2/yaml.go @@ -0,0 +1,478 @@ +// Package yaml implements YAML support for the Go language. +// +// Source code and other details for the project are available at GitHub: +// +// https://github.com/go-yaml/yaml +// +package yaml + +import ( + "errors" + "fmt" + "io" + "reflect" + "strings" + "sync" +) + +// MapSlice encodes and decodes as a YAML map. +// The order of keys is preserved when encoding and decoding. +type MapSlice []MapItem + +// MapItem is an item in a MapSlice. +type MapItem struct { + Key, Value interface{} +} + +// The Unmarshaler interface may be implemented by types to customize their +// behavior when being unmarshaled from a YAML document. The UnmarshalYAML +// method receives a function that may be called to unmarshal the original +// YAML value into a field or variable. It is safe to call the unmarshal +// function parameter more than once if necessary. +type Unmarshaler interface { + UnmarshalYAML(unmarshal func(interface{}) error) error +} + +// The Marshaler interface may be implemented by types to customize their +// behavior when being marshaled into a YAML document. The returned value +// is marshaled in place of the original value implementing Marshaler. +// +// If an error is returned by MarshalYAML, the marshaling procedure stops +// and returns with the provided error. +type Marshaler interface { + MarshalYAML() (interface{}, error) +} + +// Unmarshal decodes the first document found within the in byte slice +// and assigns decoded values into the out value. +// +// Maps and pointers (to a struct, string, int, etc) are accepted as out +// values. If an internal pointer within a struct is not initialized, +// the yaml package will initialize it if necessary for unmarshalling +// the provided data. The out parameter must not be nil. +// +// The type of the decoded values should be compatible with the respective +// values in out. If one or more values cannot be decoded due to a type +// mismatches, decoding continues partially until the end of the YAML +// content, and a *yaml.TypeError is returned with details for all +// missed values. +// +// Struct fields are only unmarshalled if they are exported (have an +// upper case first letter), and are unmarshalled using the field name +// lowercased as the default key. Custom keys may be defined via the +// "yaml" name in the field tag: the content preceding the first comma +// is used as the key, and the following comma-separated options are +// used to tweak the marshalling process (see Marshal). +// Conflicting names result in a runtime error. +// +// For example: +// +// type T struct { +// F int `yaml:"a,omitempty"` +// B int +// } +// var t T +// yaml.Unmarshal([]byte("a: 1\nb: 2"), &t) +// +// See the documentation of Marshal for the format of tags and a list of +// supported tag options. +// +func Unmarshal(in []byte, out interface{}) (err error) { + return unmarshal(in, out, false) +} + +// UnmarshalStrict is like Unmarshal except that any fields that are found +// in the data that do not have corresponding struct members, or mapping +// keys that are duplicates, will result in +// an error. +func UnmarshalStrict(in []byte, out interface{}) (err error) { + return unmarshal(in, out, true) +} + +// A Decoder reads and decodes YAML values from an input stream. +type Decoder struct { + strict bool + parser *parser +} + +// NewDecoder returns a new decoder that reads from r. +// +// The decoder introduces its own buffering and may read +// data from r beyond the YAML values requested. +func NewDecoder(r io.Reader) *Decoder { + return &Decoder{ + parser: newParserFromReader(r), + } +} + +// SetStrict sets whether strict decoding behaviour is enabled when +// decoding items in the data (see UnmarshalStrict). By default, decoding is not strict. +func (dec *Decoder) SetStrict(strict bool) { + dec.strict = strict +} + +// Decode reads the next YAML-encoded value from its input +// and stores it in the value pointed to by v. +// +// See the documentation for Unmarshal for details about the +// conversion of YAML into a Go value. +func (dec *Decoder) Decode(v interface{}) (err error) { + d := newDecoder(dec.strict) + defer handleErr(&err) + node := dec.parser.parse() + if node == nil { + return io.EOF + } + out := reflect.ValueOf(v) + if out.Kind() == reflect.Ptr && !out.IsNil() { + out = out.Elem() + } + d.unmarshal(node, out) + if len(d.terrors) > 0 { + return &TypeError{d.terrors} + } + return nil +} + +func unmarshal(in []byte, out interface{}, strict bool) (err error) { + defer handleErr(&err) + d := newDecoder(strict) + p := newParser(in) + defer p.destroy() + node := p.parse() + if node != nil { + v := reflect.ValueOf(out) + if v.Kind() == reflect.Ptr && !v.IsNil() { + v = v.Elem() + } + d.unmarshal(node, v) + } + if len(d.terrors) > 0 { + return &TypeError{d.terrors} + } + return nil +} + +// Marshal serializes the value provided into a YAML document. The structure +// of the generated document will reflect the structure of the value itself. +// Maps and pointers (to struct, string, int, etc) are accepted as the in value. +// +// Struct fields are only marshalled if they are exported (have an upper case +// first letter), and are marshalled using the field name lowercased as the +// default key. Custom keys may be defined via the "yaml" name in the field +// tag: the content preceding the first comma is used as the key, and the +// following comma-separated options are used to tweak the marshalling process. +// Conflicting names result in a runtime error. +// +// The field tag format accepted is: +// +// `(...) yaml:"[][,[,]]" (...)` +// +// The following flags are currently supported: +// +// omitempty Only include the field if it's not set to the zero +// value for the type or to empty slices or maps. +// Zero valued structs will be omitted if all their public +// fields are zero, unless they implement an IsZero +// method (see the IsZeroer interface type), in which +// case the field will be excluded if IsZero returns true. +// +// flow Marshal using a flow style (useful for structs, +// sequences and maps). +// +// inline Inline the field, which must be a struct or a map, +// causing all of its fields or keys to be processed as if +// they were part of the outer struct. For maps, keys must +// not conflict with the yaml keys of other struct fields. +// +// In addition, if the key is "-", the field is ignored. +// +// For example: +// +// type T struct { +// F int `yaml:"a,omitempty"` +// B int +// } +// yaml.Marshal(&T{B: 2}) // Returns "b: 2\n" +// yaml.Marshal(&T{F: 1}} // Returns "a: 1\nb: 0\n" +// +func Marshal(in interface{}) (out []byte, err error) { + defer handleErr(&err) + e := newEncoder() + defer e.destroy() + e.marshalDoc("", reflect.ValueOf(in)) + e.finish() + out = e.out + return +} + +// An Encoder writes YAML values to an output stream. +type Encoder struct { + encoder *encoder +} + +// NewEncoder returns a new encoder that writes to w. +// The Encoder should be closed after use to flush all data +// to w. +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{ + encoder: newEncoderWithWriter(w), + } +} + +// Encode writes the YAML encoding of v to the stream. +// If multiple items are encoded to the stream, the +// second and subsequent document will be preceded +// with a "---" document separator, but the first will not. +// +// See the documentation for Marshal for details about the conversion of Go +// values to YAML. +func (e *Encoder) Encode(v interface{}) (err error) { + defer handleErr(&err) + e.encoder.marshalDoc("", reflect.ValueOf(v)) + return nil +} + +// Close closes the encoder by writing any remaining data. +// It does not write a stream terminating string "...". +func (e *Encoder) Close() (err error) { + defer handleErr(&err) + e.encoder.finish() + return nil +} + +func handleErr(err *error) { + if v := recover(); v != nil { + if e, ok := v.(yamlError); ok { + *err = e.err + } else { + panic(v) + } + } +} + +type yamlError struct { + err error +} + +func fail(err error) { + panic(yamlError{err}) +} + +func failf(format string, args ...interface{}) { + panic(yamlError{fmt.Errorf("yaml: "+format, args...)}) +} + +// A TypeError is returned by Unmarshal when one or more fields in +// the YAML document cannot be properly decoded into the requested +// types. When this error is returned, the value is still +// unmarshaled partially. +type TypeError struct { + Errors []string +} + +func (e *TypeError) Error() string { + return fmt.Sprintf("yaml: unmarshal errors:\n %s", strings.Join(e.Errors, "\n ")) +} + +// -------------------------------------------------------------------------- +// Maintain a mapping of keys to structure field indexes + +// The code in this section was copied from mgo/bson. + +// structInfo holds details for the serialization of fields of +// a given struct. +type structInfo struct { + FieldsMap map[string]fieldInfo + FieldsList []fieldInfo + + // InlineMap is the number of the field in the struct that + // contains an ,inline map, or -1 if there's none. + InlineMap int +} + +type fieldInfo struct { + Key string + Num int + OmitEmpty bool + Flow bool + // Id holds the unique field identifier, so we can cheaply + // check for field duplicates without maintaining an extra map. + Id int + + // Inline holds the field index if the field is part of an inlined struct. + Inline []int +} + +var structMap = make(map[reflect.Type]*structInfo) +var fieldMapMutex sync.RWMutex + +func getStructInfo(st reflect.Type) (*structInfo, error) { + fieldMapMutex.RLock() + sinfo, found := structMap[st] + fieldMapMutex.RUnlock() + if found { + return sinfo, nil + } + + n := st.NumField() + fieldsMap := make(map[string]fieldInfo) + fieldsList := make([]fieldInfo, 0, n) + inlineMap := -1 + for i := 0; i != n; i++ { + field := st.Field(i) + if field.PkgPath != "" && !field.Anonymous { + continue // Private field + } + + info := fieldInfo{Num: i} + + tag := field.Tag.Get("yaml") + if tag == "" && strings.Index(string(field.Tag), ":") < 0 { + tag = string(field.Tag) + } + if tag == "-" { + continue + } + + inline := false + fields := strings.Split(tag, ",") + if len(fields) > 1 { + for _, flag := range fields[1:] { + switch flag { + case "omitempty": + info.OmitEmpty = true + case "flow": + info.Flow = true + case "inline": + inline = true + default: + return nil, errors.New(fmt.Sprintf("Unsupported flag %q in tag %q of type %s", flag, tag, st)) + } + } + tag = fields[0] + } + + if inline { + switch field.Type.Kind() { + case reflect.Map: + if inlineMap >= 0 { + return nil, errors.New("Multiple ,inline maps in struct " + st.String()) + } + if field.Type.Key() != reflect.TypeOf("") { + return nil, errors.New("Option ,inline needs a map with string keys in struct " + st.String()) + } + inlineMap = info.Num + case reflect.Struct: + sinfo, err := getStructInfo(field.Type) + if err != nil { + return nil, err + } + for _, finfo := range sinfo.FieldsList { + if _, found := fieldsMap[finfo.Key]; found { + msg := "Duplicated key '" + finfo.Key + "' in struct " + st.String() + return nil, errors.New(msg) + } + if finfo.Inline == nil { + finfo.Inline = []int{i, finfo.Num} + } else { + finfo.Inline = append([]int{i}, finfo.Inline...) + } + finfo.Id = len(fieldsList) + fieldsMap[finfo.Key] = finfo + fieldsList = append(fieldsList, finfo) + } + default: + //return nil, errors.New("Option ,inline needs a struct value or map field") + return nil, errors.New("Option ,inline needs a struct value field") + } + continue + } + + if tag != "" { + info.Key = tag + } else { + info.Key = strings.ToLower(field.Name) + } + + if _, found = fieldsMap[info.Key]; found { + msg := "Duplicated key '" + info.Key + "' in struct " + st.String() + return nil, errors.New(msg) + } + + info.Id = len(fieldsList) + fieldsList = append(fieldsList, info) + fieldsMap[info.Key] = info + } + + sinfo = &structInfo{ + FieldsMap: fieldsMap, + FieldsList: fieldsList, + InlineMap: inlineMap, + } + + fieldMapMutex.Lock() + structMap[st] = sinfo + fieldMapMutex.Unlock() + return sinfo, nil +} + +// IsZeroer is used to check whether an object is zero to +// determine whether it should be omitted when marshaling +// with the omitempty flag. One notable implementation +// is time.Time. +type IsZeroer interface { + IsZero() bool +} + +func isZero(v reflect.Value) bool { + kind := v.Kind() + if z, ok := v.Interface().(IsZeroer); ok { + if (kind == reflect.Ptr || kind == reflect.Interface) && v.IsNil() { + return true + } + return z.IsZero() + } + switch kind { + case reflect.String: + return len(v.String()) == 0 + case reflect.Interface, reflect.Ptr: + return v.IsNil() + case reflect.Slice: + return v.Len() == 0 + case reflect.Map: + return v.Len() == 0 + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Struct: + vt := v.Type() + for i := v.NumField() - 1; i >= 0; i-- { + if vt.Field(i).PkgPath != "" { + continue // Private field + } + if !isZero(v.Field(i)) { + return false + } + } + return true + } + return false +} + +// FutureLineWrap globally disables line wrapping when encoding long strings. +// This is a temporary and thus deprecated method introduced to faciliate +// migration towards v3, which offers more control of line lengths on +// individual encodings, and has a default matching the behavior introduced +// by this function. +// +// The default formatting of v2 was erroneously changed in v2.3.0 and reverted +// in v2.4.0, at which point this function was introduced to help migration. +func FutureLineWrap() { + disableLineWrapping = true +} diff --git a/vendor/gopkg.in/yaml.v2/yamlh.go b/vendor/gopkg.in/yaml.v2/yamlh.go new file mode 100644 index 000000000..f6a9c8e34 --- /dev/null +++ b/vendor/gopkg.in/yaml.v2/yamlh.go @@ -0,0 +1,739 @@ +package yaml + +import ( + "fmt" + "io" +) + +// The version directive data. +type yaml_version_directive_t struct { + major int8 // The major version number. + minor int8 // The minor version number. +} + +// The tag directive data. +type yaml_tag_directive_t struct { + handle []byte // The tag handle. + prefix []byte // The tag prefix. +} + +type yaml_encoding_t int + +// The stream encoding. +const ( + // Let the parser choose the encoding. + yaml_ANY_ENCODING yaml_encoding_t = iota + + yaml_UTF8_ENCODING // The default UTF-8 encoding. + yaml_UTF16LE_ENCODING // The UTF-16-LE encoding with BOM. + yaml_UTF16BE_ENCODING // The UTF-16-BE encoding with BOM. +) + +type yaml_break_t int + +// Line break types. +const ( + // Let the parser choose the break type. + yaml_ANY_BREAK yaml_break_t = iota + + yaml_CR_BREAK // Use CR for line breaks (Mac style). + yaml_LN_BREAK // Use LN for line breaks (Unix style). + yaml_CRLN_BREAK // Use CR LN for line breaks (DOS style). +) + +type yaml_error_type_t int + +// Many bad things could happen with the parser and emitter. +const ( + // No error is produced. + yaml_NO_ERROR yaml_error_type_t = iota + + yaml_MEMORY_ERROR // Cannot allocate or reallocate a block of memory. + yaml_READER_ERROR // Cannot read or decode the input stream. + yaml_SCANNER_ERROR // Cannot scan the input stream. + yaml_PARSER_ERROR // Cannot parse the input stream. + yaml_COMPOSER_ERROR // Cannot compose a YAML document. + yaml_WRITER_ERROR // Cannot write to the output stream. + yaml_EMITTER_ERROR // Cannot emit a YAML stream. +) + +// The pointer position. +type yaml_mark_t struct { + index int // The position index. + line int // The position line. + column int // The position column. +} + +// Node Styles + +type yaml_style_t int8 + +type yaml_scalar_style_t yaml_style_t + +// Scalar styles. +const ( + // Let the emitter choose the style. + yaml_ANY_SCALAR_STYLE yaml_scalar_style_t = iota + + yaml_PLAIN_SCALAR_STYLE // The plain scalar style. + yaml_SINGLE_QUOTED_SCALAR_STYLE // The single-quoted scalar style. + yaml_DOUBLE_QUOTED_SCALAR_STYLE // The double-quoted scalar style. + yaml_LITERAL_SCALAR_STYLE // The literal scalar style. + yaml_FOLDED_SCALAR_STYLE // The folded scalar style. +) + +type yaml_sequence_style_t yaml_style_t + +// Sequence styles. +const ( + // Let the emitter choose the style. + yaml_ANY_SEQUENCE_STYLE yaml_sequence_style_t = iota + + yaml_BLOCK_SEQUENCE_STYLE // The block sequence style. + yaml_FLOW_SEQUENCE_STYLE // The flow sequence style. +) + +type yaml_mapping_style_t yaml_style_t + +// Mapping styles. +const ( + // Let the emitter choose the style. + yaml_ANY_MAPPING_STYLE yaml_mapping_style_t = iota + + yaml_BLOCK_MAPPING_STYLE // The block mapping style. + yaml_FLOW_MAPPING_STYLE // The flow mapping style. +) + +// Tokens + +type yaml_token_type_t int + +// Token types. +const ( + // An empty token. + yaml_NO_TOKEN yaml_token_type_t = iota + + yaml_STREAM_START_TOKEN // A STREAM-START token. + yaml_STREAM_END_TOKEN // A STREAM-END token. + + yaml_VERSION_DIRECTIVE_TOKEN // A VERSION-DIRECTIVE token. + yaml_TAG_DIRECTIVE_TOKEN // A TAG-DIRECTIVE token. + yaml_DOCUMENT_START_TOKEN // A DOCUMENT-START token. + yaml_DOCUMENT_END_TOKEN // A DOCUMENT-END token. + + yaml_BLOCK_SEQUENCE_START_TOKEN // A BLOCK-SEQUENCE-START token. + yaml_BLOCK_MAPPING_START_TOKEN // A BLOCK-SEQUENCE-END token. + yaml_BLOCK_END_TOKEN // A BLOCK-END token. + + yaml_FLOW_SEQUENCE_START_TOKEN // A FLOW-SEQUENCE-START token. + yaml_FLOW_SEQUENCE_END_TOKEN // A FLOW-SEQUENCE-END token. + yaml_FLOW_MAPPING_START_TOKEN // A FLOW-MAPPING-START token. + yaml_FLOW_MAPPING_END_TOKEN // A FLOW-MAPPING-END token. + + yaml_BLOCK_ENTRY_TOKEN // A BLOCK-ENTRY token. + yaml_FLOW_ENTRY_TOKEN // A FLOW-ENTRY token. + yaml_KEY_TOKEN // A KEY token. + yaml_VALUE_TOKEN // A VALUE token. + + yaml_ALIAS_TOKEN // An ALIAS token. + yaml_ANCHOR_TOKEN // An ANCHOR token. + yaml_TAG_TOKEN // A TAG token. + yaml_SCALAR_TOKEN // A SCALAR token. +) + +func (tt yaml_token_type_t) String() string { + switch tt { + case yaml_NO_TOKEN: + return "yaml_NO_TOKEN" + case yaml_STREAM_START_TOKEN: + return "yaml_STREAM_START_TOKEN" + case yaml_STREAM_END_TOKEN: + return "yaml_STREAM_END_TOKEN" + case yaml_VERSION_DIRECTIVE_TOKEN: + return "yaml_VERSION_DIRECTIVE_TOKEN" + case yaml_TAG_DIRECTIVE_TOKEN: + return "yaml_TAG_DIRECTIVE_TOKEN" + case yaml_DOCUMENT_START_TOKEN: + return "yaml_DOCUMENT_START_TOKEN" + case yaml_DOCUMENT_END_TOKEN: + return "yaml_DOCUMENT_END_TOKEN" + case yaml_BLOCK_SEQUENCE_START_TOKEN: + return "yaml_BLOCK_SEQUENCE_START_TOKEN" + case yaml_BLOCK_MAPPING_START_TOKEN: + return "yaml_BLOCK_MAPPING_START_TOKEN" + case yaml_BLOCK_END_TOKEN: + return "yaml_BLOCK_END_TOKEN" + case yaml_FLOW_SEQUENCE_START_TOKEN: + return "yaml_FLOW_SEQUENCE_START_TOKEN" + case yaml_FLOW_SEQUENCE_END_TOKEN: + return "yaml_FLOW_SEQUENCE_END_TOKEN" + case yaml_FLOW_MAPPING_START_TOKEN: + return "yaml_FLOW_MAPPING_START_TOKEN" + case yaml_FLOW_MAPPING_END_TOKEN: + return "yaml_FLOW_MAPPING_END_TOKEN" + case yaml_BLOCK_ENTRY_TOKEN: + return "yaml_BLOCK_ENTRY_TOKEN" + case yaml_FLOW_ENTRY_TOKEN: + return "yaml_FLOW_ENTRY_TOKEN" + case yaml_KEY_TOKEN: + return "yaml_KEY_TOKEN" + case yaml_VALUE_TOKEN: + return "yaml_VALUE_TOKEN" + case yaml_ALIAS_TOKEN: + return "yaml_ALIAS_TOKEN" + case yaml_ANCHOR_TOKEN: + return "yaml_ANCHOR_TOKEN" + case yaml_TAG_TOKEN: + return "yaml_TAG_TOKEN" + case yaml_SCALAR_TOKEN: + return "yaml_SCALAR_TOKEN" + } + return "" +} + +// The token structure. +type yaml_token_t struct { + // The token type. + typ yaml_token_type_t + + // The start/end of the token. + start_mark, end_mark yaml_mark_t + + // The stream encoding (for yaml_STREAM_START_TOKEN). + encoding yaml_encoding_t + + // The alias/anchor/scalar value or tag/tag directive handle + // (for yaml_ALIAS_TOKEN, yaml_ANCHOR_TOKEN, yaml_SCALAR_TOKEN, yaml_TAG_TOKEN, yaml_TAG_DIRECTIVE_TOKEN). + value []byte + + // The tag suffix (for yaml_TAG_TOKEN). + suffix []byte + + // The tag directive prefix (for yaml_TAG_DIRECTIVE_TOKEN). + prefix []byte + + // The scalar style (for yaml_SCALAR_TOKEN). + style yaml_scalar_style_t + + // The version directive major/minor (for yaml_VERSION_DIRECTIVE_TOKEN). + major, minor int8 +} + +// Events + +type yaml_event_type_t int8 + +// Event types. +const ( + // An empty event. + yaml_NO_EVENT yaml_event_type_t = iota + + yaml_STREAM_START_EVENT // A STREAM-START event. + yaml_STREAM_END_EVENT // A STREAM-END event. + yaml_DOCUMENT_START_EVENT // A DOCUMENT-START event. + yaml_DOCUMENT_END_EVENT // A DOCUMENT-END event. + yaml_ALIAS_EVENT // An ALIAS event. + yaml_SCALAR_EVENT // A SCALAR event. + yaml_SEQUENCE_START_EVENT // A SEQUENCE-START event. + yaml_SEQUENCE_END_EVENT // A SEQUENCE-END event. + yaml_MAPPING_START_EVENT // A MAPPING-START event. + yaml_MAPPING_END_EVENT // A MAPPING-END event. +) + +var eventStrings = []string{ + yaml_NO_EVENT: "none", + yaml_STREAM_START_EVENT: "stream start", + yaml_STREAM_END_EVENT: "stream end", + yaml_DOCUMENT_START_EVENT: "document start", + yaml_DOCUMENT_END_EVENT: "document end", + yaml_ALIAS_EVENT: "alias", + yaml_SCALAR_EVENT: "scalar", + yaml_SEQUENCE_START_EVENT: "sequence start", + yaml_SEQUENCE_END_EVENT: "sequence end", + yaml_MAPPING_START_EVENT: "mapping start", + yaml_MAPPING_END_EVENT: "mapping end", +} + +func (e yaml_event_type_t) String() string { + if e < 0 || int(e) >= len(eventStrings) { + return fmt.Sprintf("unknown event %d", e) + } + return eventStrings[e] +} + +// The event structure. +type yaml_event_t struct { + + // The event type. + typ yaml_event_type_t + + // The start and end of the event. + start_mark, end_mark yaml_mark_t + + // The document encoding (for yaml_STREAM_START_EVENT). + encoding yaml_encoding_t + + // The version directive (for yaml_DOCUMENT_START_EVENT). + version_directive *yaml_version_directive_t + + // The list of tag directives (for yaml_DOCUMENT_START_EVENT). + tag_directives []yaml_tag_directive_t + + // The anchor (for yaml_SCALAR_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT, yaml_ALIAS_EVENT). + anchor []byte + + // The tag (for yaml_SCALAR_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT). + tag []byte + + // The scalar value (for yaml_SCALAR_EVENT). + value []byte + + // Is the document start/end indicator implicit, or the tag optional? + // (for yaml_DOCUMENT_START_EVENT, yaml_DOCUMENT_END_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT, yaml_SCALAR_EVENT). + implicit bool + + // Is the tag optional for any non-plain style? (for yaml_SCALAR_EVENT). + quoted_implicit bool + + // The style (for yaml_SCALAR_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT). + style yaml_style_t +} + +func (e *yaml_event_t) scalar_style() yaml_scalar_style_t { return yaml_scalar_style_t(e.style) } +func (e *yaml_event_t) sequence_style() yaml_sequence_style_t { return yaml_sequence_style_t(e.style) } +func (e *yaml_event_t) mapping_style() yaml_mapping_style_t { return yaml_mapping_style_t(e.style) } + +// Nodes + +const ( + yaml_NULL_TAG = "tag:yaml.org,2002:null" // The tag !!null with the only possible value: null. + yaml_BOOL_TAG = "tag:yaml.org,2002:bool" // The tag !!bool with the values: true and false. + yaml_STR_TAG = "tag:yaml.org,2002:str" // The tag !!str for string values. + yaml_INT_TAG = "tag:yaml.org,2002:int" // The tag !!int for integer values. + yaml_FLOAT_TAG = "tag:yaml.org,2002:float" // The tag !!float for float values. + yaml_TIMESTAMP_TAG = "tag:yaml.org,2002:timestamp" // The tag !!timestamp for date and time values. + + yaml_SEQ_TAG = "tag:yaml.org,2002:seq" // The tag !!seq is used to denote sequences. + yaml_MAP_TAG = "tag:yaml.org,2002:map" // The tag !!map is used to denote mapping. + + // Not in original libyaml. + yaml_BINARY_TAG = "tag:yaml.org,2002:binary" + yaml_MERGE_TAG = "tag:yaml.org,2002:merge" + + yaml_DEFAULT_SCALAR_TAG = yaml_STR_TAG // The default scalar tag is !!str. + yaml_DEFAULT_SEQUENCE_TAG = yaml_SEQ_TAG // The default sequence tag is !!seq. + yaml_DEFAULT_MAPPING_TAG = yaml_MAP_TAG // The default mapping tag is !!map. +) + +type yaml_node_type_t int + +// Node types. +const ( + // An empty node. + yaml_NO_NODE yaml_node_type_t = iota + + yaml_SCALAR_NODE // A scalar node. + yaml_SEQUENCE_NODE // A sequence node. + yaml_MAPPING_NODE // A mapping node. +) + +// An element of a sequence node. +type yaml_node_item_t int + +// An element of a mapping node. +type yaml_node_pair_t struct { + key int // The key of the element. + value int // The value of the element. +} + +// The node structure. +type yaml_node_t struct { + typ yaml_node_type_t // The node type. + tag []byte // The node tag. + + // The node data. + + // The scalar parameters (for yaml_SCALAR_NODE). + scalar struct { + value []byte // The scalar value. + length int // The length of the scalar value. + style yaml_scalar_style_t // The scalar style. + } + + // The sequence parameters (for YAML_SEQUENCE_NODE). + sequence struct { + items_data []yaml_node_item_t // The stack of sequence items. + style yaml_sequence_style_t // The sequence style. + } + + // The mapping parameters (for yaml_MAPPING_NODE). + mapping struct { + pairs_data []yaml_node_pair_t // The stack of mapping pairs (key, value). + pairs_start *yaml_node_pair_t // The beginning of the stack. + pairs_end *yaml_node_pair_t // The end of the stack. + pairs_top *yaml_node_pair_t // The top of the stack. + style yaml_mapping_style_t // The mapping style. + } + + start_mark yaml_mark_t // The beginning of the node. + end_mark yaml_mark_t // The end of the node. + +} + +// The document structure. +type yaml_document_t struct { + + // The document nodes. + nodes []yaml_node_t + + // The version directive. + version_directive *yaml_version_directive_t + + // The list of tag directives. + tag_directives_data []yaml_tag_directive_t + tag_directives_start int // The beginning of the tag directives list. + tag_directives_end int // The end of the tag directives list. + + start_implicit int // Is the document start indicator implicit? + end_implicit int // Is the document end indicator implicit? + + // The start/end of the document. + start_mark, end_mark yaml_mark_t +} + +// The prototype of a read handler. +// +// The read handler is called when the parser needs to read more bytes from the +// source. The handler should write not more than size bytes to the buffer. +// The number of written bytes should be set to the size_read variable. +// +// [in,out] data A pointer to an application data specified by +// yaml_parser_set_input(). +// [out] buffer The buffer to write the data from the source. +// [in] size The size of the buffer. +// [out] size_read The actual number of bytes read from the source. +// +// On success, the handler should return 1. If the handler failed, +// the returned value should be 0. On EOF, the handler should set the +// size_read to 0 and return 1. +type yaml_read_handler_t func(parser *yaml_parser_t, buffer []byte) (n int, err error) + +// This structure holds information about a potential simple key. +type yaml_simple_key_t struct { + possible bool // Is a simple key possible? + required bool // Is a simple key required? + token_number int // The number of the token. + mark yaml_mark_t // The position mark. +} + +// The states of the parser. +type yaml_parser_state_t int + +const ( + yaml_PARSE_STREAM_START_STATE yaml_parser_state_t = iota + + yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE // Expect the beginning of an implicit document. + yaml_PARSE_DOCUMENT_START_STATE // Expect DOCUMENT-START. + yaml_PARSE_DOCUMENT_CONTENT_STATE // Expect the content of a document. + yaml_PARSE_DOCUMENT_END_STATE // Expect DOCUMENT-END. + yaml_PARSE_BLOCK_NODE_STATE // Expect a block node. + yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE // Expect a block node or indentless sequence. + yaml_PARSE_FLOW_NODE_STATE // Expect a flow node. + yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE // Expect the first entry of a block sequence. + yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE // Expect an entry of a block sequence. + yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE // Expect an entry of an indentless sequence. + yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE // Expect the first key of a block mapping. + yaml_PARSE_BLOCK_MAPPING_KEY_STATE // Expect a block mapping key. + yaml_PARSE_BLOCK_MAPPING_VALUE_STATE // Expect a block mapping value. + yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE // Expect the first entry of a flow sequence. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE // Expect an entry of a flow sequence. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE // Expect a key of an ordered mapping. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE // Expect a value of an ordered mapping. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE // Expect the and of an ordered mapping entry. + yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE // Expect the first key of a flow mapping. + yaml_PARSE_FLOW_MAPPING_KEY_STATE // Expect a key of a flow mapping. + yaml_PARSE_FLOW_MAPPING_VALUE_STATE // Expect a value of a flow mapping. + yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE // Expect an empty value of a flow mapping. + yaml_PARSE_END_STATE // Expect nothing. +) + +func (ps yaml_parser_state_t) String() string { + switch ps { + case yaml_PARSE_STREAM_START_STATE: + return "yaml_PARSE_STREAM_START_STATE" + case yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE: + return "yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE" + case yaml_PARSE_DOCUMENT_START_STATE: + return "yaml_PARSE_DOCUMENT_START_STATE" + case yaml_PARSE_DOCUMENT_CONTENT_STATE: + return "yaml_PARSE_DOCUMENT_CONTENT_STATE" + case yaml_PARSE_DOCUMENT_END_STATE: + return "yaml_PARSE_DOCUMENT_END_STATE" + case yaml_PARSE_BLOCK_NODE_STATE: + return "yaml_PARSE_BLOCK_NODE_STATE" + case yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE: + return "yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE" + case yaml_PARSE_FLOW_NODE_STATE: + return "yaml_PARSE_FLOW_NODE_STATE" + case yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE: + return "yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE" + case yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE: + return "yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE" + case yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE: + return "yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE" + case yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE: + return "yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE" + case yaml_PARSE_BLOCK_MAPPING_KEY_STATE: + return "yaml_PARSE_BLOCK_MAPPING_KEY_STATE" + case yaml_PARSE_BLOCK_MAPPING_VALUE_STATE: + return "yaml_PARSE_BLOCK_MAPPING_VALUE_STATE" + case yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE" + case yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE: + return "yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE" + case yaml_PARSE_FLOW_MAPPING_KEY_STATE: + return "yaml_PARSE_FLOW_MAPPING_KEY_STATE" + case yaml_PARSE_FLOW_MAPPING_VALUE_STATE: + return "yaml_PARSE_FLOW_MAPPING_VALUE_STATE" + case yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE: + return "yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE" + case yaml_PARSE_END_STATE: + return "yaml_PARSE_END_STATE" + } + return "" +} + +// This structure holds aliases data. +type yaml_alias_data_t struct { + anchor []byte // The anchor. + index int // The node id. + mark yaml_mark_t // The anchor mark. +} + +// The parser structure. +// +// All members are internal. Manage the structure using the +// yaml_parser_ family of functions. +type yaml_parser_t struct { + + // Error handling + + error yaml_error_type_t // Error type. + + problem string // Error description. + + // The byte about which the problem occurred. + problem_offset int + problem_value int + problem_mark yaml_mark_t + + // The error context. + context string + context_mark yaml_mark_t + + // Reader stuff + + read_handler yaml_read_handler_t // Read handler. + + input_reader io.Reader // File input data. + input []byte // String input data. + input_pos int + + eof bool // EOF flag + + buffer []byte // The working buffer. + buffer_pos int // The current position of the buffer. + + unread int // The number of unread characters in the buffer. + + raw_buffer []byte // The raw buffer. + raw_buffer_pos int // The current position of the buffer. + + encoding yaml_encoding_t // The input encoding. + + offset int // The offset of the current position (in bytes). + mark yaml_mark_t // The mark of the current position. + + // Scanner stuff + + stream_start_produced bool // Have we started to scan the input stream? + stream_end_produced bool // Have we reached the end of the input stream? + + flow_level int // The number of unclosed '[' and '{' indicators. + + tokens []yaml_token_t // The tokens queue. + tokens_head int // The head of the tokens queue. + tokens_parsed int // The number of tokens fetched from the queue. + token_available bool // Does the tokens queue contain a token ready for dequeueing. + + indent int // The current indentation level. + indents []int // The indentation levels stack. + + simple_key_allowed bool // May a simple key occur at the current position? + simple_keys []yaml_simple_key_t // The stack of simple keys. + simple_keys_by_tok map[int]int // possible simple_key indexes indexed by token_number + + // Parser stuff + + state yaml_parser_state_t // The current parser state. + states []yaml_parser_state_t // The parser states stack. + marks []yaml_mark_t // The stack of marks. + tag_directives []yaml_tag_directive_t // The list of TAG directives. + + // Dumper stuff + + aliases []yaml_alias_data_t // The alias data. + + document *yaml_document_t // The currently parsed document. +} + +// Emitter Definitions + +// The prototype of a write handler. +// +// The write handler is called when the emitter needs to flush the accumulated +// characters to the output. The handler should write @a size bytes of the +// @a buffer to the output. +// +// @param[in,out] data A pointer to an application data specified by +// yaml_emitter_set_output(). +// @param[in] buffer The buffer with bytes to be written. +// @param[in] size The size of the buffer. +// +// @returns On success, the handler should return @c 1. If the handler failed, +// the returned value should be @c 0. +// +type yaml_write_handler_t func(emitter *yaml_emitter_t, buffer []byte) error + +type yaml_emitter_state_t int + +// The emitter states. +const ( + // Expect STREAM-START. + yaml_EMIT_STREAM_START_STATE yaml_emitter_state_t = iota + + yaml_EMIT_FIRST_DOCUMENT_START_STATE // Expect the first DOCUMENT-START or STREAM-END. + yaml_EMIT_DOCUMENT_START_STATE // Expect DOCUMENT-START or STREAM-END. + yaml_EMIT_DOCUMENT_CONTENT_STATE // Expect the content of a document. + yaml_EMIT_DOCUMENT_END_STATE // Expect DOCUMENT-END. + yaml_EMIT_FLOW_SEQUENCE_FIRST_ITEM_STATE // Expect the first item of a flow sequence. + yaml_EMIT_FLOW_SEQUENCE_ITEM_STATE // Expect an item of a flow sequence. + yaml_EMIT_FLOW_MAPPING_FIRST_KEY_STATE // Expect the first key of a flow mapping. + yaml_EMIT_FLOW_MAPPING_KEY_STATE // Expect a key of a flow mapping. + yaml_EMIT_FLOW_MAPPING_SIMPLE_VALUE_STATE // Expect a value for a simple key of a flow mapping. + yaml_EMIT_FLOW_MAPPING_VALUE_STATE // Expect a value of a flow mapping. + yaml_EMIT_BLOCK_SEQUENCE_FIRST_ITEM_STATE // Expect the first item of a block sequence. + yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE // Expect an item of a block sequence. + yaml_EMIT_BLOCK_MAPPING_FIRST_KEY_STATE // Expect the first key of a block mapping. + yaml_EMIT_BLOCK_MAPPING_KEY_STATE // Expect the key of a block mapping. + yaml_EMIT_BLOCK_MAPPING_SIMPLE_VALUE_STATE // Expect a value for a simple key of a block mapping. + yaml_EMIT_BLOCK_MAPPING_VALUE_STATE // Expect a value of a block mapping. + yaml_EMIT_END_STATE // Expect nothing. +) + +// The emitter structure. +// +// All members are internal. Manage the structure using the @c yaml_emitter_ +// family of functions. +type yaml_emitter_t struct { + + // Error handling + + error yaml_error_type_t // Error type. + problem string // Error description. + + // Writer stuff + + write_handler yaml_write_handler_t // Write handler. + + output_buffer *[]byte // String output data. + output_writer io.Writer // File output data. + + buffer []byte // The working buffer. + buffer_pos int // The current position of the buffer. + + raw_buffer []byte // The raw buffer. + raw_buffer_pos int // The current position of the buffer. + + encoding yaml_encoding_t // The stream encoding. + + // Emitter stuff + + canonical bool // If the output is in the canonical style? + best_indent int // The number of indentation spaces. + best_width int // The preferred width of the output lines. + unicode bool // Allow unescaped non-ASCII characters? + line_break yaml_break_t // The preferred line break. + + state yaml_emitter_state_t // The current emitter state. + states []yaml_emitter_state_t // The stack of states. + + events []yaml_event_t // The event queue. + events_head int // The head of the event queue. + + indents []int // The stack of indentation levels. + + tag_directives []yaml_tag_directive_t // The list of tag directives. + + indent int // The current indentation level. + + flow_level int // The current flow level. + + root_context bool // Is it the document root context? + sequence_context bool // Is it a sequence context? + mapping_context bool // Is it a mapping context? + simple_key_context bool // Is it a simple mapping key context? + + line int // The current line. + column int // The current column. + whitespace bool // If the last character was a whitespace? + indention bool // If the last character was an indentation character (' ', '-', '?', ':')? + open_ended bool // If an explicit document end is required? + + // Anchor analysis. + anchor_data struct { + anchor []byte // The anchor value. + alias bool // Is it an alias? + } + + // Tag analysis. + tag_data struct { + handle []byte // The tag handle. + suffix []byte // The tag suffix. + } + + // Scalar analysis. + scalar_data struct { + value []byte // The scalar value. + multiline bool // Does the scalar contain line breaks? + flow_plain_allowed bool // Can the scalar be expessed in the flow plain style? + block_plain_allowed bool // Can the scalar be expressed in the block plain style? + single_quoted_allowed bool // Can the scalar be expressed in the single quoted style? + block_allowed bool // Can the scalar be expressed in the literal or folded styles? + style yaml_scalar_style_t // The output style. + } + + // Dumper stuff + + opened bool // If the stream was already opened? + closed bool // If the stream was already closed? + + // The information associated with the document nodes. + anchors *struct { + references int // The number of references. + anchor int // The anchor id. + serialized bool // If the node has been emitted? + } + + last_anchor_id int // The last assigned anchor id. + + document *yaml_document_t // The currently emitted document. +} diff --git a/vendor/gopkg.in/yaml.v2/yamlprivateh.go b/vendor/gopkg.in/yaml.v2/yamlprivateh.go new file mode 100644 index 000000000..8110ce3c3 --- /dev/null +++ b/vendor/gopkg.in/yaml.v2/yamlprivateh.go @@ -0,0 +1,173 @@ +package yaml + +const ( + // The size of the input raw buffer. + input_raw_buffer_size = 512 + + // The size of the input buffer. + // It should be possible to decode the whole raw buffer. + input_buffer_size = input_raw_buffer_size * 3 + + // The size of the output buffer. + output_buffer_size = 128 + + // The size of the output raw buffer. + // It should be possible to encode the whole output buffer. + output_raw_buffer_size = (output_buffer_size*2 + 2) + + // The size of other stacks and queues. + initial_stack_size = 16 + initial_queue_size = 16 + initial_string_size = 16 +) + +// Check if the character at the specified position is an alphabetical +// character, a digit, '_', or '-'. +func is_alpha(b []byte, i int) bool { + return b[i] >= '0' && b[i] <= '9' || b[i] >= 'A' && b[i] <= 'Z' || b[i] >= 'a' && b[i] <= 'z' || b[i] == '_' || b[i] == '-' +} + +// Check if the character at the specified position is a digit. +func is_digit(b []byte, i int) bool { + return b[i] >= '0' && b[i] <= '9' +} + +// Get the value of a digit. +func as_digit(b []byte, i int) int { + return int(b[i]) - '0' +} + +// Check if the character at the specified position is a hex-digit. +func is_hex(b []byte, i int) bool { + return b[i] >= '0' && b[i] <= '9' || b[i] >= 'A' && b[i] <= 'F' || b[i] >= 'a' && b[i] <= 'f' +} + +// Get the value of a hex-digit. +func as_hex(b []byte, i int) int { + bi := b[i] + if bi >= 'A' && bi <= 'F' { + return int(bi) - 'A' + 10 + } + if bi >= 'a' && bi <= 'f' { + return int(bi) - 'a' + 10 + } + return int(bi) - '0' +} + +// Check if the character is ASCII. +func is_ascii(b []byte, i int) bool { + return b[i] <= 0x7F +} + +// Check if the character at the start of the buffer can be printed unescaped. +func is_printable(b []byte, i int) bool { + return ((b[i] == 0x0A) || // . == #x0A + (b[i] >= 0x20 && b[i] <= 0x7E) || // #x20 <= . <= #x7E + (b[i] == 0xC2 && b[i+1] >= 0xA0) || // #0xA0 <= . <= #xD7FF + (b[i] > 0xC2 && b[i] < 0xED) || + (b[i] == 0xED && b[i+1] < 0xA0) || + (b[i] == 0xEE) || + (b[i] == 0xEF && // #xE000 <= . <= #xFFFD + !(b[i+1] == 0xBB && b[i+2] == 0xBF) && // && . != #xFEFF + !(b[i+1] == 0xBF && (b[i+2] == 0xBE || b[i+2] == 0xBF)))) +} + +// Check if the character at the specified position is NUL. +func is_z(b []byte, i int) bool { + return b[i] == 0x00 +} + +// Check if the beginning of the buffer is a BOM. +func is_bom(b []byte, i int) bool { + return b[0] == 0xEF && b[1] == 0xBB && b[2] == 0xBF +} + +// Check if the character at the specified position is space. +func is_space(b []byte, i int) bool { + return b[i] == ' ' +} + +// Check if the character at the specified position is tab. +func is_tab(b []byte, i int) bool { + return b[i] == '\t' +} + +// Check if the character at the specified position is blank (space or tab). +func is_blank(b []byte, i int) bool { + //return is_space(b, i) || is_tab(b, i) + return b[i] == ' ' || b[i] == '\t' +} + +// Check if the character at the specified position is a line break. +func is_break(b []byte, i int) bool { + return (b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9) // PS (#x2029) +} + +func is_crlf(b []byte, i int) bool { + return b[i] == '\r' && b[i+1] == '\n' +} + +// Check if the character is a line break or NUL. +func is_breakz(b []byte, i int) bool { + //return is_break(b, i) || is_z(b, i) + return ( // is_break: + b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029) + // is_z: + b[i] == 0) +} + +// Check if the character is a line break, space, or NUL. +func is_spacez(b []byte, i int) bool { + //return is_space(b, i) || is_breakz(b, i) + return ( // is_space: + b[i] == ' ' || + // is_breakz: + b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029) + b[i] == 0) +} + +// Check if the character is a line break, space, tab, or NUL. +func is_blankz(b []byte, i int) bool { + //return is_blank(b, i) || is_breakz(b, i) + return ( // is_blank: + b[i] == ' ' || b[i] == '\t' || + // is_breakz: + b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029) + b[i] == 0) +} + +// Determine the width of the character. +func width(b byte) int { + // Don't replace these by a switch without first + // confirming that it is being inlined. + if b&0x80 == 0x00 { + return 1 + } + if b&0xE0 == 0xC0 { + return 2 + } + if b&0xF0 == 0xE0 { + return 3 + } + if b&0xF8 == 0xF0 { + return 4 + } + return 0 + +} diff --git a/vendor/modules.txt b/vendor/modules.txt index afb48c24b..ac1550733 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -569,6 +569,43 @@ github.com/docker/cli/cli/config/credentials github.com/docker/cli/cli/config/memorystore github.com/docker/cli/cli/config/types github.com/docker/cli/cli/connhelper/commandconn +# github.com/docker/distribution v2.8.3+incompatible +## explicit +github.com/docker/distribution +github.com/docker/distribution/configuration +github.com/docker/distribution/context +github.com/docker/distribution/health +github.com/docker/distribution/health/checks +github.com/docker/distribution/manifest +github.com/docker/distribution/manifest/manifestlist +github.com/docker/distribution/manifest/ocischema +github.com/docker/distribution/manifest/schema1 +github.com/docker/distribution/manifest/schema2 +github.com/docker/distribution/metrics +github.com/docker/distribution/notifications +github.com/docker/distribution/registry/api/errcode +github.com/docker/distribution/registry/api/v2 +github.com/docker/distribution/registry/auth +github.com/docker/distribution/registry/client +github.com/docker/distribution/registry/client/auth +github.com/docker/distribution/registry/client/auth/challenge +github.com/docker/distribution/registry/client/transport +github.com/docker/distribution/registry/handlers +github.com/docker/distribution/registry/listener +github.com/docker/distribution/registry/middleware/registry +github.com/docker/distribution/registry/middleware/repository +github.com/docker/distribution/registry/proxy +github.com/docker/distribution/registry/proxy/scheduler +github.com/docker/distribution/registry/storage +github.com/docker/distribution/registry/storage/cache +github.com/docker/distribution/registry/storage/cache/memory +github.com/docker/distribution/registry/storage/cache/redis +github.com/docker/distribution/registry/storage/driver +github.com/docker/distribution/registry/storage/driver/base +github.com/docker/distribution/registry/storage/driver/factory +github.com/docker/distribution/registry/storage/driver/middleware +github.com/docker/distribution/uuid +github.com/docker/distribution/version # github.com/docker/docker-credential-helpers v0.9.5 ## explicit; go 1.21 github.com/docker/docker-credential-helpers/client @@ -579,6 +616,9 @@ github.com/docker/go-metrics # github.com/docker/go-units v0.5.0 ## explicit github.com/docker/go-units +# github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 +## explicit +github.com/docker/libtrust # github.com/felixge/fgprof v0.9.3 ## explicit; go 1.14 github.com/felixge/fgprof @@ -589,6 +629,10 @@ github.com/felixge/httpsnoop ## explicit; go 1.17 github.com/fsnotify/fsnotify github.com/fsnotify/fsnotify/internal +# github.com/garyburd/redigo v1.6.4 +## explicit; go 1.17 +github.com/garyburd/redigo/internal +github.com/garyburd/redigo/redis # github.com/go-logr/logr v1.4.3 ## explicit; go 1.18 github.com/go-logr/logr @@ -733,9 +777,21 @@ github.com/google/shlex # github.com/google/uuid v1.6.0 ## explicit github.com/google/uuid +# github.com/gorilla/handlers v1.5.2 +## explicit; go 1.20 +github.com/gorilla/handlers +# github.com/gorilla/mux v1.8.1 +## explicit; go 1.20 +github.com/gorilla/mux # github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc ## explicit; go 1.21 github.com/grafana/regexp +# github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 +## explicit; go 1.14 +github.com/grpc-ecosystem/go-grpc-middleware +github.com/grpc-ecosystem/go-grpc-middleware/retry +github.com/grpc-ecosystem/go-grpc-middleware/util/backoffutils +github.com/grpc-ecosystem/go-grpc-middleware/util/metautils # github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 ## explicit; go 1.24.0 github.com/grpc-ecosystem/grpc-gateway/v2/internal/httprule @@ -1432,6 +1488,9 @@ google.golang.org/protobuf/types/known/structpb google.golang.org/protobuf/types/known/timestamppb google.golang.org/protobuf/types/known/wrapperspb google.golang.org/protobuf/types/pluginpb +# gopkg.in/yaml.v2 v2.4.0 +## explicit; go 1.15 +gopkg.in/yaml.v2 # gopkg.in/yaml.v3 v3.0.1 ## explicit gopkg.in/yaml.v3 diff --git a/worker/base/worker.go b/worker/base/worker.go index 5ea8bbc91..64b6ac109 100644 --- a/worker/base/worker.go +++ b/worker/base/worker.go @@ -23,6 +23,7 @@ import ( "github.com/moby/buildkit/executor/resources" "github.com/moby/buildkit/exporter" imageexporter "github.com/moby/buildkit/exporter/containerimage" + "github.com/moby/buildkit/exporter/earthlyoutputs" localexporter "github.com/moby/buildkit/exporter/local" ociexporter "github.com/moby/buildkit/exporter/oci" tarexporter "github.com/moby/buildkit/exporter/tar" @@ -49,12 +50,12 @@ import ( "github.com/moby/buildkit/util/network" "github.com/moby/buildkit/util/progress" "github.com/moby/buildkit/util/progress/controller" + "github.com/moby/buildkit/util/semutil" "github.com/moby/sys/user" digest "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "golang.org/x/sync/errgroup" - "golang.org/x/sync/semaphore" ) const labelCreatedAt = "buildkit/createdat" @@ -81,7 +82,7 @@ type WorkerOpt struct { IdentityMapping *user.IdentityMapping LeaseManager *leaseutil.Manager GarbageCollect func(context.Context) (gc.Stats, error) - ParallelismSem *semaphore.Weighted + ParallelismSem *semutil.Weighted MetadataStore *metadata.Store MountPoolRoot string ResourceMonitor *resources.Monitor @@ -257,6 +258,18 @@ func (w *Worker) Close() error { return stderrors.Join(errs...) } +// ParallelismStatus is Earthly-specific +func (w *Worker) ParallelismStatus() (int64, int64, int64) { + if w.ParallelismSem == nil { + return 0, 0, 0 + } + return w.ParallelismSem.Status() +} + +func (w *Worker) GCAnalytics() (cache.GCSummary, *cache.GCRunAnalytics, *cache.GCRunAnalytics) { + return w.CacheMgr.GetGCAnalytics() +} + func (w *Worker) ContentStore() *containerdsnapshot.Store { return w.WorkerOpt.ContentStore } @@ -566,6 +579,14 @@ func (w *Worker) Exporter(name string, sm *session.Manager) (exporter.Exporter, Variant: ociexporter.VariantDocker, LeaseManager: w.LeaseManager(), }) + case client.ExporterEarthly: + return earthlyoutputs.New(earthlyoutputs.Opt{ + SessionManager: sm, + ImageWriter: w.imageWriter, + Variant: ociexporter.VariantDocker, + RegistryHosts: w.RegistryHosts, + LeaseManager: w.LeaseManager(), + }) default: return nil, errors.Errorf("exporter %q could not be found", name) } diff --git a/worker/containerd/containerd.go b/worker/containerd/containerd.go index 6da1c0310..80885e8ea 100644 --- a/worker/containerd/containerd.go +++ b/worker/containerd/containerd.go @@ -19,16 +19,17 @@ import ( "github.com/moby/buildkit/solver/llbsolver/cdidevices" "github.com/moby/buildkit/util/leaseutil" "github.com/moby/buildkit/util/network/netproviders" + "github.com/moby/buildkit/util/semutil" "github.com/moby/buildkit/util/winlayers" "github.com/moby/buildkit/worker/base" wlabel "github.com/moby/buildkit/worker/label" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" - "golang.org/x/sync/semaphore" ) type RuntimeInfo = containerdexecutor.RuntimeInfo +// earthly-specific use semutil.Weighted instead of semaphore type WorkerOptions struct { Root string Address string @@ -41,7 +42,7 @@ type WorkerOptions struct { NetworkOpt netproviders.Opt ApparmorProfile string Selinux bool - ParallelismSem *semaphore.Weighted + ParallelismSem *semutil.Weighted TraceSocket string Runtime *RuntimeInfo CDIManager *cdidevices.Manager diff --git a/worker/runc/runc.go b/worker/runc/runc.go index 4b169b10b..c26aed858 100644 --- a/worker/runc/runc.go +++ b/worker/runc/runc.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strconv" + "time" "github.com/containerd/containerd/v2/core/diff/apply" ctdmetadata "github.com/containerd/containerd/v2/core/metadata" @@ -23,13 +24,13 @@ import ( "github.com/moby/buildkit/solver/llbsolver/cdidevices" "github.com/moby/buildkit/util/leaseutil" "github.com/moby/buildkit/util/network/netproviders" + "github.com/moby/buildkit/util/semutil" "github.com/moby/buildkit/util/winlayers" "github.com/moby/buildkit/worker/base" wlabel "github.com/moby/buildkit/worker/label" "github.com/moby/sys/user" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" bolt "go.etcd.io/bbolt" - "golang.org/x/sync/semaphore" ) // SnapshotterFactory instantiates a snapshotter @@ -39,7 +40,7 @@ type SnapshotterFactory struct { } // NewWorkerOpt creates a WorkerOpt. -func NewWorkerOpt(root string, snFactory SnapshotterFactory, rootless bool, processMode oci.ProcessMode, labels map[string]string, idmap *user.IdentityMapping, nopt netproviders.Opt, dns *oci.DNSConfig, binary, apparmorProfile string, selinux bool, parallelismSem *semaphore.Weighted, traceSocket, defaultCgroupParent string, cdiManager *cdidevices.Manager) (base.WorkerOpt, error) { +func NewWorkerOpt(root string, snFactory SnapshotterFactory, rootless bool, processMode oci.ProcessMode, labels map[string]string, idmap *user.IdentityMapping, nopt netproviders.Opt, dns *oci.DNSConfig, binary, apparmorProfile string, selinux bool, parallelismSem *semutil.Weighted, traceSocket, defaultCgroupParent string, hooks []oci.OciHook, sampleFrequency time.Duration, cdiManager *cdidevices.Manager) (base.WorkerOpt, error) { var opt base.WorkerOpt name := "runc-" + snFactory.Name root = filepath.Join(root, name) @@ -78,7 +79,9 @@ func NewWorkerOpt(root string, snFactory SnapshotterFactory, rootless bool, proc SELinux: selinux, TracingSocket: traceSocket, DefaultCgroupParent: defaultCgroupParent, + Hooks: hooks, // earthly-specific ResourceMonitor: rm, + SampleFrequency: sampleFrequency, // earthly-specific CDIManager: cdiManager, }, np) if err != nil { diff --git a/worker/runc/runc_test.go b/worker/runc/runc_test.go index 4eff08524..7155fc8a5 100644 --- a/worker/runc/runc_test.go +++ b/worker/runc/runc_test.go @@ -42,7 +42,7 @@ func newWorkerOpt(t *testing.T, processMode oci.ProcessMode) base.WorkerOpt { }, } rootless := false - workerOpt, err := NewWorkerOpt(tmpdir, snFactory, rootless, processMode, nil, nil, netproviders.Opt{Mode: "host"}, nil, "", "", false, nil, "", "", nil) + workerOpt, err := NewWorkerOpt(tmpdir, snFactory, rootless, processMode, nil, nil, netproviders.Opt{Mode: "host"}, nil, "", "", false, nil, "", "", []oci.OciHook{}, 0, nil) require.NoError(t, err) return workerOpt @@ -69,7 +69,11 @@ func TestRuncWorker(t *testing.T) { require.NoError(t, err) ctx := tests.NewCtx("buildkit-test") - sm, err := session.NewManager() + sm, err := session.NewManager(&session.ManagerOpt{ + HealthFrequency: 1 * time.Second, + HealthTimeout: 10 * time.Second, + HealthAllowedFailures: 1, + }) require.NoError(t, err) snap := tests.NewBusyboxSourceSnapshot(ctx, t, w, sm) @@ -190,7 +194,11 @@ func TestRuncWorkerNoProcessSandbox(t *testing.T) { require.NoError(t, err) ctx := tests.NewCtx("buildkit-test") - sm, err := session.NewManager() + sm, err := session.NewManager(&session.ManagerOpt{ + HealthFrequency: 1 * time.Second, + HealthTimeout: 10 * time.Second, + HealthAllowedFailures: 1, + }) require.NoError(t, err) snap := tests.NewBusyboxSourceSnapshot(ctx, t, w, sm) root, err := w.CacheMgr.New(ctx, snap, nil) diff --git a/worker/tests/common.go b/worker/tests/common.go index bdd61f708..f66c6151e 100644 --- a/worker/tests/common.go +++ b/worker/tests/common.go @@ -65,7 +65,11 @@ func NewCtx(s string) context.Context { func TestWorkerExec(t *testing.T, w *base.Worker) { ctx := NewCtx("buildkit-test") ctx, cancel := context.WithCancelCause(ctx) - sm, err := session.NewManager() + sm, err := session.NewManager(&session.ManagerOpt{ + HealthFrequency: 1 * time.Second, + HealthTimeout: 10 * time.Second, + HealthAllowedFailures: 1, + }) require.NoError(t, err) snap := NewBusyboxSourceSnapshot(ctx, t, w, sm) @@ -190,7 +194,11 @@ func TestWorkerExec(t *testing.T, w *base.Worker) { func TestWorkerExecFailures(t *testing.T, w *base.Worker) { ctx := NewCtx("buildkit-test") - sm, err := session.NewManager() + sm, err := session.NewManager(&session.ManagerOpt{ + HealthFrequency: 1 * time.Second, + HealthTimeout: 10 * time.Second, + HealthAllowedFailures: 1, + }) require.NoError(t, err) snap := NewBusyboxSourceSnapshot(ctx, t, w, sm) @@ -264,7 +272,11 @@ func TestWorkerExecFailures(t *testing.T, w *base.Worker) { func TestWorkerCancel(t *testing.T, w *base.Worker) { ctx := NewCtx("buildkit-test") - sm, err := session.NewManager() + sm, err := session.NewManager(&session.ManagerOpt{ + HealthFrequency: 1 * time.Second, + HealthTimeout: 10 * time.Second, + HealthAllowedFailures: 1, + }) require.NoError(t, err) snap := NewBusyboxSourceSnapshot(ctx, t, w, sm) diff --git a/worker/worker.go b/worker/worker.go index 4d4b04aa2..50d368f11 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -26,6 +26,8 @@ type Worker interface { Labels() map[string]string Platforms(noCache bool) []ocispecs.Platform BuildkitVersion() client.BuildkitVersion + ParallelismStatus() (int64, int64, int64) + GCAnalytics() (cache.GCSummary, *cache.GCRunAnalytics, *cache.GCRunAnalytics) GCPolicy() []client.PruneInfo LoadRef(ctx context.Context, id string, hidden bool) (cache.ImmutableRef, error) diff --git a/worker/workercontroller.go b/worker/workercontroller.go index 79906a07e..1ef2797e3 100644 --- a/worker/workercontroller.go +++ b/worker/workercontroller.go @@ -74,11 +74,44 @@ func (c *Controller) Get(id string) (Worker, error) { func (c *Controller) WorkerInfos() []client.WorkerInfo { out := make([]client.WorkerInfo, 0, len(c.workers)) for _, w := range c.workers { + taken, size, waiting := w.ParallelismStatus() + gcSummary, gcCurrent, gcLast := w.GCAnalytics() + gcAnalytics := client.GCAnalytics{ + NumRuns: gcSummary.NumRuns, + NumFailures: gcSummary.NumFailures, + AvgDuration: gcSummary.AvgDuration, + AvgRecordsCleared: gcSummary.AvgRecordsCleared, + AvgSizeCleared: gcSummary.AvgSizeCleared, + AvgRecordsBefore: gcSummary.AvgRecordsBefore, + AvgSizeBefore: gcSummary.AvgSizeBefore, + AllTimeRuns: gcSummary.AllTimeRuns, + AllTimeMaxDuration: gcSummary.AllTimeMaxDuration, + AllTimeDuration: gcSummary.AllTimeDuration, + } + if gcCurrent != nil { + gcAnalytics.CurrentStartTime = gcCurrent.Start + gcAnalytics.CurrentNumRecordsBefore = gcCurrent.NumRecordsBefore + gcAnalytics.CurrentSizeBefore = gcCurrent.SizeBefore + } + if gcLast != nil { + gcAnalytics.LastStartTime = gcLast.Start + gcAnalytics.LastEndTime = gcLast.End + gcAnalytics.LastNumRecordsBefore = gcLast.NumRecordsBefore + gcAnalytics.LastSizeBefore = gcLast.SizeBefore + gcAnalytics.LastNumRecordsCleared = gcLast.ClearedRecords + gcAnalytics.LastSizeCleared = gcLast.ClearedSize + gcAnalytics.LastSuccess = gcLast.Success + } out = append(out, client.WorkerInfo{ ID: w.ID(), Labels: w.Labels(), Platforms: w.Platforms(false), BuildkitVersion: w.BuildkitVersion(), + + ParallelismCurrent: int(taken), + ParallelismMax: int(size), + ParallelismWaiting: int(waiting), + GCAnalytics: gcAnalytics, }) } return out