From ac33d1e0d66f8d2106e6f14d8bf72fd68a0bf265 Mon Sep 17 00:00:00 2001 From: hectar-glitches Date: Thu, 30 Jul 2026 11:12:17 -0700 Subject: [PATCH 01/10] feat(encryption): add StandardEncryptionManager for generic file envelope encryption. Signed-off-by: hectar-glitches --- encryption/standard_manager.go | 488 ++++++++++++++++++++++++++++ encryption/standard_manager_test.go | 165 ++++++++++ 2 files changed, 653 insertions(+) create mode 100644 encryption/standard_manager.go create mode 100644 encryption/standard_manager_test.go diff --git a/encryption/standard_manager.go b/encryption/standard_manager.go new file mode 100644 index 000000000..420ffa53c --- /dev/null +++ b/encryption/standard_manager.go @@ -0,0 +1,488 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package encryption + +import ( + "context" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + + icebergio "github.com/apache/iceberg-go/io" +) + +// Defaults for [StandardEncryptionManager]. +const ( + // StandardDefaultDEKLength is the default length, in bytes, of the + // per-file data encryption key (DEK) generated for AES-256-GCM. + StandardDefaultDEKLength = 32 + + // StandardDefaultBlockSize is the default plaintext block size, in + // bytes, used to split a file into independently authenticated AES-GCM + // blocks. Blocks allow random access (Seek/ReadAt) without buffering or + // decrypting the whole file. + StandardDefaultBlockSize = 64 * 1024 +) + +// Sentinel errors returned by [StandardEncryptionManager]. +var ( + // ErrKeyIDRequired is returned by + // [StandardEncryptionManager.NewEncryptedOutputFile] when keyID is empty. + // StandardEncryptionManager always encrypts, so it requires a KEK to wrap + // the generated DEK; use [PlaintextEncryptionManager] for unencrypted + // tables instead of passing an empty keyID here. + ErrKeyIDRequired = errors.New("encryption: StandardEncryptionManager requires a non-empty keyID") + + // ErrKeyMetadataRequired is returned by + // [StandardEncryptionManager.NewDecryptedInputFile] when keyMetadata is + // empty. StandardEncryptionManager always decrypts, so it requires the + // per-file key metadata produced by [StandardEncryptionManager.NewEncryptedOutputFile]. + ErrKeyMetadataRequired = errors.New("encryption: StandardEncryptionManager requires non-empty key metadata") + + // ErrUnsupportedKeyMetadataVersion is returned when key metadata was + // produced by a newer, incompatible encoding version. + ErrUnsupportedKeyMetadataVersion = errors.New("encryption: unsupported key metadata version") +) + +// standardKeyMetadataVersion is the current encoding version written by +// [StandardEncryptionManager]. It is bumped whenever the on-disk layout of +// standardKeyMetadata or the block ciphertext format changes incompatibly. +const standardKeyMetadataVersion = 1 + +// standardKeyMetadata is the JSON-encoded structure stored as the opaque +// [EncryptionKeyMetadata] for files produced by [StandardEncryptionManager]. +type standardKeyMetadata struct { + Version int `json:"v"` + KeyID string `json:"key-id"` + WrappedKey []byte `json:"wrapped-key"` + NoncePrefix []byte `json:"nonce-prefix"` + BlockSize int `json:"block-size"` + PlaintextLength int64 `json:"plaintext-length"` +} + +// StandardEncryptionManager is a generic, format-agnostic [EncryptionManager] +// that provides envelope encryption for arbitrary files (e.g. manifests, +// manifest lists, Puffin statistics) using a [KeyManagementClient] to wrap +// and unwrap a fresh AES-256-GCM data encryption key (DEK) per file. +// +// Each file is split into fixed-size plaintext blocks, and each block is +// sealed independently with AES-GCM using a unique nonce (a per-file random +// prefix combined with the block index). This bounds memory usage and +// supports random access (Seek/ReadAt) on the decrypted file without +// buffering or decrypting more than the requested blocks. +// +// StandardEncryptionManager always encrypts and always decrypts: it fails +// closed, returning [ErrKeyIDRequired] or [ErrKeyMetadataRequired] rather +// than silently falling back to plaintext. Use [PlaintextEncryptionManager] +// for tables or files that are not encrypted. +type StandardEncryptionManager struct { + kms KeyManagementClient + dekLength int + blockSize int +} + +var _ EncryptionManager = (*StandardEncryptionManager)(nil) + +// StandardManagerOption configures a [StandardEncryptionManager] created by +// [NewStandardEncryptionManager]. +type StandardManagerOption func(*StandardEncryptionManager) + +// WithDEKLength overrides the default data encryption key length (in bytes). +// Valid AES key lengths are 16, 24, or 32 bytes. +func WithDEKLength(length int) StandardManagerOption { + return func(m *StandardEncryptionManager) { m.dekLength = length } +} + +// WithBlockSize overrides the default plaintext block size (in bytes) used +// to split files for independent block-level authentication. +func WithBlockSize(size int) StandardManagerOption { + return func(m *StandardEncryptionManager) { m.blockSize = size } +} + +// NewStandardEncryptionManager creates a [StandardEncryptionManager] backed +// by kms. kms must not be nil. +func NewStandardEncryptionManager(kms KeyManagementClient, opts ...StandardManagerOption) *StandardEncryptionManager { + m := &StandardEncryptionManager{ + kms: kms, + dekLength: StandardDefaultDEKLength, + blockSize: StandardDefaultBlockSize, + } + for _, opt := range opts { + opt(m) + } + + return m +} + +// NewEncryptedOutputFile creates a new AES-GCM block-encrypted output file. +// keyID identifies the KEK used to wrap the freshly generated per-file DEK, +// and must be non-empty; otherwise [ErrKeyIDRequired] is returned. +func (m *StandardEncryptionManager) NewEncryptedOutputFile(ctx context.Context, writer icebergio.FileWriter, keyID string) (EncryptedOutputFile, error) { + if keyID == "" { + return nil, ErrKeyIDRequired + } + + var ( + plainDEK, wrappedDEK []byte + err error + ) + if m.kms.SupportsKeyGeneration() { + plainDEK, wrappedDEK, err = m.kms.GenerateKey(ctx, keyID, m.dekLength) + if err != nil { + return nil, fmt.Errorf("encryption: failed to generate DEK: %w", err) + } + } else { + plainDEK = make([]byte, m.dekLength) + if _, err = rand.Read(plainDEK); err != nil { + return nil, fmt.Errorf("encryption: failed to generate DEK: %w", err) + } + if wrappedDEK, err = m.kms.WrapKey(ctx, keyID, plainDEK); err != nil { + return nil, fmt.Errorf("encryption: failed to wrap DEK: %w", err) + } + } + + aead, err := newStandardAEAD(plainDEK) + if err != nil { + return nil, err + } + + noncePrefix := make([]byte, 4) + if _, err := rand.Read(noncePrefix); err != nil { + return nil, fmt.Errorf("encryption: failed to generate nonce prefix: %w", err) + } + + return &standardOutputFile{ + FileWriter: writer, + aead: aead, + noncePrefix: noncePrefix, + blockSize: m.blockSize, + keyID: keyID, + wrappedKey: wrappedDEK, + }, nil +} + +// NewDecryptedInputFile wraps file for transparent block-level AES-GCM +// decryption. keyMetadata must be the non-empty blob produced by +// [StandardEncryptionManager.NewEncryptedOutputFile]; otherwise +// [ErrKeyMetadataRequired] is returned. +func (m *StandardEncryptionManager) NewDecryptedInputFile(ctx context.Context, file icebergio.File, keyMetadata EncryptionKeyMetadata) (EncryptedInputFile, error) { + if len(keyMetadata) == 0 { + return nil, ErrKeyMetadataRequired + } + + var meta standardKeyMetadata + if err := json.Unmarshal(keyMetadata, &meta); err != nil { + return nil, fmt.Errorf("encryption: failed to decode key metadata: %w", err) + } + if meta.Version != standardKeyMetadataVersion { + return nil, fmt.Errorf("%w: %d", ErrUnsupportedKeyMetadataVersion, meta.Version) + } + + plainDEK, err := m.kms.UnwrapKey(ctx, meta.KeyID, meta.WrappedKey) + if err != nil { + return nil, fmt.Errorf("encryption: failed to unwrap DEK: %w", err) + } + + aead, err := newStandardAEAD(plainDEK) + if err != nil { + return nil, err + } + + return &standardInputFile{ + underlying: file, + aead: aead, + noncePrefix: meta.NoncePrefix, + blockSize: meta.BlockSize, + plaintextLength: meta.PlaintextLength, + keyMetadata: keyMetadata, + }, nil +} + +func newStandardAEAD(key []byte) (cipher.AEAD, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrInvalidKeyLength, err) + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, fmt.Errorf("encryption: failed to create GCM: %w", err) + } + + return gcm, nil +} + +// standardBlockNonce derives the AES-GCM nonce for blockIndex: the 4-byte +// per-file random prefix followed by the 8-byte big-endian block index. The +// (key, nonce) pair is unique per block since the DEK is fresh per file and +// no two blocks in the same file share an index. +func standardBlockNonce(prefix []byte, blockIndex uint64) []byte { + nonce := make([]byte, 12) + copy(nonce, prefix) + binary.BigEndian.PutUint64(nonce[4:], blockIndex) + + return nonce +} + +// standardOutputFile is an [EncryptedOutputFile] that seals fixed-size +// plaintext blocks with AES-GCM as they are written. +type standardOutputFile struct { + icebergio.FileWriter + + aead cipher.AEAD + noncePrefix []byte + blockSize int + keyID string + wrappedKey []byte + + buf []byte + blockIndex uint64 + written int64 + closed bool + + keyMetadata EncryptionKeyMetadata +} + +var _ EncryptedOutputFile = (*standardOutputFile)(nil) + +func (f *standardOutputFile) Write(p []byte) (int, error) { + total := len(p) + for len(p) > 0 { + space := f.blockSize - len(f.buf) + n := min(space, len(p)) + f.buf = append(f.buf, p[:n]...) + p = p[n:] + f.written += int64(n) + if len(f.buf) == f.blockSize { + if err := f.flushBlock(); err != nil { + return total - len(p), err + } + } + } + + return total, nil +} + +func (f *standardOutputFile) flushBlock() error { + ciphertext := f.aead.Seal(nil, standardBlockNonce(f.noncePrefix, f.blockIndex), f.buf, nil) + if _, err := f.FileWriter.Write(ciphertext); err != nil { + return fmt.Errorf("encryption: failed to write encrypted block: %w", err) + } + f.blockIndex++ + f.buf = f.buf[:0] + + return nil +} + +// ReadFrom copies from r, encrypting as data is written, satisfying +// io.ReaderFrom (required by [icebergio.FileWriter]). +func (f *standardOutputFile) ReadFrom(r io.Reader) (int64, error) { + buf := make([]byte, 32*1024) + var total int64 + for { + n, err := r.Read(buf) + if n > 0 { + wn, werr := f.Write(buf[:n]) + total += int64(wn) + if werr != nil { + return total, werr + } + } + if err == io.EOF { + break + } + if err != nil { + return total, err + } + } + + return total, nil +} + +func (f *standardOutputFile) Close() error { + if f.closed { + return nil + } + f.closed = true + + if len(f.buf) > 0 { + if err := f.flushBlock(); err != nil { + _ = f.FileWriter.Close() + + return err + } + } + + meta := standardKeyMetadata{ + Version: standardKeyMetadataVersion, + KeyID: f.keyID, + WrappedKey: f.wrappedKey, + NoncePrefix: f.noncePrefix, + BlockSize: f.blockSize, + PlaintextLength: f.written, + } + encoded, err := json.Marshal(meta) + if err != nil { + _ = f.FileWriter.Close() + + return fmt.Errorf("encryption: failed to encode key metadata: %w", err) + } + f.keyMetadata = encoded + + return f.FileWriter.Close() +} + +// KeyMetadata returns the finalized per-file key metadata. It is only +// populated after Close has been called. +func (f *standardOutputFile) KeyMetadata() EncryptionKeyMetadata { return f.keyMetadata } + +// standardInputFile is an [EncryptedInputFile] that decrypts fixed-size +// AES-GCM blocks on demand, supporting random access via ReadAt/Seek. +type standardInputFile struct { + underlying icebergio.File + aead cipher.AEAD + noncePrefix []byte + blockSize int + plaintextLength int64 + keyMetadata EncryptionKeyMetadata + + pos int64 +} + +var _ EncryptedInputFile = (*standardInputFile)(nil) + +func (f *standardInputFile) numBlocks() int64 { + if f.plaintextLength == 0 { + return 0 + } + + return (f.plaintextLength + int64(f.blockSize) - 1) / int64(f.blockSize) +} + +func (f *standardInputFile) blockPlainLen(idx int64) int64 { + if idx == f.numBlocks()-1 { + return f.plaintextLength - idx*int64(f.blockSize) + } + + return int64(f.blockSize) +} + +func (f *standardInputFile) physicalOffset(idx int64) int64 { + return idx * int64(f.blockSize+f.aead.Overhead()) +} + +func (f *standardInputFile) readBlock(idx int64) ([]byte, error) { + plainLen := f.blockPlainLen(idx) + ciphertext := make([]byte, plainLen+int64(f.aead.Overhead())) + if _, err := f.underlying.ReadAt(ciphertext, f.physicalOffset(idx)); err != nil && !errors.Is(err, io.EOF) { + return nil, fmt.Errorf("encryption: failed to read block %d: %w", idx, err) + } + + plaintext, err := f.aead.Open(nil, standardBlockNonce(f.noncePrefix, uint64(idx)), ciphertext, nil) + if err != nil { + return nil, fmt.Errorf("%w: block %d: %v", ErrAuthenticationFailed, idx, err) + } + + return plaintext, nil +} + +func (f *standardInputFile) ReadAt(p []byte, off int64) (int, error) { + if off < 0 { + return 0, errors.New("encryption: ReadAt: negative offset") + } + if off >= f.plaintextLength { + return 0, io.EOF + } + + var read int + for read < len(p) { + curOff := off + int64(read) + if curOff >= f.plaintextLength { + break + } + idx := curOff / int64(f.blockSize) + block, err := f.readBlock(idx) + if err != nil { + return read, err + } + inBlockOff := curOff - idx*int64(f.blockSize) + read += copy(p[read:], block[inBlockOff:]) + } + + var err error + if read < len(p) { + err = io.EOF + } + + return read, err +} + +func (f *standardInputFile) Read(p []byte) (int, error) { + n, err := f.ReadAt(p, f.pos) + f.pos += int64(n) + + return n, err +} + +func (f *standardInputFile) Seek(offset int64, whence int) (int64, error) { + var newPos int64 + switch whence { + case io.SeekStart: + newPos = offset + case io.SeekCurrent: + newPos = f.pos + offset + case io.SeekEnd: + newPos = f.plaintextLength + offset + default: + return 0, fmt.Errorf("encryption: Seek: invalid whence %d", whence) + } + if newPos < 0 { + return 0, errors.New("encryption: Seek: negative position") + } + f.pos = newPos + + return newPos, nil +} + +func (f *standardInputFile) Close() error { return f.underlying.Close() } + +func (f *standardInputFile) Stat() (fs.FileInfo, error) { + info, err := f.underlying.Stat() + if err != nil { + return nil, err + } + + return standardFileInfo{FileInfo: info, size: f.plaintextLength}, nil +} + +// KeyMetadata returns the key metadata this file was decrypted with. +func (f *standardInputFile) KeyMetadata() EncryptionKeyMetadata { return f.keyMetadata } + +// standardFileInfo overrides Size() to report the plaintext length rather +// than the (larger) on-disk ciphertext length. +type standardFileInfo struct { + fs.FileInfo + size int64 +} + +func (i standardFileInfo) Size() int64 { return i.size } diff --git a/encryption/standard_manager_test.go b/encryption/standard_manager_test.go new file mode 100644 index 000000000..97127ccd2 --- /dev/null +++ b/encryption/standard_manager_test.go @@ -0,0 +1,165 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package encryption_test + +import ( + "bytes" + "errors" + "io" + "testing" + + "github.com/apache/iceberg-go/encryption" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newTestStandardManager(t *testing.T, opts ...encryption.StandardManagerOption) (*encryption.StandardEncryptionManager, *encryption.InMemoryKeyManagementClient) { + t.Helper() + kms := encryption.NewInMemoryKeyManagementClient() + require.NoError(t, kms.AddKey("kek-1", bytes.Repeat([]byte{0x42}, 32))) + + return encryption.NewStandardEncryptionManager(kms, opts...), kms +} + +func encryptAll(t *testing.T, mgr *encryption.StandardEncryptionManager, keyID string, plaintext []byte) (ciphertext []byte, keyMetadata encryption.EncryptionKeyMetadata) { + t.Helper() + fw := &memFileWriter{} + out, err := mgr.NewEncryptedOutputFile(t.Context(), fw, keyID) + require.NoError(t, err) + _, err = out.Write(plaintext) + require.NoError(t, err) + require.NoError(t, out.Close()) + + return fw.Bytes(), out.KeyMetadata() +} + +func decryptAll(t *testing.T, mgr *encryption.StandardEncryptionManager, ciphertext []byte, keyMetadata encryption.EncryptionKeyMetadata) []byte { + t.Helper() + in, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) + require.NoError(t, err) + data, err := io.ReadAll(in) + require.NoError(t, err) + + return data +} + +func TestStandardEncryptionManager_RoundTrip_SmallerThanBlock(t *testing.T) { + mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) + plaintext := []byte("hello iceberg") + + ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) + assert.NotEmpty(t, keyMetadata) + assert.NotEqual(t, plaintext, ciphertext, "ciphertext must differ from plaintext") + + got := decryptAll(t, mgr, ciphertext, keyMetadata) + assert.Equal(t, plaintext, got) +} + +func TestStandardEncryptionManager_RoundTrip_MultiBlock(t *testing.T) { + mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) + plaintext := bytes.Repeat([]byte("0123456789abcdef"), 10) + plaintext = append(plaintext, []byte("partial-tail")...) + + ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) + got := decryptAll(t, mgr, ciphertext, keyMetadata) + assert.Equal(t, plaintext, got) +} + +func TestStandardEncryptionManager_RoundTrip_ExactBlockMultiple(t *testing.T) { + mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) + plaintext := bytes.Repeat([]byte("0123456789abcdef"), 4) + + ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) + got := decryptAll(t, mgr, ciphertext, keyMetadata) + assert.Equal(t, plaintext, got) +} + +func TestStandardEncryptionManager_RoundTrip_Empty(t *testing.T) { + mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) + + ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", nil) + got := decryptAll(t, mgr, ciphertext, keyMetadata) + assert.Empty(t, got) +} + +func TestStandardEncryptionManager_RandomAccess(t *testing.T) { + mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) + plaintext := bytes.Repeat([]byte("0123456789abcdef"), 10) + plaintext = append(plaintext, []byte("tail-bytes")...) + + ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) + in, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) + require.NoError(t, err) + + // Read a slice spanning a block boundary. + buf := make([]byte, 20) + n, err := in.ReadAt(buf, 10) + require.NoError(t, err) + assert.Equal(t, plaintext[10:30], buf[:n]) + + // Seek + Read. + pos, err := in.Seek(5, io.SeekStart) + require.NoError(t, err) + assert.Equal(t, int64(5), pos) + + rest, err := io.ReadAll(in) + require.NoError(t, err) + assert.Equal(t, plaintext[5:], rest) +} + +func TestStandardEncryptionManager_TamperedBlockFailsAuthentication(t *testing.T) { + mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) + plaintext := bytes.Repeat([]byte("0123456789abcdef"), 4) + + ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) + ciphertext[0] ^= 0xFF // flip a bit in the first block's ciphertext + + in, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) + require.NoError(t, err) + + _, err = io.ReadAll(in) + require.Error(t, err) + assert.True(t, errors.Is(err, encryption.ErrAuthenticationFailed)) +} + +func TestStandardEncryptionManager_OutputFile_EmptyKeyIDRejected(t *testing.T) { + mgr, _ := newTestStandardManager(t) + _, err := mgr.NewEncryptedOutputFile(t.Context(), &memFileWriter{}, "") + require.Error(t, err) + assert.True(t, errors.Is(err, encryption.ErrKeyIDRequired)) +} + +func TestStandardEncryptionManager_InputFile_EmptyKeyMetadataRejected(t *testing.T) { + mgr, _ := newTestStandardManager(t) + _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), nil) + require.Error(t, err) + assert.True(t, errors.Is(err, encryption.ErrKeyMetadataRequired)) +} + +func TestStandardEncryptionManager_UnknownKeyIDPropagatesFromKMS(t *testing.T) { + mgr, _ := newTestStandardManager(t) + _, err := mgr.NewEncryptedOutputFile(t.Context(), &memFileWriter{}, "does-not-exist") + require.Error(t, err) + assert.True(t, errors.Is(err, encryption.ErrUnknownKeyID)) +} + +func TestStandardEncryptionManager_MalformedKeyMetadataRejected(t *testing.T) { + mgr, _ := newTestStandardManager(t) + _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), encryption.EncryptionKeyMetadata("not json")) + require.Error(t, err) +} From 638d57aa140748638f2fd46344dbe3ba109e7654 Mon Sep 17 00:00:00 2001 From: hectar-glitches Date: Mon, 3 Aug 2026 14:12:20 -0700 Subject: [PATCH 02/10] fix(encryption): validate block size and decoded key metadata in StandardEncryptionManager. Signed-off-by: hectar-glitches --- encryption/standard_manager.go | 23 +++++++++++++++++ encryption/standard_manager_test.go | 38 +++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/encryption/standard_manager.go b/encryption/standard_manager.go index 420ffa53c..4713509b9 100644 --- a/encryption/standard_manager.go +++ b/encryption/standard_manager.go @@ -63,6 +63,17 @@ var ( // ErrUnsupportedKeyMetadataVersion is returned when key metadata was // produced by a newer, incompatible encoding version. ErrUnsupportedKeyMetadataVersion = errors.New("encryption: unsupported key metadata version") + + // ErrInvalidBlockSize is returned when a configured or decoded block + // size is not positive. + ErrInvalidBlockSize = errors.New("encryption: block size must be positive") + + // ErrInvalidKeyMetadata is returned by + // [StandardEncryptionManager.NewDecryptedInputFile] when decoded key + // metadata fails basic sanity checks (e.g. a negative plaintext length + // or a nonce prefix of the wrong size). Key metadata is untrusted input + // on a crypto read path, so it is validated rather than trusted blindly. + ErrInvalidKeyMetadata = errors.New("encryption: invalid key metadata") ) // standardKeyMetadataVersion is the current encoding version written by @@ -142,6 +153,9 @@ func (m *StandardEncryptionManager) NewEncryptedOutputFile(ctx context.Context, if keyID == "" { return nil, ErrKeyIDRequired } + if m.blockSize <= 0 { + return nil, fmt.Errorf("%w: got %d", ErrInvalidBlockSize, m.blockSize) + } var ( plainDEK, wrappedDEK []byte @@ -198,6 +212,15 @@ func (m *StandardEncryptionManager) NewDecryptedInputFile(ctx context.Context, f if meta.Version != standardKeyMetadataVersion { return nil, fmt.Errorf("%w: %d", ErrUnsupportedKeyMetadataVersion, meta.Version) } + if meta.BlockSize <= 0 { + return nil, fmt.Errorf("%w: block-size must be positive, got %d", ErrInvalidKeyMetadata, meta.BlockSize) + } + if meta.PlaintextLength < 0 { + return nil, fmt.Errorf("%w: plaintext-length must be non-negative, got %d", ErrInvalidKeyMetadata, meta.PlaintextLength) + } + if len(meta.NoncePrefix) != 4 { + return nil, fmt.Errorf("%w: nonce-prefix must be 4 bytes, got %d", ErrInvalidKeyMetadata, len(meta.NoncePrefix)) + } plainDEK, err := m.kms.UnwrapKey(ctx, meta.KeyID, meta.WrappedKey) if err != nil { diff --git a/encryption/standard_manager_test.go b/encryption/standard_manager_test.go index 97127ccd2..f751ce67b 100644 --- a/encryption/standard_manager_test.go +++ b/encryption/standard_manager_test.go @@ -163,3 +163,41 @@ func TestStandardEncryptionManager_MalformedKeyMetadataRejected(t *testing.T) { _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), encryption.EncryptionKeyMetadata("not json")) require.Error(t, err) } + +func TestStandardEncryptionManager_ZeroBlockSizeRejectedOnWrite(t *testing.T) { + mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(0)) + _, err := mgr.NewEncryptedOutputFile(t.Context(), &memFileWriter{}, "kek-1") + require.Error(t, err) + assert.True(t, errors.Is(err, encryption.ErrInvalidBlockSize)) +} + +func TestStandardEncryptionManager_NegativeBlockSizeRejectedOnWrite(t *testing.T) { + mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(-1)) + _, err := mgr.NewEncryptedOutputFile(t.Context(), &memFileWriter{}, "kek-1") + require.Error(t, err) + assert.True(t, errors.Is(err, encryption.ErrInvalidBlockSize)) +} + +func TestStandardEncryptionManager_ZeroBlockSizeMetadataRejectedOnRead(t *testing.T) { + mgr, _ := newTestStandardManager(t) + meta := []byte(`{"v":1,"key-id":"kek-1","wrapped-key":"AA==","nonce-prefix":"AAAAAA==","block-size":0,"plaintext-length":10}`) + _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), meta) + require.Error(t, err) + assert.True(t, errors.Is(err, encryption.ErrInvalidKeyMetadata)) +} + +func TestStandardEncryptionManager_NegativePlaintextLengthMetadataRejectedOnRead(t *testing.T) { + mgr, _ := newTestStandardManager(t) + meta := []byte(`{"v":1,"key-id":"kek-1","wrapped-key":"AA==","nonce-prefix":"AAAAAA==","block-size":16,"plaintext-length":-1}`) + _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), meta) + require.Error(t, err) + assert.True(t, errors.Is(err, encryption.ErrInvalidKeyMetadata)) +} + +func TestStandardEncryptionManager_BadNoncePrefixLengthMetadataRejectedOnRead(t *testing.T) { + mgr, _ := newTestStandardManager(t) + meta := []byte(`{"v":1,"key-id":"kek-1","wrapped-key":"AA==","nonce-prefix":"AA==","block-size":16,"plaintext-length":10}`) + _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), meta) + require.Error(t, err) + assert.True(t, errors.Is(err, encryption.ErrInvalidKeyMetadata)) +} From a6a3b9f49751f0429918849c05e17afd24cf2490 Mon Sep 17 00:00:00 2001 From: hectar-glitches Date: Tue, 18 Aug 2026 07:20:48 -0700 Subject: [PATCH 03/10] fix(encryption): harden standard manager I/O Signed-off-by: hectar-glitches --- encryption/standard_manager.go | 118 ++++++++++++++++++++++------ encryption/standard_manager_test.go | 113 ++++++++++++++++++++++++++ 2 files changed, 208 insertions(+), 23 deletions(-) diff --git a/encryption/standard_manager.go b/encryption/standard_manager.go index 4713509b9..593ac852c 100644 --- a/encryption/standard_manager.go +++ b/encryption/standard_manager.go @@ -74,6 +74,10 @@ var ( // or a nonce prefix of the wrong size). Key metadata is untrusted input // on a crypto read path, so it is validated rather than trusted blindly. ErrInvalidKeyMetadata = errors.New("encryption: invalid key metadata") + + // ErrOutputFileClosed is returned by [standardOutputFile.Write] when + // called after Close, or after a previous flush has poisoned the writer. + ErrOutputFileClosed = errors.New("encryption: write to closed StandardEncryptionManager output file") ) // standardKeyMetadataVersion is the current encoding version written by @@ -156,7 +160,17 @@ func (m *StandardEncryptionManager) NewEncryptedOutputFile(ctx context.Context, if m.blockSize <= 0 { return nil, fmt.Errorf("%w: got %d", ErrInvalidBlockSize, m.blockSize) } + switch m.dekLength { + case 16, 24, 32: + default: + return nil, fmt.Errorf("%w: DEK length must be 16, 24, or 32 bytes; got %d", ErrInvalidKeyLength, m.dekLength) + } + // The (key, nonce) uniqueness this block format relies on requires a + // freshly generated DEK for every file: the nonce is only 4 random bytes + // plus a block index, so reusing a DEK across files would reuse (DEK, + // nonce) pairs and break AES-GCM's security guarantees. Never cache or + // reuse plainDEK/wrappedDEK across calls to NewEncryptedOutputFile. var ( plainDEK, wrappedDEK []byte err error @@ -245,7 +259,7 @@ func (m *StandardEncryptionManager) NewDecryptedInputFile(ctx context.Context, f func newStandardAEAD(key []byte) (cipher.AEAD, error) { block, err := aes.NewCipher(key) if err != nil { - return nil, fmt.Errorf("%w: %v", ErrInvalidKeyLength, err) + return nil, fmt.Errorf("%w: %w", ErrInvalidKeyLength, err) } gcm, err := cipher.NewGCM(block) if err != nil { @@ -256,9 +270,10 @@ func newStandardAEAD(key []byte) (cipher.AEAD, error) { } // standardBlockNonce derives the AES-GCM nonce for blockIndex: the 4-byte -// per-file random prefix followed by the 8-byte big-endian block index. The -// (key, nonce) pair is unique per block since the DEK is fresh per file and -// no two blocks in the same file share an index. +// per-file random prefix followed by the 8-byte big-endian block index. +// Uniqueness of the (key, nonce) pair across every block ever sealed with a +// given DEK depends entirely on the DEK being freshly generated per file; +// see the caution in [StandardEncryptionManager.NewEncryptedOutputFile]. func standardBlockNonce(prefix []byte, blockIndex uint64) []byte { nonce := make([]byte, 12) copy(nonce, prefix) @@ -282,6 +297,7 @@ type standardOutputFile struct { blockIndex uint64 written int64 closed bool + err error keyMetadata EncryptionKeyMetadata } @@ -289,15 +305,23 @@ type standardOutputFile struct { var _ EncryptedOutputFile = (*standardOutputFile)(nil) func (f *standardOutputFile) Write(p []byte) (int, error) { + if f.err != nil { + return 0, f.err + } + if f.closed { + return 0, ErrOutputFileClosed + } + total := len(p) for len(p) > 0 { space := f.blockSize - len(f.buf) n := min(space, len(p)) f.buf = append(f.buf, p[:n]...) p = p[n:] - f.written += int64(n) if len(f.buf) == f.blockSize { if err := f.flushBlock(); err != nil { + f.err = err + return total - len(p), err } } @@ -306,11 +330,15 @@ func (f *standardOutputFile) Write(p []byte) (int, error) { return total, nil } +// flushBlock seals and writes the currently buffered plaintext block. +// f.written is only advanced once the ciphertext has actually reached the +// underlying writer, so a failed flush never overcounts PlaintextLength. func (f *standardOutputFile) flushBlock() error { ciphertext := f.aead.Seal(nil, standardBlockNonce(f.noncePrefix, f.blockIndex), f.buf, nil) if _, err := f.FileWriter.Write(ciphertext); err != nil { return fmt.Errorf("encryption: failed to write encrypted block: %w", err) } + f.written += int64(len(f.buf)) f.blockIndex++ f.buf = f.buf[:0] @@ -320,7 +348,7 @@ func (f *standardOutputFile) flushBlock() error { // ReadFrom copies from r, encrypting as data is written, satisfying // io.ReaderFrom (required by [icebergio.FileWriter]). func (f *standardOutputFile) ReadFrom(r io.Reader) (int64, error) { - buf := make([]byte, 32*1024) + buf := make([]byte, max(32*1024, f.blockSize)) var total int64 for { n, err := r.Read(buf) @@ -342,37 +370,52 @@ func (f *standardOutputFile) ReadFrom(r io.Reader) (int64, error) { return total, nil } +// Close flushes any buffered partial block and finalizes the key metadata. +// closed is only set once everything, including the underlying Close, has +// succeeded; a failed Close poisons the writer (via f.err) so a retry +// reliably reports the same error instead of masking the failure as success. func (f *standardOutputFile) Close() error { + if f.err != nil { + return f.err + } if f.closed { return nil } - f.closed = true if len(f.buf) > 0 { if err := f.flushBlock(); err != nil { + f.err = err _ = f.FileWriter.Close() return err } } - meta := standardKeyMetadata{ - Version: standardKeyMetadataVersion, - KeyID: f.keyID, - WrappedKey: f.wrappedKey, - NoncePrefix: f.noncePrefix, - BlockSize: f.blockSize, - PlaintextLength: f.written, + if f.keyMetadata == nil { + meta := standardKeyMetadata{ + Version: standardKeyMetadataVersion, + KeyID: f.keyID, + WrappedKey: f.wrappedKey, + NoncePrefix: f.noncePrefix, + BlockSize: f.blockSize, + PlaintextLength: f.written, + } + encoded, err := json.Marshal(meta) + if err != nil { + f.err = fmt.Errorf("encryption: failed to encode key metadata: %w", err) + _ = f.FileWriter.Close() + + return f.err + } + f.keyMetadata = encoded } - encoded, err := json.Marshal(meta) - if err != nil { - _ = f.FileWriter.Close() - return fmt.Errorf("encryption: failed to encode key metadata: %w", err) + if err := f.FileWriter.Close(); err != nil { + return fmt.Errorf("encryption: failed to close underlying writer: %w", err) } - f.keyMetadata = encoded + f.closed = true - return f.FileWriter.Close() + return nil } // KeyMetadata returns the finalized per-file key metadata. It is only @@ -381,6 +424,11 @@ func (f *standardOutputFile) KeyMetadata() EncryptionKeyMetadata { return f.keyM // standardInputFile is an [EncryptedInputFile] that decrypts fixed-size // AES-GCM blocks on demand, supporting random access via ReadAt/Seek. +// +// ReadAt is stateless and safe for concurrent use, matching the io.ReaderAt +// contract. Read and Seek mutate the shared cursor (pos) and are not +// concurrent-safe; do not call them from multiple goroutines on the same +// instance. type standardInputFile struct { underlying icebergio.File aead cipher.AEAD @@ -414,22 +462,46 @@ func (f *standardInputFile) physicalOffset(idx int64) int64 { return idx * int64(f.blockSize+f.aead.Overhead()) } +// readBlock decrypts block idx. It validates idx and the computed block +// length before reading, since metadata (blockSize, plaintextLength) can +// originate from untrusted, JSON-decoded key metadata. It also honors the +// actual byte count returned by ReadAt: a short, non-EOF-explained read is +// reported as [ErrCiphertextTooShort] (truncated storage) rather than being +// silently zero-padded into the AEAD, which would otherwise surface as a +// misleading [ErrAuthenticationFailed]. func (f *standardInputFile) readBlock(idx int64) ([]byte, error) { + if idx < 0 || idx >= f.numBlocks() { + return nil, fmt.Errorf("%w: block index %d out of range [0, %d)", ErrInvalidKeyMetadata, idx, f.numBlocks()) + } + plainLen := f.blockPlainLen(idx) - ciphertext := make([]byte, plainLen+int64(f.aead.Overhead())) - if _, err := f.underlying.ReadAt(ciphertext, f.physicalOffset(idx)); err != nil && !errors.Is(err, io.EOF) { + if plainLen < 0 { + return nil, fmt.Errorf("%w: negative computed length for block %d", ErrInvalidKeyMetadata, idx) + } + + wantLen := plainLen + int64(f.aead.Overhead()) + ciphertext := make([]byte, wantLen) + n, err := f.underlying.ReadAt(ciphertext, f.physicalOffset(idx)) + if err != nil && !errors.Is(err, io.EOF) { return nil, fmt.Errorf("encryption: failed to read block %d: %w", idx, err) } + if int64(n) != wantLen { + return nil, fmt.Errorf("%w: block %d: read %d of %d expected ciphertext bytes", ErrCiphertextTooShort, idx, n, wantLen) + } + ciphertext = ciphertext[:n] plaintext, err := f.aead.Open(nil, standardBlockNonce(f.noncePrefix, uint64(idx)), ciphertext, nil) if err != nil { - return nil, fmt.Errorf("%w: block %d: %v", ErrAuthenticationFailed, idx, err) + return nil, fmt.Errorf("%w: block %d: %w", ErrAuthenticationFailed, idx, err) } return plaintext, nil } func (f *standardInputFile) ReadAt(p []byte, off int64) (int, error) { + if len(p) == 0 { + return 0, nil + } if off < 0 { return 0, errors.New("encryption: ReadAt: negative offset") } diff --git a/encryption/standard_manager_test.go b/encryption/standard_manager_test.go index f751ce67b..cf22740e9 100644 --- a/encryption/standard_manager_test.go +++ b/encryption/standard_manager_test.go @@ -201,3 +201,116 @@ func TestStandardEncryptionManager_BadNoncePrefixLengthMetadataRejectedOnRead(t require.Error(t, err) assert.True(t, errors.Is(err, encryption.ErrInvalidKeyMetadata)) } + +func TestStandardEncryptionManager_InvalidDEKLengthRejected(t *testing.T) { + mgr, _ := newTestStandardManager(t, encryption.WithDEKLength(15)) + _, err := mgr.NewEncryptedOutputFile(t.Context(), &memFileWriter{}, "kek-1") + require.Error(t, err) + assert.True(t, errors.Is(err, encryption.ErrInvalidKeyLength)) +} + +// shortReadFile is an icebergio.File stub whose ReadAt returns fewer bytes +// than requested along with io.EOF once truncateAt is reached, mimicking a +// real backend (S3, local fs) reading a genuinely truncated file. A plain +// bytes.Reader always fills the requested slice, which is why the round-trip +// tests never exercise this path. +type shortReadFile struct { + *memFile + truncateAt int64 +} + +func (f *shortReadFile) ReadAt(p []byte, off int64) (int, error) { + if off >= f.truncateAt { + return 0, io.EOF + } + if off+int64(len(p)) > f.truncateAt { + p = p[:f.truncateAt-off] + } + n, err := f.memFile.ReadAt(p, off) + if err == nil && int64(n) < int64(len(p)) { + err = io.EOF + } + + return n, err +} + +func TestStandardEncryptionManager_TruncatedBackendReadReportsCiphertextTooShort(t *testing.T) { + mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) + plaintext := bytes.Repeat([]byte("0123456789abcdef"), 4) + + ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) + truncated := &shortReadFile{memFile: newMemFile(ciphertext), truncateAt: int64(len(ciphertext) - 5)} + + in, err := mgr.NewDecryptedInputFile(t.Context(), truncated, keyMetadata) + require.NoError(t, err) + + _, err = io.ReadAll(in) + require.Error(t, err) + assert.True(t, errors.Is(err, encryption.ErrCiphertextTooShort)) + assert.False(t, errors.Is(err, encryption.ErrAuthenticationFailed), "a truncated read must not be misreported as tampering") +} + +func TestStandardEncryptionManager_EmptyFile_ZeroLengthReadAt(t *testing.T) { + mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) + ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", nil) + + in, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) + require.NoError(t, err) + + n, err := in.ReadAt(nil, 0) + assert.NoError(t, err, "a zero-length ReadAt on an empty file should behave like bytes.Reader, not report io.EOF") + assert.Equal(t, 0, n) +} + +// failAfterNWriter is an icebergio.FileWriter stub that fails the Nth call +// to Write, simulating a mid-stream flush failure on the underlying storage. +type failAfterNWriter struct { + memFileWriter + failAt int + writes int +} + +func (w *failAfterNWriter) Write(p []byte) (int, error) { + w.writes++ + if w.writes == w.failAt { + return 0, errors.New("simulated write failure") + } + + return w.memFileWriter.Write(p) +} + +func TestStandardEncryptionManager_FlushFailurePoisonsWriterAndClose(t *testing.T) { + mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(4)) + fw := &failAfterNWriter{failAt: 2} + out, err := mgr.NewEncryptedOutputFile(t.Context(), fw, "kek-1") + require.NoError(t, err) + + // First block (4 bytes) flushes fine; the second block's flush fails. + _, err = out.Write([]byte("aaaabbbb")) + require.Error(t, err) + + // A subsequent Write must return the same sticky error, not attempt more I/O. + _, err2 := out.Write([]byte("c")) + require.Error(t, err2) + assert.ErrorIs(t, err2, err) + + // Close must report the failure rather than silently succeeding. + closeErr := out.Close() + require.Error(t, closeErr) + assert.Nil(t, out.KeyMetadata(), "key metadata must not be finalized when the file failed to write") + + // A retried Close must keep reporting the same error, not nil. + closeErr2 := out.Close() + require.Error(t, closeErr2) +} + +func TestStandardEncryptionManager_WriteAfterCloseRejected(t *testing.T) { + mgr, _ := newTestStandardManager(t) + out, err := mgr.NewEncryptedOutputFile(t.Context(), &memFileWriter{}, "kek-1") + require.NoError(t, err) + require.NoError(t, out.Close()) + + _, err = out.Write([]byte("late")) + require.Error(t, err) + assert.True(t, errors.Is(err, encryption.ErrOutputFileClosed)) +} From ec907ac94b85c8531faf75c405c6d3bea4aee290 Mon Sep 17 00:00:00 2001 From: hectar-glitches Date: Wed, 26 Aug 2026 17:32:09 -0400 Subject: [PATCH 04/10] Address blockers --- encryption/standard_manager.go | 381 ++++++++++++++++++++-------- encryption/standard_manager_test.go | 84 +++++- 2 files changed, 354 insertions(+), 111 deletions(-) diff --git a/encryption/standard_manager.go b/encryption/standard_manager.go index 593ac852c..d0eb94546 100644 --- a/encryption/standard_manager.go +++ b/encryption/standard_manager.go @@ -28,6 +28,7 @@ import ( "fmt" "io" "io/fs" + "math" icebergio "github.com/apache/iceberg-go/io" ) @@ -43,6 +44,15 @@ const ( // blocks. Blocks allow random access (Seek/ReadAt) without buffering or // decrypting the whole file. StandardDefaultBlockSize = 64 * 1024 + + // StandardMaxBlockSize is the largest plaintext block size accepted + // from the AES GCM Stream header on read, and the largest that + // [WithBlockSize] will configure for writing. The header's block-length + // field is unauthenticated, untrusted data (the Iceberg AES GCM Stream + // spec's "File length" note applies equally here); without a ceiling, a + // crafted header claiming e.g. 1<<40 bytes would force a huge + // allocation (see readBlock) before any authentication check can run. + StandardMaxBlockSize = 128 * 1024 * 1024 ) // Sentinel errors returned by [StandardEncryptionManager]. @@ -64,15 +74,22 @@ var ( // produced by a newer, incompatible encoding version. ErrUnsupportedKeyMetadataVersion = errors.New("encryption: unsupported key metadata version") - // ErrInvalidBlockSize is returned when a configured or decoded block - // size is not positive. - ErrInvalidBlockSize = errors.New("encryption: block size must be positive") + // ErrInvalidBlockSize is returned when a configured block size is not + // positive or exceeds [StandardMaxBlockSize]. + ErrInvalidBlockSize = errors.New("encryption: block size must be positive and at most StandardMaxBlockSize") + + // ErrInvalidStreamHeader is returned when the AES GCM Stream header + // (the "AGS1" magic and little-endian block-length fields at the start + // of an encrypted file, per the Iceberg AES GCM Stream spec) is + // missing, truncated, or specifies a block length outside the + // supported range. + ErrInvalidStreamHeader = errors.New("encryption: invalid AES GCM Stream header") // ErrInvalidKeyMetadata is returned by // [StandardEncryptionManager.NewDecryptedInputFile] when decoded key // metadata fails basic sanity checks (e.g. a negative plaintext length - // or a nonce prefix of the wrong size). Key metadata is untrusted input - // on a crypto read path, so it is validated rather than trusted blindly. + // or a missing AAD prefix). Key metadata is untrusted input on a crypto + // read path, so it is validated rather than trusted blindly. ErrInvalidKeyMetadata = errors.New("encryption: invalid key metadata") // ErrOutputFileClosed is returned by [standardOutputFile.Write] when @@ -80,20 +97,65 @@ var ( ErrOutputFileClosed = errors.New("encryption: write to closed StandardEncryptionManager output file") ) +// Constants describing the Iceberg AES GCM Stream ("AGS1") wire format used +// for the ciphertext produced by [StandardEncryptionManager]. See +// https://iceberg.apache.org/gcm-stream-spec/ for the full specification. +const ( + // standardStreamMagic identifies an AES GCM Stream version 1 file. + standardStreamMagic = "AGS1" + + // standardHeaderLength is the length, in bytes, of the magic plus the + // little-endian block-length field written at the start of every file. + standardHeaderLength = len(standardStreamMagic) + 4 + + // standardNonceLength is the length, in bytes, of the random AES-GCM + // nonce stored at the start of every cipher block. + standardNonceLength = 12 + + // standardTagLength is the length, in bytes, of the AES-GCM + // authentication tag appended to every cipher block's ciphertext. + standardTagLength = 16 + + // standardBlockOverhead is the number of ciphertext bytes added to + // each block beyond its plaintext length (nonce + tag). + standardBlockOverhead = standardNonceLength + standardTagLength + + // standardAADPrefixLength is the length, in bytes, of the random + // per-file AAD prefix generated for new output files. + standardAADPrefixLength = 16 +) + // standardKeyMetadataVersion is the current encoding version written by // [StandardEncryptionManager]. It is bumped whenever the on-disk layout of -// standardKeyMetadata or the block ciphertext format changes incompatibly. +// standardKeyMetadata changes incompatibly. const standardKeyMetadataVersion = 1 // standardKeyMetadata is the JSON-encoded structure stored as the opaque // [EncryptionKeyMetadata] for files produced by [StandardEncryptionManager]. +// +// Per the table spec, DataFile/ManifestFile key_metadata is explicitly +// "implementation-specific"; what must be Iceberg AES GCM Stream compliant - +// and is - is the wire format of the encrypted byte stream itself (magic, +// block framing, nonce placement, and AAD; see the constants above). type standardKeyMetadata struct { - Version int `json:"v"` - KeyID string `json:"key-id"` - WrappedKey []byte `json:"wrapped-key"` - NoncePrefix []byte `json:"nonce-prefix"` - BlockSize int `json:"block-size"` - PlaintextLength int64 `json:"plaintext-length"` + Version int `json:"v"` + KeyID string `json:"key-id"` + WrappedKey []byte `json:"wrapped-key"` + + // AADPrefix is combined with each block's little-endian index to form + // the AES GCM Stream additional authenticated data, binding every + // ciphertext block to this file and to its position so that blocks + // cannot be silently reordered, replayed from another file, or spliced + // in from elsewhere in the same file. It is not secret. + AADPrefix []byte `json:"aad-prefix"` + + // PlaintextLength is the trusted total plaintext size used to compute + // the block count on read. Per the AES GCM Stream spec's "File length" + // note, a reader must use a length from a trusted source rather than + // the underlying storage's reported size, since storage size alone + // cannot distinguish a genuinely short file from one truncated by an + // attacker who does not also control this metadata. + PlaintextLength int64 `json:"plaintext-length"` } // StandardEncryptionManager is a generic, format-agnostic [EncryptionManager] @@ -101,11 +163,13 @@ type standardKeyMetadata struct { // manifest lists, Puffin statistics) using a [KeyManagementClient] to wrap // and unwrap a fresh AES-256-GCM data encryption key (DEK) per file. // -// Each file is split into fixed-size plaintext blocks, and each block is -// sealed independently with AES-GCM using a unique nonce (a per-file random -// prefix combined with the block index). This bounds memory usage and -// supports random access (Seek/ReadAt) on the decrypted file without -// buffering or decrypting more than the requested blocks. +// Each file is split into fixed-size plaintext blocks and written using the +// Iceberg AES GCM Stream ("AGS1") format: a magic/block-length header +// followed by independently authenticated blocks, each carrying its own +// random nonce and authenticated with an AAD that binds it to the file and +// to its position. This bounds memory usage and supports random access +// (Seek/ReadAt) on the decrypted file without buffering or decrypting more +// than the requested blocks. // // StandardEncryptionManager always encrypts and always decrypts: it fails // closed, returning [ErrKeyIDRequired] or [ErrKeyMetadataRequired] rather @@ -130,7 +194,8 @@ func WithDEKLength(length int) StandardManagerOption { } // WithBlockSize overrides the default plaintext block size (in bytes) used -// to split files for independent block-level authentication. +// to split files for independent block-level authentication. size must be +// positive and at most [StandardMaxBlockSize]. func WithBlockSize(size int) StandardManagerOption { return func(m *StandardEncryptionManager) { m.blockSize = size } } @@ -157,7 +222,7 @@ func (m *StandardEncryptionManager) NewEncryptedOutputFile(ctx context.Context, if keyID == "" { return nil, ErrKeyIDRequired } - if m.blockSize <= 0 { + if m.blockSize <= 0 || m.blockSize > StandardMaxBlockSize { return nil, fmt.Errorf("%w: got %d", ErrInvalidBlockSize, m.blockSize) } switch m.dekLength { @@ -167,10 +232,8 @@ func (m *StandardEncryptionManager) NewEncryptedOutputFile(ctx context.Context, } // The (key, nonce) uniqueness this block format relies on requires a - // freshly generated DEK for every file: the nonce is only 4 random bytes - // plus a block index, so reusing a DEK across files would reuse (DEK, - // nonce) pairs and break AES-GCM's security guarantees. Never cache or - // reuse plainDEK/wrappedDEK across calls to NewEncryptedOutputFile. + // freshly generated DEK for every file: never cache or reuse + // plainDEK/wrappedDEK across calls to NewEncryptedOutputFile. var ( plainDEK, wrappedDEK []byte err error @@ -195,18 +258,27 @@ func (m *StandardEncryptionManager) NewEncryptedOutputFile(ctx context.Context, return nil, err } - noncePrefix := make([]byte, 4) - if _, err := rand.Read(noncePrefix); err != nil { - return nil, fmt.Errorf("encryption: failed to generate nonce prefix: %w", err) + aadPrefix := make([]byte, standardAADPrefixLength) + if _, err := rand.Read(aadPrefix); err != nil { + return nil, fmt.Errorf("encryption: failed to generate AAD prefix: %w", err) + } + + // Write the AES GCM Stream header (magic + little-endian block length) + // up front, before any ciphertext blocks, per the format spec. + header := make([]byte, standardHeaderLength) + copy(header, standardStreamMagic) + binary.LittleEndian.PutUint32(header[len(standardStreamMagic):], uint32(m.blockSize)) //nolint:gosec // bounded by ErrInvalidBlockSize above + if _, err := writer.Write(header); err != nil { + return nil, fmt.Errorf("encryption: failed to write stream header: %w", err) } return &standardOutputFile{ - FileWriter: writer, - aead: aead, - noncePrefix: noncePrefix, - blockSize: m.blockSize, - keyID: keyID, - wrappedKey: wrappedDEK, + FileWriter: writer, + aead: aead, + aadPrefix: aadPrefix, + blockSize: m.blockSize, + keyID: keyID, + wrappedKey: wrappedDEK, }, nil } @@ -226,14 +298,11 @@ func (m *StandardEncryptionManager) NewDecryptedInputFile(ctx context.Context, f if meta.Version != standardKeyMetadataVersion { return nil, fmt.Errorf("%w: %d", ErrUnsupportedKeyMetadataVersion, meta.Version) } - if meta.BlockSize <= 0 { - return nil, fmt.Errorf("%w: block-size must be positive, got %d", ErrInvalidKeyMetadata, meta.BlockSize) - } if meta.PlaintextLength < 0 { return nil, fmt.Errorf("%w: plaintext-length must be non-negative, got %d", ErrInvalidKeyMetadata, meta.PlaintextLength) } - if len(meta.NoncePrefix) != 4 { - return nil, fmt.Errorf("%w: nonce-prefix must be 4 bytes, got %d", ErrInvalidKeyMetadata, len(meta.NoncePrefix)) + if len(meta.AADPrefix) == 0 { + return nil, fmt.Errorf("%w: aad-prefix must not be empty", ErrInvalidKeyMetadata) } plainDEK, err := m.kms.UnwrapKey(ctx, meta.KeyID, meta.WrappedKey) @@ -246,11 +315,16 @@ func (m *StandardEncryptionManager) NewDecryptedInputFile(ctx context.Context, f return nil, err } + blockSize, err := readStandardStreamHeader(file) + if err != nil { + return nil, err + } + return &standardInputFile{ underlying: file, aead: aead, - noncePrefix: meta.NoncePrefix, - blockSize: meta.BlockSize, + aadPrefix: meta.AADPrefix, + blockSize: blockSize, plaintextLength: meta.PlaintextLength, keyMetadata: keyMetadata, }, nil @@ -269,32 +343,86 @@ func newStandardAEAD(key []byte) (cipher.AEAD, error) { return gcm, nil } -// standardBlockNonce derives the AES-GCM nonce for blockIndex: the 4-byte -// per-file random prefix followed by the 8-byte big-endian block index. -// Uniqueness of the (key, nonce) pair across every block ever sealed with a -// given DEK depends entirely on the DEK being freshly generated per file; -// see the caution in [StandardEncryptionManager.NewEncryptedOutputFile]. -func standardBlockNonce(prefix []byte, blockIndex uint64) []byte { - nonce := make([]byte, 12) - copy(nonce, prefix) - binary.BigEndian.PutUint64(nonce[4:], blockIndex) - - return nonce +// readStandardStreamHeader reads and validates the AES GCM Stream magic and +// block-length header at the start of file, returning the plaintext block +// length. The header is untrusted, unauthenticated data, so the returned +// length is bounded to [StandardMaxBlockSize] before any allocation sized by +// it takes place. +func readStandardStreamHeader(file icebergio.File) (int64, error) { + header := make([]byte, standardHeaderLength) + n, err := file.ReadAt(header, 0) + if err != nil && !errors.Is(err, io.EOF) { + return 0, fmt.Errorf("encryption: failed to read stream header: %w", err) + } + if n != standardHeaderLength { + return 0, fmt.Errorf("%w: expected %d header bytes, got %d", ErrInvalidStreamHeader, standardHeaderLength, n) + } + if string(header[:len(standardStreamMagic)]) != standardStreamMagic { + return 0, fmt.Errorf("%w: missing %q magic", ErrInvalidStreamHeader, standardStreamMagic) + } + + blockSize := int64(binary.LittleEndian.Uint32(header[len(standardStreamMagic):])) + if blockSize <= 0 || blockSize > StandardMaxBlockSize { + return 0, fmt.Errorf("%w: block length %d out of supported range (0, %d]", ErrInvalidStreamHeader, blockSize, StandardMaxBlockSize) + } + + return blockSize, nil +} + +// standardBlockAAD derives the AES-GCM additional authenticated data for +// blockIndex: the per-file AAD prefix followed by the 4-byte little-endian +// block index, per the Iceberg AES GCM Stream spec. This binds every +// ciphertext block to this file and to its position, which matters because +// blocks carry independent random nonces (rather than a nonce derived from +// the block index): without the index in the AAD, an attacker able to +// tamper with ciphertext at rest could silently reorder or splice blocks. +func standardBlockAAD(prefix []byte, blockIndex uint32) []byte { + aad := make([]byte, len(prefix)+4) + copy(aad, prefix) + binary.LittleEndian.PutUint32(aad[len(prefix):], blockIndex) + + return aad +} + +// checkedMulInt64 returns a*b and true, or (0, false) if the multiplication +// overflows int64. +func checkedMulInt64(a, b int64) (int64, bool) { + if a == 0 || b == 0 { + return 0, true + } + result := a * b + if result/b != a { + return 0, false + } + + return result, true +} + +// checkedAddInt64 returns a+b and true, or (0, false) if the addition +// overflows int64. +func checkedAddInt64(a, b int64) (int64, bool) { + result := a + b + if (b > 0 && result < a) || (b < 0 && result > a) { + return 0, false + } + + return result, true } // standardOutputFile is an [EncryptedOutputFile] that seals fixed-size -// plaintext blocks with AES-GCM as they are written. +// plaintext blocks with AES-GCM as they are written, using the Iceberg AES +// GCM Stream ("AGS1") wire format. type standardOutputFile struct { icebergio.FileWriter - aead cipher.AEAD - noncePrefix []byte - blockSize int - keyID string - wrappedKey []byte + aead cipher.AEAD + aadPrefix []byte + blockSize int + keyID string + wrappedKey []byte buf []byte - blockIndex uint64 + blockIndex uint32 written int64 closed bool err error @@ -313,29 +441,44 @@ func (f *standardOutputFile) Write(p []byte) (int, error) { } total := len(p) + consumed := 0 // bytes of p appended into f.buf so far in this call + accepted := 0 // bytes of p known to be durably flushed; reported on failure for len(p) > 0 { space := f.blockSize - len(f.buf) n := min(space, len(p)) f.buf = append(f.buf, p[:n]...) p = p[n:] + consumed += n if len(f.buf) == f.blockSize { if err := f.flushBlock(); err != nil { f.err = err - return total - len(p), err + return accepted, err } + accepted = consumed } } return total, nil } -// flushBlock seals and writes the currently buffered plaintext block. -// f.written is only advanced once the ciphertext has actually reached the -// underlying writer, so a failed flush never overcounts PlaintextLength. +// flushBlock seals and writes the currently buffered plaintext block using a +// fresh random nonce, per the Iceberg AES GCM Stream format. f.written is +// only advanced once the ciphertext has actually reached the underlying +// writer, so a failed flush never overcounts PlaintextLength. func (f *standardOutputFile) flushBlock() error { - ciphertext := f.aead.Seal(nil, standardBlockNonce(f.noncePrefix, f.blockIndex), f.buf, nil) - if _, err := f.FileWriter.Write(ciphertext); err != nil { + if f.blockIndex == math.MaxUint32 { + return errors.New("encryption: cannot write block: exceeded maximum block count") + } + + nonce := make([]byte, standardNonceLength) + if _, err := rand.Read(nonce); err != nil { + return fmt.Errorf("encryption: failed to generate block nonce: %w", err) + } + + aad := standardBlockAAD(f.aadPrefix, f.blockIndex) + sealed := f.aead.Seal(nonce, nonce, f.buf, aad) + if _, err := f.FileWriter.Write(sealed); err != nil { return fmt.Errorf("encryption: failed to write encrypted block: %w", err) } f.written += int64(len(f.buf)) @@ -371,9 +514,11 @@ func (f *standardOutputFile) ReadFrom(r io.Reader) (int64, error) { } // Close flushes any buffered partial block and finalizes the key metadata. -// closed is only set once everything, including the underlying Close, has -// succeeded; a failed Close poisons the writer (via f.err) so a retry -// reliably reports the same error instead of masking the failure as success. +// closed is only set, and keyMetadata only published, once everything - +// including the underlying Close - has succeeded; a failed Close poisons the +// writer (via f.err) so a retry reliably reports the same error instead of +// masking the failure as success or exposing metadata for an output that +// never finished. func (f *standardOutputFile) Close() error { if f.err != nil { return f.err @@ -391,35 +536,35 @@ func (f *standardOutputFile) Close() error { } } - if f.keyMetadata == nil { - meta := standardKeyMetadata{ - Version: standardKeyMetadataVersion, - KeyID: f.keyID, - WrappedKey: f.wrappedKey, - NoncePrefix: f.noncePrefix, - BlockSize: f.blockSize, - PlaintextLength: f.written, - } - encoded, err := json.Marshal(meta) - if err != nil { - f.err = fmt.Errorf("encryption: failed to encode key metadata: %w", err) - _ = f.FileWriter.Close() + meta := standardKeyMetadata{ + Version: standardKeyMetadataVersion, + KeyID: f.keyID, + WrappedKey: f.wrappedKey, + AADPrefix: f.aadPrefix, + PlaintextLength: f.written, + } + encoded, err := json.Marshal(meta) + if err != nil { + f.err = fmt.Errorf("encryption: failed to encode key metadata: %w", err) + _ = f.FileWriter.Close() - return f.err - } - f.keyMetadata = encoded + return f.err } if err := f.FileWriter.Close(); err != nil { - return fmt.Errorf("encryption: failed to close underlying writer: %w", err) + f.err = fmt.Errorf("encryption: failed to close underlying writer: %w", err) + + return f.err } + + f.keyMetadata = encoded f.closed = true return nil } // KeyMetadata returns the finalized per-file key metadata. It is only -// populated after Close has been called. +// populated after Close has succeeded. func (f *standardOutputFile) KeyMetadata() EncryptionKeyMetadata { return f.keyMetadata } // standardInputFile is an [EncryptedInputFile] that decrypts fixed-size @@ -432,8 +577,8 @@ func (f *standardOutputFile) KeyMetadata() EncryptionKeyMetadata { return f.keyM type standardInputFile struct { underlying icebergio.File aead cipher.AEAD - noncePrefix []byte - blockSize int + aadPrefix []byte + blockSize int64 plaintextLength int64 keyMetadata EncryptionKeyMetadata @@ -447,31 +592,59 @@ func (f *standardInputFile) numBlocks() int64 { return 0 } - return (f.plaintextLength + int64(f.blockSize) - 1) / int64(f.blockSize) + // Overflow-safe ceiling division: plaintextLength and blockSize both + // originate from data outside this reader's control (key metadata and + // the untrusted stream header, respectively), so avoid computing + // (plaintextLength + blockSize - 1), which can overflow near + // math.MaxInt64. + return 1 + (f.plaintextLength-1)/f.blockSize } func (f *standardInputFile) blockPlainLen(idx int64) int64 { if idx == f.numBlocks()-1 { - return f.plaintextLength - idx*int64(f.blockSize) + return f.plaintextLength - idx*f.blockSize } - return int64(f.blockSize) + return f.blockSize } -func (f *standardInputFile) physicalOffset(idx int64) int64 { - return idx * int64(f.blockSize+f.aead.Overhead()) +// blockPhysicalOffset computes the physical offset of block idx in the +// underlying ciphertext, using overflow-checked arithmetic. blockSize +// originates from the untrusted stream header and idx is derived from the +// untrusted plaintext-length in key metadata; a small blockSize combined +// with a huge plaintext-length could otherwise overflow int64 before the +// resulting (implausible) offset is ever used in a read. +func blockPhysicalOffset(idx, blockSize int64) (int64, error) { + physicalBlockSize, ok := checkedAddInt64(blockSize, standardBlockOverhead) + if !ok { + return 0, fmt.Errorf("%w: block size %d overflows physical layout", ErrInvalidKeyMetadata, blockSize) + } + product, ok := checkedMulInt64(idx, physicalBlockSize) + if !ok { + return 0, fmt.Errorf("%w: block %d physical offset overflows", ErrInvalidKeyMetadata, idx) + } + offset, ok := checkedAddInt64(product, int64(standardHeaderLength)) + if !ok { + return 0, fmt.Errorf("%w: block %d physical offset overflows", ErrInvalidKeyMetadata, idx) + } + + return offset, nil } // readBlock decrypts block idx. It validates idx and the computed block -// length before reading, since metadata (blockSize, plaintextLength) can -// originate from untrusted, JSON-decoded key metadata. It also honors the +// length before reading, since metadata (plaintextLength) and the stream +// header (blockSize) can originate from untrusted input. It also honors the // actual byte count returned by ReadAt: a short, non-EOF-explained read is // reported as [ErrCiphertextTooShort] (truncated storage) rather than being // silently zero-padded into the AEAD, which would otherwise surface as a // misleading [ErrAuthenticationFailed]. func (f *standardInputFile) readBlock(idx int64) ([]byte, error) { - if idx < 0 || idx >= f.numBlocks() { - return nil, fmt.Errorf("%w: block index %d out of range [0, %d)", ErrInvalidKeyMetadata, idx, f.numBlocks()) + numBlocks := f.numBlocks() + if idx < 0 || idx >= numBlocks { + return nil, fmt.Errorf("%w: block index %d out of range [0, %d)", ErrInvalidKeyMetadata, idx, numBlocks) + } + if idx > math.MaxUint32 { + return nil, fmt.Errorf("%w: block index %d exceeds the maximum supported block count", ErrInvalidKeyMetadata, idx) } plainLen := f.blockPlainLen(idx) @@ -479,9 +652,14 @@ func (f *standardInputFile) readBlock(idx int64) ([]byte, error) { return nil, fmt.Errorf("%w: negative computed length for block %d", ErrInvalidKeyMetadata, idx) } - wantLen := plainLen + int64(f.aead.Overhead()) + offset, err := blockPhysicalOffset(idx, f.blockSize) + if err != nil { + return nil, err + } + + wantLen := plainLen + standardBlockOverhead ciphertext := make([]byte, wantLen) - n, err := f.underlying.ReadAt(ciphertext, f.physicalOffset(idx)) + n, err := f.underlying.ReadAt(ciphertext, offset) if err != nil && !errors.Is(err, io.EOF) { return nil, fmt.Errorf("encryption: failed to read block %d: %w", idx, err) } @@ -490,7 +668,10 @@ func (f *standardInputFile) readBlock(idx int64) ([]byte, error) { } ciphertext = ciphertext[:n] - plaintext, err := f.aead.Open(nil, standardBlockNonce(f.noncePrefix, uint64(idx)), ciphertext, nil) + nonce, sealed := ciphertext[:standardNonceLength], ciphertext[standardNonceLength:] + aad := standardBlockAAD(f.aadPrefix, uint32(idx)) + + plaintext, err := f.aead.Open(nil, nonce, sealed, aad) if err != nil { return nil, fmt.Errorf("%w: block %d: %w", ErrAuthenticationFailed, idx, err) } @@ -515,12 +696,12 @@ func (f *standardInputFile) ReadAt(p []byte, off int64) (int, error) { if curOff >= f.plaintextLength { break } - idx := curOff / int64(f.blockSize) + idx := curOff / f.blockSize block, err := f.readBlock(idx) if err != nil { return read, err } - inBlockOff := curOff - idx*int64(f.blockSize) + inBlockOff := curOff - idx*f.blockSize read += copy(p[read:], block[inBlockOff:]) } diff --git a/encryption/standard_manager_test.go b/encryption/standard_manager_test.go index cf22740e9..a83b6e22a 100644 --- a/encryption/standard_manager_test.go +++ b/encryption/standard_manager_test.go @@ -19,8 +19,10 @@ package encryption_test import ( "bytes" + "encoding/binary" "errors" "io" + "math" "testing" "github.com/apache/iceberg-go/encryption" @@ -127,7 +129,10 @@ func TestStandardEncryptionManager_TamperedBlockFailsAuthentication(t *testing.T plaintext := bytes.Repeat([]byte("0123456789abcdef"), 4) ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) - ciphertext[0] ^= 0xFF // flip a bit in the first block's ciphertext + // Flip a bit inside the first block's ciphertext, past the 8-byte AES + // GCM Stream header and 12-byte nonce; flipping the header instead + // would fail earlier, at stream-header validation. + ciphertext[8+12] ^= 0xFF in, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) require.NoError(t, err) @@ -137,6 +142,60 @@ func TestStandardEncryptionManager_TamperedBlockFailsAuthentication(t *testing.T assert.True(t, errors.Is(err, encryption.ErrAuthenticationFailed)) } +func TestStandardEncryptionManager_ReorderedBlocksFailAuthentication(t *testing.T) { + mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) + plaintext := bytes.Repeat([]byte("0123456789abcdef"), 2) // exactly two full 16-byte blocks + + ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) + + // The AES GCM Stream header (magic + block length) is 8 bytes; each + // cipher block here is 16 (block size) + 12 (nonce) + 16 (tag) = 44 + // bytes. Swap the two blocks: with a random nonce per block, only the + // AAD (which binds a block to its position) can catch this. + const headerLen, blockPhysicalLen = 8, 44 + block1 := append([]byte(nil), ciphertext[headerLen:headerLen+blockPhysicalLen]...) + block2 := append([]byte(nil), ciphertext[headerLen+blockPhysicalLen:headerLen+2*blockPhysicalLen]...) + copy(ciphertext[headerLen:headerLen+blockPhysicalLen], block2) + copy(ciphertext[headerLen+blockPhysicalLen:headerLen+2*blockPhysicalLen], block1) + + in, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) + require.NoError(t, err) + + _, err = io.ReadAll(in) + require.Error(t, err) + assert.True(t, errors.Is(err, encryption.ErrAuthenticationFailed), "swapping blocks must be detected: the AAD binds each block to its position") +} + +func TestStandardEncryptionManager_StreamHeaderMagicMismatchRejected(t *testing.T) { + mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) + ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", []byte("hello iceberg")) + copy(ciphertext[:4], "XXXX") + + _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) + require.Error(t, err) + assert.True(t, errors.Is(err, encryption.ErrInvalidStreamHeader)) +} + +func TestStandardEncryptionManager_StreamHeaderZeroBlockLengthRejected(t *testing.T) { + mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) + ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", []byte("hello iceberg")) + binary.LittleEndian.PutUint32(ciphertext[4:8], 0) + + _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) + require.Error(t, err) + assert.True(t, errors.Is(err, encryption.ErrInvalidStreamHeader)) +} + +func TestStandardEncryptionManager_StreamHeaderBlockLengthExceedsMaxRejected(t *testing.T) { + mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) + ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", []byte("hello iceberg")) + binary.LittleEndian.PutUint32(ciphertext[4:8], math.MaxUint32) + + _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) + require.Error(t, err) + assert.True(t, errors.Is(err, encryption.ErrInvalidStreamHeader)) +} + func TestStandardEncryptionManager_OutputFile_EmptyKeyIDRejected(t *testing.T) { mgr, _ := newTestStandardManager(t) _, err := mgr.NewEncryptedOutputFile(t.Context(), &memFileWriter{}, "") @@ -178,25 +237,24 @@ func TestStandardEncryptionManager_NegativeBlockSizeRejectedOnWrite(t *testing.T assert.True(t, errors.Is(err, encryption.ErrInvalidBlockSize)) } -func TestStandardEncryptionManager_ZeroBlockSizeMetadataRejectedOnRead(t *testing.T) { - mgr, _ := newTestStandardManager(t) - meta := []byte(`{"v":1,"key-id":"kek-1","wrapped-key":"AA==","nonce-prefix":"AAAAAA==","block-size":0,"plaintext-length":10}`) - _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), meta) +func TestStandardEncryptionManager_BlockSizeExceedingMaxRejectedOnWrite(t *testing.T) { + mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(encryption.StandardMaxBlockSize+1)) + _, err := mgr.NewEncryptedOutputFile(t.Context(), &memFileWriter{}, "kek-1") require.Error(t, err) - assert.True(t, errors.Is(err, encryption.ErrInvalidKeyMetadata)) + assert.True(t, errors.Is(err, encryption.ErrInvalidBlockSize)) } func TestStandardEncryptionManager_NegativePlaintextLengthMetadataRejectedOnRead(t *testing.T) { mgr, _ := newTestStandardManager(t) - meta := []byte(`{"v":1,"key-id":"kek-1","wrapped-key":"AA==","nonce-prefix":"AAAAAA==","block-size":16,"plaintext-length":-1}`) + meta := []byte(`{"v":1,"key-id":"kek-1","wrapped-key":"AA==","aad-prefix":"MTIzNDU2Nzg5MDEyMzQ1Ng==","plaintext-length":-1}`) _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), meta) require.Error(t, err) assert.True(t, errors.Is(err, encryption.ErrInvalidKeyMetadata)) } -func TestStandardEncryptionManager_BadNoncePrefixLengthMetadataRejectedOnRead(t *testing.T) { +func TestStandardEncryptionManager_EmptyAADPrefixMetadataRejectedOnRead(t *testing.T) { mgr, _ := newTestStandardManager(t) - meta := []byte(`{"v":1,"key-id":"kek-1","wrapped-key":"AA==","nonce-prefix":"AA==","block-size":16,"plaintext-length":10}`) + meta := []byte(`{"v":1,"key-id":"kek-1","wrapped-key":"AA==","aad-prefix":"","plaintext-length":10}`) _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), meta) require.Error(t, err) assert.True(t, errors.Is(err, encryption.ErrInvalidKeyMetadata)) @@ -281,13 +339,17 @@ func (w *failAfterNWriter) Write(p []byte) (int, error) { func TestStandardEncryptionManager_FlushFailurePoisonsWriterAndClose(t *testing.T) { mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(4)) - fw := &failAfterNWriter{failAt: 2} + // Write call 1 is the AES GCM Stream header written by + // NewEncryptedOutputFile; call 2 flushes the first (4-byte) block; + // call 3 flushes the second block and fails. + fw := &failAfterNWriter{failAt: 3} out, err := mgr.NewEncryptedOutputFile(t.Context(), fw, "kek-1") require.NoError(t, err) // First block (4 bytes) flushes fine; the second block's flush fails. - _, err = out.Write([]byte("aaaabbbb")) + n, err := out.Write([]byte("aaaabbbb")) require.Error(t, err) + assert.Equal(t, 4, n, "only the first block reached storage") // A subsequent Write must return the same sticky error, not attempt more I/O. _, err2 := out.Write([]byte("c")) From 8108dc5b3274d95c8c0d3b9bd092d7b89996cd4e Mon Sep 17 00:00:00 2001 From: hectar-glitches Date: Tue, 1 Sep 2026 17:55:50 -0400 Subject: [PATCH 05/10] fix major concerns with filewriter and standardinputfile --- encryption/standard_manager.go | 98 ++++++++++++++++++++++++----- encryption/standard_manager_test.go | 66 +++++++++++++++++++ 2 files changed, 147 insertions(+), 17 deletions(-) diff --git a/encryption/standard_manager.go b/encryption/standard_manager.go index d0eb94546..bd1f900ff 100644 --- a/encryption/standard_manager.go +++ b/encryption/standard_manager.go @@ -29,6 +29,7 @@ import ( "io" "io/fs" "math" + "sync" icebergio "github.com/apache/iceberg-go/io" ) @@ -94,7 +95,8 @@ var ( // ErrOutputFileClosed is returned by [standardOutputFile.Write] when // called after Close, or after a previous flush has poisoned the writer. - ErrOutputFileClosed = errors.New("encryption: write to closed StandardEncryptionManager output file") + // It wraps [fs.ErrClosed] so callers can test with errors.Is(err, fs.ErrClosed). + ErrOutputFileClosed = fmt.Errorf("encryption: write to closed StandardEncryptionManager output file: %w", fs.ErrClosed) ) // Constants describing the Iceberg AES GCM Stream ("AGS1") wire format used @@ -142,6 +144,14 @@ type standardKeyMetadata struct { KeyID string `json:"key-id"` WrappedKey []byte `json:"wrapped-key"` + // BlockSize is the trusted plaintext block size the file was written + // with. It is validated bounded before any file I/O, and is what's + // actually used to size reads; the stream header's block-length field + // is untrusted (unauthenticated, attacker-influenced storage bytes) and + // is only compared against this value, never used directly to size an + // allocation. + BlockSize int64 `json:"block-size"` + // AADPrefix is combined with each block's little-endian index to form // the AES GCM Stream additional authenticated data, binding every // ciphertext block to this file and to its position so that blocks @@ -201,8 +211,12 @@ func WithBlockSize(size int) StandardManagerOption { } // NewStandardEncryptionManager creates a [StandardEncryptionManager] backed -// by kms. kms must not be nil. +// by kms. kms must not be nil; NewStandardEncryptionManager panics if it is. func NewStandardEncryptionManager(kms KeyManagementClient, opts ...StandardManagerOption) *StandardEncryptionManager { + if kms == nil { + panic("encryption: NewStandardEncryptionManager: kms must not be nil") + } + m := &StandardEncryptionManager{ kms: kms, dekLength: StandardDefaultDEKLength, @@ -245,7 +259,7 @@ func (m *StandardEncryptionManager) NewEncryptedOutputFile(ctx context.Context, } } else { plainDEK = make([]byte, m.dekLength) - if _, err = rand.Read(plainDEK); err != nil { + if _, err = io.ReadFull(rand.Reader, plainDEK); err != nil { return nil, fmt.Errorf("encryption: failed to generate DEK: %w", err) } if wrappedDEK, err = m.kms.WrapKey(ctx, keyID, plainDEK); err != nil { @@ -259,7 +273,7 @@ func (m *StandardEncryptionManager) NewEncryptedOutputFile(ctx context.Context, } aadPrefix := make([]byte, standardAADPrefixLength) - if _, err := rand.Read(aadPrefix); err != nil { + if _, err := io.ReadFull(rand.Reader, aadPrefix); err != nil { return nil, fmt.Errorf("encryption: failed to generate AAD prefix: %w", err) } @@ -298,6 +312,9 @@ func (m *StandardEncryptionManager) NewDecryptedInputFile(ctx context.Context, f if meta.Version != standardKeyMetadataVersion { return nil, fmt.Errorf("%w: %d", ErrUnsupportedKeyMetadataVersion, meta.Version) } + if meta.BlockSize <= 0 || meta.BlockSize > StandardMaxBlockSize { + return nil, fmt.Errorf("%w: block-size must be positive and at most %d, got %d", ErrInvalidKeyMetadata, StandardMaxBlockSize, meta.BlockSize) + } if meta.PlaintextLength < 0 { return nil, fmt.Errorf("%w: plaintext-length must be non-negative, got %d", ErrInvalidKeyMetadata, meta.PlaintextLength) } @@ -315,16 +332,22 @@ func (m *StandardEncryptionManager) NewDecryptedInputFile(ctx context.Context, f return nil, err } - blockSize, err := readStandardStreamHeader(file) + // headerBlockSize is untrusted (unauthenticated bytes read from + // storage): it is only ever compared against the trusted, already + // bounded meta.BlockSize below, never used to size a read or allocation. + headerBlockSize, err := readStandardStreamHeader(file) if err != nil { return nil, err } + if headerBlockSize != meta.BlockSize { + return nil, fmt.Errorf("%w: stream header block length %d does not match key metadata block-size %d", ErrInvalidStreamHeader, headerBlockSize, meta.BlockSize) + } return &standardInputFile{ underlying: file, aead: aead, aadPrefix: meta.AADPrefix, - blockSize: blockSize, + blockSize: meta.BlockSize, plaintextLength: meta.PlaintextLength, keyMetadata: keyMetadata, }, nil @@ -427,11 +450,28 @@ type standardOutputFile struct { closed bool err error + // underlyingClosed tracks whether FileWriter.Close has already been + // attempted, so a poisoned writer is closed exactly once regardless of + // whether the failure is first observed in Write or in Close. + underlyingClosed bool + keyMetadata EncryptionKeyMetadata } var _ EncryptedOutputFile = (*standardOutputFile)(nil) +// closeUnderlyingIgnoringError closes the underlying writer at most once. +// The error is ignored: the caller is already reporting a more specific +// failure (a flush or encode error), and this is best-effort cleanup so a +// poisoned writer never leaks its underlying file descriptor or connection. +func (f *standardOutputFile) closeUnderlyingIgnoringError() { + if f.underlyingClosed { + return + } + f.underlyingClosed = true + _ = f.FileWriter.Close() +} + func (f *standardOutputFile) Write(p []byte) (int, error) { if f.err != nil { return 0, f.err @@ -452,6 +492,7 @@ func (f *standardOutputFile) Write(p []byte) (int, error) { if len(f.buf) == f.blockSize { if err := f.flushBlock(); err != nil { f.err = err + f.closeUnderlyingIgnoringError() return accepted, err } @@ -472,7 +513,7 @@ func (f *standardOutputFile) flushBlock() error { } nonce := make([]byte, standardNonceLength) - if _, err := rand.Read(nonce); err != nil { + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { return fmt.Errorf("encryption: failed to generate block nonce: %w", err) } @@ -530,7 +571,7 @@ func (f *standardOutputFile) Close() error { if len(f.buf) > 0 { if err := f.flushBlock(); err != nil { f.err = err - _ = f.FileWriter.Close() + f.closeUnderlyingIgnoringError() return err } @@ -540,22 +581,25 @@ func (f *standardOutputFile) Close() error { Version: standardKeyMetadataVersion, KeyID: f.keyID, WrappedKey: f.wrappedKey, + BlockSize: int64(f.blockSize), AADPrefix: f.aadPrefix, PlaintextLength: f.written, } encoded, err := json.Marshal(meta) if err != nil { f.err = fmt.Errorf("encryption: failed to encode key metadata: %w", err) - _ = f.FileWriter.Close() + f.closeUnderlyingIgnoringError() return f.err } if err := f.FileWriter.Close(); err != nil { + f.underlyingClosed = true f.err = fmt.Errorf("encryption: failed to close underlying writer: %w", err) return f.err } + f.underlyingClosed = true f.keyMetadata = encoded f.closed = true @@ -583,6 +627,15 @@ type standardInputFile struct { keyMetadata EncryptionKeyMetadata pos int64 + + // cacheMu guards cacheIdx/cacheBlock/cacheValid below. Serializing + // readBlock keeps the single-entry cache correct under the concurrent + // ReadAt usage this type documents, and also avoids re-decrypting the + // same block for runs of small, sequential reads that land in it. + cacheMu sync.Mutex + cacheIdx int64 + cacheBlock []byte + cacheValid bool } var _ EncryptedInputFile = (*standardInputFile)(nil) @@ -609,11 +662,10 @@ func (f *standardInputFile) blockPlainLen(idx int64) int64 { } // blockPhysicalOffset computes the physical offset of block idx in the -// underlying ciphertext, using overflow-checked arithmetic. blockSize -// originates from the untrusted stream header and idx is derived from the -// untrusted plaintext-length in key metadata; a small blockSize combined -// with a huge plaintext-length could otherwise overflow int64 before the -// resulting (implausible) offset is ever used in a read. +// underlying ciphertext, using overflow-checked arithmetic. idx is derived +// from the untrusted plaintext-length in key metadata, so a small blockSize +// combined with a huge plaintext-length could otherwise overflow int64 +// before the resulting (implausible) offset is ever used in a read. func blockPhysicalOffset(idx, blockSize int64) (int64, error) { physicalBlockSize, ok := checkedAddInt64(blockSize, standardBlockOverhead) if !ok { @@ -631,14 +683,22 @@ func blockPhysicalOffset(idx, blockSize int64) (int64, error) { return offset, nil } -// readBlock decrypts block idx. It validates idx and the computed block -// length before reading, since metadata (plaintextLength) and the stream -// header (blockSize) can originate from untrusted input. It also honors the +// readBlock decrypts block idx, or returns it from the single-entry cache +// if it was the most recently decrypted block. It validates idx and the +// computed block length before reading, since metadata (blockSize, +// plaintextLength) can originate from untrusted input. It also honors the // actual byte count returned by ReadAt: a short, non-EOF-explained read is // reported as [ErrCiphertextTooShort] (truncated storage) rather than being // silently zero-padded into the AEAD, which would otherwise surface as a // misleading [ErrAuthenticationFailed]. func (f *standardInputFile) readBlock(idx int64) ([]byte, error) { + f.cacheMu.Lock() + defer f.cacheMu.Unlock() + + if f.cacheValid && f.cacheIdx == idx { + return f.cacheBlock, nil + } + numBlocks := f.numBlocks() if idx < 0 || idx >= numBlocks { return nil, fmt.Errorf("%w: block index %d out of range [0, %d)", ErrInvalidKeyMetadata, idx, numBlocks) @@ -676,6 +736,10 @@ func (f *standardInputFile) readBlock(idx int64) ([]byte, error) { return nil, fmt.Errorf("%w: block %d: %w", ErrAuthenticationFailed, idx, err) } + f.cacheIdx = idx + f.cacheBlock = plaintext + f.cacheValid = true + return plaintext, nil } diff --git a/encryption/standard_manager_test.go b/encryption/standard_manager_test.go index a83b6e22a..107ab29d6 100644 --- a/encryption/standard_manager_test.go +++ b/encryption/standard_manager_test.go @@ -22,6 +22,7 @@ import ( "encoding/binary" "errors" "io" + "io/fs" "math" "testing" @@ -322,10 +323,13 @@ func TestStandardEncryptionManager_EmptyFile_ZeroLengthReadAt(t *testing.T) { // failAfterNWriter is an icebergio.FileWriter stub that fails the Nth call // to Write, simulating a mid-stream flush failure on the underlying storage. +// It also counts Close calls, since memFileWriter.Close is a no-op and would +// otherwise mask a leaked/never-closed underlying writer. type failAfterNWriter struct { memFileWriter failAt int writes int + closes int } func (w *failAfterNWriter) Write(p []byte) (int, error) { @@ -337,6 +341,12 @@ func (w *failAfterNWriter) Write(p []byte) (int, error) { return w.memFileWriter.Write(p) } +func (w *failAfterNWriter) Close() error { + w.closes++ + + return w.memFileWriter.Close() +} + func TestStandardEncryptionManager_FlushFailurePoisonsWriterAndClose(t *testing.T) { mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(4)) // Write call 1 is the AES GCM Stream header written by @@ -350,6 +360,7 @@ func TestStandardEncryptionManager_FlushFailurePoisonsWriterAndClose(t *testing. n, err := out.Write([]byte("aaaabbbb")) require.Error(t, err) assert.Equal(t, 4, n, "only the first block reached storage") + assert.Equal(t, 1, fw.closes, "a poisoned flush must close the underlying writer, not leak it") // A subsequent Write must return the same sticky error, not attempt more I/O. _, err2 := out.Write([]byte("c")) @@ -364,6 +375,7 @@ func TestStandardEncryptionManager_FlushFailurePoisonsWriterAndClose(t *testing. // A retried Close must keep reporting the same error, not nil. closeErr2 := out.Close() require.Error(t, closeErr2) + assert.Equal(t, 1, fw.closes, "a retried Close must not close the underlying writer again") } func TestStandardEncryptionManager_WriteAfterCloseRejected(t *testing.T) { @@ -375,4 +387,58 @@ func TestStandardEncryptionManager_WriteAfterCloseRejected(t *testing.T) { _, err = out.Write([]byte("late")) require.Error(t, err) assert.True(t, errors.Is(err, encryption.ErrOutputFileClosed)) + assert.True(t, errors.Is(err, fs.ErrClosed)) +} + +func TestNewStandardEncryptionManager_NilKMSPanics(t *testing.T) { + assert.PanicsWithValue(t, "encryption: NewStandardEncryptionManager: kms must not be nil", func() { + encryption.NewStandardEncryptionManager(nil) + }) +} + +func TestStandardEncryptionManager_StreamHeaderBlockLengthMismatchRejected(t *testing.T) { + mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) + ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", []byte("hello iceberg")) + // A header block length that is itself in-range, but does not match the + // (trusted) block-size recorded in key metadata. + binary.LittleEndian.PutUint32(ciphertext[4:8], 32) + + _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) + require.Error(t, err) + assert.True(t, errors.Is(err, encryption.ErrInvalidStreamHeader)) +} + +// countingReadAtFile wraps a memFile and counts ReadAt calls, to verify that +// repeated small reads landing in the same block reuse the decrypted-block +// cache instead of re-decrypting on every call. +type countingReadAtFile struct { + *memFile + reads int +} + +func (f *countingReadAtFile) ReadAt(p []byte, off int64) (int, error) { + f.reads++ + + return f.memFile.ReadAt(p, off) +} + +func TestStandardEncryptionManager_RepeatedSmallReadsReuseDecryptedBlockCache(t *testing.T) { + mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(4096)) + plaintext := bytes.Repeat([]byte("x"), 5000) // spans exactly two blocks (4096 + 904) + + ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) + counting := &countingReadAtFile{memFile: newMemFile(ciphertext)} + + in, err := mgr.NewDecryptedInputFile(t.Context(), counting, keyMetadata) + require.NoError(t, err) + baseline := counting.reads // the stream header was already read once above + + buf := make([]byte, 1) + for range plaintext { + n, err := in.Read(buf) + require.NoError(t, err) + require.Equal(t, 1, n) + } + + assert.LessOrEqual(t, counting.reads-baseline, 2, "5000 one-byte reads across two blocks must decrypt each block at most once") } From 6f2fb5436498fe8e3e0ea3bea28cce1dad38834c Mon Sep 17 00:00:00 2001 From: hectar-glitches Date: Sun, 6 Sep 2026 13:13:28 -0700 Subject: [PATCH 06/10] address the major and minor issues, rename the identifier from standard to go envelope --- encryption/standard_manager.go | 343 ++++++++++++++++------------ encryption/standard_manager_test.go | 277 +++++++++++++++++----- 2 files changed, 414 insertions(+), 206 deletions(-) diff --git a/encryption/standard_manager.go b/encryption/standard_manager.go index bd1f900ff..6ba23d16f 100644 --- a/encryption/standard_manager.go +++ b/encryption/standard_manager.go @@ -34,50 +34,54 @@ import ( icebergio "github.com/apache/iceberg-go/io" ) -// Defaults for [StandardEncryptionManager]. +// Defaults for [GoEnvelopeEncryptionManager]. const ( - // StandardDefaultDEKLength is the default length, in bytes, of the + // GoEnvelopeDefaultDEKLength is the default length, in bytes, of the // per-file data encryption key (DEK) generated for AES-256-GCM. - StandardDefaultDEKLength = 32 + GoEnvelopeDefaultDEKLength = 32 - // StandardDefaultBlockSize is the default plaintext block size, in + // GoEnvelopeDefaultBlockSize is the default plaintext block size, in // bytes, used to split a file into independently authenticated AES-GCM // blocks. Blocks allow random access (Seek/ReadAt) without buffering or // decrypting the whole file. - StandardDefaultBlockSize = 64 * 1024 + // + // This is not Java's Ciphers.PLAIN_BLOCK_SIZE (1 MiB, a compile-time + // constant there); see the [GoEnvelopeEncryptionManager] doc comment. + GoEnvelopeDefaultBlockSize = 64 * 1024 - // StandardMaxBlockSize is the largest plaintext block size accepted + // GoEnvelopeMaxBlockSize is the largest plaintext block size accepted // from the AES GCM Stream header on read, and the largest that // [WithBlockSize] will configure for writing. The header's block-length // field is unauthenticated, untrusted data (the Iceberg AES GCM Stream // spec's "File length" note applies equally here); without a ceiling, a // crafted header claiming e.g. 1<<40 bytes would force a huge // allocation (see readBlock) before any authentication check can run. - StandardMaxBlockSize = 128 * 1024 * 1024 + GoEnvelopeMaxBlockSize = 128 * 1024 * 1024 ) -// Sentinel errors returned by [StandardEncryptionManager]. +// Sentinel errors returned by [GoEnvelopeEncryptionManager]. var ( // ErrKeyIDRequired is returned by - // [StandardEncryptionManager.NewEncryptedOutputFile] when keyID is empty. - // StandardEncryptionManager always encrypts, so it requires a KEK to wrap - // the generated DEK; use [PlaintextEncryptionManager] for unencrypted - // tables instead of passing an empty keyID here. - ErrKeyIDRequired = errors.New("encryption: StandardEncryptionManager requires a non-empty keyID") + // [GoEnvelopeEncryptionManager.NewEncryptedOutputFile] when keyID is + // empty. GoEnvelopeEncryptionManager always encrypts, so it requires a + // KEK to wrap the generated DEK; use [PlaintextEncryptionManager] for + // unencrypted tables instead of passing an empty keyID here. + ErrKeyIDRequired = errors.New("encryption: GoEnvelopeEncryptionManager requires a non-empty keyID") // ErrKeyMetadataRequired is returned by - // [StandardEncryptionManager.NewDecryptedInputFile] when keyMetadata is - // empty. StandardEncryptionManager always decrypts, so it requires the - // per-file key metadata produced by [StandardEncryptionManager.NewEncryptedOutputFile]. - ErrKeyMetadataRequired = errors.New("encryption: StandardEncryptionManager requires non-empty key metadata") + // [GoEnvelopeEncryptionManager.NewDecryptedInputFile] when keyMetadata + // is empty. GoEnvelopeEncryptionManager always decrypts, so it requires + // the per-file key metadata produced by + // [GoEnvelopeEncryptionManager.NewEncryptedOutputFile]. + ErrKeyMetadataRequired = errors.New("encryption: GoEnvelopeEncryptionManager requires non-empty key metadata") // ErrUnsupportedKeyMetadataVersion is returned when key metadata was // produced by a newer, incompatible encoding version. ErrUnsupportedKeyMetadataVersion = errors.New("encryption: unsupported key metadata version") // ErrInvalidBlockSize is returned when a configured block size is not - // positive or exceeds [StandardMaxBlockSize]. - ErrInvalidBlockSize = errors.New("encryption: block size must be positive and at most StandardMaxBlockSize") + // positive or exceeds [GoEnvelopeMaxBlockSize]. + ErrInvalidBlockSize = errors.New("encryption: block size must be positive and at most GoEnvelopeMaxBlockSize") // ErrInvalidStreamHeader is returned when the AES GCM Stream header // (the "AGS1" magic and little-endian block-length fields at the start @@ -87,59 +91,70 @@ var ( ErrInvalidStreamHeader = errors.New("encryption: invalid AES GCM Stream header") // ErrInvalidKeyMetadata is returned by - // [StandardEncryptionManager.NewDecryptedInputFile] when decoded key + // [GoEnvelopeEncryptionManager.NewDecryptedInputFile] when decoded key // metadata fails basic sanity checks (e.g. a negative plaintext length // or a missing AAD prefix). Key metadata is untrusted input on a crypto // read path, so it is validated rather than trusted blindly. ErrInvalidKeyMetadata = errors.New("encryption: invalid key metadata") - // ErrOutputFileClosed is returned by [standardOutputFile.Write] when + // ErrOutputFileClosed is returned by [goEnvelopeOutputFile.Write] when // called after Close, or after a previous flush has poisoned the writer. // It wraps [fs.ErrClosed] so callers can test with errors.Is(err, fs.ErrClosed). - ErrOutputFileClosed = fmt.Errorf("encryption: write to closed StandardEncryptionManager output file: %w", fs.ErrClosed) + ErrOutputFileClosed = fmt.Errorf("encryption: write to closed GoEnvelopeEncryptionManager output file: %w", fs.ErrClosed) + + // ErrBlockTruncated is returned by [goEnvelopeInputFile.ReadAt] when the + // underlying storage returns fewer ciphertext bytes for a block than its + // recorded length requires, indicating the file was truncated at rest. + // This is distinct from [ErrCiphertextTooShort], which [KeyManagementClient] + // implementations use for a too-short wrapped key or KMS-encrypted + // payload; keeping them separate lets a caller tell a malformed KMS blob + // apart from a short block read. + ErrBlockTruncated = errors.New("encryption: block truncated: read fewer ciphertext bytes than expected") ) // Constants describing the Iceberg AES GCM Stream ("AGS1") wire format used -// for the ciphertext produced by [StandardEncryptionManager]. See +// for the ciphertext produced by [GoEnvelopeEncryptionManager]. See // https://iceberg.apache.org/gcm-stream-spec/ for the full specification. const ( - // standardStreamMagic identifies an AES GCM Stream version 1 file. - standardStreamMagic = "AGS1" + // gcmStreamMagic identifies an AES GCM Stream version 1 file. + gcmStreamMagic = "AGS1" - // standardHeaderLength is the length, in bytes, of the magic plus the + // gcmStreamHeaderLength is the length, in bytes, of the magic plus the // little-endian block-length field written at the start of every file. - standardHeaderLength = len(standardStreamMagic) + 4 + gcmStreamHeaderLength = len(gcmStreamMagic) + 4 - // standardNonceLength is the length, in bytes, of the random AES-GCM + // gcmStreamNonceLength is the length, in bytes, of the random AES-GCM // nonce stored at the start of every cipher block. - standardNonceLength = 12 + gcmStreamNonceLength = 12 - // standardTagLength is the length, in bytes, of the AES-GCM + // gcmStreamTagLength is the length, in bytes, of the AES-GCM // authentication tag appended to every cipher block's ciphertext. - standardTagLength = 16 + gcmStreamTagLength = 16 - // standardBlockOverhead is the number of ciphertext bytes added to + // gcmStreamBlockOverhead is the number of ciphertext bytes added to // each block beyond its plaintext length (nonce + tag). - standardBlockOverhead = standardNonceLength + standardTagLength + gcmStreamBlockOverhead = gcmStreamNonceLength + gcmStreamTagLength - // standardAADPrefixLength is the length, in bytes, of the random + // gcmStreamAADPrefixLength is the length, in bytes, of the random // per-file AAD prefix generated for new output files. - standardAADPrefixLength = 16 + gcmStreamAADPrefixLength = 16 ) -// standardKeyMetadataVersion is the current encoding version written by -// [StandardEncryptionManager]. It is bumped whenever the on-disk layout of -// standardKeyMetadata changes incompatibly. -const standardKeyMetadataVersion = 1 +// goEnvelopeKeyMetadataVersion is the current encoding version written by +// [GoEnvelopeEncryptionManager]. It is bumped whenever the on-disk layout of +// goEnvelopeKeyMetadata changes incompatibly. +const goEnvelopeKeyMetadataVersion = 1 -// standardKeyMetadata is the JSON-encoded structure stored as the opaque -// [EncryptionKeyMetadata] for files produced by [StandardEncryptionManager]. +// goEnvelopeKeyMetadata is the JSON-encoded structure stored as the opaque +// [EncryptionKeyMetadata] for files produced by [GoEnvelopeEncryptionManager]. // -// Per the table spec, DataFile/ManifestFile key_metadata is explicitly -// "implementation-specific"; what must be Iceberg AES GCM Stream compliant - -// and is - is the wire format of the encrypted byte stream itself (magic, -// block framing, nonce placement, and AAD; see the constants above). -type standardKeyMetadata struct { +// This encoding is Go-specific, not Java's Avro-encoded StandardKeyMetadata; +// see the [GoEnvelopeEncryptionManager] doc comment. Per the table spec, +// DataFile/ManifestFile key_metadata is explicitly "implementation-specific"; +// what must be Iceberg AES GCM Stream compliant - and is - is the wire +// format of the encrypted byte stream itself (magic, block framing, nonce +// placement, and AAD; see the constants above). +type goEnvelopeKeyMetadata struct { Version int `json:"v"` KeyID string `json:"key-id"` WrappedKey []byte `json:"wrapped-key"` @@ -168,7 +183,7 @@ type standardKeyMetadata struct { PlaintextLength int64 `json:"plaintext-length"` } -// StandardEncryptionManager is a generic, format-agnostic [EncryptionManager] +// GoEnvelopeEncryptionManager is a generic, format-agnostic [EncryptionManager] // that provides envelope encryption for arbitrary files (e.g. manifests, // manifest lists, Puffin statistics) using a [KeyManagementClient] to wrap // and unwrap a fresh AES-256-GCM data encryption key (DEK) per file. @@ -181,46 +196,56 @@ type standardKeyMetadata struct { // (Seek/ReadAt) on the decrypted file without buffering or decrypting more // than the requested blocks. // -// StandardEncryptionManager always encrypts and always decrypts: it fails +// GoEnvelopeEncryptionManager always encrypts and always decrypts: it fails // closed, returning [ErrKeyIDRequired] or [ErrKeyMetadataRequired] rather // than silently falling back to plaintext. Use [PlaintextEncryptionManager] // for tables or files that are not encrypted. -type StandardEncryptionManager struct { +// +// Not interoperable with Java's StandardEncryptionManager: the AGS1 byte +// stream framing (magic, block layout, nonce placement, AAD) matches the +// spec, but the two implementations are not cross-readable. Java's reader +// hard-requires a 1 MiB header block size (a compile-time constant there), +// while [GoEnvelopeDefaultBlockSize] is 64 KiB, and Java decodes key +// metadata as a 1-byte version plus Avro-encoded StandardKeyMetadata, while +// this type encodes key metadata as JSON (goEnvelopeKeyMetadata). Files +// written by one are not readable by the other. +type GoEnvelopeEncryptionManager struct { kms KeyManagementClient dekLength int blockSize int } -var _ EncryptionManager = (*StandardEncryptionManager)(nil) +var _ EncryptionManager = (*GoEnvelopeEncryptionManager)(nil) -// StandardManagerOption configures a [StandardEncryptionManager] created by -// [NewStandardEncryptionManager]. -type StandardManagerOption func(*StandardEncryptionManager) +// GoEnvelopeManagerOption configures a [GoEnvelopeEncryptionManager] created +// by [NewGoEnvelopeEncryptionManager]. +type GoEnvelopeManagerOption func(*GoEnvelopeEncryptionManager) // WithDEKLength overrides the default data encryption key length (in bytes). // Valid AES key lengths are 16, 24, or 32 bytes. -func WithDEKLength(length int) StandardManagerOption { - return func(m *StandardEncryptionManager) { m.dekLength = length } +func WithDEKLength(length int) GoEnvelopeManagerOption { + return func(m *GoEnvelopeEncryptionManager) { m.dekLength = length } } // WithBlockSize overrides the default plaintext block size (in bytes) used // to split files for independent block-level authentication. size must be -// positive and at most [StandardMaxBlockSize]. -func WithBlockSize(size int) StandardManagerOption { - return func(m *StandardEncryptionManager) { m.blockSize = size } +// positive and at most [GoEnvelopeMaxBlockSize]. +func WithBlockSize(size int) GoEnvelopeManagerOption { + return func(m *GoEnvelopeEncryptionManager) { m.blockSize = size } } -// NewStandardEncryptionManager creates a [StandardEncryptionManager] backed -// by kms. kms must not be nil; NewStandardEncryptionManager panics if it is. -func NewStandardEncryptionManager(kms KeyManagementClient, opts ...StandardManagerOption) *StandardEncryptionManager { +// NewGoEnvelopeEncryptionManager creates a [GoEnvelopeEncryptionManager] +// backed by kms. kms must not be nil; NewGoEnvelopeEncryptionManager panics +// if it is. +func NewGoEnvelopeEncryptionManager(kms KeyManagementClient, opts ...GoEnvelopeManagerOption) *GoEnvelopeEncryptionManager { if kms == nil { - panic("encryption: NewStandardEncryptionManager: kms must not be nil") + panic("encryption: NewGoEnvelopeEncryptionManager: kms must not be nil") } - m := &StandardEncryptionManager{ + m := &GoEnvelopeEncryptionManager{ kms: kms, - dekLength: StandardDefaultDEKLength, - blockSize: StandardDefaultBlockSize, + dekLength: GoEnvelopeDefaultDEKLength, + blockSize: GoEnvelopeDefaultBlockSize, } for _, opt := range opts { opt(m) @@ -232,11 +257,11 @@ func NewStandardEncryptionManager(kms KeyManagementClient, opts ...StandardManag // NewEncryptedOutputFile creates a new AES-GCM block-encrypted output file. // keyID identifies the KEK used to wrap the freshly generated per-file DEK, // and must be non-empty; otherwise [ErrKeyIDRequired] is returned. -func (m *StandardEncryptionManager) NewEncryptedOutputFile(ctx context.Context, writer icebergio.FileWriter, keyID string) (EncryptedOutputFile, error) { +func (m *GoEnvelopeEncryptionManager) NewEncryptedOutputFile(ctx context.Context, writer icebergio.FileWriter, keyID string) (EncryptedOutputFile, error) { if keyID == "" { return nil, ErrKeyIDRequired } - if m.blockSize <= 0 || m.blockSize > StandardMaxBlockSize { + if m.blockSize <= 0 || m.blockSize > GoEnvelopeMaxBlockSize { return nil, fmt.Errorf("%w: got %d", ErrInvalidBlockSize, m.blockSize) } switch m.dekLength { @@ -267,26 +292,26 @@ func (m *StandardEncryptionManager) NewEncryptedOutputFile(ctx context.Context, } } - aead, err := newStandardAEAD(plainDEK) + aead, err := newGoEnvelopeAEAD(plainDEK) if err != nil { return nil, err } - aadPrefix := make([]byte, standardAADPrefixLength) + aadPrefix := make([]byte, gcmStreamAADPrefixLength) if _, err := io.ReadFull(rand.Reader, aadPrefix); err != nil { return nil, fmt.Errorf("encryption: failed to generate AAD prefix: %w", err) } // Write the AES GCM Stream header (magic + little-endian block length) // up front, before any ciphertext blocks, per the format spec. - header := make([]byte, standardHeaderLength) - copy(header, standardStreamMagic) - binary.LittleEndian.PutUint32(header[len(standardStreamMagic):], uint32(m.blockSize)) //nolint:gosec // bounded by ErrInvalidBlockSize above + header := make([]byte, gcmStreamHeaderLength) + copy(header, gcmStreamMagic) + binary.LittleEndian.PutUint32(header[len(gcmStreamMagic):], uint32(m.blockSize)) //nolint:gosec // bounded by ErrInvalidBlockSize above if _, err := writer.Write(header); err != nil { return nil, fmt.Errorf("encryption: failed to write stream header: %w", err) } - return &standardOutputFile{ + return &goEnvelopeOutputFile{ FileWriter: writer, aead: aead, aadPrefix: aadPrefix, @@ -298,22 +323,22 @@ func (m *StandardEncryptionManager) NewEncryptedOutputFile(ctx context.Context, // NewDecryptedInputFile wraps file for transparent block-level AES-GCM // decryption. keyMetadata must be the non-empty blob produced by -// [StandardEncryptionManager.NewEncryptedOutputFile]; otherwise +// [GoEnvelopeEncryptionManager.NewEncryptedOutputFile]; otherwise // [ErrKeyMetadataRequired] is returned. -func (m *StandardEncryptionManager) NewDecryptedInputFile(ctx context.Context, file icebergio.File, keyMetadata EncryptionKeyMetadata) (EncryptedInputFile, error) { +func (m *GoEnvelopeEncryptionManager) NewDecryptedInputFile(ctx context.Context, file icebergio.File, keyMetadata EncryptionKeyMetadata) (EncryptedInputFile, error) { if len(keyMetadata) == 0 { return nil, ErrKeyMetadataRequired } - var meta standardKeyMetadata + var meta goEnvelopeKeyMetadata if err := json.Unmarshal(keyMetadata, &meta); err != nil { return nil, fmt.Errorf("encryption: failed to decode key metadata: %w", err) } - if meta.Version != standardKeyMetadataVersion { + if meta.Version != goEnvelopeKeyMetadataVersion { return nil, fmt.Errorf("%w: %d", ErrUnsupportedKeyMetadataVersion, meta.Version) } - if meta.BlockSize <= 0 || meta.BlockSize > StandardMaxBlockSize { - return nil, fmt.Errorf("%w: block-size must be positive and at most %d, got %d", ErrInvalidKeyMetadata, StandardMaxBlockSize, meta.BlockSize) + if meta.BlockSize <= 0 || meta.BlockSize > GoEnvelopeMaxBlockSize { + return nil, fmt.Errorf("%w: block-size must be positive and at most %d, got %d", ErrInvalidKeyMetadata, GoEnvelopeMaxBlockSize, meta.BlockSize) } if meta.PlaintextLength < 0 { return nil, fmt.Errorf("%w: plaintext-length must be non-negative, got %d", ErrInvalidKeyMetadata, meta.PlaintextLength) @@ -327,7 +352,7 @@ func (m *StandardEncryptionManager) NewDecryptedInputFile(ctx context.Context, f return nil, fmt.Errorf("encryption: failed to unwrap DEK: %w", err) } - aead, err := newStandardAEAD(plainDEK) + aead, err := newGoEnvelopeAEAD(plainDEK) if err != nil { return nil, err } @@ -335,7 +360,7 @@ func (m *StandardEncryptionManager) NewDecryptedInputFile(ctx context.Context, f // headerBlockSize is untrusted (unauthenticated bytes read from // storage): it is only ever compared against the trusted, already // bounded meta.BlockSize below, never used to size a read or allocation. - headerBlockSize, err := readStandardStreamHeader(file) + headerBlockSize, err := readGCMStreamHeader(file) if err != nil { return nil, err } @@ -343,7 +368,7 @@ func (m *StandardEncryptionManager) NewDecryptedInputFile(ctx context.Context, f return nil, fmt.Errorf("%w: stream header block length %d does not match key metadata block-size %d", ErrInvalidStreamHeader, headerBlockSize, meta.BlockSize) } - return &standardInputFile{ + return &goEnvelopeInputFile{ underlying: file, aead: aead, aadPrefix: meta.AADPrefix, @@ -353,7 +378,7 @@ func (m *StandardEncryptionManager) NewDecryptedInputFile(ctx context.Context, f }, nil } -func newStandardAEAD(key []byte) (cipher.AEAD, error) { +func newGoEnvelopeAEAD(key []byte) (cipher.AEAD, error) { block, err := aes.NewCipher(key) if err != nil { return nil, fmt.Errorf("%w: %w", ErrInvalidKeyLength, err) @@ -366,40 +391,40 @@ func newStandardAEAD(key []byte) (cipher.AEAD, error) { return gcm, nil } -// readStandardStreamHeader reads and validates the AES GCM Stream magic and +// readGCMStreamHeader reads and validates the AES GCM Stream magic and // block-length header at the start of file, returning the plaintext block // length. The header is untrusted, unauthenticated data, so the returned -// length is bounded to [StandardMaxBlockSize] before any allocation sized by -// it takes place. -func readStandardStreamHeader(file icebergio.File) (int64, error) { - header := make([]byte, standardHeaderLength) +// length is bounded to [GoEnvelopeMaxBlockSize] before any allocation sized +// by it takes place. +func readGCMStreamHeader(file icebergio.File) (int64, error) { + header := make([]byte, gcmStreamHeaderLength) n, err := file.ReadAt(header, 0) if err != nil && !errors.Is(err, io.EOF) { return 0, fmt.Errorf("encryption: failed to read stream header: %w", err) } - if n != standardHeaderLength { - return 0, fmt.Errorf("%w: expected %d header bytes, got %d", ErrInvalidStreamHeader, standardHeaderLength, n) + if n != gcmStreamHeaderLength { + return 0, fmt.Errorf("%w: expected %d header bytes, got %d", ErrInvalidStreamHeader, gcmStreamHeaderLength, n) } - if string(header[:len(standardStreamMagic)]) != standardStreamMagic { - return 0, fmt.Errorf("%w: missing %q magic", ErrInvalidStreamHeader, standardStreamMagic) + if string(header[:len(gcmStreamMagic)]) != gcmStreamMagic { + return 0, fmt.Errorf("%w: missing %q magic", ErrInvalidStreamHeader, gcmStreamMagic) } - blockSize := int64(binary.LittleEndian.Uint32(header[len(standardStreamMagic):])) - if blockSize <= 0 || blockSize > StandardMaxBlockSize { - return 0, fmt.Errorf("%w: block length %d out of supported range (0, %d]", ErrInvalidStreamHeader, blockSize, StandardMaxBlockSize) + blockSize := int64(binary.LittleEndian.Uint32(header[len(gcmStreamMagic):])) + if blockSize <= 0 || blockSize > GoEnvelopeMaxBlockSize { + return 0, fmt.Errorf("%w: block length %d out of supported range (0, %d]", ErrInvalidStreamHeader, blockSize, GoEnvelopeMaxBlockSize) } return blockSize, nil } -// standardBlockAAD derives the AES-GCM additional authenticated data for +// gcmStreamBlockAAD derives the AES-GCM additional authenticated data for // blockIndex: the per-file AAD prefix followed by the 4-byte little-endian // block index, per the Iceberg AES GCM Stream spec. This binds every // ciphertext block to this file and to its position, which matters because // blocks carry independent random nonces (rather than a nonce derived from // the block index): without the index in the AAD, an attacker able to // tamper with ciphertext at rest could silently reorder or splice blocks. -func standardBlockAAD(prefix []byte, blockIndex uint32) []byte { +func gcmStreamBlockAAD(prefix []byte, blockIndex uint32) []byte { aad := make([]byte, len(prefix)+4) copy(aad, prefix) binary.LittleEndian.PutUint32(aad[len(prefix):], blockIndex) @@ -432,10 +457,10 @@ func checkedAddInt64(a, b int64) (int64, bool) { return result, true } -// standardOutputFile is an [EncryptedOutputFile] that seals fixed-size +// goEnvelopeOutputFile is an [EncryptedOutputFile] that seals fixed-size // plaintext blocks with AES-GCM as they are written, using the Iceberg AES // GCM Stream ("AGS1") wire format. -type standardOutputFile struct { +type goEnvelopeOutputFile struct { icebergio.FileWriter aead cipher.AEAD @@ -458,13 +483,19 @@ type standardOutputFile struct { keyMetadata EncryptionKeyMetadata } -var _ EncryptedOutputFile = (*standardOutputFile)(nil) +var _ EncryptedOutputFile = (*goEnvelopeOutputFile)(nil) // closeUnderlyingIgnoringError closes the underlying writer at most once. // The error is ignored: the caller is already reporting a more specific // failure (a flush or encode error), and this is best-effort cleanup so a // poisoned writer never leaks its underlying file descriptor or connection. -func (f *standardOutputFile) closeUnderlyingIgnoringError() { +// +// defensive: today every call site sets f.err before calling this, and +// Write/Close both bail out early once f.err is set, so the +// underlyingClosed guard never actually stops a second Close in practice. +// It stays as a hard invariant in case a future call site is added that +// doesn't follow that pattern. +func (f *goEnvelopeOutputFile) closeUnderlyingIgnoringError() { if f.underlyingClosed { return } @@ -472,7 +503,7 @@ func (f *standardOutputFile) closeUnderlyingIgnoringError() { _ = f.FileWriter.Close() } -func (f *standardOutputFile) Write(p []byte) (int, error) { +func (f *goEnvelopeOutputFile) Write(p []byte) (int, error) { if f.err != nil { return 0, f.err } @@ -507,17 +538,17 @@ func (f *standardOutputFile) Write(p []byte) (int, error) { // fresh random nonce, per the Iceberg AES GCM Stream format. f.written is // only advanced once the ciphertext has actually reached the underlying // writer, so a failed flush never overcounts PlaintextLength. -func (f *standardOutputFile) flushBlock() error { +func (f *goEnvelopeOutputFile) flushBlock() error { if f.blockIndex == math.MaxUint32 { return errors.New("encryption: cannot write block: exceeded maximum block count") } - nonce := make([]byte, standardNonceLength) + nonce := make([]byte, gcmStreamNonceLength) if _, err := io.ReadFull(rand.Reader, nonce); err != nil { return fmt.Errorf("encryption: failed to generate block nonce: %w", err) } - aad := standardBlockAAD(f.aadPrefix, f.blockIndex) + aad := gcmStreamBlockAAD(f.aadPrefix, f.blockIndex) sealed := f.aead.Seal(nonce, nonce, f.buf, aad) if _, err := f.FileWriter.Write(sealed); err != nil { return fmt.Errorf("encryption: failed to write encrypted block: %w", err) @@ -531,7 +562,7 @@ func (f *standardOutputFile) flushBlock() error { // ReadFrom copies from r, encrypting as data is written, satisfying // io.ReaderFrom (required by [icebergio.FileWriter]). -func (f *standardOutputFile) ReadFrom(r io.Reader) (int64, error) { +func (f *goEnvelopeOutputFile) ReadFrom(r io.Reader) (int64, error) { buf := make([]byte, max(32*1024, f.blockSize)) var total int64 for { @@ -560,7 +591,7 @@ func (f *standardOutputFile) ReadFrom(r io.Reader) (int64, error) { // writer (via f.err) so a retry reliably reports the same error instead of // masking the failure as success or exposing metadata for an output that // never finished. -func (f *standardOutputFile) Close() error { +func (f *goEnvelopeOutputFile) Close() error { if f.err != nil { return f.err } @@ -577,8 +608,8 @@ func (f *standardOutputFile) Close() error { } } - meta := standardKeyMetadata{ - Version: standardKeyMetadataVersion, + meta := goEnvelopeKeyMetadata{ + Version: goEnvelopeKeyMetadataVersion, KeyID: f.keyID, WrappedKey: f.wrappedKey, BlockSize: int64(f.blockSize), @@ -609,16 +640,24 @@ func (f *standardOutputFile) Close() error { // KeyMetadata returns the finalized per-file key metadata. It is only // populated after Close has succeeded. -func (f *standardOutputFile) KeyMetadata() EncryptionKeyMetadata { return f.keyMetadata } +func (f *goEnvelopeOutputFile) KeyMetadata() EncryptionKeyMetadata { return f.keyMetadata } -// standardInputFile is an [EncryptedInputFile] that decrypts fixed-size +// goEnvelopeInputFile is an [EncryptedInputFile] that decrypts fixed-size // AES-GCM blocks on demand, supporting random access via ReadAt/Seek. // -// ReadAt is stateless and safe for concurrent use, matching the io.ReaderAt -// contract. Read and Seek mutate the shared cursor (pos) and are not +// ReadAt is safe, and intended, for concurrent use, matching the io.ReaderAt +// contract: readBlock only holds cacheMu around the cache check/publish, not +// across the underlying ReadAt or AEAD decryption, so concurrent calls for +// distinct blocks actually run in parallel rather than being serialized. A +// lost cache race (two goroutines decrypting the same block) is harmless, +// just wasted work. Read and Seek mutate the shared cursor (pos) and are not // concurrent-safe; do not call them from multiple goroutines on the same // instance. -type standardInputFile struct { +// +// The single-entry cache retains one full decrypted block (up to +// [GoEnvelopeMaxBlockSize], 128 MiB) for the lifetime of the input file, not +// just the lifetime of a single read call. +type goEnvelopeInputFile struct { underlying icebergio.File aead cipher.AEAD aadPrefix []byte @@ -628,19 +667,18 @@ type standardInputFile struct { pos int64 - // cacheMu guards cacheIdx/cacheBlock/cacheValid below. Serializing - // readBlock keeps the single-entry cache correct under the concurrent - // ReadAt usage this type documents, and also avoids re-decrypting the - // same block for runs of small, sequential reads that land in it. + // cacheMu guards cacheIdx/cacheBlock/cacheValid below. It is only held + // around the cache check and the cache publish, never across the + // underlying ReadAt or AEAD decryption in readBlock. cacheMu sync.Mutex cacheIdx int64 cacheBlock []byte cacheValid bool } -var _ EncryptedInputFile = (*standardInputFile)(nil) +var _ EncryptedInputFile = (*goEnvelopeInputFile)(nil) -func (f *standardInputFile) numBlocks() int64 { +func (f *goEnvelopeInputFile) numBlocks() int64 { if f.plaintextLength == 0 { return 0 } @@ -653,8 +691,11 @@ func (f *standardInputFile) numBlocks() int64 { return 1 + (f.plaintextLength-1)/f.blockSize } -func (f *standardInputFile) blockPlainLen(idx int64) int64 { - if idx == f.numBlocks()-1 { +// blockPlainLen returns the plaintext length of block idx, given the total +// numBlocks (passed in rather than recomputed, since every caller has +// already computed it). +func (f *goEnvelopeInputFile) blockPlainLen(idx, numBlocks int64) int64 { + if idx == numBlocks-1 { return f.plaintextLength - idx*f.blockSize } @@ -667,7 +708,7 @@ func (f *standardInputFile) blockPlainLen(idx int64) int64 { // combined with a huge plaintext-length could otherwise overflow int64 // before the resulting (implausible) offset is ever used in a read. func blockPhysicalOffset(idx, blockSize int64) (int64, error) { - physicalBlockSize, ok := checkedAddInt64(blockSize, standardBlockOverhead) + physicalBlockSize, ok := checkedAddInt64(blockSize, gcmStreamBlockOverhead) if !ok { return 0, fmt.Errorf("%w: block size %d overflows physical layout", ErrInvalidKeyMetadata, blockSize) } @@ -675,7 +716,7 @@ func blockPhysicalOffset(idx, blockSize int64) (int64, error) { if !ok { return 0, fmt.Errorf("%w: block %d physical offset overflows", ErrInvalidKeyMetadata, idx) } - offset, ok := checkedAddInt64(product, int64(standardHeaderLength)) + offset, ok := checkedAddInt64(product, int64(gcmStreamHeaderLength)) if !ok { return 0, fmt.Errorf("%w: block %d physical offset overflows", ErrInvalidKeyMetadata, idx) } @@ -688,18 +729,30 @@ func blockPhysicalOffset(idx, blockSize int64) (int64, error) { // computed block length before reading, since metadata (blockSize, // plaintextLength) can originate from untrusted input. It also honors the // actual byte count returned by ReadAt: a short, non-EOF-explained read is -// reported as [ErrCiphertextTooShort] (truncated storage) rather than being +// reported as [ErrBlockTruncated] (truncated storage) rather than being // silently zero-padded into the AEAD, which would otherwise surface as a // misleading [ErrAuthenticationFailed]. -func (f *standardInputFile) readBlock(idx int64) ([]byte, error) { +// +// cacheMu is only held around the cache check and the cache publish at the +// end, never across the underlying ReadAt or aead.Open below: holding it +// across I/O would serialize all concurrent readers, including remote +// backends. A lost race between two goroutines decrypting the same block is +// harmless (redundant work, not a correctness issue). +func (f *goEnvelopeInputFile) readBlock(idx int64) ([]byte, error) { f.cacheMu.Lock() - defer f.cacheMu.Unlock() - if f.cacheValid && f.cacheIdx == idx { - return f.cacheBlock, nil + block := f.cacheBlock + f.cacheMu.Unlock() + + return block, nil } + f.cacheMu.Unlock() numBlocks := f.numBlocks() + // defensive: idx is only ever derived from ReadAt's off, which already + // rejects off >= f.plaintextLength before calling readBlock; kept as a + // hard boundary check since idx ultimately traces back to untrusted + // key metadata (plaintextLength, blockSize). if idx < 0 || idx >= numBlocks { return nil, fmt.Errorf("%w: block index %d out of range [0, %d)", ErrInvalidKeyMetadata, idx, numBlocks) } @@ -707,7 +760,7 @@ func (f *standardInputFile) readBlock(idx int64) ([]byte, error) { return nil, fmt.Errorf("%w: block index %d exceeds the maximum supported block count", ErrInvalidKeyMetadata, idx) } - plainLen := f.blockPlainLen(idx) + plainLen := f.blockPlainLen(idx, numBlocks) if plainLen < 0 { return nil, fmt.Errorf("%w: negative computed length for block %d", ErrInvalidKeyMetadata, idx) } @@ -717,33 +770,35 @@ func (f *standardInputFile) readBlock(idx int64) ([]byte, error) { return nil, err } - wantLen := plainLen + standardBlockOverhead + wantLen := plainLen + gcmStreamBlockOverhead ciphertext := make([]byte, wantLen) n, err := f.underlying.ReadAt(ciphertext, offset) if err != nil && !errors.Is(err, io.EOF) { return nil, fmt.Errorf("encryption: failed to read block %d: %w", idx, err) } if int64(n) != wantLen { - return nil, fmt.Errorf("%w: block %d: read %d of %d expected ciphertext bytes", ErrCiphertextTooShort, idx, n, wantLen) + return nil, fmt.Errorf("%w: block %d: read %d of %d expected ciphertext bytes", ErrBlockTruncated, idx, n, wantLen) } ciphertext = ciphertext[:n] - nonce, sealed := ciphertext[:standardNonceLength], ciphertext[standardNonceLength:] - aad := standardBlockAAD(f.aadPrefix, uint32(idx)) + nonce, sealed := ciphertext[:gcmStreamNonceLength], ciphertext[gcmStreamNonceLength:] + aad := gcmStreamBlockAAD(f.aadPrefix, uint32(idx)) plaintext, err := f.aead.Open(nil, nonce, sealed, aad) if err != nil { return nil, fmt.Errorf("%w: block %d: %w", ErrAuthenticationFailed, idx, err) } + f.cacheMu.Lock() f.cacheIdx = idx f.cacheBlock = plaintext f.cacheValid = true + f.cacheMu.Unlock() return plaintext, nil } -func (f *standardInputFile) ReadAt(p []byte, off int64) (int, error) { +func (f *goEnvelopeInputFile) ReadAt(p []byte, off int64) (int, error) { if len(p) == 0 { return 0, nil } @@ -777,14 +832,14 @@ func (f *standardInputFile) ReadAt(p []byte, off int64) (int, error) { return read, err } -func (f *standardInputFile) Read(p []byte) (int, error) { +func (f *goEnvelopeInputFile) Read(p []byte) (int, error) { n, err := f.ReadAt(p, f.pos) f.pos += int64(n) return n, err } -func (f *standardInputFile) Seek(offset int64, whence int) (int64, error) { +func (f *goEnvelopeInputFile) Seek(offset int64, whence int) (int64, error) { var newPos int64 switch whence { case io.SeekStart: @@ -804,25 +859,25 @@ func (f *standardInputFile) Seek(offset int64, whence int) (int64, error) { return newPos, nil } -func (f *standardInputFile) Close() error { return f.underlying.Close() } +func (f *goEnvelopeInputFile) Close() error { return f.underlying.Close() } -func (f *standardInputFile) Stat() (fs.FileInfo, error) { +func (f *goEnvelopeInputFile) Stat() (fs.FileInfo, error) { info, err := f.underlying.Stat() if err != nil { return nil, err } - return standardFileInfo{FileInfo: info, size: f.plaintextLength}, nil + return goEnvelopeFileInfo{FileInfo: info, size: f.plaintextLength}, nil } // KeyMetadata returns the key metadata this file was decrypted with. -func (f *standardInputFile) KeyMetadata() EncryptionKeyMetadata { return f.keyMetadata } +func (f *goEnvelopeInputFile) KeyMetadata() EncryptionKeyMetadata { return f.keyMetadata } -// standardFileInfo overrides Size() to report the plaintext length rather +// goEnvelopeFileInfo overrides Size() to report the plaintext length rather // than the (larger) on-disk ciphertext length. -type standardFileInfo struct { +type goEnvelopeFileInfo struct { fs.FileInfo size int64 } -func (i standardFileInfo) Size() int64 { return i.size } +func (i goEnvelopeFileInfo) Size() int64 { return i.size } diff --git a/encryption/standard_manager_test.go b/encryption/standard_manager_test.go index 107ab29d6..3c52955f3 100644 --- a/encryption/standard_manager_test.go +++ b/encryption/standard_manager_test.go @@ -21,25 +21,28 @@ import ( "bytes" "encoding/binary" "errors" + "fmt" "io" "io/fs" "math" "testing" + "time" "github.com/apache/iceberg-go/encryption" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" ) -func newTestStandardManager(t *testing.T, opts ...encryption.StandardManagerOption) (*encryption.StandardEncryptionManager, *encryption.InMemoryKeyManagementClient) { +func newTestGoEnvelopeManager(t *testing.T, opts ...encryption.GoEnvelopeManagerOption) (*encryption.GoEnvelopeEncryptionManager, *encryption.InMemoryKeyManagementClient) { t.Helper() kms := encryption.NewInMemoryKeyManagementClient() require.NoError(t, kms.AddKey("kek-1", bytes.Repeat([]byte{0x42}, 32))) - return encryption.NewStandardEncryptionManager(kms, opts...), kms + return encryption.NewGoEnvelopeEncryptionManager(kms, opts...), kms } -func encryptAll(t *testing.T, mgr *encryption.StandardEncryptionManager, keyID string, plaintext []byte) (ciphertext []byte, keyMetadata encryption.EncryptionKeyMetadata) { +func encryptAll(t *testing.T, mgr *encryption.GoEnvelopeEncryptionManager, keyID string, plaintext []byte) (ciphertext []byte, keyMetadata encryption.EncryptionKeyMetadata) { t.Helper() fw := &memFileWriter{} out, err := mgr.NewEncryptedOutputFile(t.Context(), fw, keyID) @@ -51,7 +54,7 @@ func encryptAll(t *testing.T, mgr *encryption.StandardEncryptionManager, keyID s return fw.Bytes(), out.KeyMetadata() } -func decryptAll(t *testing.T, mgr *encryption.StandardEncryptionManager, ciphertext []byte, keyMetadata encryption.EncryptionKeyMetadata) []byte { +func decryptAll(t *testing.T, mgr *encryption.GoEnvelopeEncryptionManager, ciphertext []byte, keyMetadata encryption.EncryptionKeyMetadata) []byte { t.Helper() in, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) require.NoError(t, err) @@ -61,8 +64,8 @@ func decryptAll(t *testing.T, mgr *encryption.StandardEncryptionManager, ciphert return data } -func TestStandardEncryptionManager_RoundTrip_SmallerThanBlock(t *testing.T) { - mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) +func TestGoEnvelopeEncryptionManager_RoundTrip_SmallerThanBlock(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) plaintext := []byte("hello iceberg") ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) @@ -73,8 +76,8 @@ func TestStandardEncryptionManager_RoundTrip_SmallerThanBlock(t *testing.T) { assert.Equal(t, plaintext, got) } -func TestStandardEncryptionManager_RoundTrip_MultiBlock(t *testing.T) { - mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) +func TestGoEnvelopeEncryptionManager_RoundTrip_MultiBlock(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) plaintext := bytes.Repeat([]byte("0123456789abcdef"), 10) plaintext = append(plaintext, []byte("partial-tail")...) @@ -83,8 +86,8 @@ func TestStandardEncryptionManager_RoundTrip_MultiBlock(t *testing.T) { assert.Equal(t, plaintext, got) } -func TestStandardEncryptionManager_RoundTrip_ExactBlockMultiple(t *testing.T) { - mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) +func TestGoEnvelopeEncryptionManager_RoundTrip_ExactBlockMultiple(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) plaintext := bytes.Repeat([]byte("0123456789abcdef"), 4) ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) @@ -92,16 +95,16 @@ func TestStandardEncryptionManager_RoundTrip_ExactBlockMultiple(t *testing.T) { assert.Equal(t, plaintext, got) } -func TestStandardEncryptionManager_RoundTrip_Empty(t *testing.T) { - mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) +func TestGoEnvelopeEncryptionManager_RoundTrip_Empty(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", nil) got := decryptAll(t, mgr, ciphertext, keyMetadata) assert.Empty(t, got) } -func TestStandardEncryptionManager_RandomAccess(t *testing.T) { - mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) +func TestGoEnvelopeEncryptionManager_RandomAccess(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) plaintext := bytes.Repeat([]byte("0123456789abcdef"), 10) plaintext = append(plaintext, []byte("tail-bytes")...) @@ -125,8 +128,8 @@ func TestStandardEncryptionManager_RandomAccess(t *testing.T) { assert.Equal(t, plaintext[5:], rest) } -func TestStandardEncryptionManager_TamperedBlockFailsAuthentication(t *testing.T) { - mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) +func TestGoEnvelopeEncryptionManager_TamperedBlockFailsAuthentication(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) plaintext := bytes.Repeat([]byte("0123456789abcdef"), 4) ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) @@ -143,8 +146,8 @@ func TestStandardEncryptionManager_TamperedBlockFailsAuthentication(t *testing.T assert.True(t, errors.Is(err, encryption.ErrAuthenticationFailed)) } -func TestStandardEncryptionManager_ReorderedBlocksFailAuthentication(t *testing.T) { - mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) +func TestGoEnvelopeEncryptionManager_ReorderedBlocksFailAuthentication(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) plaintext := bytes.Repeat([]byte("0123456789abcdef"), 2) // exactly two full 16-byte blocks ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) @@ -167,8 +170,8 @@ func TestStandardEncryptionManager_ReorderedBlocksFailAuthentication(t *testing. assert.True(t, errors.Is(err, encryption.ErrAuthenticationFailed), "swapping blocks must be detected: the AAD binds each block to its position") } -func TestStandardEncryptionManager_StreamHeaderMagicMismatchRejected(t *testing.T) { - mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) +func TestGoEnvelopeEncryptionManager_StreamHeaderMagicMismatchRejected(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", []byte("hello iceberg")) copy(ciphertext[:4], "XXXX") @@ -177,8 +180,8 @@ func TestStandardEncryptionManager_StreamHeaderMagicMismatchRejected(t *testing. assert.True(t, errors.Is(err, encryption.ErrInvalidStreamHeader)) } -func TestStandardEncryptionManager_StreamHeaderZeroBlockLengthRejected(t *testing.T) { - mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) +func TestGoEnvelopeEncryptionManager_StreamHeaderZeroBlockLengthRejected(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", []byte("hello iceberg")) binary.LittleEndian.PutUint32(ciphertext[4:8], 0) @@ -187,8 +190,8 @@ func TestStandardEncryptionManager_StreamHeaderZeroBlockLengthRejected(t *testin assert.True(t, errors.Is(err, encryption.ErrInvalidStreamHeader)) } -func TestStandardEncryptionManager_StreamHeaderBlockLengthExceedsMaxRejected(t *testing.T) { - mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) +func TestGoEnvelopeEncryptionManager_StreamHeaderBlockLengthExceedsMaxRejected(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", []byte("hello iceberg")) binary.LittleEndian.PutUint32(ciphertext[4:8], math.MaxUint32) @@ -197,72 +200,92 @@ func TestStandardEncryptionManager_StreamHeaderBlockLengthExceedsMaxRejected(t * assert.True(t, errors.Is(err, encryption.ErrInvalidStreamHeader)) } -func TestStandardEncryptionManager_OutputFile_EmptyKeyIDRejected(t *testing.T) { - mgr, _ := newTestStandardManager(t) +func TestGoEnvelopeEncryptionManager_OutputFile_EmptyKeyIDRejected(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t) _, err := mgr.NewEncryptedOutputFile(t.Context(), &memFileWriter{}, "") require.Error(t, err) assert.True(t, errors.Is(err, encryption.ErrKeyIDRequired)) } -func TestStandardEncryptionManager_InputFile_EmptyKeyMetadataRejected(t *testing.T) { - mgr, _ := newTestStandardManager(t) +func TestGoEnvelopeEncryptionManager_InputFile_EmptyKeyMetadataRejected(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t) _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), nil) require.Error(t, err) assert.True(t, errors.Is(err, encryption.ErrKeyMetadataRequired)) } -func TestStandardEncryptionManager_UnknownKeyIDPropagatesFromKMS(t *testing.T) { - mgr, _ := newTestStandardManager(t) +func TestGoEnvelopeEncryptionManager_UnknownKeyIDPropagatesFromKMS(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t) _, err := mgr.NewEncryptedOutputFile(t.Context(), &memFileWriter{}, "does-not-exist") require.Error(t, err) assert.True(t, errors.Is(err, encryption.ErrUnknownKeyID)) } -func TestStandardEncryptionManager_MalformedKeyMetadataRejected(t *testing.T) { - mgr, _ := newTestStandardManager(t) +func TestGoEnvelopeEncryptionManager_MalformedKeyMetadataRejected(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t) _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), encryption.EncryptionKeyMetadata("not json")) require.Error(t, err) } -func TestStandardEncryptionManager_ZeroBlockSizeRejectedOnWrite(t *testing.T) { - mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(0)) +func TestGoEnvelopeEncryptionManager_ZeroBlockSizeRejectedOnWrite(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(0)) _, err := mgr.NewEncryptedOutputFile(t.Context(), &memFileWriter{}, "kek-1") require.Error(t, err) assert.True(t, errors.Is(err, encryption.ErrInvalidBlockSize)) } -func TestStandardEncryptionManager_NegativeBlockSizeRejectedOnWrite(t *testing.T) { - mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(-1)) +func TestGoEnvelopeEncryptionManager_NegativeBlockSizeRejectedOnWrite(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(-1)) _, err := mgr.NewEncryptedOutputFile(t.Context(), &memFileWriter{}, "kek-1") require.Error(t, err) assert.True(t, errors.Is(err, encryption.ErrInvalidBlockSize)) } -func TestStandardEncryptionManager_BlockSizeExceedingMaxRejectedOnWrite(t *testing.T) { - mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(encryption.StandardMaxBlockSize+1)) +func TestGoEnvelopeEncryptionManager_BlockSizeExceedingMaxRejectedOnWrite(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(encryption.GoEnvelopeMaxBlockSize+1)) _, err := mgr.NewEncryptedOutputFile(t.Context(), &memFileWriter{}, "kek-1") require.Error(t, err) assert.True(t, errors.Is(err, encryption.ErrInvalidBlockSize)) } -func TestStandardEncryptionManager_NegativePlaintextLengthMetadataRejectedOnRead(t *testing.T) { - mgr, _ := newTestStandardManager(t) - meta := []byte(`{"v":1,"key-id":"kek-1","wrapped-key":"AA==","aad-prefix":"MTIzNDU2Nzg5MDEyMzQ1Ng==","plaintext-length":-1}`) +func TestGoEnvelopeEncryptionManager_NegativePlaintextLengthMetadataRejectedOnRead(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t) + meta := []byte(`{"v":1,"key-id":"kek-1","wrapped-key":"AA==","block-size":16,"aad-prefix":"MTIzNDU2Nzg5MDEyMzQ1Ng==","plaintext-length":-1}`) _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), meta) require.Error(t, err) assert.True(t, errors.Is(err, encryption.ErrInvalidKeyMetadata)) + assert.ErrorContains(t, err, "plaintext-length must be non-negative") } -func TestStandardEncryptionManager_EmptyAADPrefixMetadataRejectedOnRead(t *testing.T) { - mgr, _ := newTestStandardManager(t) - meta := []byte(`{"v":1,"key-id":"kek-1","wrapped-key":"AA==","aad-prefix":"","plaintext-length":10}`) +func TestGoEnvelopeEncryptionManager_EmptyAADPrefixMetadataRejectedOnRead(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t) + meta := []byte(`{"v":1,"key-id":"kek-1","wrapped-key":"AA==","block-size":16,"aad-prefix":"","plaintext-length":10}`) _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), meta) require.Error(t, err) assert.True(t, errors.Is(err, encryption.ErrInvalidKeyMetadata)) + assert.ErrorContains(t, err, "aad-prefix must not be empty") } -func TestStandardEncryptionManager_InvalidDEKLengthRejected(t *testing.T) { - mgr, _ := newTestStandardManager(t, encryption.WithDEKLength(15)) +func TestGoEnvelopeEncryptionManager_ZeroBlockSizeMetadataRejectedOnRead(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t) + meta := []byte(`{"v":1,"key-id":"kek-1","wrapped-key":"AA==","block-size":0,"aad-prefix":"MTIzNDU2Nzg5MDEyMzQ1Ng==","plaintext-length":10}`) + _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), meta) + require.Error(t, err) + assert.True(t, errors.Is(err, encryption.ErrInvalidKeyMetadata)) + assert.ErrorContains(t, err, "block-size must be positive") +} + +func TestGoEnvelopeEncryptionManager_BlockSizeExceedingMaxMetadataRejectedOnRead(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t) + meta := []byte(fmt.Sprintf(`{"v":1,"key-id":"kek-1","wrapped-key":"AA==","block-size":%d,"aad-prefix":"MTIzNDU2Nzg5MDEyMzQ1Ng==","plaintext-length":10}`, encryption.GoEnvelopeMaxBlockSize+1)) + _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), meta) + require.Error(t, err) + assert.True(t, errors.Is(err, encryption.ErrInvalidKeyMetadata)) + assert.ErrorContains(t, err, "block-size must be positive") +} + +func TestGoEnvelopeEncryptionManager_InvalidDEKLengthRejected(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t, encryption.WithDEKLength(15)) _, err := mgr.NewEncryptedOutputFile(t.Context(), &memFileWriter{}, "kek-1") require.Error(t, err) assert.True(t, errors.Is(err, encryption.ErrInvalidKeyLength)) @@ -293,8 +316,8 @@ func (f *shortReadFile) ReadAt(p []byte, off int64) (int, error) { return n, err } -func TestStandardEncryptionManager_TruncatedBackendReadReportsCiphertextTooShort(t *testing.T) { - mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) +func TestGoEnvelopeEncryptionManager_TruncatedBackendReadReportsBlockTruncated(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) plaintext := bytes.Repeat([]byte("0123456789abcdef"), 4) ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) @@ -305,12 +328,13 @@ func TestStandardEncryptionManager_TruncatedBackendReadReportsCiphertextTooShort _, err = io.ReadAll(in) require.Error(t, err) - assert.True(t, errors.Is(err, encryption.ErrCiphertextTooShort)) + assert.True(t, errors.Is(err, encryption.ErrBlockTruncated)) + assert.False(t, errors.Is(err, encryption.ErrCiphertextTooShort), "a truncated block read is distinct from a too-short KMS-wrapped key/payload") assert.False(t, errors.Is(err, encryption.ErrAuthenticationFailed), "a truncated read must not be misreported as tampering") } -func TestStandardEncryptionManager_EmptyFile_ZeroLengthReadAt(t *testing.T) { - mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) +func TestGoEnvelopeEncryptionManager_EmptyFile_ZeroLengthReadAt(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", nil) in, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) @@ -347,8 +371,8 @@ func (w *failAfterNWriter) Close() error { return w.memFileWriter.Close() } -func TestStandardEncryptionManager_FlushFailurePoisonsWriterAndClose(t *testing.T) { - mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(4)) +func TestGoEnvelopeEncryptionManager_FlushFailurePoisonsWriterAndClose(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(4)) // Write call 1 is the AES GCM Stream header written by // NewEncryptedOutputFile; call 2 flushes the first (4-byte) block; // call 3 flushes the second block and fails. @@ -378,8 +402,8 @@ func TestStandardEncryptionManager_FlushFailurePoisonsWriterAndClose(t *testing. assert.Equal(t, 1, fw.closes, "a retried Close must not close the underlying writer again") } -func TestStandardEncryptionManager_WriteAfterCloseRejected(t *testing.T) { - mgr, _ := newTestStandardManager(t) +func TestGoEnvelopeEncryptionManager_WriteAfterCloseRejected(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t) out, err := mgr.NewEncryptedOutputFile(t.Context(), &memFileWriter{}, "kek-1") require.NoError(t, err) require.NoError(t, out.Close()) @@ -390,14 +414,14 @@ func TestStandardEncryptionManager_WriteAfterCloseRejected(t *testing.T) { assert.True(t, errors.Is(err, fs.ErrClosed)) } -func TestNewStandardEncryptionManager_NilKMSPanics(t *testing.T) { - assert.PanicsWithValue(t, "encryption: NewStandardEncryptionManager: kms must not be nil", func() { - encryption.NewStandardEncryptionManager(nil) +func TestNewGoEnvelopeEncryptionManager_NilKMSPanics(t *testing.T) { + assert.PanicsWithValue(t, "encryption: NewGoEnvelopeEncryptionManager: kms must not be nil", func() { + encryption.NewGoEnvelopeEncryptionManager(nil) }) } -func TestStandardEncryptionManager_StreamHeaderBlockLengthMismatchRejected(t *testing.T) { - mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(16)) +func TestGoEnvelopeEncryptionManager_StreamHeaderBlockLengthMismatchRejected(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", []byte("hello iceberg")) // A header block length that is itself in-range, but does not match the // (trusted) block-size recorded in key metadata. @@ -422,8 +446,8 @@ func (f *countingReadAtFile) ReadAt(p []byte, off int64) (int, error) { return f.memFile.ReadAt(p, off) } -func TestStandardEncryptionManager_RepeatedSmallReadsReuseDecryptedBlockCache(t *testing.T) { - mgr, _ := newTestStandardManager(t, encryption.WithBlockSize(4096)) +func TestGoEnvelopeEncryptionManager_RepeatedSmallReadsReuseDecryptedBlockCache(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(4096)) plaintext := bytes.Repeat([]byte("x"), 5000) // spans exactly two blocks (4096 + 904) ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) @@ -442,3 +466,132 @@ func TestStandardEncryptionManager_RepeatedSmallReadsReuseDecryptedBlockCache(t assert.LessOrEqual(t, counting.reads-baseline, 2, "5000 one-byte reads across two blocks must decrypt each block at most once") } + +func TestGoEnvelopeEncryptionManager_RoundTrip_DefaultBlockSize(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t) // exercises GoEnvelopeDefaultBlockSize (64 KiB), not an explicit small size + plaintext := bytes.Repeat([]byte("iceberg-default-block-size-test-"), 4000) + require.Greater(t, len(plaintext), 64*1024, "plaintext must span multiple default-size blocks") + + ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) + got := decryptAll(t, mgr, ciphertext, keyMetadata) + assert.Equal(t, plaintext, got) +} + +func TestGoEnvelopeEncryptionManager_ReadFrom(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(4)) + fw := &memFileWriter{} + out, err := mgr.NewEncryptedOutputFile(t.Context(), fw, "kek-1") + require.NoError(t, err) + + plaintext := []byte("hello iceberg world") + n, err := out.ReadFrom(bytes.NewReader(plaintext)) + require.NoError(t, err) + assert.EqualValues(t, len(plaintext), n) + require.NoError(t, out.Close()) + + got := decryptAll(t, mgr, fw.Bytes(), out.KeyMetadata()) + assert.Equal(t, plaintext, got) +} + +func TestGoEnvelopeEncryptionManager_Stat(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) + plaintext := []byte("hello iceberg world, this is a stat test") + ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) + + in, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) + require.NoError(t, err) + + info, err := in.Stat() + require.NoError(t, err) + assert.EqualValues(t, len(plaintext), info.Size(), "Stat must report the plaintext length, not the on-disk ciphertext length") + assert.Less(t, info.Size(), int64(len(ciphertext)), "ciphertext must be larger than plaintext due to per-block nonce/tag overhead") +} + +// closeFailWriter is an icebergio.FileWriter stub whose Close always fails +// even though every Write succeeds, simulating a backend where the final +// flush/commit (e.g. an S3 multipart completion) fails independently of the +// preceding writes. +type closeFailWriter struct { + memFileWriter + closes int +} + +func (w *closeFailWriter) Close() error { + w.closes++ + + return errors.New("simulated close failure") +} + +func TestGoEnvelopeEncryptionManager_UnderlyingCloseFailurePoisonsWriter(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t) + fw := &closeFailWriter{} + out, err := mgr.NewEncryptedOutputFile(t.Context(), fw, "kek-1") + require.NoError(t, err) + + _, err = out.Write([]byte("hello")) + require.NoError(t, err) + + closeErr := out.Close() + require.Error(t, closeErr) + assert.Nil(t, out.KeyMetadata(), "key metadata must not be finalized when the underlying Close fails") + + // A retried Close must keep reporting the same error, not nil, and must + // not attempt to close the underlying writer again. + closeErr2 := out.Close() + require.Error(t, closeErr2) + assert.ErrorIs(t, closeErr2, closeErr) + assert.Equal(t, 1, fw.closes, "a retried Close must not re-close the underlying writer") + + _, err = out.Write([]byte("late")) + require.Error(t, err) + assert.ErrorIs(t, err, closeErr, "a subsequent Write must report the same sticky error") +} + +// slowReadFile wraps a memFile and sleeps before every ReadAt, simulating a +// remote backend (e.g. S3) with real network latency. It is used to prove +// that concurrent ReadAt calls for distinct blocks actually overlap rather +// than being serialized by readBlock's internal cache lock. +type slowReadFile struct { + *memFile + delay time.Duration +} + +func (f *slowReadFile) ReadAt(p []byte, off int64) (int, error) { + time.Sleep(f.delay) + + return f.memFile.ReadAt(p, off) +} + +func TestGoEnvelopeEncryptionManager_ConcurrentReadAtOverlaps(t *testing.T) { + const ( + numBlocks = 8 + blockSize = 16 + delay = 20 * time.Millisecond + ) + mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(blockSize)) + plaintext := bytes.Repeat([]byte("0123456789abcdef"), numBlocks) + ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) + + slow := &slowReadFile{memFile: newMemFile(ciphertext), delay: delay} + in, err := mgr.NewDecryptedInputFile(t.Context(), slow, keyMetadata) // reads the header; not timed below + require.NoError(t, err) + + start := time.Now() + var g errgroup.Group + for i := range numBlocks { + g.Go(func() error { + buf := make([]byte, blockSize) + _, err := in.ReadAt(buf, int64(i*blockSize)) + + return err + }) + } + require.NoError(t, g.Wait()) + elapsed := time.Since(start) + + // Serial execution (the pre-fix behavior, holding cacheMu across ReadAt) + // would take roughly numBlocks*delay. True concurrency keeps it close to + // a single delay; the threshold below is generous to stay robust on + // slow or loaded CI machines while still failing on a serialized regression. + assert.Less(t, elapsed, numBlocks/2*delay, "concurrent ReadAt calls for distinct blocks must overlap, not serialize") +} From af92a15f5f25df7e11a1a37169fda62eebfef05e Mon Sep 17 00:00:00 2001 From: hectar-glitches Date: Wed, 9 Sep 2026 13:15:29 -0700 Subject: [PATCH 07/10] address memory limit issue and nits --- encryption/{standard_manager.go => go_envelope_manager.go} | 0 .../{standard_manager_test.go => go_envelope_manager_test.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename encryption/{standard_manager.go => go_envelope_manager.go} (100%) rename encryption/{standard_manager_test.go => go_envelope_manager_test.go} (100%) diff --git a/encryption/standard_manager.go b/encryption/go_envelope_manager.go similarity index 100% rename from encryption/standard_manager.go rename to encryption/go_envelope_manager.go diff --git a/encryption/standard_manager_test.go b/encryption/go_envelope_manager_test.go similarity index 100% rename from encryption/standard_manager_test.go rename to encryption/go_envelope_manager_test.go From 2ab1ae5cdd065402c92cd439c4aa97d57e483f94 Mon Sep 17 00:00:00 2001 From: hectar-glitches Date: Mon, 21 Sep 2026 20:35:33 -0700 Subject: [PATCH 08/10] fix(encryption): add nonce-uniqueness test, bound aad-prefix length --- encryption/go_envelope_manager.go | 10 ++++- encryption/go_envelope_manager_test.go | 59 +++++++++++++++++++++++++- 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/encryption/go_envelope_manager.go b/encryption/go_envelope_manager.go index 6ba23d16f..b327599a6 100644 --- a/encryption/go_envelope_manager.go +++ b/encryption/go_envelope_manager.go @@ -343,8 +343,14 @@ func (m *GoEnvelopeEncryptionManager) NewDecryptedInputFile(ctx context.Context, if meta.PlaintextLength < 0 { return nil, fmt.Errorf("%w: plaintext-length must be non-negative, got %d", ErrInvalidKeyMetadata, meta.PlaintextLength) } - if len(meta.AADPrefix) == 0 { - return nil, fmt.Errorf("%w: aad-prefix must not be empty", ErrInvalidKeyMetadata) + // AADPrefix is bounded to exactly the length this implementation's own + // writer emits (gcmStreamAADPrefixLength): key metadata is Go-specific + // (not cross-implementation compatible; see the type doc comment), so + // there is no other length a valid file could carry. Without this + // ceiling, an oversized prefix would be re-allocated on every + // gcmStreamBlockAAD call in readBlock. + if len(meta.AADPrefix) != gcmStreamAADPrefixLength { + return nil, fmt.Errorf("%w: aad-prefix must be exactly %d bytes, got %d", ErrInvalidKeyMetadata, gcmStreamAADPrefixLength, len(meta.AADPrefix)) } plainDEK, err := m.kms.UnwrapKey(ctx, meta.KeyID, meta.WrappedKey) diff --git a/encryption/go_envelope_manager_test.go b/encryption/go_envelope_manager_test.go index 3c52955f3..72c99d319 100644 --- a/encryption/go_envelope_manager_test.go +++ b/encryption/go_envelope_manager_test.go @@ -19,6 +19,7 @@ package encryption_test import ( "bytes" + "encoding/base64" "encoding/binary" "errors" "fmt" @@ -263,7 +264,18 @@ func TestGoEnvelopeEncryptionManager_EmptyAADPrefixMetadataRejectedOnRead(t *tes _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), meta) require.Error(t, err) assert.True(t, errors.Is(err, encryption.ErrInvalidKeyMetadata)) - assert.ErrorContains(t, err, "aad-prefix must not be empty") + assert.ErrorContains(t, err, "aad-prefix must be exactly 16 bytes") +} + +func TestGoEnvelopeEncryptionManager_OversizedAADPrefixMetadataRejectedOnRead(t *testing.T) { + mgr, _ := newTestGoEnvelopeManager(t) + // 17 zero bytes, base64-encoded: one byte longer than gcmStreamAADPrefixLength. + oversized := base64.StdEncoding.EncodeToString(make([]byte, 17)) + meta := []byte(fmt.Sprintf(`{"v":1,"key-id":"kek-1","wrapped-key":"AA==","block-size":16,"aad-prefix":"%s","plaintext-length":10}`, oversized)) + _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), meta) + require.Error(t, err) + assert.True(t, errors.Is(err, encryption.ErrInvalidKeyMetadata)) + assert.ErrorContains(t, err, "aad-prefix must be exactly 16 bytes") } func TestGoEnvelopeEncryptionManager_ZeroBlockSizeMetadataRejectedOnRead(t *testing.T) { @@ -595,3 +607,48 @@ func TestGoEnvelopeEncryptionManager_ConcurrentReadAtOverlaps(t *testing.T) { // slow or loaded CI machines while still failing on a serialized regression. assert.Less(t, elapsed, numBlocks/2*delay, "concurrent ReadAt calls for distinct blocks must overlap, not serialize") } + +// TestGoEnvelopeEncryptionManager_BlockNoncesAreUnique pins the one GCM +// property whose loss is catastrophic and silent: reusing a nonce under a +// single key leaks the plaintext XOR of the two blocks and the GHASH +// authentication key, yet a file sealed with a reused (or constant) nonce +// still round-trips and decrypts correctly, so no round-trip, tamper, or +// reorder test would ever catch a regression here. Sealing identical +// plaintext in every block makes any nonce collision directly visible as +// matching ciphertext bytes. +func TestGoEnvelopeEncryptionManager_BlockNoncesAreUnique(t *testing.T) { + const ( + blockSize = 16 + numBlocks = 8 + // AGS1 wire layout (see the GoEnvelopeEncryptionManager doc comment): + // an 8-byte header (4-byte "AGS1" magic + 4-byte little-endian block + // length), then each block as nonce(12) || ciphertext(blockSize) || tag(16). + headerLength = 8 + nonceLength = 12 + blockOverhead = 28 // nonceLength + 16-byte GCM tag + ) + mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(blockSize)) + // Every block carries byte-for-byte identical plaintext. + plaintext := bytes.Repeat([]byte("0123456789abcdef"), numBlocks) + + ciphertext, _ := encryptAll(t, mgr, "kek-1", plaintext) + require.Len(t, ciphertext, headerLength+numBlocks*(blockSize+blockOverhead)) + + seenNonceAt := make(map[string]int, numBlocks) + cipherBlocks := make([][]byte, numBlocks) + for i := range numBlocks { + offset := headerLength + i*(blockSize+blockOverhead) + block := ciphertext[offset : offset+blockSize+blockOverhead] + nonce := string(block[:nonceLength]) + + if prev, ok := seenNonceAt[nonce]; ok { + t.Fatalf("block %d reused the nonce from block %d: GCM nonce reuse under a single key leaks the plaintext XOR and the GHASH authentication key", i, prev) + } + seenNonceAt[nonce] = i + cipherBlocks[i] = block + } + + for i := 1; i < numBlocks; i++ { + assert.NotEqual(t, cipherBlocks[0], cipherBlocks[i], "identical plaintext in blocks 0 and %d must not produce identical ciphertext", i) + } +} From 7772b3d5a8cf71be84d210e4ecb2a389e371e1fb Mon Sep 17 00:00:00 2001 From: Matt Topol Date: Thu, 1 Oct 2026 13:34:10 -0400 Subject: [PATCH 09/10] chore(encryption): rebase onto main, fix modernize lint --- encryption/go_envelope_manager_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/encryption/go_envelope_manager_test.go b/encryption/go_envelope_manager_test.go index 72c99d319..06b638571 100644 --- a/encryption/go_envelope_manager_test.go +++ b/encryption/go_envelope_manager_test.go @@ -271,7 +271,7 @@ func TestGoEnvelopeEncryptionManager_OversizedAADPrefixMetadataRejectedOnRead(t mgr, _ := newTestGoEnvelopeManager(t) // 17 zero bytes, base64-encoded: one byte longer than gcmStreamAADPrefixLength. oversized := base64.StdEncoding.EncodeToString(make([]byte, 17)) - meta := []byte(fmt.Sprintf(`{"v":1,"key-id":"kek-1","wrapped-key":"AA==","block-size":16,"aad-prefix":"%s","plaintext-length":10}`, oversized)) + meta := fmt.Appendf(nil, `{"v":1,"key-id":"kek-1","wrapped-key":"AA==","block-size":16,"aad-prefix":"%s","plaintext-length":10}`, oversized) _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), meta) require.Error(t, err) assert.True(t, errors.Is(err, encryption.ErrInvalidKeyMetadata)) @@ -289,7 +289,7 @@ func TestGoEnvelopeEncryptionManager_ZeroBlockSizeMetadataRejectedOnRead(t *test func TestGoEnvelopeEncryptionManager_BlockSizeExceedingMaxMetadataRejectedOnRead(t *testing.T) { mgr, _ := newTestGoEnvelopeManager(t) - meta := []byte(fmt.Sprintf(`{"v":1,"key-id":"kek-1","wrapped-key":"AA==","block-size":%d,"aad-prefix":"MTIzNDU2Nzg5MDEyMzQ1Ng==","plaintext-length":10}`, encryption.GoEnvelopeMaxBlockSize+1)) + meta := fmt.Appendf(nil, `{"v":1,"key-id":"kek-1","wrapped-key":"AA==","block-size":%d,"aad-prefix":"MTIzNDU2Nzg5MDEyMzQ1Ng==","plaintext-length":10}`, encryption.GoEnvelopeMaxBlockSize+1) _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), meta) require.Error(t, err) assert.True(t, errors.Is(err, encryption.ErrInvalidKeyMetadata)) From 05f427e14d375697437bd10ec61bc94680480b5f Mon Sep 17 00:00:00 2001 From: hectar-glitches Date: Wed, 7 Oct 2026 15:22:14 -0700 Subject: [PATCH 10/10] feat(encryption): make StandardEncryptionManager Java-compatible Read and write the same bytes as Java's StandardEncryptionManager and iceberg-rust, so files can be exchanged between implementations. - Use a fixed 1 MiB plaintext block size. Remove WithBlockSize, GoEnvelopeMaxBlockSize and ErrInvalidBlockSize. The reader requires the stream header to declare 1 MiB. - Encode key metadata as StandardKeyMetadata: a version byte followed by an Avro record {encryption_key, aad_prefix, file_length}. The DEK is generated locally and stored raw; there is no KMS call per file. The reader derives the plaintext length from file_length. - Write block 0 for an empty file, giving the 36-byte layout Java and iceberg-rust use, and accept it on read. - Default the data key length to 16 bytes and rename the option to WithStandardDataKeyLength. - Rename GoEnvelopeEncryptionManager back to StandardEncryptionManager. - Document that ErrOutputFileClosed is only returned after a successful Close. Tests decode the writer's output with a stdlib-only AGS1 decoder, read a stream built the same way, and check a hand-encoded key metadata vector. Block-size tests now use multi-MiB payloads. --- encryption/go_envelope_manager.go | 889 ------------------------- encryption/go_envelope_manager_test.go | 654 ------------------ encryption/standard_manager.go | 862 ++++++++++++++++++++++++ encryption/standard_manager_test.go | 869 ++++++++++++++++++++++++ 4 files changed, 1731 insertions(+), 1543 deletions(-) delete mode 100644 encryption/go_envelope_manager.go delete mode 100644 encryption/go_envelope_manager_test.go create mode 100644 encryption/standard_manager.go create mode 100644 encryption/standard_manager_test.go diff --git a/encryption/go_envelope_manager.go b/encryption/go_envelope_manager.go deleted file mode 100644 index b327599a6..000000000 --- a/encryption/go_envelope_manager.go +++ /dev/null @@ -1,889 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package encryption - -import ( - "context" - "crypto/aes" - "crypto/cipher" - "crypto/rand" - "encoding/binary" - "encoding/json" - "errors" - "fmt" - "io" - "io/fs" - "math" - "sync" - - icebergio "github.com/apache/iceberg-go/io" -) - -// Defaults for [GoEnvelopeEncryptionManager]. -const ( - // GoEnvelopeDefaultDEKLength is the default length, in bytes, of the - // per-file data encryption key (DEK) generated for AES-256-GCM. - GoEnvelopeDefaultDEKLength = 32 - - // GoEnvelopeDefaultBlockSize is the default plaintext block size, in - // bytes, used to split a file into independently authenticated AES-GCM - // blocks. Blocks allow random access (Seek/ReadAt) without buffering or - // decrypting the whole file. - // - // This is not Java's Ciphers.PLAIN_BLOCK_SIZE (1 MiB, a compile-time - // constant there); see the [GoEnvelopeEncryptionManager] doc comment. - GoEnvelopeDefaultBlockSize = 64 * 1024 - - // GoEnvelopeMaxBlockSize is the largest plaintext block size accepted - // from the AES GCM Stream header on read, and the largest that - // [WithBlockSize] will configure for writing. The header's block-length - // field is unauthenticated, untrusted data (the Iceberg AES GCM Stream - // spec's "File length" note applies equally here); without a ceiling, a - // crafted header claiming e.g. 1<<40 bytes would force a huge - // allocation (see readBlock) before any authentication check can run. - GoEnvelopeMaxBlockSize = 128 * 1024 * 1024 -) - -// Sentinel errors returned by [GoEnvelopeEncryptionManager]. -var ( - // ErrKeyIDRequired is returned by - // [GoEnvelopeEncryptionManager.NewEncryptedOutputFile] when keyID is - // empty. GoEnvelopeEncryptionManager always encrypts, so it requires a - // KEK to wrap the generated DEK; use [PlaintextEncryptionManager] for - // unencrypted tables instead of passing an empty keyID here. - ErrKeyIDRequired = errors.New("encryption: GoEnvelopeEncryptionManager requires a non-empty keyID") - - // ErrKeyMetadataRequired is returned by - // [GoEnvelopeEncryptionManager.NewDecryptedInputFile] when keyMetadata - // is empty. GoEnvelopeEncryptionManager always decrypts, so it requires - // the per-file key metadata produced by - // [GoEnvelopeEncryptionManager.NewEncryptedOutputFile]. - ErrKeyMetadataRequired = errors.New("encryption: GoEnvelopeEncryptionManager requires non-empty key metadata") - - // ErrUnsupportedKeyMetadataVersion is returned when key metadata was - // produced by a newer, incompatible encoding version. - ErrUnsupportedKeyMetadataVersion = errors.New("encryption: unsupported key metadata version") - - // ErrInvalidBlockSize is returned when a configured block size is not - // positive or exceeds [GoEnvelopeMaxBlockSize]. - ErrInvalidBlockSize = errors.New("encryption: block size must be positive and at most GoEnvelopeMaxBlockSize") - - // ErrInvalidStreamHeader is returned when the AES GCM Stream header - // (the "AGS1" magic and little-endian block-length fields at the start - // of an encrypted file, per the Iceberg AES GCM Stream spec) is - // missing, truncated, or specifies a block length outside the - // supported range. - ErrInvalidStreamHeader = errors.New("encryption: invalid AES GCM Stream header") - - // ErrInvalidKeyMetadata is returned by - // [GoEnvelopeEncryptionManager.NewDecryptedInputFile] when decoded key - // metadata fails basic sanity checks (e.g. a negative plaintext length - // or a missing AAD prefix). Key metadata is untrusted input on a crypto - // read path, so it is validated rather than trusted blindly. - ErrInvalidKeyMetadata = errors.New("encryption: invalid key metadata") - - // ErrOutputFileClosed is returned by [goEnvelopeOutputFile.Write] when - // called after Close, or after a previous flush has poisoned the writer. - // It wraps [fs.ErrClosed] so callers can test with errors.Is(err, fs.ErrClosed). - ErrOutputFileClosed = fmt.Errorf("encryption: write to closed GoEnvelopeEncryptionManager output file: %w", fs.ErrClosed) - - // ErrBlockTruncated is returned by [goEnvelopeInputFile.ReadAt] when the - // underlying storage returns fewer ciphertext bytes for a block than its - // recorded length requires, indicating the file was truncated at rest. - // This is distinct from [ErrCiphertextTooShort], which [KeyManagementClient] - // implementations use for a too-short wrapped key or KMS-encrypted - // payload; keeping them separate lets a caller tell a malformed KMS blob - // apart from a short block read. - ErrBlockTruncated = errors.New("encryption: block truncated: read fewer ciphertext bytes than expected") -) - -// Constants describing the Iceberg AES GCM Stream ("AGS1") wire format used -// for the ciphertext produced by [GoEnvelopeEncryptionManager]. See -// https://iceberg.apache.org/gcm-stream-spec/ for the full specification. -const ( - // gcmStreamMagic identifies an AES GCM Stream version 1 file. - gcmStreamMagic = "AGS1" - - // gcmStreamHeaderLength is the length, in bytes, of the magic plus the - // little-endian block-length field written at the start of every file. - gcmStreamHeaderLength = len(gcmStreamMagic) + 4 - - // gcmStreamNonceLength is the length, in bytes, of the random AES-GCM - // nonce stored at the start of every cipher block. - gcmStreamNonceLength = 12 - - // gcmStreamTagLength is the length, in bytes, of the AES-GCM - // authentication tag appended to every cipher block's ciphertext. - gcmStreamTagLength = 16 - - // gcmStreamBlockOverhead is the number of ciphertext bytes added to - // each block beyond its plaintext length (nonce + tag). - gcmStreamBlockOverhead = gcmStreamNonceLength + gcmStreamTagLength - - // gcmStreamAADPrefixLength is the length, in bytes, of the random - // per-file AAD prefix generated for new output files. - gcmStreamAADPrefixLength = 16 -) - -// goEnvelopeKeyMetadataVersion is the current encoding version written by -// [GoEnvelopeEncryptionManager]. It is bumped whenever the on-disk layout of -// goEnvelopeKeyMetadata changes incompatibly. -const goEnvelopeKeyMetadataVersion = 1 - -// goEnvelopeKeyMetadata is the JSON-encoded structure stored as the opaque -// [EncryptionKeyMetadata] for files produced by [GoEnvelopeEncryptionManager]. -// -// This encoding is Go-specific, not Java's Avro-encoded StandardKeyMetadata; -// see the [GoEnvelopeEncryptionManager] doc comment. Per the table spec, -// DataFile/ManifestFile key_metadata is explicitly "implementation-specific"; -// what must be Iceberg AES GCM Stream compliant - and is - is the wire -// format of the encrypted byte stream itself (magic, block framing, nonce -// placement, and AAD; see the constants above). -type goEnvelopeKeyMetadata struct { - Version int `json:"v"` - KeyID string `json:"key-id"` - WrappedKey []byte `json:"wrapped-key"` - - // BlockSize is the trusted plaintext block size the file was written - // with. It is validated bounded before any file I/O, and is what's - // actually used to size reads; the stream header's block-length field - // is untrusted (unauthenticated, attacker-influenced storage bytes) and - // is only compared against this value, never used directly to size an - // allocation. - BlockSize int64 `json:"block-size"` - - // AADPrefix is combined with each block's little-endian index to form - // the AES GCM Stream additional authenticated data, binding every - // ciphertext block to this file and to its position so that blocks - // cannot be silently reordered, replayed from another file, or spliced - // in from elsewhere in the same file. It is not secret. - AADPrefix []byte `json:"aad-prefix"` - - // PlaintextLength is the trusted total plaintext size used to compute - // the block count on read. Per the AES GCM Stream spec's "File length" - // note, a reader must use a length from a trusted source rather than - // the underlying storage's reported size, since storage size alone - // cannot distinguish a genuinely short file from one truncated by an - // attacker who does not also control this metadata. - PlaintextLength int64 `json:"plaintext-length"` -} - -// GoEnvelopeEncryptionManager is a generic, format-agnostic [EncryptionManager] -// that provides envelope encryption for arbitrary files (e.g. manifests, -// manifest lists, Puffin statistics) using a [KeyManagementClient] to wrap -// and unwrap a fresh AES-256-GCM data encryption key (DEK) per file. -// -// Each file is split into fixed-size plaintext blocks and written using the -// Iceberg AES GCM Stream ("AGS1") format: a magic/block-length header -// followed by independently authenticated blocks, each carrying its own -// random nonce and authenticated with an AAD that binds it to the file and -// to its position. This bounds memory usage and supports random access -// (Seek/ReadAt) on the decrypted file without buffering or decrypting more -// than the requested blocks. -// -// GoEnvelopeEncryptionManager always encrypts and always decrypts: it fails -// closed, returning [ErrKeyIDRequired] or [ErrKeyMetadataRequired] rather -// than silently falling back to plaintext. Use [PlaintextEncryptionManager] -// for tables or files that are not encrypted. -// -// Not interoperable with Java's StandardEncryptionManager: the AGS1 byte -// stream framing (magic, block layout, nonce placement, AAD) matches the -// spec, but the two implementations are not cross-readable. Java's reader -// hard-requires a 1 MiB header block size (a compile-time constant there), -// while [GoEnvelopeDefaultBlockSize] is 64 KiB, and Java decodes key -// metadata as a 1-byte version plus Avro-encoded StandardKeyMetadata, while -// this type encodes key metadata as JSON (goEnvelopeKeyMetadata). Files -// written by one are not readable by the other. -type GoEnvelopeEncryptionManager struct { - kms KeyManagementClient - dekLength int - blockSize int -} - -var _ EncryptionManager = (*GoEnvelopeEncryptionManager)(nil) - -// GoEnvelopeManagerOption configures a [GoEnvelopeEncryptionManager] created -// by [NewGoEnvelopeEncryptionManager]. -type GoEnvelopeManagerOption func(*GoEnvelopeEncryptionManager) - -// WithDEKLength overrides the default data encryption key length (in bytes). -// Valid AES key lengths are 16, 24, or 32 bytes. -func WithDEKLength(length int) GoEnvelopeManagerOption { - return func(m *GoEnvelopeEncryptionManager) { m.dekLength = length } -} - -// WithBlockSize overrides the default plaintext block size (in bytes) used -// to split files for independent block-level authentication. size must be -// positive and at most [GoEnvelopeMaxBlockSize]. -func WithBlockSize(size int) GoEnvelopeManagerOption { - return func(m *GoEnvelopeEncryptionManager) { m.blockSize = size } -} - -// NewGoEnvelopeEncryptionManager creates a [GoEnvelopeEncryptionManager] -// backed by kms. kms must not be nil; NewGoEnvelopeEncryptionManager panics -// if it is. -func NewGoEnvelopeEncryptionManager(kms KeyManagementClient, opts ...GoEnvelopeManagerOption) *GoEnvelopeEncryptionManager { - if kms == nil { - panic("encryption: NewGoEnvelopeEncryptionManager: kms must not be nil") - } - - m := &GoEnvelopeEncryptionManager{ - kms: kms, - dekLength: GoEnvelopeDefaultDEKLength, - blockSize: GoEnvelopeDefaultBlockSize, - } - for _, opt := range opts { - opt(m) - } - - return m -} - -// NewEncryptedOutputFile creates a new AES-GCM block-encrypted output file. -// keyID identifies the KEK used to wrap the freshly generated per-file DEK, -// and must be non-empty; otherwise [ErrKeyIDRequired] is returned. -func (m *GoEnvelopeEncryptionManager) NewEncryptedOutputFile(ctx context.Context, writer icebergio.FileWriter, keyID string) (EncryptedOutputFile, error) { - if keyID == "" { - return nil, ErrKeyIDRequired - } - if m.blockSize <= 0 || m.blockSize > GoEnvelopeMaxBlockSize { - return nil, fmt.Errorf("%w: got %d", ErrInvalidBlockSize, m.blockSize) - } - switch m.dekLength { - case 16, 24, 32: - default: - return nil, fmt.Errorf("%w: DEK length must be 16, 24, or 32 bytes; got %d", ErrInvalidKeyLength, m.dekLength) - } - - // The (key, nonce) uniqueness this block format relies on requires a - // freshly generated DEK for every file: never cache or reuse - // plainDEK/wrappedDEK across calls to NewEncryptedOutputFile. - var ( - plainDEK, wrappedDEK []byte - err error - ) - if m.kms.SupportsKeyGeneration() { - plainDEK, wrappedDEK, err = m.kms.GenerateKey(ctx, keyID, m.dekLength) - if err != nil { - return nil, fmt.Errorf("encryption: failed to generate DEK: %w", err) - } - } else { - plainDEK = make([]byte, m.dekLength) - if _, err = io.ReadFull(rand.Reader, plainDEK); err != nil { - return nil, fmt.Errorf("encryption: failed to generate DEK: %w", err) - } - if wrappedDEK, err = m.kms.WrapKey(ctx, keyID, plainDEK); err != nil { - return nil, fmt.Errorf("encryption: failed to wrap DEK: %w", err) - } - } - - aead, err := newGoEnvelopeAEAD(plainDEK) - if err != nil { - return nil, err - } - - aadPrefix := make([]byte, gcmStreamAADPrefixLength) - if _, err := io.ReadFull(rand.Reader, aadPrefix); err != nil { - return nil, fmt.Errorf("encryption: failed to generate AAD prefix: %w", err) - } - - // Write the AES GCM Stream header (magic + little-endian block length) - // up front, before any ciphertext blocks, per the format spec. - header := make([]byte, gcmStreamHeaderLength) - copy(header, gcmStreamMagic) - binary.LittleEndian.PutUint32(header[len(gcmStreamMagic):], uint32(m.blockSize)) //nolint:gosec // bounded by ErrInvalidBlockSize above - if _, err := writer.Write(header); err != nil { - return nil, fmt.Errorf("encryption: failed to write stream header: %w", err) - } - - return &goEnvelopeOutputFile{ - FileWriter: writer, - aead: aead, - aadPrefix: aadPrefix, - blockSize: m.blockSize, - keyID: keyID, - wrappedKey: wrappedDEK, - }, nil -} - -// NewDecryptedInputFile wraps file for transparent block-level AES-GCM -// decryption. keyMetadata must be the non-empty blob produced by -// [GoEnvelopeEncryptionManager.NewEncryptedOutputFile]; otherwise -// [ErrKeyMetadataRequired] is returned. -func (m *GoEnvelopeEncryptionManager) NewDecryptedInputFile(ctx context.Context, file icebergio.File, keyMetadata EncryptionKeyMetadata) (EncryptedInputFile, error) { - if len(keyMetadata) == 0 { - return nil, ErrKeyMetadataRequired - } - - var meta goEnvelopeKeyMetadata - if err := json.Unmarshal(keyMetadata, &meta); err != nil { - return nil, fmt.Errorf("encryption: failed to decode key metadata: %w", err) - } - if meta.Version != goEnvelopeKeyMetadataVersion { - return nil, fmt.Errorf("%w: %d", ErrUnsupportedKeyMetadataVersion, meta.Version) - } - if meta.BlockSize <= 0 || meta.BlockSize > GoEnvelopeMaxBlockSize { - return nil, fmt.Errorf("%w: block-size must be positive and at most %d, got %d", ErrInvalidKeyMetadata, GoEnvelopeMaxBlockSize, meta.BlockSize) - } - if meta.PlaintextLength < 0 { - return nil, fmt.Errorf("%w: plaintext-length must be non-negative, got %d", ErrInvalidKeyMetadata, meta.PlaintextLength) - } - // AADPrefix is bounded to exactly the length this implementation's own - // writer emits (gcmStreamAADPrefixLength): key metadata is Go-specific - // (not cross-implementation compatible; see the type doc comment), so - // there is no other length a valid file could carry. Without this - // ceiling, an oversized prefix would be re-allocated on every - // gcmStreamBlockAAD call in readBlock. - if len(meta.AADPrefix) != gcmStreamAADPrefixLength { - return nil, fmt.Errorf("%w: aad-prefix must be exactly %d bytes, got %d", ErrInvalidKeyMetadata, gcmStreamAADPrefixLength, len(meta.AADPrefix)) - } - - plainDEK, err := m.kms.UnwrapKey(ctx, meta.KeyID, meta.WrappedKey) - if err != nil { - return nil, fmt.Errorf("encryption: failed to unwrap DEK: %w", err) - } - - aead, err := newGoEnvelopeAEAD(plainDEK) - if err != nil { - return nil, err - } - - // headerBlockSize is untrusted (unauthenticated bytes read from - // storage): it is only ever compared against the trusted, already - // bounded meta.BlockSize below, never used to size a read or allocation. - headerBlockSize, err := readGCMStreamHeader(file) - if err != nil { - return nil, err - } - if headerBlockSize != meta.BlockSize { - return nil, fmt.Errorf("%w: stream header block length %d does not match key metadata block-size %d", ErrInvalidStreamHeader, headerBlockSize, meta.BlockSize) - } - - return &goEnvelopeInputFile{ - underlying: file, - aead: aead, - aadPrefix: meta.AADPrefix, - blockSize: meta.BlockSize, - plaintextLength: meta.PlaintextLength, - keyMetadata: keyMetadata, - }, nil -} - -func newGoEnvelopeAEAD(key []byte) (cipher.AEAD, error) { - block, err := aes.NewCipher(key) - if err != nil { - return nil, fmt.Errorf("%w: %w", ErrInvalidKeyLength, err) - } - gcm, err := cipher.NewGCM(block) - if err != nil { - return nil, fmt.Errorf("encryption: failed to create GCM: %w", err) - } - - return gcm, nil -} - -// readGCMStreamHeader reads and validates the AES GCM Stream magic and -// block-length header at the start of file, returning the plaintext block -// length. The header is untrusted, unauthenticated data, so the returned -// length is bounded to [GoEnvelopeMaxBlockSize] before any allocation sized -// by it takes place. -func readGCMStreamHeader(file icebergio.File) (int64, error) { - header := make([]byte, gcmStreamHeaderLength) - n, err := file.ReadAt(header, 0) - if err != nil && !errors.Is(err, io.EOF) { - return 0, fmt.Errorf("encryption: failed to read stream header: %w", err) - } - if n != gcmStreamHeaderLength { - return 0, fmt.Errorf("%w: expected %d header bytes, got %d", ErrInvalidStreamHeader, gcmStreamHeaderLength, n) - } - if string(header[:len(gcmStreamMagic)]) != gcmStreamMagic { - return 0, fmt.Errorf("%w: missing %q magic", ErrInvalidStreamHeader, gcmStreamMagic) - } - - blockSize := int64(binary.LittleEndian.Uint32(header[len(gcmStreamMagic):])) - if blockSize <= 0 || blockSize > GoEnvelopeMaxBlockSize { - return 0, fmt.Errorf("%w: block length %d out of supported range (0, %d]", ErrInvalidStreamHeader, blockSize, GoEnvelopeMaxBlockSize) - } - - return blockSize, nil -} - -// gcmStreamBlockAAD derives the AES-GCM additional authenticated data for -// blockIndex: the per-file AAD prefix followed by the 4-byte little-endian -// block index, per the Iceberg AES GCM Stream spec. This binds every -// ciphertext block to this file and to its position, which matters because -// blocks carry independent random nonces (rather than a nonce derived from -// the block index): without the index in the AAD, an attacker able to -// tamper with ciphertext at rest could silently reorder or splice blocks. -func gcmStreamBlockAAD(prefix []byte, blockIndex uint32) []byte { - aad := make([]byte, len(prefix)+4) - copy(aad, prefix) - binary.LittleEndian.PutUint32(aad[len(prefix):], blockIndex) - - return aad -} - -// checkedMulInt64 returns a*b and true, or (0, false) if the multiplication -// overflows int64. -func checkedMulInt64(a, b int64) (int64, bool) { - if a == 0 || b == 0 { - return 0, true - } - result := a * b - if result/b != a { - return 0, false - } - - return result, true -} - -// checkedAddInt64 returns a+b and true, or (0, false) if the addition -// overflows int64. -func checkedAddInt64(a, b int64) (int64, bool) { - result := a + b - if (b > 0 && result < a) || (b < 0 && result > a) { - return 0, false - } - - return result, true -} - -// goEnvelopeOutputFile is an [EncryptedOutputFile] that seals fixed-size -// plaintext blocks with AES-GCM as they are written, using the Iceberg AES -// GCM Stream ("AGS1") wire format. -type goEnvelopeOutputFile struct { - icebergio.FileWriter - - aead cipher.AEAD - aadPrefix []byte - blockSize int - keyID string - wrappedKey []byte - - buf []byte - blockIndex uint32 - written int64 - closed bool - err error - - // underlyingClosed tracks whether FileWriter.Close has already been - // attempted, so a poisoned writer is closed exactly once regardless of - // whether the failure is first observed in Write or in Close. - underlyingClosed bool - - keyMetadata EncryptionKeyMetadata -} - -var _ EncryptedOutputFile = (*goEnvelopeOutputFile)(nil) - -// closeUnderlyingIgnoringError closes the underlying writer at most once. -// The error is ignored: the caller is already reporting a more specific -// failure (a flush or encode error), and this is best-effort cleanup so a -// poisoned writer never leaks its underlying file descriptor or connection. -// -// defensive: today every call site sets f.err before calling this, and -// Write/Close both bail out early once f.err is set, so the -// underlyingClosed guard never actually stops a second Close in practice. -// It stays as a hard invariant in case a future call site is added that -// doesn't follow that pattern. -func (f *goEnvelopeOutputFile) closeUnderlyingIgnoringError() { - if f.underlyingClosed { - return - } - f.underlyingClosed = true - _ = f.FileWriter.Close() -} - -func (f *goEnvelopeOutputFile) Write(p []byte) (int, error) { - if f.err != nil { - return 0, f.err - } - if f.closed { - return 0, ErrOutputFileClosed - } - - total := len(p) - consumed := 0 // bytes of p appended into f.buf so far in this call - accepted := 0 // bytes of p known to be durably flushed; reported on failure - for len(p) > 0 { - space := f.blockSize - len(f.buf) - n := min(space, len(p)) - f.buf = append(f.buf, p[:n]...) - p = p[n:] - consumed += n - if len(f.buf) == f.blockSize { - if err := f.flushBlock(); err != nil { - f.err = err - f.closeUnderlyingIgnoringError() - - return accepted, err - } - accepted = consumed - } - } - - return total, nil -} - -// flushBlock seals and writes the currently buffered plaintext block using a -// fresh random nonce, per the Iceberg AES GCM Stream format. f.written is -// only advanced once the ciphertext has actually reached the underlying -// writer, so a failed flush never overcounts PlaintextLength. -func (f *goEnvelopeOutputFile) flushBlock() error { - if f.blockIndex == math.MaxUint32 { - return errors.New("encryption: cannot write block: exceeded maximum block count") - } - - nonce := make([]byte, gcmStreamNonceLength) - if _, err := io.ReadFull(rand.Reader, nonce); err != nil { - return fmt.Errorf("encryption: failed to generate block nonce: %w", err) - } - - aad := gcmStreamBlockAAD(f.aadPrefix, f.blockIndex) - sealed := f.aead.Seal(nonce, nonce, f.buf, aad) - if _, err := f.FileWriter.Write(sealed); err != nil { - return fmt.Errorf("encryption: failed to write encrypted block: %w", err) - } - f.written += int64(len(f.buf)) - f.blockIndex++ - f.buf = f.buf[:0] - - return nil -} - -// ReadFrom copies from r, encrypting as data is written, satisfying -// io.ReaderFrom (required by [icebergio.FileWriter]). -func (f *goEnvelopeOutputFile) ReadFrom(r io.Reader) (int64, error) { - buf := make([]byte, max(32*1024, f.blockSize)) - var total int64 - for { - n, err := r.Read(buf) - if n > 0 { - wn, werr := f.Write(buf[:n]) - total += int64(wn) - if werr != nil { - return total, werr - } - } - if err == io.EOF { - break - } - if err != nil { - return total, err - } - } - - return total, nil -} - -// Close flushes any buffered partial block and finalizes the key metadata. -// closed is only set, and keyMetadata only published, once everything - -// including the underlying Close - has succeeded; a failed Close poisons the -// writer (via f.err) so a retry reliably reports the same error instead of -// masking the failure as success or exposing metadata for an output that -// never finished. -func (f *goEnvelopeOutputFile) Close() error { - if f.err != nil { - return f.err - } - if f.closed { - return nil - } - - if len(f.buf) > 0 { - if err := f.flushBlock(); err != nil { - f.err = err - f.closeUnderlyingIgnoringError() - - return err - } - } - - meta := goEnvelopeKeyMetadata{ - Version: goEnvelopeKeyMetadataVersion, - KeyID: f.keyID, - WrappedKey: f.wrappedKey, - BlockSize: int64(f.blockSize), - AADPrefix: f.aadPrefix, - PlaintextLength: f.written, - } - encoded, err := json.Marshal(meta) - if err != nil { - f.err = fmt.Errorf("encryption: failed to encode key metadata: %w", err) - f.closeUnderlyingIgnoringError() - - return f.err - } - - if err := f.FileWriter.Close(); err != nil { - f.underlyingClosed = true - f.err = fmt.Errorf("encryption: failed to close underlying writer: %w", err) - - return f.err - } - f.underlyingClosed = true - - f.keyMetadata = encoded - f.closed = true - - return nil -} - -// KeyMetadata returns the finalized per-file key metadata. It is only -// populated after Close has succeeded. -func (f *goEnvelopeOutputFile) KeyMetadata() EncryptionKeyMetadata { return f.keyMetadata } - -// goEnvelopeInputFile is an [EncryptedInputFile] that decrypts fixed-size -// AES-GCM blocks on demand, supporting random access via ReadAt/Seek. -// -// ReadAt is safe, and intended, for concurrent use, matching the io.ReaderAt -// contract: readBlock only holds cacheMu around the cache check/publish, not -// across the underlying ReadAt or AEAD decryption, so concurrent calls for -// distinct blocks actually run in parallel rather than being serialized. A -// lost cache race (two goroutines decrypting the same block) is harmless, -// just wasted work. Read and Seek mutate the shared cursor (pos) and are not -// concurrent-safe; do not call them from multiple goroutines on the same -// instance. -// -// The single-entry cache retains one full decrypted block (up to -// [GoEnvelopeMaxBlockSize], 128 MiB) for the lifetime of the input file, not -// just the lifetime of a single read call. -type goEnvelopeInputFile struct { - underlying icebergio.File - aead cipher.AEAD - aadPrefix []byte - blockSize int64 - plaintextLength int64 - keyMetadata EncryptionKeyMetadata - - pos int64 - - // cacheMu guards cacheIdx/cacheBlock/cacheValid below. It is only held - // around the cache check and the cache publish, never across the - // underlying ReadAt or AEAD decryption in readBlock. - cacheMu sync.Mutex - cacheIdx int64 - cacheBlock []byte - cacheValid bool -} - -var _ EncryptedInputFile = (*goEnvelopeInputFile)(nil) - -func (f *goEnvelopeInputFile) numBlocks() int64 { - if f.plaintextLength == 0 { - return 0 - } - - // Overflow-safe ceiling division: plaintextLength and blockSize both - // originate from data outside this reader's control (key metadata and - // the untrusted stream header, respectively), so avoid computing - // (plaintextLength + blockSize - 1), which can overflow near - // math.MaxInt64. - return 1 + (f.plaintextLength-1)/f.blockSize -} - -// blockPlainLen returns the plaintext length of block idx, given the total -// numBlocks (passed in rather than recomputed, since every caller has -// already computed it). -func (f *goEnvelopeInputFile) blockPlainLen(idx, numBlocks int64) int64 { - if idx == numBlocks-1 { - return f.plaintextLength - idx*f.blockSize - } - - return f.blockSize -} - -// blockPhysicalOffset computes the physical offset of block idx in the -// underlying ciphertext, using overflow-checked arithmetic. idx is derived -// from the untrusted plaintext-length in key metadata, so a small blockSize -// combined with a huge plaintext-length could otherwise overflow int64 -// before the resulting (implausible) offset is ever used in a read. -func blockPhysicalOffset(idx, blockSize int64) (int64, error) { - physicalBlockSize, ok := checkedAddInt64(blockSize, gcmStreamBlockOverhead) - if !ok { - return 0, fmt.Errorf("%w: block size %d overflows physical layout", ErrInvalidKeyMetadata, blockSize) - } - product, ok := checkedMulInt64(idx, physicalBlockSize) - if !ok { - return 0, fmt.Errorf("%w: block %d physical offset overflows", ErrInvalidKeyMetadata, idx) - } - offset, ok := checkedAddInt64(product, int64(gcmStreamHeaderLength)) - if !ok { - return 0, fmt.Errorf("%w: block %d physical offset overflows", ErrInvalidKeyMetadata, idx) - } - - return offset, nil -} - -// readBlock decrypts block idx, or returns it from the single-entry cache -// if it was the most recently decrypted block. It validates idx and the -// computed block length before reading, since metadata (blockSize, -// plaintextLength) can originate from untrusted input. It also honors the -// actual byte count returned by ReadAt: a short, non-EOF-explained read is -// reported as [ErrBlockTruncated] (truncated storage) rather than being -// silently zero-padded into the AEAD, which would otherwise surface as a -// misleading [ErrAuthenticationFailed]. -// -// cacheMu is only held around the cache check and the cache publish at the -// end, never across the underlying ReadAt or aead.Open below: holding it -// across I/O would serialize all concurrent readers, including remote -// backends. A lost race between two goroutines decrypting the same block is -// harmless (redundant work, not a correctness issue). -func (f *goEnvelopeInputFile) readBlock(idx int64) ([]byte, error) { - f.cacheMu.Lock() - if f.cacheValid && f.cacheIdx == idx { - block := f.cacheBlock - f.cacheMu.Unlock() - - return block, nil - } - f.cacheMu.Unlock() - - numBlocks := f.numBlocks() - // defensive: idx is only ever derived from ReadAt's off, which already - // rejects off >= f.plaintextLength before calling readBlock; kept as a - // hard boundary check since idx ultimately traces back to untrusted - // key metadata (plaintextLength, blockSize). - if idx < 0 || idx >= numBlocks { - return nil, fmt.Errorf("%w: block index %d out of range [0, %d)", ErrInvalidKeyMetadata, idx, numBlocks) - } - if idx > math.MaxUint32 { - return nil, fmt.Errorf("%w: block index %d exceeds the maximum supported block count", ErrInvalidKeyMetadata, idx) - } - - plainLen := f.blockPlainLen(idx, numBlocks) - if plainLen < 0 { - return nil, fmt.Errorf("%w: negative computed length for block %d", ErrInvalidKeyMetadata, idx) - } - - offset, err := blockPhysicalOffset(idx, f.blockSize) - if err != nil { - return nil, err - } - - wantLen := plainLen + gcmStreamBlockOverhead - ciphertext := make([]byte, wantLen) - n, err := f.underlying.ReadAt(ciphertext, offset) - if err != nil && !errors.Is(err, io.EOF) { - return nil, fmt.Errorf("encryption: failed to read block %d: %w", idx, err) - } - if int64(n) != wantLen { - return nil, fmt.Errorf("%w: block %d: read %d of %d expected ciphertext bytes", ErrBlockTruncated, idx, n, wantLen) - } - ciphertext = ciphertext[:n] - - nonce, sealed := ciphertext[:gcmStreamNonceLength], ciphertext[gcmStreamNonceLength:] - aad := gcmStreamBlockAAD(f.aadPrefix, uint32(idx)) - - plaintext, err := f.aead.Open(nil, nonce, sealed, aad) - if err != nil { - return nil, fmt.Errorf("%w: block %d: %w", ErrAuthenticationFailed, idx, err) - } - - f.cacheMu.Lock() - f.cacheIdx = idx - f.cacheBlock = plaintext - f.cacheValid = true - f.cacheMu.Unlock() - - return plaintext, nil -} - -func (f *goEnvelopeInputFile) ReadAt(p []byte, off int64) (int, error) { - if len(p) == 0 { - return 0, nil - } - if off < 0 { - return 0, errors.New("encryption: ReadAt: negative offset") - } - if off >= f.plaintextLength { - return 0, io.EOF - } - - var read int - for read < len(p) { - curOff := off + int64(read) - if curOff >= f.plaintextLength { - break - } - idx := curOff / f.blockSize - block, err := f.readBlock(idx) - if err != nil { - return read, err - } - inBlockOff := curOff - idx*f.blockSize - read += copy(p[read:], block[inBlockOff:]) - } - - var err error - if read < len(p) { - err = io.EOF - } - - return read, err -} - -func (f *goEnvelopeInputFile) Read(p []byte) (int, error) { - n, err := f.ReadAt(p, f.pos) - f.pos += int64(n) - - return n, err -} - -func (f *goEnvelopeInputFile) Seek(offset int64, whence int) (int64, error) { - var newPos int64 - switch whence { - case io.SeekStart: - newPos = offset - case io.SeekCurrent: - newPos = f.pos + offset - case io.SeekEnd: - newPos = f.plaintextLength + offset - default: - return 0, fmt.Errorf("encryption: Seek: invalid whence %d", whence) - } - if newPos < 0 { - return 0, errors.New("encryption: Seek: negative position") - } - f.pos = newPos - - return newPos, nil -} - -func (f *goEnvelopeInputFile) Close() error { return f.underlying.Close() } - -func (f *goEnvelopeInputFile) Stat() (fs.FileInfo, error) { - info, err := f.underlying.Stat() - if err != nil { - return nil, err - } - - return goEnvelopeFileInfo{FileInfo: info, size: f.plaintextLength}, nil -} - -// KeyMetadata returns the key metadata this file was decrypted with. -func (f *goEnvelopeInputFile) KeyMetadata() EncryptionKeyMetadata { return f.keyMetadata } - -// goEnvelopeFileInfo overrides Size() to report the plaintext length rather -// than the (larger) on-disk ciphertext length. -type goEnvelopeFileInfo struct { - fs.FileInfo - size int64 -} - -func (i goEnvelopeFileInfo) Size() int64 { return i.size } diff --git a/encryption/go_envelope_manager_test.go b/encryption/go_envelope_manager_test.go deleted file mode 100644 index 06b638571..000000000 --- a/encryption/go_envelope_manager_test.go +++ /dev/null @@ -1,654 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package encryption_test - -import ( - "bytes" - "encoding/base64" - "encoding/binary" - "errors" - "fmt" - "io" - "io/fs" - "math" - "testing" - "time" - - "github.com/apache/iceberg-go/encryption" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "golang.org/x/sync/errgroup" -) - -func newTestGoEnvelopeManager(t *testing.T, opts ...encryption.GoEnvelopeManagerOption) (*encryption.GoEnvelopeEncryptionManager, *encryption.InMemoryKeyManagementClient) { - t.Helper() - kms := encryption.NewInMemoryKeyManagementClient() - require.NoError(t, kms.AddKey("kek-1", bytes.Repeat([]byte{0x42}, 32))) - - return encryption.NewGoEnvelopeEncryptionManager(kms, opts...), kms -} - -func encryptAll(t *testing.T, mgr *encryption.GoEnvelopeEncryptionManager, keyID string, plaintext []byte) (ciphertext []byte, keyMetadata encryption.EncryptionKeyMetadata) { - t.Helper() - fw := &memFileWriter{} - out, err := mgr.NewEncryptedOutputFile(t.Context(), fw, keyID) - require.NoError(t, err) - _, err = out.Write(plaintext) - require.NoError(t, err) - require.NoError(t, out.Close()) - - return fw.Bytes(), out.KeyMetadata() -} - -func decryptAll(t *testing.T, mgr *encryption.GoEnvelopeEncryptionManager, ciphertext []byte, keyMetadata encryption.EncryptionKeyMetadata) []byte { - t.Helper() - in, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) - require.NoError(t, err) - data, err := io.ReadAll(in) - require.NoError(t, err) - - return data -} - -func TestGoEnvelopeEncryptionManager_RoundTrip_SmallerThanBlock(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) - plaintext := []byte("hello iceberg") - - ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) - assert.NotEmpty(t, keyMetadata) - assert.NotEqual(t, plaintext, ciphertext, "ciphertext must differ from plaintext") - - got := decryptAll(t, mgr, ciphertext, keyMetadata) - assert.Equal(t, plaintext, got) -} - -func TestGoEnvelopeEncryptionManager_RoundTrip_MultiBlock(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) - plaintext := bytes.Repeat([]byte("0123456789abcdef"), 10) - plaintext = append(plaintext, []byte("partial-tail")...) - - ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) - got := decryptAll(t, mgr, ciphertext, keyMetadata) - assert.Equal(t, plaintext, got) -} - -func TestGoEnvelopeEncryptionManager_RoundTrip_ExactBlockMultiple(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) - plaintext := bytes.Repeat([]byte("0123456789abcdef"), 4) - - ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) - got := decryptAll(t, mgr, ciphertext, keyMetadata) - assert.Equal(t, plaintext, got) -} - -func TestGoEnvelopeEncryptionManager_RoundTrip_Empty(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) - - ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", nil) - got := decryptAll(t, mgr, ciphertext, keyMetadata) - assert.Empty(t, got) -} - -func TestGoEnvelopeEncryptionManager_RandomAccess(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) - plaintext := bytes.Repeat([]byte("0123456789abcdef"), 10) - plaintext = append(plaintext, []byte("tail-bytes")...) - - ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) - in, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) - require.NoError(t, err) - - // Read a slice spanning a block boundary. - buf := make([]byte, 20) - n, err := in.ReadAt(buf, 10) - require.NoError(t, err) - assert.Equal(t, plaintext[10:30], buf[:n]) - - // Seek + Read. - pos, err := in.Seek(5, io.SeekStart) - require.NoError(t, err) - assert.Equal(t, int64(5), pos) - - rest, err := io.ReadAll(in) - require.NoError(t, err) - assert.Equal(t, plaintext[5:], rest) -} - -func TestGoEnvelopeEncryptionManager_TamperedBlockFailsAuthentication(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) - plaintext := bytes.Repeat([]byte("0123456789abcdef"), 4) - - ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) - // Flip a bit inside the first block's ciphertext, past the 8-byte AES - // GCM Stream header and 12-byte nonce; flipping the header instead - // would fail earlier, at stream-header validation. - ciphertext[8+12] ^= 0xFF - - in, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) - require.NoError(t, err) - - _, err = io.ReadAll(in) - require.Error(t, err) - assert.True(t, errors.Is(err, encryption.ErrAuthenticationFailed)) -} - -func TestGoEnvelopeEncryptionManager_ReorderedBlocksFailAuthentication(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) - plaintext := bytes.Repeat([]byte("0123456789abcdef"), 2) // exactly two full 16-byte blocks - - ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) - - // The AES GCM Stream header (magic + block length) is 8 bytes; each - // cipher block here is 16 (block size) + 12 (nonce) + 16 (tag) = 44 - // bytes. Swap the two blocks: with a random nonce per block, only the - // AAD (which binds a block to its position) can catch this. - const headerLen, blockPhysicalLen = 8, 44 - block1 := append([]byte(nil), ciphertext[headerLen:headerLen+blockPhysicalLen]...) - block2 := append([]byte(nil), ciphertext[headerLen+blockPhysicalLen:headerLen+2*blockPhysicalLen]...) - copy(ciphertext[headerLen:headerLen+blockPhysicalLen], block2) - copy(ciphertext[headerLen+blockPhysicalLen:headerLen+2*blockPhysicalLen], block1) - - in, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) - require.NoError(t, err) - - _, err = io.ReadAll(in) - require.Error(t, err) - assert.True(t, errors.Is(err, encryption.ErrAuthenticationFailed), "swapping blocks must be detected: the AAD binds each block to its position") -} - -func TestGoEnvelopeEncryptionManager_StreamHeaderMagicMismatchRejected(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) - ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", []byte("hello iceberg")) - copy(ciphertext[:4], "XXXX") - - _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) - require.Error(t, err) - assert.True(t, errors.Is(err, encryption.ErrInvalidStreamHeader)) -} - -func TestGoEnvelopeEncryptionManager_StreamHeaderZeroBlockLengthRejected(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) - ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", []byte("hello iceberg")) - binary.LittleEndian.PutUint32(ciphertext[4:8], 0) - - _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) - require.Error(t, err) - assert.True(t, errors.Is(err, encryption.ErrInvalidStreamHeader)) -} - -func TestGoEnvelopeEncryptionManager_StreamHeaderBlockLengthExceedsMaxRejected(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) - ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", []byte("hello iceberg")) - binary.LittleEndian.PutUint32(ciphertext[4:8], math.MaxUint32) - - _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) - require.Error(t, err) - assert.True(t, errors.Is(err, encryption.ErrInvalidStreamHeader)) -} - -func TestGoEnvelopeEncryptionManager_OutputFile_EmptyKeyIDRejected(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t) - _, err := mgr.NewEncryptedOutputFile(t.Context(), &memFileWriter{}, "") - require.Error(t, err) - assert.True(t, errors.Is(err, encryption.ErrKeyIDRequired)) -} - -func TestGoEnvelopeEncryptionManager_InputFile_EmptyKeyMetadataRejected(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t) - _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), nil) - require.Error(t, err) - assert.True(t, errors.Is(err, encryption.ErrKeyMetadataRequired)) -} - -func TestGoEnvelopeEncryptionManager_UnknownKeyIDPropagatesFromKMS(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t) - _, err := mgr.NewEncryptedOutputFile(t.Context(), &memFileWriter{}, "does-not-exist") - require.Error(t, err) - assert.True(t, errors.Is(err, encryption.ErrUnknownKeyID)) -} - -func TestGoEnvelopeEncryptionManager_MalformedKeyMetadataRejected(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t) - _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), encryption.EncryptionKeyMetadata("not json")) - require.Error(t, err) -} - -func TestGoEnvelopeEncryptionManager_ZeroBlockSizeRejectedOnWrite(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(0)) - _, err := mgr.NewEncryptedOutputFile(t.Context(), &memFileWriter{}, "kek-1") - require.Error(t, err) - assert.True(t, errors.Is(err, encryption.ErrInvalidBlockSize)) -} - -func TestGoEnvelopeEncryptionManager_NegativeBlockSizeRejectedOnWrite(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(-1)) - _, err := mgr.NewEncryptedOutputFile(t.Context(), &memFileWriter{}, "kek-1") - require.Error(t, err) - assert.True(t, errors.Is(err, encryption.ErrInvalidBlockSize)) -} - -func TestGoEnvelopeEncryptionManager_BlockSizeExceedingMaxRejectedOnWrite(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(encryption.GoEnvelopeMaxBlockSize+1)) - _, err := mgr.NewEncryptedOutputFile(t.Context(), &memFileWriter{}, "kek-1") - require.Error(t, err) - assert.True(t, errors.Is(err, encryption.ErrInvalidBlockSize)) -} - -func TestGoEnvelopeEncryptionManager_NegativePlaintextLengthMetadataRejectedOnRead(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t) - meta := []byte(`{"v":1,"key-id":"kek-1","wrapped-key":"AA==","block-size":16,"aad-prefix":"MTIzNDU2Nzg5MDEyMzQ1Ng==","plaintext-length":-1}`) - _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), meta) - require.Error(t, err) - assert.True(t, errors.Is(err, encryption.ErrInvalidKeyMetadata)) - assert.ErrorContains(t, err, "plaintext-length must be non-negative") -} - -func TestGoEnvelopeEncryptionManager_EmptyAADPrefixMetadataRejectedOnRead(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t) - meta := []byte(`{"v":1,"key-id":"kek-1","wrapped-key":"AA==","block-size":16,"aad-prefix":"","plaintext-length":10}`) - _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), meta) - require.Error(t, err) - assert.True(t, errors.Is(err, encryption.ErrInvalidKeyMetadata)) - assert.ErrorContains(t, err, "aad-prefix must be exactly 16 bytes") -} - -func TestGoEnvelopeEncryptionManager_OversizedAADPrefixMetadataRejectedOnRead(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t) - // 17 zero bytes, base64-encoded: one byte longer than gcmStreamAADPrefixLength. - oversized := base64.StdEncoding.EncodeToString(make([]byte, 17)) - meta := fmt.Appendf(nil, `{"v":1,"key-id":"kek-1","wrapped-key":"AA==","block-size":16,"aad-prefix":"%s","plaintext-length":10}`, oversized) - _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), meta) - require.Error(t, err) - assert.True(t, errors.Is(err, encryption.ErrInvalidKeyMetadata)) - assert.ErrorContains(t, err, "aad-prefix must be exactly 16 bytes") -} - -func TestGoEnvelopeEncryptionManager_ZeroBlockSizeMetadataRejectedOnRead(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t) - meta := []byte(`{"v":1,"key-id":"kek-1","wrapped-key":"AA==","block-size":0,"aad-prefix":"MTIzNDU2Nzg5MDEyMzQ1Ng==","plaintext-length":10}`) - _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), meta) - require.Error(t, err) - assert.True(t, errors.Is(err, encryption.ErrInvalidKeyMetadata)) - assert.ErrorContains(t, err, "block-size must be positive") -} - -func TestGoEnvelopeEncryptionManager_BlockSizeExceedingMaxMetadataRejectedOnRead(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t) - meta := fmt.Appendf(nil, `{"v":1,"key-id":"kek-1","wrapped-key":"AA==","block-size":%d,"aad-prefix":"MTIzNDU2Nzg5MDEyMzQ1Ng==","plaintext-length":10}`, encryption.GoEnvelopeMaxBlockSize+1) - _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(nil), meta) - require.Error(t, err) - assert.True(t, errors.Is(err, encryption.ErrInvalidKeyMetadata)) - assert.ErrorContains(t, err, "block-size must be positive") -} - -func TestGoEnvelopeEncryptionManager_InvalidDEKLengthRejected(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t, encryption.WithDEKLength(15)) - _, err := mgr.NewEncryptedOutputFile(t.Context(), &memFileWriter{}, "kek-1") - require.Error(t, err) - assert.True(t, errors.Is(err, encryption.ErrInvalidKeyLength)) -} - -// shortReadFile is an icebergio.File stub whose ReadAt returns fewer bytes -// than requested along with io.EOF once truncateAt is reached, mimicking a -// real backend (S3, local fs) reading a genuinely truncated file. A plain -// bytes.Reader always fills the requested slice, which is why the round-trip -// tests never exercise this path. -type shortReadFile struct { - *memFile - truncateAt int64 -} - -func (f *shortReadFile) ReadAt(p []byte, off int64) (int, error) { - if off >= f.truncateAt { - return 0, io.EOF - } - if off+int64(len(p)) > f.truncateAt { - p = p[:f.truncateAt-off] - } - n, err := f.memFile.ReadAt(p, off) - if err == nil && int64(n) < int64(len(p)) { - err = io.EOF - } - - return n, err -} - -func TestGoEnvelopeEncryptionManager_TruncatedBackendReadReportsBlockTruncated(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) - plaintext := bytes.Repeat([]byte("0123456789abcdef"), 4) - - ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) - truncated := &shortReadFile{memFile: newMemFile(ciphertext), truncateAt: int64(len(ciphertext) - 5)} - - in, err := mgr.NewDecryptedInputFile(t.Context(), truncated, keyMetadata) - require.NoError(t, err) - - _, err = io.ReadAll(in) - require.Error(t, err) - assert.True(t, errors.Is(err, encryption.ErrBlockTruncated)) - assert.False(t, errors.Is(err, encryption.ErrCiphertextTooShort), "a truncated block read is distinct from a too-short KMS-wrapped key/payload") - assert.False(t, errors.Is(err, encryption.ErrAuthenticationFailed), "a truncated read must not be misreported as tampering") -} - -func TestGoEnvelopeEncryptionManager_EmptyFile_ZeroLengthReadAt(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) - ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", nil) - - in, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) - require.NoError(t, err) - - n, err := in.ReadAt(nil, 0) - assert.NoError(t, err, "a zero-length ReadAt on an empty file should behave like bytes.Reader, not report io.EOF") - assert.Equal(t, 0, n) -} - -// failAfterNWriter is an icebergio.FileWriter stub that fails the Nth call -// to Write, simulating a mid-stream flush failure on the underlying storage. -// It also counts Close calls, since memFileWriter.Close is a no-op and would -// otherwise mask a leaked/never-closed underlying writer. -type failAfterNWriter struct { - memFileWriter - failAt int - writes int - closes int -} - -func (w *failAfterNWriter) Write(p []byte) (int, error) { - w.writes++ - if w.writes == w.failAt { - return 0, errors.New("simulated write failure") - } - - return w.memFileWriter.Write(p) -} - -func (w *failAfterNWriter) Close() error { - w.closes++ - - return w.memFileWriter.Close() -} - -func TestGoEnvelopeEncryptionManager_FlushFailurePoisonsWriterAndClose(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(4)) - // Write call 1 is the AES GCM Stream header written by - // NewEncryptedOutputFile; call 2 flushes the first (4-byte) block; - // call 3 flushes the second block and fails. - fw := &failAfterNWriter{failAt: 3} - out, err := mgr.NewEncryptedOutputFile(t.Context(), fw, "kek-1") - require.NoError(t, err) - - // First block (4 bytes) flushes fine; the second block's flush fails. - n, err := out.Write([]byte("aaaabbbb")) - require.Error(t, err) - assert.Equal(t, 4, n, "only the first block reached storage") - assert.Equal(t, 1, fw.closes, "a poisoned flush must close the underlying writer, not leak it") - - // A subsequent Write must return the same sticky error, not attempt more I/O. - _, err2 := out.Write([]byte("c")) - require.Error(t, err2) - assert.ErrorIs(t, err2, err) - - // Close must report the failure rather than silently succeeding. - closeErr := out.Close() - require.Error(t, closeErr) - assert.Nil(t, out.KeyMetadata(), "key metadata must not be finalized when the file failed to write") - - // A retried Close must keep reporting the same error, not nil. - closeErr2 := out.Close() - require.Error(t, closeErr2) - assert.Equal(t, 1, fw.closes, "a retried Close must not close the underlying writer again") -} - -func TestGoEnvelopeEncryptionManager_WriteAfterCloseRejected(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t) - out, err := mgr.NewEncryptedOutputFile(t.Context(), &memFileWriter{}, "kek-1") - require.NoError(t, err) - require.NoError(t, out.Close()) - - _, err = out.Write([]byte("late")) - require.Error(t, err) - assert.True(t, errors.Is(err, encryption.ErrOutputFileClosed)) - assert.True(t, errors.Is(err, fs.ErrClosed)) -} - -func TestNewGoEnvelopeEncryptionManager_NilKMSPanics(t *testing.T) { - assert.PanicsWithValue(t, "encryption: NewGoEnvelopeEncryptionManager: kms must not be nil", func() { - encryption.NewGoEnvelopeEncryptionManager(nil) - }) -} - -func TestGoEnvelopeEncryptionManager_StreamHeaderBlockLengthMismatchRejected(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) - ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", []byte("hello iceberg")) - // A header block length that is itself in-range, but does not match the - // (trusted) block-size recorded in key metadata. - binary.LittleEndian.PutUint32(ciphertext[4:8], 32) - - _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) - require.Error(t, err) - assert.True(t, errors.Is(err, encryption.ErrInvalidStreamHeader)) -} - -// countingReadAtFile wraps a memFile and counts ReadAt calls, to verify that -// repeated small reads landing in the same block reuse the decrypted-block -// cache instead of re-decrypting on every call. -type countingReadAtFile struct { - *memFile - reads int -} - -func (f *countingReadAtFile) ReadAt(p []byte, off int64) (int, error) { - f.reads++ - - return f.memFile.ReadAt(p, off) -} - -func TestGoEnvelopeEncryptionManager_RepeatedSmallReadsReuseDecryptedBlockCache(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(4096)) - plaintext := bytes.Repeat([]byte("x"), 5000) // spans exactly two blocks (4096 + 904) - - ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) - counting := &countingReadAtFile{memFile: newMemFile(ciphertext)} - - in, err := mgr.NewDecryptedInputFile(t.Context(), counting, keyMetadata) - require.NoError(t, err) - baseline := counting.reads // the stream header was already read once above - - buf := make([]byte, 1) - for range plaintext { - n, err := in.Read(buf) - require.NoError(t, err) - require.Equal(t, 1, n) - } - - assert.LessOrEqual(t, counting.reads-baseline, 2, "5000 one-byte reads across two blocks must decrypt each block at most once") -} - -func TestGoEnvelopeEncryptionManager_RoundTrip_DefaultBlockSize(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t) // exercises GoEnvelopeDefaultBlockSize (64 KiB), not an explicit small size - plaintext := bytes.Repeat([]byte("iceberg-default-block-size-test-"), 4000) - require.Greater(t, len(plaintext), 64*1024, "plaintext must span multiple default-size blocks") - - ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) - got := decryptAll(t, mgr, ciphertext, keyMetadata) - assert.Equal(t, plaintext, got) -} - -func TestGoEnvelopeEncryptionManager_ReadFrom(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(4)) - fw := &memFileWriter{} - out, err := mgr.NewEncryptedOutputFile(t.Context(), fw, "kek-1") - require.NoError(t, err) - - plaintext := []byte("hello iceberg world") - n, err := out.ReadFrom(bytes.NewReader(plaintext)) - require.NoError(t, err) - assert.EqualValues(t, len(plaintext), n) - require.NoError(t, out.Close()) - - got := decryptAll(t, mgr, fw.Bytes(), out.KeyMetadata()) - assert.Equal(t, plaintext, got) -} - -func TestGoEnvelopeEncryptionManager_Stat(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(16)) - plaintext := []byte("hello iceberg world, this is a stat test") - ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) - - in, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) - require.NoError(t, err) - - info, err := in.Stat() - require.NoError(t, err) - assert.EqualValues(t, len(plaintext), info.Size(), "Stat must report the plaintext length, not the on-disk ciphertext length") - assert.Less(t, info.Size(), int64(len(ciphertext)), "ciphertext must be larger than plaintext due to per-block nonce/tag overhead") -} - -// closeFailWriter is an icebergio.FileWriter stub whose Close always fails -// even though every Write succeeds, simulating a backend where the final -// flush/commit (e.g. an S3 multipart completion) fails independently of the -// preceding writes. -type closeFailWriter struct { - memFileWriter - closes int -} - -func (w *closeFailWriter) Close() error { - w.closes++ - - return errors.New("simulated close failure") -} - -func TestGoEnvelopeEncryptionManager_UnderlyingCloseFailurePoisonsWriter(t *testing.T) { - mgr, _ := newTestGoEnvelopeManager(t) - fw := &closeFailWriter{} - out, err := mgr.NewEncryptedOutputFile(t.Context(), fw, "kek-1") - require.NoError(t, err) - - _, err = out.Write([]byte("hello")) - require.NoError(t, err) - - closeErr := out.Close() - require.Error(t, closeErr) - assert.Nil(t, out.KeyMetadata(), "key metadata must not be finalized when the underlying Close fails") - - // A retried Close must keep reporting the same error, not nil, and must - // not attempt to close the underlying writer again. - closeErr2 := out.Close() - require.Error(t, closeErr2) - assert.ErrorIs(t, closeErr2, closeErr) - assert.Equal(t, 1, fw.closes, "a retried Close must not re-close the underlying writer") - - _, err = out.Write([]byte("late")) - require.Error(t, err) - assert.ErrorIs(t, err, closeErr, "a subsequent Write must report the same sticky error") -} - -// slowReadFile wraps a memFile and sleeps before every ReadAt, simulating a -// remote backend (e.g. S3) with real network latency. It is used to prove -// that concurrent ReadAt calls for distinct blocks actually overlap rather -// than being serialized by readBlock's internal cache lock. -type slowReadFile struct { - *memFile - delay time.Duration -} - -func (f *slowReadFile) ReadAt(p []byte, off int64) (int, error) { - time.Sleep(f.delay) - - return f.memFile.ReadAt(p, off) -} - -func TestGoEnvelopeEncryptionManager_ConcurrentReadAtOverlaps(t *testing.T) { - const ( - numBlocks = 8 - blockSize = 16 - delay = 20 * time.Millisecond - ) - mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(blockSize)) - plaintext := bytes.Repeat([]byte("0123456789abcdef"), numBlocks) - ciphertext, keyMetadata := encryptAll(t, mgr, "kek-1", plaintext) - - slow := &slowReadFile{memFile: newMemFile(ciphertext), delay: delay} - in, err := mgr.NewDecryptedInputFile(t.Context(), slow, keyMetadata) // reads the header; not timed below - require.NoError(t, err) - - start := time.Now() - var g errgroup.Group - for i := range numBlocks { - g.Go(func() error { - buf := make([]byte, blockSize) - _, err := in.ReadAt(buf, int64(i*blockSize)) - - return err - }) - } - require.NoError(t, g.Wait()) - elapsed := time.Since(start) - - // Serial execution (the pre-fix behavior, holding cacheMu across ReadAt) - // would take roughly numBlocks*delay. True concurrency keeps it close to - // a single delay; the threshold below is generous to stay robust on - // slow or loaded CI machines while still failing on a serialized regression. - assert.Less(t, elapsed, numBlocks/2*delay, "concurrent ReadAt calls for distinct blocks must overlap, not serialize") -} - -// TestGoEnvelopeEncryptionManager_BlockNoncesAreUnique pins the one GCM -// property whose loss is catastrophic and silent: reusing a nonce under a -// single key leaks the plaintext XOR of the two blocks and the GHASH -// authentication key, yet a file sealed with a reused (or constant) nonce -// still round-trips and decrypts correctly, so no round-trip, tamper, or -// reorder test would ever catch a regression here. Sealing identical -// plaintext in every block makes any nonce collision directly visible as -// matching ciphertext bytes. -func TestGoEnvelopeEncryptionManager_BlockNoncesAreUnique(t *testing.T) { - const ( - blockSize = 16 - numBlocks = 8 - // AGS1 wire layout (see the GoEnvelopeEncryptionManager doc comment): - // an 8-byte header (4-byte "AGS1" magic + 4-byte little-endian block - // length), then each block as nonce(12) || ciphertext(blockSize) || tag(16). - headerLength = 8 - nonceLength = 12 - blockOverhead = 28 // nonceLength + 16-byte GCM tag - ) - mgr, _ := newTestGoEnvelopeManager(t, encryption.WithBlockSize(blockSize)) - // Every block carries byte-for-byte identical plaintext. - plaintext := bytes.Repeat([]byte("0123456789abcdef"), numBlocks) - - ciphertext, _ := encryptAll(t, mgr, "kek-1", plaintext) - require.Len(t, ciphertext, headerLength+numBlocks*(blockSize+blockOverhead)) - - seenNonceAt := make(map[string]int, numBlocks) - cipherBlocks := make([][]byte, numBlocks) - for i := range numBlocks { - offset := headerLength + i*(blockSize+blockOverhead) - block := ciphertext[offset : offset+blockSize+blockOverhead] - nonce := string(block[:nonceLength]) - - if prev, ok := seenNonceAt[nonce]; ok { - t.Fatalf("block %d reused the nonce from block %d: GCM nonce reuse under a single key leaks the plaintext XOR and the GHASH authentication key", i, prev) - } - seenNonceAt[nonce] = i - cipherBlocks[i] = block - } - - for i := 1; i < numBlocks; i++ { - assert.NotEqual(t, cipherBlocks[0], cipherBlocks[i], "identical plaintext in blocks 0 and %d must not produce identical ciphertext", i) - } -} diff --git a/encryption/standard_manager.go b/encryption/standard_manager.go new file mode 100644 index 000000000..55a75c5c3 --- /dev/null +++ b/encryption/standard_manager.go @@ -0,0 +1,862 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package encryption + +import ( + "context" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/binary" + "errors" + "fmt" + "io" + "io/fs" + "math" + "slices" + "sync" + + icebergio "github.com/apache/iceberg-go/io" +) + +// StandardDefaultDataKeyLength is the default length, in bytes, of the +// per-file data encryption key (DEK). It is 128-bit, matching the default of +// the encryption.data-key-length table property in the Java and Rust +// implementations. +const StandardDefaultDataKeyLength = 16 + +// Sentinel errors returned by [StandardEncryptionManager]. +var ( + // ErrKeyIDRequired is returned by + // [StandardEncryptionManager.NewEncryptedOutputFile] when keyID is empty. + // StandardEncryptionManager always encrypts, so it requires the table's + // encryption key ID; use [PlaintextEncryptionManager] for unencrypted + // tables instead of passing an empty keyID here. + ErrKeyIDRequired = errors.New("encryption: StandardEncryptionManager requires a non-empty keyID") + + // ErrKeyMetadataRequired is returned by + // [StandardEncryptionManager.NewDecryptedInputFile] when keyMetadata is + // empty. StandardEncryptionManager always decrypts, so it requires the + // per-file key metadata produced by + // [StandardEncryptionManager.NewEncryptedOutputFile]. + ErrKeyMetadataRequired = errors.New("encryption: StandardEncryptionManager requires non-empty key metadata") + + // ErrUnsupportedKeyMetadataVersion is returned when key metadata starts + // with a version byte other than the one this package understands. + ErrUnsupportedKeyMetadataVersion = errors.New("encryption: unsupported key metadata version") + + // ErrInvalidStreamHeader is returned when the AES GCM Stream header (the + // "AGS1" magic and little-endian plain block length at the start of an + // encrypted file, per the Iceberg AES GCM Stream spec) is missing, + // truncated, or declares a block length other than the fixed 1 MiB. + ErrInvalidStreamHeader = errors.New("encryption: invalid AES GCM Stream header") + + // ErrInvalidKeyMetadata is returned by + // [StandardEncryptionManager.NewDecryptedInputFile] when key metadata is + // not a well-formed StandardKeyMetadata record or fails basic sanity + // checks (e.g. a truncated record, a missing or wrong-sized AAD prefix, or + // a missing or impossible file length). Key metadata is untrusted input on + // a crypto read path, so it is validated rather than trusted blindly. + ErrInvalidKeyMetadata = errors.New("encryption: invalid key metadata") + + // ErrOutputFileClosed is returned by [standardOutputFile.Write] when + // called after a successful Close. It wraps [fs.ErrClosed] so callers can + // test with errors.Is(err, fs.ErrClosed). A Write after a failed flush or + // a failed Close returns that original error instead. + ErrOutputFileClosed = fmt.Errorf("encryption: write to closed StandardEncryptionManager output file: %w", fs.ErrClosed) + + // ErrBlockTruncated is returned by [standardInputFile.ReadAt] when the + // underlying storage returns fewer ciphertext bytes for a block than its + // recorded length requires, indicating the file was truncated at rest. + // This is distinct from [ErrCiphertextTooShort], which [KeyManagementClient] + // implementations use for a too-short wrapped key or KMS-encrypted + // payload; keeping them separate lets a caller tell a malformed KMS blob + // apart from a short block read. + ErrBlockTruncated = errors.New("encryption: block truncated: read fewer ciphertext bytes than expected") +) + +// Constants describing the Iceberg AES GCM Stream ("AGS1") wire format used +// for the ciphertext produced by [StandardEncryptionManager]. See +// https://iceberg.apache.org/gcm-stream-spec/ for the full specification. +const ( + // gcmStreamMagic identifies an AES GCM Stream version 1 file. + gcmStreamMagic = "AGS1" + + // gcmStreamHeaderLength is the length, in bytes, of the magic plus the + // 4-byte little-endian plain block length written at the start of every + // file. + gcmStreamHeaderLength = 8 + + // gcmStreamPlainBlockSize is the plaintext block size. It is fixed: + // Java's Ciphers.PLAIN_BLOCK_SIZE and iceberg-rust's PLAIN_BLOCK_SIZE are + // both 1 MiB, and Java's AesGcmInputStream rejects any other value in the + // header, so a stream with a different block size is unreadable there. + gcmStreamPlainBlockSize = 1024 * 1024 + + // gcmStreamNonceLength is the length, in bytes, of the random AES-GCM + // nonce stored at the start of every cipher block. + gcmStreamNonceLength = 12 + + // gcmStreamTagLength is the length, in bytes, of the AES-GCM + // authentication tag appended to every cipher block's ciphertext. + gcmStreamTagLength = 16 + + // gcmStreamBlockOverhead is the number of ciphertext bytes added to + // each block beyond its plaintext length (nonce + tag). + gcmStreamBlockOverhead = gcmStreamNonceLength + gcmStreamTagLength + + // gcmStreamCipherBlockSize is the on-disk length of a full block. + gcmStreamCipherBlockSize = gcmStreamPlainBlockSize + gcmStreamBlockOverhead + + // gcmStreamMinLength is the length of the smallest valid stream: the + // header followed by one block holding no plaintext. Java and iceberg-rust + // always write block 0, even for an empty file, and iceberg-rust rejects + // anything shorter. + gcmStreamMinLength = gcmStreamHeaderLength + gcmStreamBlockOverhead + + // gcmStreamAADPrefixLength is the length, in bytes, of the random + // per-file AAD prefix. It matches Java's ENCRYPTION_AAD_LENGTH_DEFAULT. + gcmStreamAADPrefixLength = 16 +) + +// standardKeyMetadataVersion is the leading version byte of Iceberg's +// StandardKeyMetadata encoding. +const standardKeyMetadataVersion = 1 + +// standardKeyMetadata is the decoded form of Iceberg's StandardKeyMetadata: +// a version byte followed by an Avro binary record +// +// {encryption_key: bytes, aad_prefix: ["null", bytes], file_length: ["null", long]} +// +// This is the key_metadata encoding Java's StandardEncryptionManager, +// iceberg-rust and pyiceberg read and write. It holds the raw DEK, so it is a +// secret and must be protected by whatever stores it. +type standardKeyMetadata struct { + encryptionKey []byte + + // aadPrefix is combined with each block's little-endian index to form + // the AES GCM Stream additional authenticated data, binding every + // ciphertext block to this file and to its position so that blocks + // cannot be silently reordered, replayed from another file, or spliced + // in from elsewhere in the same file. It is not secret. It is nil when + // the record omits it. + aadPrefix []byte + + // fileLength is the length of the encrypted stream, header included. + // Per the AES GCM Stream spec's "File length" note, a reader must take + // the length from a trusted source rather than the underlying storage's + // reported size, since storage size alone cannot distinguish a genuinely + // short file from one truncated by an attacker who does not also control + // this metadata. hasFileLength is false when the record omits it. + fileLength int64 + hasFileLength bool +} + +// encodeStandardKeyMetadata encodes the version byte and Avro record for a +// file, with both optional fields present. +func encodeStandardKeyMetadata(encryptionKey, aadPrefix []byte, fileLength int64) EncryptionKeyMetadata { + b := []byte{standardKeyMetadataVersion} + b = binary.AppendVarint(b, int64(len(encryptionKey))) + b = append(b, encryptionKey...) + b = binary.AppendVarint(b, 1) // union branch 1 (bytes): aad_prefix is present + b = binary.AppendVarint(b, int64(len(aadPrefix))) + b = append(b, aadPrefix...) + b = binary.AppendVarint(b, 1) // union branch 1 (long): file_length is present + b = binary.AppendVarint(b, fileLength) + + return b +} + +// decodeStandardKeyMetadata decodes the version byte and Avro record written +// by [encodeStandardKeyMetadata] or by Java's StandardKeyMetadata.buffer(). +func decodeStandardKeyMetadata(b []byte) (standardKeyMetadata, error) { + if b[0] != standardKeyMetadataVersion { + return standardKeyMetadata{}, fmt.Errorf("%w: %d", ErrUnsupportedKeyMetadataVersion, b[0]) + } + + r := avroReader{buf: b[1:]} + var k standardKeyMetadata + k.encryptionKey = r.bytes() + if r.present() { + k.aadPrefix = r.bytes() + } + if r.present() { + k.fileLength = r.long() + k.hasFileLength = true + } + if r.err == nil && len(r.buf) != 0 { + r.err = fmt.Errorf("%d unexpected trailing bytes", len(r.buf)) + } + if r.err != nil { + return standardKeyMetadata{}, fmt.Errorf("%w: %w", ErrInvalidKeyMetadata, r.err) + } + + return k, nil +} + +// avroReader decodes the few Avro binary primitives StandardKeyMetadata uses, +// remembering the first error. Avro longs are zig-zag varints, which is what +// encoding/binary's Varint implements. +type avroReader struct { + buf []byte + err error +} + +func (r *avroReader) long() int64 { + if r.err != nil { + return 0 + } + v, n := binary.Varint(r.buf) + if n <= 0 { + r.err = errors.New("truncated or oversized varint") + + return 0 + } + r.buf = r.buf[n:] + + return v +} + +func (r *avroReader) bytes() []byte { + n := r.long() + if r.err != nil { + return nil + } + if n < 0 || n > int64(len(r.buf)) { + r.err = fmt.Errorf("bytes length %d out of range (%d bytes remain)", n, len(r.buf)) + + return nil + } + out := slices.Clone(r.buf[:n]) + r.buf = r.buf[n:] + + return out +} + +// present reads the branch index of a ["null", T] union and reports whether +// the value is present (branch 1) rather than null (branch 0). +func (r *avroReader) present() bool { + branch := r.long() + if r.err != nil { + return false + } + if branch != 0 && branch != 1 { + r.err = fmt.Errorf("unexpected union branch %d", branch) + + return false + } + + return branch == 1 +} + +// plaintextLengthFromStreamLength derives the plaintext length from the +// length of the encrypted stream, the way Java's +// AesGcmInputStream.calculatePlaintextLength does. It accepts the 36-byte +// form of an empty file (header plus one empty block) and rejects lengths no +// writer could have produced. +func plaintextLengthFromStreamLength(streamLength int64) (int64, error) { + if streamLength < gcmStreamMinLength { + return 0, fmt.Errorf("%w: file-length %d is shorter than the %d-byte minimum stream", ErrInvalidKeyMetadata, streamLength, gcmStreamMinLength) + } + + body := streamLength - gcmStreamHeaderLength + plainLength := body / gcmStreamCipherBlockSize * gcmStreamPlainBlockSize + if lastBlock := body % gcmStreamCipherBlockSize; lastBlock != 0 { + if lastBlock < gcmStreamBlockOverhead { + return 0, fmt.Errorf("%w: file-length %d ends in a %d-byte block, shorter than its %d-byte nonce and tag", ErrInvalidKeyMetadata, streamLength, lastBlock, gcmStreamBlockOverhead) + } + plainLength += lastBlock - gcmStreamBlockOverhead + } + + return plainLength, nil +} + +// StandardEncryptionManager is a generic, format-agnostic [EncryptionManager] +// for arbitrary files (e.g. manifests, manifest lists, Puffin statistics). It +// reads and writes the same bytes as Java's StandardEncryptionManager, +// iceberg-rust and pyiceberg: an Iceberg AES GCM Stream ("AGS1") with fixed +// 1 MiB plaintext blocks, and key metadata in Iceberg's StandardKeyMetadata +// encoding. +// +// Each file is encrypted under a fresh random data encryption key (DEK), +// generated locally with no KMS call. Each block carries its own random nonce +// and is authenticated with an AAD that binds it to the file and to its +// position. This bounds memory usage and supports random access +// (Seek/ReadAt) on the decrypted file without buffering or decrypting more +// than the requested blocks. +// +// The key metadata returned by Close contains the raw DEK, so it is a secret: +// persist it only where something protects it, such as a manifest or manifest +// list that is itself encrypted under a key encryption key. Wrapping the DEK +// that way is not done by this type. +// +// StandardEncryptionManager always encrypts and always decrypts: it fails +// closed, returning [ErrKeyIDRequired] or [ErrKeyMetadataRequired] rather +// than silently falling back to plaintext. Use [PlaintextEncryptionManager] +// for tables or files that are not encrypted. +type StandardEncryptionManager struct { + dataKeyLength int +} + +var _ EncryptionManager = (*StandardEncryptionManager)(nil) + +// StandardManagerOption configures a [StandardEncryptionManager] created by +// [NewStandardEncryptionManager]. +type StandardManagerOption func(*StandardEncryptionManager) + +// WithStandardDataKeyLength sets the length, in bytes, of the per-file data +// encryption key. It corresponds to the encryption.data-key-length table +// property. Valid AES key lengths are 16, 24, or 32 bytes; the default is +// [StandardDefaultDataKeyLength]. +func WithStandardDataKeyLength(length int) StandardManagerOption { + return func(m *StandardEncryptionManager) { m.dataKeyLength = length } +} + +// NewStandardEncryptionManager creates a [StandardEncryptionManager]. +func NewStandardEncryptionManager(opts ...StandardManagerOption) *StandardEncryptionManager { + m := &StandardEncryptionManager{dataKeyLength: StandardDefaultDataKeyLength} + for _, opt := range opts { + opt(m) + } + + return m +} + +// NewEncryptedOutputFile creates a new AES-GCM block-encrypted output file. +// keyID is the table's encryption key ID and must be non-empty; otherwise +// [ErrKeyIDRequired] is returned. It is not recorded in the file's key +// metadata. +func (m *StandardEncryptionManager) NewEncryptedOutputFile(_ context.Context, writer icebergio.FileWriter, keyID string) (EncryptedOutputFile, error) { + if keyID == "" { + return nil, ErrKeyIDRequired + } + switch m.dataKeyLength { + case 16, 24, 32: + default: + return nil, fmt.Errorf("%w: DEK length must be 16, 24, or 32 bytes; got %d", ErrInvalidKeyLength, m.dataKeyLength) + } + + // A fresh DEK per file keeps the number of random-nonce encryptions under + // any one key within the 2^32 limit NIST SP 800-38D sets for random + // 96-bit nonces: a file holds at most 2^32-1 blocks. + dek := make([]byte, m.dataKeyLength) + if _, err := io.ReadFull(rand.Reader, dek); err != nil { + return nil, fmt.Errorf("encryption: failed to generate DEK: %w", err) + } + + aead, err := newStandardAEAD(dek) + if err != nil { + return nil, err + } + + aadPrefix := make([]byte, gcmStreamAADPrefixLength) + if _, err := io.ReadFull(rand.Reader, aadPrefix); err != nil { + return nil, fmt.Errorf("encryption: failed to generate AAD prefix: %w", err) + } + + // Write the AES GCM Stream header (magic + little-endian block length) + // up front, before any ciphertext blocks, per the format spec. + header := make([]byte, gcmStreamHeaderLength) + copy(header, gcmStreamMagic) + binary.LittleEndian.PutUint32(header[len(gcmStreamMagic):], gcmStreamPlainBlockSize) + if _, err := writer.Write(header); err != nil { + return nil, fmt.Errorf("encryption: failed to write stream header: %w", err) + } + + return &standardOutputFile{ + FileWriter: writer, + aead: aead, + dek: dek, + aadPrefix: aadPrefix, + encryptedLength: gcmStreamHeaderLength, + }, nil +} + +// NewDecryptedInputFile wraps file for transparent block-level AES-GCM +// decryption. keyMetadata must be the non-empty StandardKeyMetadata blob +// produced by [StandardEncryptionManager.NewEncryptedOutputFile] or by another +// Iceberg implementation; otherwise [ErrKeyMetadataRequired] is returned. +// +// The plaintext length is derived from the file length recorded in key +// metadata, never from the underlying file's size. Key metadata without a file +// length is rejected, because there is no other trusted source for it here. +// Key metadata without an AAD prefix, which the encoding permits, is rejected +// too: the prefix is what binds a block to its file. +func (m *StandardEncryptionManager) NewDecryptedInputFile(_ context.Context, file icebergio.File, keyMetadata EncryptionKeyMetadata) (EncryptedInputFile, error) { + if len(keyMetadata) == 0 { + return nil, ErrKeyMetadataRequired + } + + meta, err := decodeStandardKeyMetadata(keyMetadata) + if err != nil { + return nil, err + } + // Java, iceberg-rust and this writer all emit 16-byte prefixes. Bounding + // it also keeps the per-block AAD allocation in readBlock small. + if len(meta.aadPrefix) != gcmStreamAADPrefixLength { + return nil, fmt.Errorf("%w: aad-prefix must be exactly %d bytes, got %d", ErrInvalidKeyMetadata, gcmStreamAADPrefixLength, len(meta.aadPrefix)) + } + if !meta.hasFileLength { + return nil, fmt.Errorf("%w: file-length is required; the plaintext length is not taken from the underlying file's size", ErrInvalidKeyMetadata) + } + plaintextLength, err := plaintextLengthFromStreamLength(meta.fileLength) + if err != nil { + return nil, err + } + + aead, err := newStandardAEAD(meta.encryptionKey) + if err != nil { + return nil, err + } + + if err := validateGCMStreamHeader(file); err != nil { + return nil, err + } + + return &standardInputFile{ + underlying: file, + aead: aead, + aadPrefix: meta.aadPrefix, + plaintextLength: plaintextLength, + keyMetadata: keyMetadata, + }, nil +} + +func newStandardAEAD(key []byte) (cipher.AEAD, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrInvalidKeyLength, err) + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, fmt.Errorf("encryption: failed to create GCM: %w", err) + } + + return gcm, nil +} + +// validateGCMStreamHeader reads the AES GCM Stream header at the start of +// file and checks that it is the "AGS1" magic followed by the fixed 1 MiB +// plain block length. The header is unauthenticated storage data, so it is +// only validated here: the block size comes from [gcmStreamPlainBlockSize], +// never from the header. +func validateGCMStreamHeader(file icebergio.File) error { + header := make([]byte, gcmStreamHeaderLength) + n, err := file.ReadAt(header, 0) + if err != nil && !errors.Is(err, io.EOF) { + return fmt.Errorf("encryption: failed to read stream header: %w", err) + } + if n != gcmStreamHeaderLength { + return fmt.Errorf("%w: expected %d header bytes, got %d", ErrInvalidStreamHeader, gcmStreamHeaderLength, n) + } + if string(header[:len(gcmStreamMagic)]) != gcmStreamMagic { + return fmt.Errorf("%w: missing %q magic", ErrInvalidStreamHeader, gcmStreamMagic) + } + + if blockSize := binary.LittleEndian.Uint32(header[len(gcmStreamMagic):]); blockSize != gcmStreamPlainBlockSize { + return fmt.Errorf("%w: block length %d, want %d", ErrInvalidStreamHeader, blockSize, gcmStreamPlainBlockSize) + } + + return nil +} + +// gcmStreamBlockAAD derives the AES-GCM additional authenticated data for +// blockIndex: the per-file AAD prefix followed by the 4-byte little-endian +// block index, per the Iceberg AES GCM Stream spec. GCM authenticates the AAD +// along with the ciphertext, so a block only verifies at its own index in its +// own file: reordered, duplicated, or spliced-in blocks fail authentication. +func gcmStreamBlockAAD(prefix []byte, blockIndex uint32) []byte { + aad := make([]byte, len(prefix)+4) + copy(aad, prefix) + binary.LittleEndian.PutUint32(aad[len(prefix):], blockIndex) + + return aad +} + +// standardOutputFile is an [EncryptedOutputFile] that seals fixed-size +// plaintext blocks with AES-GCM as they are written, using the Iceberg AES +// GCM Stream ("AGS1") wire format. +type standardOutputFile struct { + icebergio.FileWriter + + aead cipher.AEAD + dek []byte + aadPrefix []byte + + buf []byte + blockIndex uint32 + closed bool + err error + + // encryptedLength is the number of bytes written to the underlying + // writer so far, header included. It becomes file_length in the key + // metadata, and only advances once a block has been written. + encryptedLength int64 + + // underlyingClosed records that FileWriter.Close has been called, so the + // underlying writer is closed at most once. + underlyingClosed bool + + keyMetadata EncryptionKeyMetadata +} + +var _ EncryptedOutputFile = (*standardOutputFile)(nil) + +// closeUnderlyingIgnoringError closes the underlying writer at most once. +// The error is ignored: the caller is already reporting a more specific +// failure (a flush error), and this is best-effort cleanup so a poisoned +// writer never leaks its underlying file descriptor or connection. +func (f *standardOutputFile) closeUnderlyingIgnoringError() { + if f.underlyingClosed { + return + } + f.underlyingClosed = true + _ = f.FileWriter.Close() +} + +func (f *standardOutputFile) Write(p []byte) (int, error) { + if f.err != nil { + return 0, f.err + } + if f.closed { + return 0, ErrOutputFileClosed + } + + total := len(p) + consumed := 0 // bytes of p appended into f.buf so far in this call + accepted := 0 // bytes of p known to be durably flushed; reported on failure + for len(p) > 0 { + space := gcmStreamPlainBlockSize - len(f.buf) + n := min(space, len(p)) + f.buf = append(f.buf, p[:n]...) + p = p[n:] + consumed += n + if len(f.buf) == gcmStreamPlainBlockSize { + if err := f.flushBlock(); err != nil { + f.err = err + f.closeUnderlyingIgnoringError() + + return accepted, err + } + accepted = consumed + } + } + + return total, nil +} + +// flushBlock seals and writes the currently buffered plaintext block using a +// fresh random nonce, per the Iceberg AES GCM Stream format. +// f.encryptedLength only advances once the ciphertext has reached the +// underlying writer, so file_length never counts a block that failed to write. +func (f *standardOutputFile) flushBlock() error { + if f.blockIndex == math.MaxUint32 { + return errors.New("encryption: cannot write block: exceeded maximum block count") + } + + nonce := make([]byte, gcmStreamNonceLength) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return fmt.Errorf("encryption: failed to generate block nonce: %w", err) + } + + aad := gcmStreamBlockAAD(f.aadPrefix, f.blockIndex) + sealed := f.aead.Seal(nonce, nonce, f.buf, aad) + if _, err := f.FileWriter.Write(sealed); err != nil { + return fmt.Errorf("encryption: failed to write encrypted block: %w", err) + } + f.encryptedLength += int64(len(sealed)) + f.blockIndex++ + f.buf = f.buf[:0] + + return nil +} + +// ReadFrom copies from r, encrypting as data is written, satisfying +// io.ReaderFrom (required by [icebergio.FileWriter]). +func (f *standardOutputFile) ReadFrom(r io.Reader) (int64, error) { + buf := make([]byte, gcmStreamPlainBlockSize) + var total int64 + for { + n, err := r.Read(buf) + if n > 0 { + wn, werr := f.Write(buf[:n]) + total += int64(wn) + if werr != nil { + return total, werr + } + } + if err == io.EOF { + break + } + if err != nil { + return total, err + } + } + + return total, nil +} + +// Close flushes any buffered partial block, writes an empty block 0 if the +// file is empty, and finalizes the key metadata. closed is set, and +// keyMetadata published, only once everything, including the underlying +// Close, has succeeded. A failed Close poisons the writer (via f.err): a retry +// returns the same error, and no key metadata is exposed for an output that +// never finished. +func (f *standardOutputFile) Close() error { + if f.err != nil { + return f.err + } + if f.closed { + return nil + } + + // Java and iceberg-rust write block 0 even when the file is empty, so an + // empty file is 36 bytes (a header plus one 28-byte block), and + // iceberg-rust rejects anything shorter. Match them. + if len(f.buf) > 0 || f.blockIndex == 0 { + if err := f.flushBlock(); err != nil { + f.err = err + f.closeUnderlyingIgnoringError() + + return err + } + } + + if err := f.FileWriter.Close(); err != nil { + f.underlyingClosed = true + f.err = fmt.Errorf("encryption: failed to close underlying writer: %w", err) + + return f.err + } + f.underlyingClosed = true + + f.keyMetadata = encodeStandardKeyMetadata(f.dek, f.aadPrefix, f.encryptedLength) + f.closed = true + + return nil +} + +// KeyMetadata returns the finalized per-file key metadata. It is only +// populated after Close has succeeded. It contains the raw DEK. +func (f *standardOutputFile) KeyMetadata() EncryptionKeyMetadata { return f.keyMetadata } + +// standardInputFile is an [EncryptedInputFile] that decrypts fixed-size +// AES-GCM blocks on demand, supporting random access via ReadAt/Seek. +// +// The plaintext length comes from the trusted file length in key metadata, so +// ciphertext appended after the last block is ignored, and truncating the +// ciphertext while also editing that metadata cannot be detected. Both follow +// from the AES GCM Stream spec's "File length" rule. +// +// ReadAt is safe, and intended, for concurrent use, matching the io.ReaderAt +// contract: readBlock only holds cacheMu around the cache check/publish, not +// across the underlying ReadAt or AEAD decryption, so concurrent calls for +// distinct blocks run in parallel. A lost cache race (two goroutines +// decrypting the same block) is harmless, just wasted work. Read and Seek +// mutate the shared cursor (pos) and are not concurrent-safe; do not call them +// from multiple goroutines on the same instance. +// +// The single-entry cache retains one decrypted block (1 MiB) for the lifetime +// of the input file, not just the lifetime of a single read call. +type standardInputFile struct { + underlying icebergio.File + aead cipher.AEAD + aadPrefix []byte + plaintextLength int64 + keyMetadata EncryptionKeyMetadata + + pos int64 + + // cacheMu guards cacheIdx/cacheBlock/cacheValid below. It is only held + // around the cache check and the cache publish, never across the + // underlying ReadAt or AEAD decryption in readBlock. + cacheMu sync.Mutex + cacheIdx int64 + cacheBlock []byte + cacheValid bool +} + +var _ EncryptedInputFile = (*standardInputFile)(nil) + +func (f *standardInputFile) numBlocks() int64 { + if f.plaintextLength == 0 { + return 0 + } + + // plaintextLength is derived from a file length in key metadata, so it + // can be close to math.MaxInt64; 1 + (n-1)/size avoids the overflow that + // (n + size - 1)/size would hit there. + return 1 + (f.plaintextLength-1)/gcmStreamPlainBlockSize +} + +// blockPlainLen returns the plaintext length of block idx, given the total +// number of blocks. +func (f *standardInputFile) blockPlainLen(idx, numBlocks int64) int64 { + if idx == numBlocks-1 { + return f.plaintextLength - idx*gcmStreamPlainBlockSize + } + + return gcmStreamPlainBlockSize +} + +// readBlock decrypts block idx, or returns it from the single-entry cache +// if it was the most recently decrypted block. It checks the byte count +// returned by ReadAt: a short read is reported as [ErrBlockTruncated] +// (truncated storage), since zero-padding it into the AEAD would surface as a +// misleading [ErrAuthenticationFailed]. +// +// cacheMu is only held around the cache check and the cache publish at the +// end, never across the underlying ReadAt or aead.Open below: holding it +// across I/O would serialize all concurrent readers, including remote +// backends. A lost race between two goroutines decrypting the same block is +// harmless (redundant work, not a correctness issue). +func (f *standardInputFile) readBlock(idx int64) ([]byte, error) { + f.cacheMu.Lock() + if f.cacheValid && f.cacheIdx == idx { + block := f.cacheBlock + f.cacheMu.Unlock() + + return block, nil + } + f.cacheMu.Unlock() + + numBlocks := f.numBlocks() + if idx < 0 || idx >= numBlocks { + return nil, fmt.Errorf("%w: block index %d out of range [0, %d)", ErrInvalidKeyMetadata, idx, numBlocks) + } + // The AAD carries the block index as 32 bits, and a file-length near + // math.MaxInt64 would otherwise imply more blocks than that. + if idx > math.MaxUint32 { + return nil, fmt.Errorf("%w: block index %d exceeds the maximum supported block count", ErrInvalidKeyMetadata, idx) + } + + // idx < numBlocks and plaintextLength was derived from an int64 stream + // length, so this offset is no larger than that length and cannot + // overflow. + offset := gcmStreamHeaderLength + idx*gcmStreamCipherBlockSize + + wantLen := f.blockPlainLen(idx, numBlocks) + gcmStreamBlockOverhead + ciphertext := make([]byte, wantLen) + n, err := f.underlying.ReadAt(ciphertext, offset) + if err != nil && !errors.Is(err, io.EOF) { + return nil, fmt.Errorf("encryption: failed to read block %d: %w", idx, err) + } + if int64(n) != wantLen { + return nil, fmt.Errorf("%w: block %d: read %d of %d expected ciphertext bytes", ErrBlockTruncated, idx, n, wantLen) + } + + nonce, sealed := ciphertext[:gcmStreamNonceLength], ciphertext[gcmStreamNonceLength:] + aad := gcmStreamBlockAAD(f.aadPrefix, uint32(idx)) + + plaintext, err := f.aead.Open(nil, nonce, sealed, aad) + if err != nil { + return nil, fmt.Errorf("%w: block %d: %w", ErrAuthenticationFailed, idx, err) + } + + f.cacheMu.Lock() + f.cacheIdx = idx + f.cacheBlock = plaintext + f.cacheValid = true + f.cacheMu.Unlock() + + return plaintext, nil +} + +func (f *standardInputFile) ReadAt(p []byte, off int64) (int, error) { + if len(p) == 0 { + return 0, nil + } + if off < 0 { + return 0, errors.New("encryption: ReadAt: negative offset") + } + if off >= f.plaintextLength { + return 0, io.EOF + } + + var read int + for read < len(p) { + curOff := off + int64(read) + if curOff >= f.plaintextLength { + break + } + idx := curOff / gcmStreamPlainBlockSize + block, err := f.readBlock(idx) + if err != nil { + return read, err + } + inBlockOff := curOff - idx*gcmStreamPlainBlockSize + read += copy(p[read:], block[inBlockOff:]) + } + + var err error + if read < len(p) { + err = io.EOF + } + + return read, err +} + +func (f *standardInputFile) Read(p []byte) (int, error) { + n, err := f.ReadAt(p, f.pos) + f.pos += int64(n) + + return n, err +} + +func (f *standardInputFile) Seek(offset int64, whence int) (int64, error) { + var newPos int64 + switch whence { + case io.SeekStart: + newPos = offset + case io.SeekCurrent: + newPos = f.pos + offset + case io.SeekEnd: + newPos = f.plaintextLength + offset + default: + return 0, fmt.Errorf("encryption: Seek: invalid whence %d", whence) + } + if newPos < 0 { + return 0, errors.New("encryption: Seek: negative position") + } + f.pos = newPos + + return newPos, nil +} + +func (f *standardInputFile) Close() error { return f.underlying.Close() } + +func (f *standardInputFile) Stat() (fs.FileInfo, error) { + info, err := f.underlying.Stat() + if err != nil { + return nil, err + } + + return standardFileInfo{FileInfo: info, size: f.plaintextLength}, nil +} + +// KeyMetadata returns the key metadata this file was decrypted with. +func (f *standardInputFile) KeyMetadata() EncryptionKeyMetadata { return f.keyMetadata } + +// standardFileInfo overrides Size() to report the plaintext length rather +// than the (larger) on-disk ciphertext length. +type standardFileInfo struct { + fs.FileInfo + size int64 +} + +func (i standardFileInfo) Size() int64 { return i.size } diff --git a/encryption/standard_manager_test.go b/encryption/standard_manager_test.go new file mode 100644 index 000000000..8b8266271 --- /dev/null +++ b/encryption/standard_manager_test.go @@ -0,0 +1,869 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package encryption_test + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "math" + mrand "math/rand/v2" + "slices" + "testing" + "time" + + "github.com/apache/iceberg-go/encryption" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" +) + +// AES GCM Stream layout constants, restated here from the spec rather than +// imported, so the tests do not share them with the code under test. +const ( + plainBlockSize = 1 << 20 // the one block size Java and iceberg-rust accept + blockOverhead = 12 + 16 // nonce + GCM tag + headerLength = 4 + 4 // "AGS1" magic + little-endian block length + testKeyID = "table-key" +) + +func newTestManager(opts ...encryption.StandardManagerOption) *encryption.StandardEncryptionManager { + return encryption.NewStandardEncryptionManager(opts...) +} + +// testData returns n deterministic pseudo-random bytes, so that blocks differ +// from one another and a swapped or misplaced block is detectable. +func testData(n int) []byte { + r := mrand.New(mrand.NewPCG(1, 2)) + b := make([]byte, n) + for i := range b { + b[i] = byte(r.Uint32()) + } + + return b +} + +func encryptAll(t *testing.T, mgr *encryption.StandardEncryptionManager, plaintext []byte) (ciphertext []byte, keyMetadata encryption.EncryptionKeyMetadata) { + t.Helper() + fw := &memFileWriter{} + out, err := mgr.NewEncryptedOutputFile(t.Context(), fw, testKeyID) + require.NoError(t, err) + _, err = out.Write(plaintext) + require.NoError(t, err) + require.NoError(t, out.Close()) + + return fw.Bytes(), out.KeyMetadata() +} + +func decryptAll(t *testing.T, mgr *encryption.StandardEncryptionManager, ciphertext []byte, keyMetadata encryption.EncryptionKeyMetadata) []byte { + t.Helper() + in, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) + require.NoError(t, err) + data, err := io.ReadAll(in) + require.NoError(t, err) + + return data +} + +// expectedStreamLength is the length of the encrypted stream for n plaintext +// bytes: the header, then ceil(n/block) blocks of nonce, ciphertext and tag, +// and always at least block 0 (an empty file is a header plus one empty block). +func expectedStreamLength(n int) int { + blocks := max(1, (n+plainBlockSize-1)/plainBlockSize) + + return headerLength + n + blocks*blockOverhead +} + +// --------------------------------------------------------------------------- +// Independent AGS1 / StandardKeyMetadata codecs, built only from the standard +// library and the spec text so that they share no code, and no bugs, with the +// package under test. +// --------------------------------------------------------------------------- + +func avroLong(v int64) []byte { return binary.AppendVarint(nil, v) } + +func avroBytes(b []byte) []byte { return append(avroLong(int64(len(b))), b...) } + +// avroKeyMetadata hand-encodes StandardKeyMetadata with both optional fields +// present: version byte 1, then the Avro record +// {encryption_key: bytes, aad_prefix: ["null", bytes], file_length: ["null", long]}. +func avroKeyMetadata(dek, aadPrefix []byte, fileLength int64) []byte { + return slices.Concat([]byte{1}, avroBytes(dek), avroLong(1), avroBytes(aadPrefix), avroLong(1), avroLong(fileLength)) +} + +type parsedKeyMetadata struct { + dek, aadPrefix []byte + fileLength int64 +} + +func parseKeyMetadata(t *testing.T, km []byte) parsedKeyMetadata { + t.Helper() + require.NotEmpty(t, km) + require.Equal(t, byte(1), km[0], "version byte") + rest := km[1:] + + readLong := func() int64 { + v, n := binary.Varint(rest) + require.Positive(t, n, "malformed varint") + rest = rest[n:] + + return v + } + readBytes := func() []byte { + n := readLong() + require.GreaterOrEqual(t, n, int64(0)) + require.LessOrEqual(t, n, int64(len(rest))) + b := rest[:n] + rest = rest[n:] + + return b + } + + var p parsedKeyMetadata + p.dek = readBytes() + require.EqualValues(t, 1, readLong(), "aad_prefix must be present (union branch 1)") + p.aadPrefix = readBytes() + require.EqualValues(t, 1, readLong(), "file_length must be present (union branch 1)") + p.fileLength = readLong() + require.Empty(t, rest, "trailing bytes after the record") + + return p +} + +func newGCM(t *testing.T, dek []byte) cipher.AEAD { + t.Helper() + block, err := aes.NewCipher(dek) + require.NoError(t, err) + gcm, err := cipher.NewGCM(block) + require.NoError(t, err) + + return gcm +} + +// blockAAD builds the spec's AAD, aadPrefix || little-endian uint32 block index. +func blockAAD(aadPrefix []byte, index uint32) []byte { + return binary.LittleEndian.AppendUint32(slices.Clone(aadPrefix), index) +} + +// decodeAGS1 reads an AGS1 stream the way the spec describes. +func decodeAGS1(t *testing.T, stream, dek, aadPrefix []byte) []byte { + t.Helper() + require.GreaterOrEqual(t, len(stream), headerLength) + require.Equal(t, "AGS1", string(stream[:4]), "magic") + require.EqualValues(t, plainBlockSize, binary.LittleEndian.Uint32(stream[4:8]), "plain block length") + + gcm := newGCM(t, dek) + var plaintext []byte + body := stream[headerLength:] + for index := uint32(0); len(body) > 0; index++ { + block := body[:min(len(body), plainBlockSize+blockOverhead)] + body = body[len(block):] + require.GreaterOrEqual(t, len(block), blockOverhead, "block %d", index) + + nonce, sealed := block[:12], block[12:] + opened, err := gcm.Open(nil, nonce, sealed, blockAAD(aadPrefix, index)) + require.NoError(t, err, "block %d", index) + plaintext = append(plaintext, opened...) + } + + return plaintext +} + +// encodeAGS1 writes an AGS1 stream the way Java does: block 0 is always +// written, even for empty plaintext. The real writer draws a random nonce per +// block; this one derives each block's nonce from its index (so it still +// differs from block to block) to make the output reproducible. It is only +// fit for tests. +func encodeAGS1(t *testing.T, plaintext, dek, aadPrefix []byte) []byte { + t.Helper() + gcm := newGCM(t, dek) + + stream := append([]byte("AGS1"), binary.LittleEndian.AppendUint32(nil, plainBlockSize)...) + for index := uint32(0); index == 0 || len(plaintext) > 0; index++ { + n := min(len(plaintext), plainBlockSize) + nonce := bytes.Repeat([]byte{byte(index) + 1}, 12) + stream = gcm.Seal(append(stream, nonce...), nonce, plaintext[:n], blockAAD(aadPrefix, index)) + plaintext = plaintext[n:] + } + + return stream +} + +// --------------------------------------------------------------------------- +// Round trips and layout +// --------------------------------------------------------------------------- + +func TestStandardEncryptionManager_RoundTrip(t *testing.T) { + sizes := []int{ + 0, + 1, + 13, + plainBlockSize - 1, + plainBlockSize, + plainBlockSize + 1, + 2 * plainBlockSize, + 2*plainBlockSize + 12345, + } + for _, size := range sizes { + t.Run(fmt.Sprintf("%d bytes", size), func(t *testing.T) { + mgr := newTestManager() + plaintext := testData(size) + + ciphertext, keyMetadata := encryptAll(t, mgr, plaintext) + assert.Len(t, ciphertext, expectedStreamLength(size), "no empty trailing block, and block 0 even when empty") + assert.EqualValues(t, len(ciphertext), parseKeyMetadata(t, keyMetadata).fileLength, "file_length is the encrypted length") + + got := decryptAll(t, mgr, ciphertext, keyMetadata) + assert.True(t, bytes.Equal(plaintext, got), "decrypted bytes differ from the plaintext") + }) + } +} + +// Java writes block 0 even for an empty file, and iceberg-rust rejects any +// stream shorter than the resulting 8 + 28 = 36 bytes. +func TestStandardEncryptionManager_EmptyFileUsesJavaLayout(t *testing.T) { + mgr := newTestManager() + ciphertext, keyMetadata := encryptAll(t, mgr, nil) + + assert.Len(t, ciphertext, 36) + assert.EqualValues(t, 36, parseKeyMetadata(t, keyMetadata).fileLength) + + in, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) + require.NoError(t, err) + info, err := in.Stat() + require.NoError(t, err) + assert.EqualValues(t, 0, info.Size()) + got, err := io.ReadAll(in) + require.NoError(t, err) + assert.Empty(t, got) +} + +func TestStandardEncryptionManager_RandomAccess(t *testing.T) { + mgr := newTestManager() + plaintext := testData(2*plainBlockSize + 100) + + ciphertext, keyMetadata := encryptAll(t, mgr, plaintext) + in, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) + require.NoError(t, err) + + // Read a slice spanning a block boundary. + buf := make([]byte, 20) + n, err := in.ReadAt(buf, plainBlockSize-10) + require.NoError(t, err) + assert.Equal(t, plaintext[plainBlockSize-10:plainBlockSize+10], buf[:n]) + + // A read running off the end returns what is left, with io.EOF. + tail := make([]byte, 200) + n, err = in.ReadAt(tail, 2*plainBlockSize) + assert.Equal(t, io.EOF, err) + assert.Equal(t, plaintext[2*plainBlockSize:], tail[:n]) + + // Seek + Read. + pos, err := in.Seek(5, io.SeekStart) + require.NoError(t, err) + assert.Equal(t, int64(5), pos) + + rest, err := io.ReadAll(in) + require.NoError(t, err) + assert.True(t, bytes.Equal(plaintext[5:], rest)) +} + +func TestStandardEncryptionManager_EmptyFile_ZeroLengthReadAt(t *testing.T) { + mgr := newTestManager() + ciphertext, keyMetadata := encryptAll(t, mgr, nil) + + in, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) + require.NoError(t, err) + + n, err := in.ReadAt(nil, 0) + assert.NoError(t, err, "a zero-length ReadAt must return (0, nil), like os.File") + assert.Equal(t, 0, n) +} + +func TestStandardEncryptionManager_ReadFrom(t *testing.T) { + mgr := newTestManager() + fw := &memFileWriter{} + out, err := mgr.NewEncryptedOutputFile(t.Context(), fw, testKeyID) + require.NoError(t, err) + + plaintext := testData(2*plainBlockSize + 7) + n, err := out.ReadFrom(bytes.NewReader(plaintext)) + require.NoError(t, err) + assert.EqualValues(t, len(plaintext), n) + require.NoError(t, out.Close()) + + got := decryptAll(t, mgr, fw.Bytes(), out.KeyMetadata()) + assert.True(t, bytes.Equal(plaintext, got)) +} + +func TestStandardEncryptionManager_Stat(t *testing.T) { + mgr := newTestManager() + plaintext := []byte("hello iceberg world, this is a stat test") + ciphertext, keyMetadata := encryptAll(t, mgr, plaintext) + + in, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) + require.NoError(t, err) + + info, err := in.Stat() + require.NoError(t, err) + assert.EqualValues(t, len(plaintext), info.Size(), "Stat must report the plaintext length, not the on-disk ciphertext length") + assert.Less(t, info.Size(), int64(len(ciphertext)), "ciphertext must be larger than plaintext due to the header and per-block nonce/tag overhead") +} + +// --------------------------------------------------------------------------- +// Conformance with the AGS1 spec and StandardKeyMetadata +// --------------------------------------------------------------------------- + +// Round trips through the package can't catch a wrong AAD encoding (say, a +// big-endian block index), because writer and reader share gcmStreamBlockAAD. +// This decodes with crypto/aes and crypto/cipher alone, so such an encoding +// fails at block 1. +func TestStandardEncryptionManager_OutputDecodesWithSpecDecoder(t *testing.T) { + for _, size := range []int{0, 1, plainBlockSize, 2*plainBlockSize + 17} { + t.Run(fmt.Sprintf("%d bytes", size), func(t *testing.T) { + plaintext := testData(size) + ciphertext, keyMetadata := encryptAll(t, newTestManager(), plaintext) + + meta := parseKeyMetadata(t, keyMetadata) + assert.Len(t, meta.dek, 16, "the default data key length is 128-bit") + assert.Len(t, meta.aadPrefix, 16) + assert.EqualValues(t, len(ciphertext), meta.fileLength) + + got := decodeAGS1(t, ciphertext, meta.dek, meta.aadPrefix) + assert.True(t, bytes.Equal(plaintext, got)) + }) + } +} + +// The reverse direction: a stream and key metadata produced by the +// independent codecs above must be readable by the manager. +func TestStandardEncryptionManager_ReadsSpecBuiltStream(t *testing.T) { + dek, err := hex.DecodeString("000102030405060708090a0b0c0d0e0f") + require.NoError(t, err) + aadPrefix, err := hex.DecodeString("a0a1a2a3a4a5a6a7a8a9aaabacadaeaf") + require.NoError(t, err) + + // Known-answer vector: StandardKeyMetadata for an empty file, written out + // by hand from the Avro binary encoding. 0x01 is the version, 0x20 the + // zig-zag length 16, each 0x02 selects the non-null union branch, and 0x48 + // is the zig-zag encoding of the 36-byte file length. + const wantEmpty = "01" + "20" + "000102030405060708090a0b0c0d0e0f" + "02" + "20" + "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf" + "02" + "48" + assert.Equal(t, wantEmpty, hex.EncodeToString(avroKeyMetadata(dek, aadPrefix, 36)), "the test encoder must match the hand-written vector") + + for _, size := range []int{0, 1, plainBlockSize, 2*plainBlockSize + 17} { + t.Run(fmt.Sprintf("%d bytes", size), func(t *testing.T) { + plaintext := testData(size) + stream := encodeAGS1(t, plaintext, dek, aadPrefix) + require.Len(t, stream, expectedStreamLength(size)) + + got := decryptAll(t, newTestManager(), stream, avroKeyMetadata(dek, aadPrefix, int64(len(stream)))) + assert.True(t, bytes.Equal(plaintext, got)) + }) + } +} + +// TestStandardEncryptionManager_BlockNoncesAreUnique pins the one GCM +// property whose loss is catastrophic and silent: reusing a nonce under a +// single key leaks the plaintext XOR of the two blocks and the GHASH +// authentication key, yet a file sealed with a reused (or constant) nonce +// still round-trips and decrypts correctly, so no round-trip, tamper, or +// reorder test would ever catch a regression here. Sealing identical +// plaintext in every block makes any nonce collision directly visible as +// matching ciphertext bytes. +func TestStandardEncryptionManager_BlockNoncesAreUnique(t *testing.T) { + const numBlocks = 4 + // Every block carries byte-for-byte identical plaintext. + plaintext := bytes.Repeat([]byte("0123456789abcdef"), numBlocks*plainBlockSize/16) + + ciphertext, _ := encryptAll(t, newTestManager(), plaintext) + require.Len(t, ciphertext, headerLength+numBlocks*(plainBlockSize+blockOverhead)) + + seenNonceAt := make(map[string]int, numBlocks) + cipherBlocks := make([][]byte, numBlocks) + for i := range numBlocks { + offset := headerLength + i*(plainBlockSize+blockOverhead) + block := ciphertext[offset : offset+plainBlockSize+blockOverhead] + nonce := string(block[:12]) + + if prev, ok := seenNonceAt[nonce]; ok { + t.Fatalf("block %d reused the nonce from block %d: GCM nonce reuse under a single key leaks the plaintext XOR and the GHASH authentication key", i, prev) + } + seenNonceAt[nonce] = i + cipherBlocks[i] = block + } + + for i := 1; i < numBlocks; i++ { + assert.False(t, bytes.Equal(cipherBlocks[0], cipherBlocks[i]), "identical plaintext in blocks 0 and %d must not produce identical ciphertext", i) + } +} + +func TestStandardEncryptionManager_DataKeyLength(t *testing.T) { + t.Run("valid lengths", func(t *testing.T) { + for _, length := range []int{16, 24, 32} { + mgr := newTestManager(encryption.WithStandardDataKeyLength(length)) + plaintext := testData(1000) + + ciphertext, keyMetadata := encryptAll(t, mgr, plaintext) + assert.Len(t, parseKeyMetadata(t, keyMetadata).dek, length) + assert.Equal(t, plaintext, decryptAll(t, mgr, ciphertext, keyMetadata)) + } + }) + + t.Run("invalid lengths", func(t *testing.T) { + for _, length := range []int{-1, 0, 15, 17, 33} { + mgr := newTestManager(encryption.WithStandardDataKeyLength(length)) + _, err := mgr.NewEncryptedOutputFile(t.Context(), &memFileWriter{}, testKeyID) + require.Error(t, err, "length %d", length) + assert.ErrorIs(t, err, encryption.ErrInvalidKeyLength) + } + }) +} + +// --------------------------------------------------------------------------- +// Fail-closed behaviour and untrusted input +// --------------------------------------------------------------------------- + +func TestStandardEncryptionManager_OutputFile_EmptyKeyIDRejected(t *testing.T) { + _, err := newTestManager().NewEncryptedOutputFile(t.Context(), &memFileWriter{}, "") + require.Error(t, err) + assert.ErrorIs(t, err, encryption.ErrKeyIDRequired) +} + +func TestStandardEncryptionManager_InputFile_EmptyKeyMetadataRejected(t *testing.T) { + _, err := newTestManager().NewDecryptedInputFile(t.Context(), newMemFile(nil), nil) + require.Error(t, err) + assert.ErrorIs(t, err, encryption.ErrKeyMetadataRequired) +} + +func TestStandardEncryptionManager_TamperedBlockFailsAuthentication(t *testing.T) { + mgr := newTestManager() + ciphertext, keyMetadata := encryptAll(t, mgr, testData(2*plainBlockSize)) + // Flip a bit inside the first block's ciphertext, past the 8-byte AES + // GCM Stream header and 12-byte nonce; flipping the header instead + // would fail earlier, at stream-header validation. + ciphertext[headerLength+12] ^= 0xFF + + in, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) + require.NoError(t, err) + + _, err = io.ReadAll(in) + require.Error(t, err) + assert.ErrorIs(t, err, encryption.ErrAuthenticationFailed) +} + +func TestStandardEncryptionManager_ReorderedBlocksFailAuthentication(t *testing.T) { + mgr := newTestManager() + ciphertext, keyMetadata := encryptAll(t, mgr, testData(2*plainBlockSize)) // exactly two full blocks + + // Swap the two blocks: with a random nonce per block, only the AAD (which + // binds a block to its position) can catch this. + const physical = plainBlockSize + blockOverhead + block0 := slices.Clone(ciphertext[headerLength : headerLength+physical]) + block1 := slices.Clone(ciphertext[headerLength+physical : headerLength+2*physical]) + copy(ciphertext[headerLength:], block1) + copy(ciphertext[headerLength+physical:], block0) + + in, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), keyMetadata) + require.NoError(t, err) + + _, err = io.ReadAll(in) + require.Error(t, err) + assert.ErrorIs(t, err, encryption.ErrAuthenticationFailed, "swapping blocks must be detected: the AAD binds each block to its position") +} + +func TestStandardEncryptionManager_WrongKeyMaterialFailsAuthentication(t *testing.T) { + mgr := newTestManager() + ciphertext, keyMetadata := encryptAll(t, mgr, []byte("hello iceberg")) + good := parseKeyMetadata(t, keyMetadata) + + tests := map[string][]byte{ + "wrong DEK": avroKeyMetadata(bytes.Repeat([]byte{0x55}, len(good.dek)), good.aadPrefix, good.fileLength), + "wrong AAD prefix": avroKeyMetadata(good.dek, bytes.Repeat([]byte{0x55}, len(good.aadPrefix)), good.fileLength), + } + for name, meta := range tests { + t.Run(name, func(t *testing.T) { + in, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext), meta) + require.NoError(t, err) + + _, err = io.ReadAll(in) + require.Error(t, err) + assert.ErrorIs(t, err, encryption.ErrAuthenticationFailed) + }) + } +} + +func TestStandardEncryptionManager_InvalidStreamHeaderRejected(t *testing.T) { + mgr := newTestManager() + ciphertext, keyMetadata := encryptAll(t, mgr, []byte("hello iceberg")) + + setBlockLength := func(n uint32) func([]byte) { + return func(b []byte) { binary.LittleEndian.PutUint32(b[4:8], n) } + } + tests := map[string]func([]byte){ + "magic mismatch": func(b []byte) { copy(b[:4], "XXXX") }, + "zero block length": setBlockLength(0), + "64 KiB block length": setBlockLength(64 * 1024), + "2 MiB block length": setBlockLength(2 * plainBlockSize), + "maximum uint32 block length": setBlockLength(math.MaxUint32), + } + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + stream := slices.Clone(ciphertext) + mutate(stream) + + _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(stream), keyMetadata) + require.Error(t, err) + assert.ErrorIs(t, err, encryption.ErrInvalidStreamHeader) + }) + } + + t.Run("truncated header", func(t *testing.T) { + _, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(ciphertext[:5]), keyMetadata) + require.Error(t, err) + assert.ErrorIs(t, err, encryption.ErrInvalidStreamHeader) + }) +} + +func TestStandardEncryptionManager_InvalidKeyMetadataRejectedOnRead(t *testing.T) { + dek := bytes.Repeat([]byte{0x11}, 16) + aad := bytes.Repeat([]byte{0x22}, 16) + valid := func(fileLength int64) []byte { return avroKeyMetadata(dek, aad, fileLength) } + const physical = plainBlockSize + blockOverhead + + tests := []struct { + name string + meta []byte + wantErr error + wantMsg string + }{ + {"unsupported version", slices.Concat([]byte{2}, valid(36)[1:]), encryption.ErrUnsupportedKeyMetadataVersion, ""}, + {"version byte only", []byte{1}, encryption.ErrInvalidKeyMetadata, "varint"}, + {"truncated mid-record", valid(36)[:20], encryption.ErrInvalidKeyMetadata, "out of range"}, + {"trailing bytes", append(valid(36), 0), encryption.ErrInvalidKeyMetadata, "trailing"}, + {"oversized varint", slices.Concat([]byte{1}, bytes.Repeat([]byte{0xff}, 11)), encryption.ErrInvalidKeyMetadata, "varint"}, + {"negative bytes length", slices.Concat([]byte{1}, avroLong(-1)), encryption.ErrInvalidKeyMetadata, "out of range"}, + {"bytes length beyond the record", slices.Concat([]byte{1}, avroLong(1<<40)), encryption.ErrInvalidKeyMetadata, "out of range"}, + {"unknown union branch", slices.Concat([]byte{1}, avroBytes(dek), avroLong(2)), encryption.ErrInvalidKeyMetadata, "union branch"}, + {"missing aad prefix", slices.Concat([]byte{1}, avroBytes(dek), avroLong(0), avroLong(1), avroLong(36)), encryption.ErrInvalidKeyMetadata, "aad-prefix must be exactly 16 bytes, got 0"}, + {"empty aad prefix", avroKeyMetadata(dek, nil, 36), encryption.ErrInvalidKeyMetadata, "aad-prefix must be exactly 16 bytes, got 0"}, + {"short aad prefix", avroKeyMetadata(dek, aad[:15], 36), encryption.ErrInvalidKeyMetadata, "aad-prefix must be exactly 16 bytes, got 15"}, + {"long aad prefix", avroKeyMetadata(dek, append(slices.Clone(aad), 0), 36), encryption.ErrInvalidKeyMetadata, "aad-prefix must be exactly 16 bytes, got 17"}, + {"missing file length", slices.Concat([]byte{1}, avroBytes(dek), avroLong(1), avroBytes(aad), avroLong(0)), encryption.ErrInvalidKeyMetadata, "file-length is required"}, + {"negative file length", valid(-1), encryption.ErrInvalidKeyMetadata, "shorter than"}, + {"file length below the 36-byte minimum", valid(35), encryption.ErrInvalidKeyMetadata, "shorter than"}, + {"file length ends inside a nonce", valid(headerLength + physical + 5), encryption.ErrInvalidKeyMetadata, "nonce and tag"}, + {"invalid DEK length", avroKeyMetadata(dek[:15], aad, 36), encryption.ErrInvalidKeyLength, ""}, + {"empty DEK", avroKeyMetadata(nil, aad, 36), encryption.ErrInvalidKeyLength, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := newTestManager().NewDecryptedInputFile(t.Context(), newMemFile(nil), tt.meta) + require.Error(t, err) + assert.ErrorIs(t, err, tt.wantErr) + if tt.wantMsg != "" { + assert.ErrorContains(t, err, tt.wantMsg) + } + }) + } +} + +// A file length near math.MaxInt64 is well-formed, so construction succeeds, +// but reading must fail closed without overflowing or allocating more than +// one block. +func TestStandardEncryptionManager_HugeFileLengthFailsClosedOnRead(t *testing.T) { + mgr := newTestManager() + dek := bytes.Repeat([]byte{0x11}, 16) + aad := bytes.Repeat([]byte{0x22}, 16) + header := encodeAGS1(t, nil, dek, aad)[:headerLength] + + in, err := mgr.NewDecryptedInputFile(t.Context(), newMemFile(header), avroKeyMetadata(dek, aad, math.MaxInt64)) + require.NoError(t, err) + + buf := make([]byte, 16) + _, err = in.ReadAt(buf, 0) + assert.ErrorIs(t, err, encryption.ErrBlockTruncated, "block 0 lies beyond the real file") + + _, err = in.ReadAt(buf, (math.MaxUint32+1)*plainBlockSize) + assert.ErrorIs(t, err, encryption.ErrInvalidKeyMetadata, "the AAD only has 32 bits for the block index") +} + +// shortReadFile is an icebergio.File stub whose ReadAt returns fewer bytes +// than requested along with io.EOF once truncateAt is reached, mimicking a +// real backend (S3, local fs) reading a genuinely truncated file. A plain +// bytes.Reader always fills the requested slice, which is why the round-trip +// tests never exercise this path. +type shortReadFile struct { + *memFile + truncateAt int64 +} + +func (f *shortReadFile) ReadAt(p []byte, off int64) (int, error) { + if off >= f.truncateAt { + return 0, io.EOF + } + if off+int64(len(p)) > f.truncateAt { + p = p[:f.truncateAt-off] + } + n, err := f.memFile.ReadAt(p, off) + if err == nil && int64(n) < int64(len(p)) { + err = io.EOF + } + + return n, err +} + +func TestStandardEncryptionManager_TruncatedBackendReadReportsBlockTruncated(t *testing.T) { + mgr := newTestManager() + ciphertext, keyMetadata := encryptAll(t, mgr, testData(100)) + truncated := &shortReadFile{memFile: newMemFile(ciphertext), truncateAt: int64(len(ciphertext) - 5)} + + in, err := mgr.NewDecryptedInputFile(t.Context(), truncated, keyMetadata) + require.NoError(t, err) + + _, err = io.ReadAll(in) + require.Error(t, err) + assert.ErrorIs(t, err, encryption.ErrBlockTruncated) + assert.NotErrorIs(t, err, encryption.ErrCiphertextTooShort, "a truncated block read is distinct from a too-short KMS-wrapped key/payload") + assert.NotErrorIs(t, err, encryption.ErrAuthenticationFailed, "a truncated read must not be misreported as tampering") +} + +// The plaintext length comes from the file length in key metadata, not from +// the storage size, so bytes appended after the last block are ignored, as +// the AES GCM Stream spec's "File length" rule implies. +func TestStandardEncryptionManager_TrailingBytesAfterLastBlockIgnored(t *testing.T) { + mgr := newTestManager() + plaintext := testData(plainBlockSize + 10) + ciphertext, keyMetadata := encryptAll(t, mgr, plaintext) + + got := decryptAll(t, mgr, append(ciphertext, []byte("trailing garbage")...), keyMetadata) + assert.True(t, bytes.Equal(plaintext, got)) +} + +// --------------------------------------------------------------------------- +// Write and close failures +// --------------------------------------------------------------------------- + +// failAfterNWriter is an icebergio.FileWriter stub that fails the Nth call +// to Write, simulating a mid-stream flush failure on the underlying storage. +// It also counts Close calls, since memFileWriter.Close is a no-op and would +// otherwise mask a leaked/never-closed underlying writer. +type failAfterNWriter struct { + memFileWriter + failAt int + writes int + closes int +} + +func (w *failAfterNWriter) Write(p []byte) (int, error) { + w.writes++ + if w.writes == w.failAt { + return 0, errors.New("simulated write failure") + } + + return w.memFileWriter.Write(p) +} + +func (w *failAfterNWriter) Close() error { + w.closes++ + + return w.memFileWriter.Close() +} + +func TestStandardEncryptionManager_FlushFailurePoisonsWriterAndClose(t *testing.T) { + mgr := newTestManager() + // Write call 1 is the AES GCM Stream header written by + // NewEncryptedOutputFile; call 2 flushes the first block; call 3 flushes + // the second block and fails. + fw := &failAfterNWriter{failAt: 3} + out, err := mgr.NewEncryptedOutputFile(t.Context(), fw, testKeyID) + require.NoError(t, err) + + // The first block flushes fine; the second block's flush fails. + n, err := out.Write(make([]byte, 2*plainBlockSize)) + require.Error(t, err) + assert.Equal(t, plainBlockSize, n, "only the first block reached storage") + assert.Equal(t, 1, fw.closes, "a poisoned flush must close the underlying writer, not leak it") + + // A subsequent Write must return the same sticky error, not attempt more + // I/O, and not the post-Close error: the writer was poisoned, not closed. + _, err2 := out.Write([]byte("c")) + require.Error(t, err2) + assert.ErrorIs(t, err2, err) + assert.NotErrorIs(t, err2, encryption.ErrOutputFileClosed) + assert.NotErrorIs(t, err2, fs.ErrClosed) + + // Close must report the failure rather than silently succeeding. + closeErr := out.Close() + require.Error(t, closeErr) + assert.Nil(t, out.KeyMetadata(), "key metadata must not be finalized when the file failed to write") + + // A retried Close must keep reporting the same error, not nil. + closeErr2 := out.Close() + require.Error(t, closeErr2) + assert.Equal(t, 1, fw.closes, "a retried Close must not close the underlying writer again") +} + +func TestStandardEncryptionManager_WriteAfterCloseRejected(t *testing.T) { + out, err := newTestManager().NewEncryptedOutputFile(t.Context(), &memFileWriter{}, testKeyID) + require.NoError(t, err) + require.NoError(t, out.Close()) + + _, err = out.Write([]byte("late")) + require.Error(t, err) + assert.ErrorIs(t, err, encryption.ErrOutputFileClosed) + assert.ErrorIs(t, err, fs.ErrClosed) +} + +// closeFailWriter is an icebergio.FileWriter stub whose Close always fails +// even though every Write succeeds, simulating a backend where the final +// flush/commit (e.g. an S3 multipart completion) fails independently of the +// preceding writes. +type closeFailWriter struct { + memFileWriter + closes int +} + +func (w *closeFailWriter) Close() error { + w.closes++ + + return errors.New("simulated close failure") +} + +func TestStandardEncryptionManager_UnderlyingCloseFailurePoisonsWriter(t *testing.T) { + fw := &closeFailWriter{} + out, err := newTestManager().NewEncryptedOutputFile(t.Context(), fw, testKeyID) + require.NoError(t, err) + + _, err = out.Write([]byte("hello")) + require.NoError(t, err) + + closeErr := out.Close() + require.Error(t, closeErr) + assert.Nil(t, out.KeyMetadata(), "key metadata must not be finalized when the underlying Close fails") + + // A retried Close must keep reporting the same error, not nil, and must + // not attempt to close the underlying writer again. + closeErr2 := out.Close() + require.Error(t, closeErr2) + assert.ErrorIs(t, closeErr2, closeErr) + assert.Equal(t, 1, fw.closes, "a retried Close must not re-close the underlying writer") + + _, err = out.Write([]byte("late")) + require.Error(t, err) + assert.ErrorIs(t, err, closeErr, "a subsequent Write must report the same sticky error") +} + +// --------------------------------------------------------------------------- +// Block cache and concurrency +// --------------------------------------------------------------------------- + +// countingReadAtFile wraps a memFile and counts ReadAt calls, to verify that +// repeated small reads landing in the same block reuse the decrypted-block +// cache instead of re-decrypting on every call. +type countingReadAtFile struct { + *memFile + reads int +} + +func (f *countingReadAtFile) ReadAt(p []byte, off int64) (int, error) { + f.reads++ + + return f.memFile.ReadAt(p, off) +} + +func TestStandardEncryptionManager_RepeatedSmallReadsReuseDecryptedBlockCache(t *testing.T) { + mgr := newTestManager() + plaintext := testData(plainBlockSize + 5000) // a full block, then a 5000-byte tail block + + ciphertext, keyMetadata := encryptAll(t, mgr, plaintext) + counting := &countingReadAtFile{memFile: newMemFile(ciphertext)} + + in, err := mgr.NewDecryptedInputFile(t.Context(), counting, keyMetadata) + require.NoError(t, err) + baseline := counting.reads // the stream header was already read once above + + buf := make([]byte, 1) + readOneByteAt := func(off int64) { + _, err := in.Seek(off, io.SeekStart) + require.NoError(t, err) + n, err := in.Read(buf) + require.NoError(t, err) + require.Equal(t, 1, n) + } + for off := range int64(5000) { + readOneByteAt(off) // all in block 0 + } + for off := range int64(5000) { + readOneByteAt(plainBlockSize + off) // all in block 1 + } + + assert.LessOrEqual(t, counting.reads-baseline, 2, "10000 one-byte reads across two blocks must decrypt each block at most once") +} + +// slowReadFile wraps a memFile and sleeps before every ReadAt, simulating a +// remote backend (e.g. S3) with real network latency. It is used to check +// that concurrent ReadAt calls for distinct blocks overlap instead of queuing +// behind readBlock's cache lock. +type slowReadFile struct { + *memFile + delay time.Duration +} + +func (f *slowReadFile) ReadAt(p []byte, off int64) (int, error) { + time.Sleep(f.delay) + + return f.memFile.ReadAt(p, off) +} + +func TestStandardEncryptionManager_ConcurrentReadAtOverlaps(t *testing.T) { + const ( + numBlocks = 8 + delay = 20 * time.Millisecond + ) + mgr := newTestManager() + ciphertext, keyMetadata := encryptAll(t, mgr, testData(numBlocks*plainBlockSize)) + + slow := &slowReadFile{memFile: newMemFile(ciphertext), delay: delay} + in, err := mgr.NewDecryptedInputFile(t.Context(), slow, keyMetadata) // reads the header; not timed below + require.NoError(t, err) + + start := time.Now() + var g errgroup.Group + for i := range numBlocks { + g.Go(func() error { + buf := make([]byte, 16) + _, err := in.ReadAt(buf, int64(i)*plainBlockSize) + + return err + }) + } + require.NoError(t, g.Wait()) + elapsed := time.Since(start) + + // Serial execution (for example, holding cacheMu across ReadAt) would take + // roughly numBlocks*delay. True concurrency keeps it close to a single + // delay; the threshold below is generous to stay robust on slow or loaded + // CI machines while still failing on a serialized implementation. + assert.Less(t, elapsed, numBlocks/2*delay, "concurrent ReadAt calls for distinct blocks must overlap, not serialize") +}