diff --git a/.changeset/azure-template-upload-headers.md b/.changeset/azure-template-upload-headers.md new file mode 100644 index 0000000000..2295a0b468 --- /dev/null +++ b/.changeset/azure-template-upload-headers.md @@ -0,0 +1,6 @@ +--- +"e2b": patch +"@e2b/python-sdk": patch +--- + +Apply the request headers the API returns with a template layer-file upload link. Azure Blob Storage requires `x-ms-blob-type` on the upload request, which its signed URL cannot carry, so `COPY` instructions failed on Azure-backed clusters. GCS- and S3-backed clusters return no headers and are unaffected. diff --git a/packages/js-sdk/src/template/buildApi.ts b/packages/js-sdk/src/template/buildApi.ts index 489445c4ee..5b8aba70ea 100644 --- a/packages/js-sdk/src/template/buildApi.ts +++ b/packages/js-sdk/src/template/buildApi.ts @@ -113,6 +113,7 @@ export async function uploadFile( fileName: string fileContextPath: string url: string + headers?: Record ignorePatterns: string[] resolveSymlinks: boolean gzip: boolean @@ -128,6 +129,7 @@ export async function uploadFile( const { fileName, url, + headers, fileContextPath, ignorePatterns, resolveSymlinks, @@ -154,7 +156,7 @@ export async function uploadFile( abortOpts?.signal ) - const res = await putFileStream(url, tar.path, tar.size, signal) + const res = await putFileStream(url, tar.path, tar.size, signal, headers) if (!res.ok) { throw new FileUploadError( @@ -176,7 +178,8 @@ async function putFileStream( url: string, filePath: string, size: number, - signal: AbortSignal | undefined + signal: AbortSignal | undefined, + headers?: Record ): Promise<{ ok: boolean; statusText: string }> { // Prefer undici's fetch: it honors the explicit Content-Length on stream // bodies on every runtime, while Deno's native fetch ignores the header and @@ -192,7 +195,13 @@ async function putFileStream( body: stream.Readable.toWeb( fs.createReadStream(filePath) ) as ReadableStream, + // API-returned headers applied as given (Azure needs x-ms-blob-type, which a SAS cannot carry); Content-Length stays ours, dropped case-insensitively since fetch header names are not case-sensitive. headers: { + ...Object.fromEntries( + Object.entries(headers ?? {}).filter( + ([name]) => name.toLowerCase() !== 'content-length' + ) + ), 'Content-Length': size.toString(), }, // Streaming request bodies require half-duplex mode. diff --git a/packages/js-sdk/src/template/index.ts b/packages/js-sdk/src/template/index.ts index 1cc6401b28..12ba6caeb6 100644 --- a/packages/js-sdk/src/template/index.ts +++ b/packages/js-sdk/src/template/index.ts @@ -1112,7 +1112,7 @@ export class TemplateBase stackTrace = this.stackTraces[index + 1] } - const { present, url } = await getFileUploadLink( + const { present, url, headers } = await getFileUploadLink( client, { templateID, @@ -1131,6 +1131,7 @@ export class TemplateBase fileName: src, fileContextPath: this.fileContextPath.toString(), url, + headers, ignorePatterns: [ ...this.fileIgnorePatterns, ...readDockerignore(this.fileContextPath.toString()), diff --git a/packages/js-sdk/tests/template/uploadFile.test.ts b/packages/js-sdk/tests/template/uploadFile.test.ts index e9f7492c9c..fe4b59124c 100644 --- a/packages/js-sdk/tests/template/uploadFile.test.ts +++ b/packages/js-sdk/tests/template/uploadFile.test.ts @@ -73,5 +73,44 @@ describe('uploadFile transfer encoding', () => { // Content-Type (e.g. inferred from the archive's file extension) makes // the storage backend reject the upload with 403 Forbidden. expect(capturedHeaders['content-type']).toBeUndefined() + + // S3/GCS presigned PUTs sign the header set — the upload must add nothing the API did not ask for. + expect(capturedHeaders['x-ms-blob-type']).toBeUndefined() + }) + + test('sends the headers the API returned with the upload link', async () => { + await uploadFile( + { + fileName: '*.txt', + fileContextPath: testDir, + url: baseUrl, + headers: { 'x-ms-blob-type': 'BlockBlob' }, + ignorePatterns: [], + resolveSymlinks: false, + gzip: true, + }, + undefined + ) + + // Azure's Put Blob needs a request header a SAS cannot carry, so the API hands it back instead. + expect(capturedHeaders['x-ms-blob-type']).toBe('BlockBlob') + }) + + test('keeps its own Content-Length when the API returns one', async () => { + await uploadFile( + { + fileName: '*.txt', + fileContextPath: testDir, + url: baseUrl, + // lowercase on purpose: header names are case-insensitive, object keys are not + headers: { 'content-length': '1' }, + ignorePatterns: [], + resolveSymlinks: false, + gzip: true, + }, + undefined + ) + + expect(Number(capturedHeaders['content-length'])).toBe(capturedBodyLength) }) }) diff --git a/packages/python-sdk/e2b/template_async/build_api.py b/packages/python-sdk/e2b/template_async/build_api.py index 328a002f9a..8c8733b621 100644 --- a/packages/python-sdk/e2b/template_async/build_api.py +++ b/packages/python-sdk/e2b/template_async/build_api.py @@ -1,7 +1,7 @@ import asyncio import os from types import TracebackType -from typing import Callable, Optional, List, Union +from typing import Callable, Dict, Optional, List, Union import httpx from pyqwest import HTTPTransport @@ -115,6 +115,8 @@ async def upload_file( resolve_symlinks: bool, gzip: bool, stack_trace: Optional[TracebackType], + *, + headers: Optional[Dict[str, str]] = None, request_timeout: Optional[float] = None, ): # Uploading a large build-context archive can take far longer than the 60s @@ -152,14 +154,18 @@ async def upload_file( ) ), ) as client: - # Stream the archive from disk via an async iterator. The - # explicit Content-Length suppresses chunked transfer - # encoding, which S3 presigned URLs reject; reqwest keeps the - # Content-Length framing for the streamed body. + # API-returned headers applied as given, but Content-Length stays ours — explicit so S3 presigned URLs see no chunked encoding. response = await client.put( url, content=aiter_io_chunks(tar_file), - headers={"Content-Length": str(size)}, + headers={ + **{ + k: v + for k, v in (headers or {}).items() + if k.lower() != "content-length" + }, + "Content-Length": str(size), + }, ) response.raise_for_status() finally: diff --git a/packages/python-sdk/e2b/template_async/main.py b/packages/python-sdk/e2b/template_async/main.py index 2247f8b0ae..f21f07f5a5 100644 --- a/packages/python-sdk/e2b/template_async/main.py +++ b/packages/python-sdk/e2b/template_async/main.py @@ -4,6 +4,7 @@ from typing_extensions import Unpack from e2b.api.client.client import AuthenticatedClient +from e2b.api.client.types import Unset from e2b.connection_config import ApiParams, ConnectionConfig from e2b.template.consts import GZIP, RESOLVE_SYMLINKS from e2b.template.logger import LogEntry, LogEntryEnd, LogEntryStart @@ -137,6 +138,11 @@ async def _build( resolve_symlinks, gzip, stack_trace, + headers=( + file_info.headers.to_dict() + if not isinstance(file_info.headers, Unset) + else None + ), request_timeout=request_timeout, ) if on_build_logs: diff --git a/packages/python-sdk/e2b/template_sync/build_api.py b/packages/python-sdk/e2b/template_sync/build_api.py index 735dcf7970..e931c03b38 100644 --- a/packages/python-sdk/e2b/template_sync/build_api.py +++ b/packages/python-sdk/e2b/template_sync/build_api.py @@ -1,6 +1,7 @@ +import os import time from types import TracebackType -from typing import Callable, Optional, List, Union +from typing import Callable, Dict, Optional, List, Union import httpx from pyqwest import SyncHTTPTransport @@ -113,6 +114,8 @@ def upload_file( resolve_symlinks: bool, gzip: bool, stack_trace: Optional[TracebackType], + *, + headers: Optional[Dict[str, str]] = None, request_timeout: Optional[float] = None, ): # Uploading a large build-context archive can take far longer than the 60s @@ -127,6 +130,7 @@ def upload_file( tar_file = tar_file_stream( file_name, context_path, ignore_patterns, resolve_symlinks, gzip ) + size = os.fstat(tar_file.fileno()).st_size try: # Through the pyqwest adapter the upload timeout is a # whole-request deadline for the entire transfer, not a per-write @@ -148,11 +152,19 @@ def upload_file( ) ), ) as client: - # httpx streams the archive from disk in chunks and sets - # Content-Length from the file size—S3 presigned URLs reject - # chunked transfer encoding, and reqwest keeps the - # Content-Length framing for the streamed body. - response = client.put(url, content=tar_file) + # API-returned headers applied as given, but Content-Length stays ours — explicit so S3 presigned URLs see no chunked encoding. + response = client.put( + url, + content=tar_file, + headers={ + **{ + k: v + for k, v in (headers or {}).items() + if k.lower() != "content-length" + }, + "Content-Length": str(size), + }, + ) response.raise_for_status() finally: # Closing the spooled temp file is best-effort: a failure here diff --git a/packages/python-sdk/e2b/template_sync/main.py b/packages/python-sdk/e2b/template_sync/main.py index 1481966e04..7316422f0e 100644 --- a/packages/python-sdk/e2b/template_sync/main.py +++ b/packages/python-sdk/e2b/template_sync/main.py @@ -4,6 +4,7 @@ from typing_extensions import Unpack from e2b.api.client.client import AuthenticatedClient +from e2b.api.client.types import Unset from e2b.connection_config import ApiParams, ConnectionConfig from e2b.api.client_sync import get_api_client @@ -137,6 +138,11 @@ def _build( resolve_symlinks, gzip, stack_trace, + headers=( + file_info.headers.to_dict() + if not isinstance(file_info.headers, Unset) + else None + ), request_timeout=request_timeout, ) if on_build_logs: diff --git a/packages/python-sdk/tests/async/template_async/test_upload_file.py b/packages/python-sdk/tests/async/template_async/test_upload_file.py index a8d2993427..3754639f8b 100644 --- a/packages/python-sdk/tests/async/template_async/test_upload_file.py +++ b/packages/python-sdk/tests/async/template_async/test_upload_file.py @@ -240,3 +240,88 @@ def failing_close_stream(*args, **kwargs): thread.join(timeout=5) assert state["headers"] is not None + + +async def test_upload_file_sends_the_headers_the_api_returned(tmp_path): + # Azure's Put Blob needs a request header a SAS cannot carry, so the API hands it back with the upload link. + (tmp_path / "hello.txt").write_text("hello world") + + server, thread, state = _make_server() + host, port = server.server_address + + try: + client = AuthenticatedClient(base_url="http://test", token="test") + await upload_file( + api_client=client, + file_name="*.txt", + context_path=str(tmp_path), + url=f"http://{host}:{port}/upload", + ignore_patterns=[], + resolve_symlinks=False, + gzip=True, + stack_trace=None, + headers={"x-ms-blob-type": "BlockBlob"}, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + assert state["headers"]["x-ms-blob-type"] == "BlockBlob" + + +async def test_upload_file_adds_no_headers_when_the_api_returns_none(tmp_path): + # S3/GCS presigned PUTs sign the header set — the upload must add nothing the API did not ask for. + (tmp_path / "hello.txt").write_text("hello world") + + server, thread, state = _make_server() + host, port = server.server_address + + try: + client = AuthenticatedClient(base_url="http://test", token="test") + await upload_file( + api_client=client, + file_name="*.txt", + context_path=str(tmp_path), + url=f"http://{host}:{port}/upload", + ignore_patterns=[], + resolve_symlinks=False, + gzip=True, + stack_trace=None, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + assert "x-ms-blob-type" not in state["headers"] + + +async def test_upload_file_keeps_its_own_content_length(tmp_path): + # An API-returned Content-Length must never override the real archive size. + (tmp_path / "hello.txt").write_text("hello world") + + server, thread, state = _make_server() + host, port = server.server_address + + try: + client = AuthenticatedClient(base_url="http://test", token="test") + await upload_file( + api_client=client, + file_name="*.txt", + context_path=str(tmp_path), + url=f"http://{host}:{port}/upload", + ignore_patterns=[], + resolve_symlinks=False, + gzip=True, + stack_trace=None, + # lowercase on purpose: header names are case-insensitive, dict keys are not + headers={"x-ms-blob-type": "BlockBlob", "content-length": "1"}, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + assert state["body_length"] > 1 + assert int(state["headers"]["content-length"]) == state["body_length"] diff --git a/packages/python-sdk/tests/sync/template_sync/test_upload_file.py b/packages/python-sdk/tests/sync/template_sync/test_upload_file.py index a08ae9906a..52bbf7e527 100644 --- a/packages/python-sdk/tests/sync/template_sync/test_upload_file.py +++ b/packages/python-sdk/tests/sync/template_sync/test_upload_file.py @@ -236,3 +236,88 @@ def failing_close_stream(*args, **kwargs): thread.join(timeout=5) assert state["headers"] is not None + + +def test_upload_file_sends_the_headers_the_api_returned(tmp_path): + # Azure's Put Blob needs a request header a SAS cannot carry, so the API hands it back with the upload link. + (tmp_path / "hello.txt").write_text("hello world") + + server, thread, state = _make_server() + host, port = server.server_address + + try: + client = AuthenticatedClient(base_url="http://test", token="test") + upload_file( + api_client=client, + file_name="*.txt", + context_path=str(tmp_path), + url=f"http://{host}:{port}/upload", + ignore_patterns=[], + resolve_symlinks=False, + gzip=True, + stack_trace=None, + headers={"x-ms-blob-type": "BlockBlob"}, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + assert state["headers"]["x-ms-blob-type"] == "BlockBlob" + + +def test_upload_file_adds_no_headers_when_the_api_returns_none(tmp_path): + # S3/GCS presigned PUTs sign the header set — the upload must add nothing the API did not ask for. + (tmp_path / "hello.txt").write_text("hello world") + + server, thread, state = _make_server() + host, port = server.server_address + + try: + client = AuthenticatedClient(base_url="http://test", token="test") + upload_file( + api_client=client, + file_name="*.txt", + context_path=str(tmp_path), + url=f"http://{host}:{port}/upload", + ignore_patterns=[], + resolve_symlinks=False, + gzip=True, + stack_trace=None, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + assert "x-ms-blob-type" not in state["headers"] + + +def test_upload_file_keeps_its_own_content_length(tmp_path): + # An API-returned Content-Length must never override the real archive size. + (tmp_path / "hello.txt").write_text("hello world") + + server, thread, state = _make_server() + host, port = server.server_address + + try: + client = AuthenticatedClient(base_url="http://test", token="test") + upload_file( + api_client=client, + file_name="*.txt", + context_path=str(tmp_path), + url=f"http://{host}:{port}/upload", + ignore_patterns=[], + resolve_symlinks=False, + gzip=True, + stack_trace=None, + # lowercase on purpose: header names are case-insensitive, dict keys are not + headers={"x-ms-blob-type": "BlockBlob", "content-length": "1"}, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + assert state["body_length"] > 1 + assert int(state["headers"]["content-length"]) == state["body_length"]