Merge upstream main into fix-3713-empty-property-helpers

Author: daviddallakyan2005

.github/workflows/codeql.yml

@@ -46,11 +46,11 @@ jobs:
         persist-credentials: false
 
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4
+      uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6
       with:
         languages: actions
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4
+      uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6
       with:
         category: "/language:actions"

.github/workflows/pypi-build-artifacts.yml

@@ -69,7 +69,7 @@ jobs:
         if: matrix.os == 'ubuntu-latest'
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@4726cd35bb13f7bde50cf2761f2499ac7b3aa32c # v4.1.1
+        uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0
         with:
           output-dir: wheelhouse
           config-file: "pyproject.toml"

.github/workflows/stale.yml

@@ -32,7 +32,7 @@ jobs:
     if: github.repository_owner == 'apache'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@1e223db275d687790206a7acac4d1a11bd6fe629 # v10.4.0
+      - uses: actions/stale@4391f3da665fdf50b6810c1a66712fb9ba21aa93 # v11.0.0
         with:
           # stale issues
           stale-issue-label: 'stale'

.github/workflows/svn-build-artifacts.yml

@@ -63,7 +63,7 @@ jobs:
         if: matrix.os == 'ubuntu-latest'
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@4726cd35bb13f7bde50cf2761f2499ac7b3aa32c # v4.1.1
+        uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0
         with:
           output-dir: wheelhouse
           config-file: "pyproject.toml"

mkdocs/docs/configuration.md

@@ -172,6 +172,8 @@ For the FileIO there are several configuration options available:
 | s3.force-virtual-addressing | False                      | Whether to use virtual addressing of buckets. If true, then virtual addressing is always enabled. If false, then virtual addressing is only enabled if endpoint_override is empty. This can be used for non-AWS backends that only support virtual hosted-style access. |
 | s3.retry-strategy-impl      | None                       | Ability to set a custom S3 retry strategy. A full path to a class needs to be given that extends the [S3RetryStrategy](https://github.com/apache/arrow/blob/639201bfa412db26ce45e73851432018af6c945e/python/pyarrow/_s3fs.pyx#L110) base class.                         |
 | s3.anonymous                | True                       | Configure whether to use anonymous connection. If False (default), uses key/secret if configured or boto's credential resolver.                                                                                                                                         |
+| s3.server-side-encryption   | aws:kms                    | Configure server-side encryption (e.g. `AES256` or `aws:kms`). Only supported by `FsspecFileIO`.                                                                                                                                     |
+| s3.sse-kms-key-id           | alias/my-key               | Configure the SSE-KMS key id (or ARN) for multipart uploads. Only supported by `FsspecFileIO`.                                                                                                                                                               |
 
 <!-- markdown-link-check-enable-->
 

pyiceberg/catalog/dynamodb.py

@@ -837,7 +837,10 @@ def _get_update_database_item(namespace_item: dict[str, Any], updated_properties
 
 
 def _get_namespace_properties(namespace_dict: dict[str, str]) -> Properties:
-    return {_remove_property_prefix(key): val for key, val in namespace_dict.items() if key.startswith(PROPERTY_KEY_PREFIX)}
+    # removeprefix removes the literal prefix, unlike lstrip which removes any leading prefix characters
+    return {
+        key.removeprefix(PROPERTY_KEY_PREFIX): val for key, val in namespace_dict.items() if key.startswith(PROPERTY_KEY_PREFIX)
+    }
 
 
 def _convert_dynamo_item_to_regular_dict(dynamo_json: dict[str, Any]) -> dict[str, str]:
@@ -888,7 +891,3 @@ def _convert_dynamo_item_to_regular_dict(dynamo_json: dict[str, Any]) -> dict[st
 
 def _add_property_prefix(prop: str) -> str:
     return PROPERTY_KEY_PREFIX + prop
-
-
-def _remove_property_prefix(prop: str) -> str:
-    return prop.lstrip(PROPERTY_KEY_PREFIX)

pyiceberg/expressions/literals.py

@@ -251,7 +251,7 @@ def __init__(self) -> None:
 
     @singledispatchmethod
     def to(self, type_var: IcebergType) -> Literal:  # type: ignore
-        raise TypeError("Cannot change the type of IntAboveMax")
+        raise TypeError("Cannot change the type of LongAboveMax")
 
     @to.register(LongType)
     def _(self, _: LongType) -> Literal[int]:
@@ -264,7 +264,7 @@ def __init__(self) -> None:
 
     @singledispatchmethod
     def to(self, type_var: IcebergType) -> Literal:  # type: ignore
-        raise TypeError("Cannot change the type of IntBelowMin")
+        raise TypeError("Cannot change the type of LongBelowMin")
 
     @to.register(LongType)
     def _(self, _: LongType) -> Literal[int]:

pyiceberg/io/__init__.py

@@ -80,6 +80,8 @@ def _is_local_path(path: str) -> bool:
 S3_ROLE_SESSION_NAME = "s3.role-session-name"
 S3_FORCE_VIRTUAL_ADDRESSING = "s3.force-virtual-addressing"
 S3_RETRY_STRATEGY_IMPL = "s3.retry-strategy-impl"
+S3_SERVER_SIDE_ENCRYPTION = "s3.server-side-encryption"
+S3_SSE_KMS_KEY_ID = "s3.sse-kms-key-id"
 HDFS_HOST = "hdfs.host"
 HDFS_PORT = "hdfs.port"
 HDFS_USER = "hdfs.user"

pyiceberg/io/fsspec.py

@@ -79,11 +79,13 @@
     S3_REGION,
     S3_REQUEST_TIMEOUT,
     S3_SECRET_ACCESS_KEY,
+    S3_SERVER_SIDE_ENCRYPTION,
     S3_SESSION_TOKEN,
     S3_SIGNER,
     S3_SIGNER_ENDPOINT,
     S3_SIGNER_ENDPOINT_DEFAULT,
     S3_SIGNER_URI,
+    S3_SSE_KMS_KEY_ID,
     FileIO,
     InputFile,
     InputStream,
@@ -178,6 +180,7 @@ def _s3(properties: Properties) -> AbstractFileSystem:
         "region_name": get_first_property_value(properties, S3_REGION, AWS_REGION),
     }
     config_kwargs = {}
+    s3_additional_kwargs = {}
     register_events: dict[str, Callable[[AWSRequest], None]] = {}
 
     if signer := properties.get(S3_SIGNER):
@@ -202,14 +205,20 @@ def _s3(properties: Properties) -> AbstractFileSystem:
     if request_timeout := properties.get(S3_REQUEST_TIMEOUT):
         config_kwargs["read_timeout"] = float(request_timeout)
 
-    if _force_virtual_addressing := properties.get(S3_FORCE_VIRTUAL_ADDRESSING):
+    if property_as_bool(properties, S3_FORCE_VIRTUAL_ADDRESSING, False):
         config_kwargs["s3"] = {"addressing_style": "virtual"}
 
     if s3_anonymous := properties.get(S3_ANONYMOUS):
         anon = strtobool(s3_anonymous)
     else:
         anon = False
 
+    if server_side_encryption := properties.get(S3_SERVER_SIDE_ENCRYPTION):
+        s3_additional_kwargs["ServerSideEncryption"] = server_side_encryption
+
+    if sse_kms_key_id := properties.get(S3_SSE_KMS_KEY_ID):
+        s3_additional_kwargs["SSEKMSKeyId"] = sse_kms_key_id
+
     s3_fs_kwargs = {
         "anon": anon,
         "client_kwargs": client_kwargs,
@@ -219,6 +228,9 @@ def _s3(properties: Properties) -> AbstractFileSystem:
     if profile_name := get_first_property_value(properties, S3_PROFILE_NAME, AWS_PROFILE_NAME):
         s3_fs_kwargs["profile"] = profile_name
 
+    if s3_additional_kwargs:
+        s3_fs_kwargs["s3_additional_kwargs"] = s3_additional_kwargs
+
     fs = S3FileSystem(**s3_fs_kwargs)
 
     for event_name, event_function in register_events.items():

pyiceberg/partitioning.py

@@ -500,7 +500,7 @@ def _to_partition_representation(type: IcebergType, value: Any) -> Any:
     can return date that still needs to be transformed into an int (days
     since epoch).
     """
-    return TypeError(f"Unsupported partition field type: {type}")
+    raise TypeError(f"Unsupported partition field type: {type}")
 
 
 @_to_partition_representation.register(TimestampType)