diff --git a/justfile b/justfile index 5ebd4695..38bd19e7 100644 --- a/justfile +++ b/justfile @@ -1,17 +1,23 @@ -generate-clients: update-openapi-management update-openapi-catalog generate-management-client generate-iceberg-client +generate-clients: update-openapi-management update-openapi-catalog update-openapi-generic-table generate-management-client generate-iceberg-client generate-generic-table-client update-openapi-management: curl -o openapi/management-open-api.yaml https://raw.githubusercontent.com/lakekeeper/lakekeeper/refs/heads/main/docs/docs/api/management-open-api.yaml update-openapi-catalog: curl -o openapi/rest-catalog-open-api.yaml https://raw.githubusercontent.com/lakekeeper/lakekeeper/refs/heads/main/docs/docs/api/rest-catalog-open-api.yaml - -generate-management-client: + +update-openapi-generic-table: + curl -o openapi/generic-table-open-api.yaml https://raw.githubusercontent.com/lakekeeper/lakekeeper/refs/heads/main/docs/docs/api/generic-table-open-api.yaml + +generate-management-client: npx @hey-api/openapi-ts -i ./openapi/management-open-api.yaml -o ./src/gen/management -c @hey-api/client-fetch -generate-iceberg-client: +generate-iceberg-client: npx @hey-api/openapi-ts -i ./openapi/rest-catalog-open-api.yaml -o ./src/gen/iceberg -c @hey-api/client-fetch +generate-generic-table-client: + npx @hey-api/openapi-ts -i ./openapi/generic-table-open-api.yaml -o ./src/gen/generic-table -c @hey-api/client-fetch + reviewable: install fix-all build install: diff --git a/openapi/generic-table-open-api.yaml b/openapi/generic-table-open-api.yaml new file mode 100644 index 00000000..ac246dd6 --- /dev/null +++ b/openapi/generic-table-open-api.yaml @@ -0,0 +1,432 @@ +openapi: 3.1.0 +info: + title: Lakekeeper Generic Table API + description: Lakekeeper data-plane API for non-Iceberg formats (Lance, Delta, ...). + license: + name: Apache-2.0 + identifier: Apache-2.0 + version: 0.0.0 +servers: + - url: '{scheme}://{host}{basePath}' + description: Lakekeeper Generic Table API + variables: + basePath: + default: '' + description: Optional path prefix (starting with '/') to be prepended to all routes + host: + default: localhost + description: The host (and optional port) for the specified server + scheme: + default: https + description: The scheme of the URI, either http or https +paths: + /lakekeeper/v1/{prefix}/generic-tables/rename: + post: + tags: + - generic-table + summary: Rename a generic table + operationId: rename_generic_table + parameters: + - name: prefix + in: path + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/RenameGenericTableRequest' + required: true + responses: + '204': + description: Generic table renamed successfully + 4XX: + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + /lakekeeper/v1/{prefix}/namespaces/{namespace}/generic-tables: + get: + tags: + - generic-table + summary: List generic tables in a namespace + operationId: list_generic_tables + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: namespace + in: path + required: true + schema: + type: string + - name: pageToken + in: query + required: false + schema: + type: + - string + - 'null' + - name: pageSize + in: query + required: false + schema: + type: + - integer + - 'null' + format: int64 + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/ListGenericTablesResponse' + 4XX: + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + post: + tags: + - generic-table + summary: Create a generic table + operationId: create_generic_table + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: namespace + in: path + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/CreateGenericTableRequest' + required: true + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/LoadGenericTableResponse' + 4XX: + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + /lakekeeper/v1/{prefix}/namespaces/{namespace}/generic-tables/{table}: + get: + tags: + - generic-table + summary: Load a generic table + operationId: load_generic_table + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: namespace + in: path + required: true + schema: + type: string + - name: table + in: path + required: true + schema: + type: string + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/LoadGenericTableResponse' + 4XX: + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + delete: + tags: + - generic-table + summary: Drop a generic table + operationId: drop_generic_table + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: namespace + in: path + required: true + schema: + type: string + - name: table + in: path + required: true + schema: + type: string + responses: + '204': + description: Generic table dropped successfully + 4XX: + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + /lakekeeper/v1/{prefix}/namespaces/{namespace}/generic-tables/{table}/credentials: + get: + tags: + - generic-table + summary: Load credentials for a generic table + operationId: load_generic_table_credentials + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: namespace + in: path + required: true + schema: + type: string + - name: table + in: path + required: true + schema: + type: string + - name: referenced-by + in: query + required: false + schema: + type: string + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/LoadGenericTableCredentialsResponse' + 4XX: + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' +components: + schemas: + CreateGenericTableRequest: + type: object + required: + - name + - format + properties: + base-location: + type: + - string + - 'null' + doc: + type: + - string + - 'null' + format: + type: string + name: + type: string + properties: + type: object + additionalProperties: + type: string + propertyNames: + type: string + schema: {} + statistics: {} + ErrorModel: + type: object + description: JSON error payload returned in a response with further details on the error + required: + - message + - type + - code + properties: + code: + type: integer + format: int32 + description: HTTP response code + minimum: 0 + message: + type: string + description: Human-readable error message + stack: + type: array + items: + type: string + type: + type: string + description: Internal type definition of the error + GenericTableData: + type: object + required: + - name + - format + - base-location + - protected + properties: + base-location: + type: string + doc: + type: + - string + - 'null' + format: + type: string + name: + type: string + properties: + type: object + additionalProperties: + type: string + propertyNames: + type: string + protected: + type: boolean + description: Whether the generic table is protected from being deleted. + schema: {} + statistics: {} + GenericTableIdentifier: + type: object + required: + - namespace + - name + properties: + format: + type: + - string + - 'null' + id: + type: + - string + - 'null' + format: uuid + name: + type: string + namespace: + type: array + items: + type: string + IcebergErrorResponse: + type: object + description: JSON wrapper for all error responses (non-2xx) + required: + - error + properties: + error: + $ref: '#/components/schemas/ErrorModel' + ListGenericTablesResponse: + type: object + required: + - identifiers + properties: + identifiers: + type: array + items: + $ref: '#/components/schemas/GenericTableIdentifier' + next-page-token: + type: + - string + - 'null' + LoadGenericTableCredentialsResponse: + type: object + required: + - storage-credentials + properties: + storage-credentials: + type: array + items: + $ref: '#/components/schemas/StorageCredential' + LoadGenericTableResponse: + type: object + required: + - table + properties: + config: + type: + - object + - 'null' + additionalProperties: + type: string + propertyNames: + type: string + storage-credentials: + type: + - array + - 'null' + items: + $ref: '#/components/schemas/StorageCredential' + table: + $ref: '#/components/schemas/GenericTableData' + RenameGenericTableRequest: + type: object + required: + - source + - destination + properties: + destination: + $ref: '#/components/schemas/RenameGenericTableTarget' + source: + $ref: '#/components/schemas/RenameGenericTableTarget' + RenameGenericTableTarget: + type: object + required: + - namespace + - name + properties: + name: + type: string + namespace: + type: array + items: + type: string + StorageCredential: + type: object + required: + - prefix + - config + properties: + config: + type: object + additionalProperties: + type: string + propertyNames: + type: string + prefix: + type: string + securitySchemes: + bearerAuth: + type: http + scheme: bearer + bearerFormat: JWT +security: + - bearerAuth: [] +tags: + - name: generic-table + description: Manage generic (non-Iceberg) tables diff --git a/openapi/management-open-api.yaml b/openapi/management-open-api.yaml index bdd4b5fd..fce599a7 100644 --- a/openapi/management-open-api.yaml +++ b/openapi/management-open-api.yaml @@ -959,6 +959,119 @@ paths: application/json: schema: $ref: '#/components/schemas/GetOpenFGAWarehouseActionsResponse' + /management/v1/permissions/warehouse/{warehouse_id}/generic-table/{generic_table_id}/assignments: + get: + tags: + - permissions-openfga + summary: Get user and role assignments for a generic table + operationId: get_generic_table_assignments_by_id + parameters: + - name: relations + in: query + description: Relations to be loaded. If not specified, all relations are returned. + required: false + schema: + type: array + items: + $ref: '#/components/schemas/GenericTableRelation' + - name: warehouse_id + in: path + description: Warehouse ID + required: true + schema: + type: string + format: uuid + - name: generic_table_id + in: path + description: Generic Table ID + required: true + schema: + type: string + format: uuid + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/GetGenericTableAssignmentsResponse' + post: + tags: + - permissions-openfga + summary: Update permissions for a generic table + operationId: update_generic_table_assignments_by_id + parameters: + - name: warehouse_id + in: path + description: Warehouse ID + required: true + schema: + type: string + format: uuid + - name: generic_table_id + in: path + description: Generic Table ID + required: true + schema: + type: string + format: uuid + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/UpdateGenericTableAssignmentsRequest' + required: true + responses: + '204': + description: Permissions updated successfully + /management/v1/permissions/warehouse/{warehouse_id}/generic-table/{generic_table_id}/authorizer-actions: + get: + tags: + - permissions-openfga + summary: Get allowed Authorizer actions on a generic table + description: |- + Returns Authorizer permissions (OpenFGA relations) for the specified generic table. + For Catalog permissions, use `/management/v1/warehouse/{warehouse_id}/generic-table/{generic_table_id}/actions` instead. + operationId: get_authorizer_generic_table_actions + parameters: + - name: principalUser + in: query + description: |- + The user to show actions for. + If neither user nor role is specified, shows actions for the current user. + required: false + schema: + type: string + - name: principalRole + in: query + description: |- + The role to show actions for. + If neither user nor role is specified, shows actions for the current user. + required: false + schema: + type: string + format: uuid + - name: warehouse_id + in: path + description: Warehouse ID + required: true + schema: + type: string + format: uuid + - name: generic_table_id + in: path + description: Generic Table ID + required: true + schema: + type: string + format: uuid + responses: + '200': + description: Generic Table Authorizer Actions + content: + application/json: + schema: + $ref: '#/components/schemas/GetOpenFGAGenericTableActionsResponse' /management/v1/permissions/warehouse/{warehouse_id}/managed-access: post: tags: @@ -2712,6 +2825,162 @@ paths: application/json: schema: $ref: '#/components/schemas/IcebergErrorResponse' + /management/v1/warehouse/{warehouse_id}/format-version-policy: + post: + tags: + - warehouse + summary: Update Format Version Policy + description: |- + Configures which Iceberg table format versions may be created in, or + upgraded to, within a warehouse, and the default version applied when a + create-table request does not specify one. + operationId: update_warehouse_format_version_policy + parameters: + - name: warehouse_id + in: path + required: true + schema: + type: string + format: uuid + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/UpdateWarehouseFormatVersionPolicyRequest' + required: true + responses: + '200': + description: Format version policy updated successfully + content: + application/json: + schema: + $ref: '#/components/schemas/GetWarehouseResponse' + 4XX: + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + /management/v1/warehouse/{warehouse_id}/generic-table/{generic_table_id}/actions: + get: + tags: + - warehouse + summary: Get allowed actions for a generic table + operationId: get_generic_table_actions + parameters: + - name: principalUser + in: query + description: |- + The user to show actions for. + If neither user nor role is specified, shows actions for the current user. + required: false + schema: + type: string + - name: principalRole + in: query + description: |- + The role to show actions for. + If neither user nor role is specified, shows actions for the current user. + required: false + schema: + type: string + format: uuid + - name: warehouse_id + in: path + required: true + schema: + type: string + format: uuid + - name: generic_table_id + in: path + required: true + schema: + type: string + format: uuid + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/GetLakekeeperGenericTableActionsResponse' + 4XX: + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + /management/v1/warehouse/{warehouse_id}/generic-table/{generic_table_id}/protection: + get: + tags: + - warehouse + summary: Get Generic Table Protection + description: Retrieves whether a generic table is protected from deletion. + operationId: get_generic_table_protection + parameters: + - name: warehouse_id + in: path + required: true + schema: + type: string + format: uuid + - name: generic_table_id + in: path + required: true + schema: + type: string + format: uuid + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/ProtectionResponse' + 4XX: + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + post: + tags: + - warehouse + summary: Set Generic Table Protection + description: Configures whether a generic table should be protected from deletion. + operationId: set_generic_table_protection + parameters: + - name: warehouse_id + in: path + required: true + schema: + type: string + format: uuid + - name: generic_table_id + in: path + required: true + schema: + type: string + format: uuid + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/SetProtectionRequest' + required: true + responses: + '200': + description: Generic table protection set successfully + content: + application/json: + schema: + $ref: '#/components/schemas/ProtectionResponse' + 4XX: + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' /management/v1/warehouse/{warehouse_id}/namespace/{namespace_id}/actions: get: tags: @@ -3791,6 +4060,19 @@ components: properties: action: $ref: '#/components/schemas/LakekeeperViewAction' + - type: object + required: + - generic-table + properties: + generic-table: + allOf: + - $ref: '#/components/schemas/TabularIdentOrUuid' + - type: object + required: + - action + properties: + action: + $ref: '#/components/schemas/LakekeeperGenericTableAction' description: Represents an action on an object CatalogActionsBatchCheckRequest: type: object @@ -3911,6 +4193,19 @@ components: properties: action: $ref: '#/components/schemas/ViewAction' + - type: object + required: + - generic-table + properties: + generic-table: + allOf: + - $ref: '#/components/schemas/TabularIdentOrUuid' + - type: object + required: + - action + properties: + action: + $ref: '#/components/schemas/GenericTableAction' description: Represents an action on an object CheckRequest: type: object @@ -4121,6 +4416,26 @@ components: - warehouse-name - storage-profile properties: + allowed-format-versions: + type: + - array + - 'null' + items: + type: integer + format: int32 + description: |- + Iceberg table format versions that may be created in, or upgraded to, + within this warehouse. Must be a non-empty subset of `[1, 2, 3]`. + Defaults to all supported versions when omitted. + default-format-version: + type: + - integer + - 'null' + format: int32 + description: |- + Default Iceberg table format version applied when a create-table request + does not specify one. Must be a member of `allowed-format-versions`. When + omitted, resolves to v2 if allowed, otherwise the highest allowed version. delete-profile: $ref: '#/components/schemas/TabularDeleteProfile' description: 'Profile to determine behavior upon dropping of tabulars. Default: hard deletion.' @@ -4438,6 +4753,103 @@ components: type: string universe_domain: type: string + GenericTableAction: + type: string + enum: + - drop + - undrop + - write_data + - read_data + - get_metadata + - rename + - include_in_list + - get_tasks + - control_tasks + - set_protection + - read_assignments + - grant_pass_grants + - grant_manage_grants + - grant_describe + - grant_select + - grant_modify + - change_ownership + GenericTableAssignment: + oneOf: + - allOf: + - $ref: '#/components/schemas/UserOrRole' + - type: object + required: + - type + properties: + type: + type: string + enum: + - ownership + title: GenericTableAssignmentOwnership + - allOf: + - $ref: '#/components/schemas/UserOrRole' + - type: object + required: + - type + properties: + type: + type: string + enum: + - pass_grants + title: GenericTableAssignmentPassGrants + - allOf: + - $ref: '#/components/schemas/UserOrRole' + - type: object + required: + - type + properties: + type: + type: string + enum: + - manage_grants + title: GenericTableAssignmentManageGrants + - allOf: + - $ref: '#/components/schemas/UserOrRole' + - type: object + required: + - type + properties: + type: + type: string + enum: + - describe + title: GenericTableAssignmentDescribe + - allOf: + - $ref: '#/components/schemas/UserOrRole' + - type: object + required: + - type + properties: + type: + type: string + enum: + - select + title: GenericTableAssignmentSelect + - allOf: + - $ref: '#/components/schemas/UserOrRole' + - type: object + required: + - type + properties: + type: + type: string + enum: + - modify + title: GenericTableAssignmentModify + GenericTableRelation: + type: string + enum: + - ownership + - pass_grants + - manage_grants + - describe + - select + - modify GetEndpointStatisticsRequest: type: object required: @@ -4471,6 +4883,24 @@ components: Can return statistics for a specific warehouse, all warehouses or requests that could not be associated to any warehouse. + GetGenericTableAssignmentsResponse: + type: object + required: + - assignments + properties: + assignments: + type: array + items: + $ref: '#/components/schemas/GenericTableAssignment' + GetLakekeeperGenericTableActionsResponse: + type: object + required: + - allowed-actions + properties: + allowed-actions: + type: array + items: + $ref: '#/components/schemas/LakekeeperGenericTableAction' GetLakekeeperNamespaceActionsResponse: type: object required: @@ -4571,6 +5001,15 @@ components: type: boolean managed-access-inherited: type: boolean + GetOpenFGAGenericTableActionsResponse: + type: object + required: + - allowed-actions + properties: + allowed-actions: + type: array + items: + $ref: '#/components/schemas/OpenFGAGenericTableAction' GetOpenFGANamespaceActionsResponse: type: object required: @@ -4856,7 +5295,25 @@ components: - delete-profile - status - protected + - allowed-format-versions properties: + allowed-format-versions: + type: array + items: + type: integer + format: int32 + description: |- + Iceberg table format versions that may be created in, or upgraded to, + within this warehouse. + default-format-version: + type: + - integer + - 'null' + format: int32 + description: |- + Default Iceberg table format version applied when a create-table request + does not specify one. When absent, resolves to v2 if allowed, otherwise + the highest allowed version. delete-profile: $ref: '#/components/schemas/TabularDeleteProfile' description: Delete profile used for the warehouse. @@ -4907,6 +5364,88 @@ components: properties: error: $ref: '#/components/schemas/ErrorModel' + LakekeeperGenericTableAction: + oneOf: + - type: object + required: + - action + properties: + action: + type: string + enum: + - drop + - type: object + required: + - action + properties: + action: + type: string + enum: + - read_data + - type: object + required: + - action + properties: + action: + type: string + enum: + - write_data + - type: object + required: + - action + properties: + action: + type: string + enum: + - get_metadata + - type: object + required: + - action + properties: + action: + type: string + enum: + - rename + - type: object + required: + - action + properties: + action: + type: string + enum: + - include_in_list + - type: object + required: + - action + properties: + action: + type: string + enum: + - undrop + - type: object + required: + - action + properties: + action: + type: string + enum: + - get_tasks + - type: object + required: + - action + properties: + action: + type: string + enum: + - control_tasks + - type: object + required: + - action + properties: + action: + type: string + enum: + - set_protection LakekeeperNamespaceAction: oneOf: - type: object @@ -5054,6 +5593,53 @@ components: type: string enum: - include_in_list + - type: object + required: + - action + properties: + action: + type: string + enum: + - create_generic_table + base_location: + type: + - string + - 'null' + description: |- + User-supplied base location override — primary lever for + path-based authorization policy. + format: + type: + - string + - 'null' + description: |- + Generic table format (e.g. "lance", "delta") — primary lever for + format-based authorization policy. + generic_table_id: + type: + - string + - 'null' + format: uuid + description: Generic table ID, if externally provided. + name: + type: + - string + - 'null' + description: Name of the generic table to create. + properties: + type: object + additionalProperties: + type: string + propertyNames: + type: string + - type: object + required: + - action + properties: + action: + type: string + enum: + - list_generic_tables LakekeeperProjectAction: oneOf: - type: object @@ -5212,6 +5798,22 @@ components: type: string enum: - update + - type: object + required: + - action + properties: + action: + type: string + enum: + - manage_role_assignments + - type: object + required: + - action + properties: + action: + type: string + enum: + - read_role_assignments LakekeeperServerAction: oneOf: - type: object @@ -5395,6 +5997,14 @@ components: type: string enum: - delete + - type: object + required: + - action + properties: + action: + type: string + enum: + - read_role_assignments LakekeeperViewAction: oneOf: - type: object @@ -5660,6 +6270,14 @@ components: type: string enum: - set_protection + - type: object + required: + - action + properties: + action: + type: string + enum: + - set_format_version_policy - type: object required: - action @@ -5905,6 +6523,7 @@ components: enum: - create_table - create_view + - create_generic_table - create_namespace - delete - update_properties @@ -6032,6 +6651,16 @@ components: - select - create - modify + OpenFGAGenericTableAction: + type: string + enum: + - read_assignments + - grant_pass_grants + - grant_manage_grants + - grant_describe + - grant_select + - grant_modify + - change_ownership OpenFGANamespaceAction: type: string enum: @@ -6693,6 +7322,16 @@ components: - path - virtual_host - auto + ScheduleTaskResponse: + type: object + description: Response returned on a successful schedule call. + required: + - task-id + properties: + task-id: + type: string + format: uuid + description: The id of the newly scheduled task. SearchRoleRequest: type: object required: @@ -7444,11 +8083,15 @@ components: type: string table: type: string - description: Name of the table or view + description: Name of the table, view, or generic table. warehouse-id: type: string format: uuid - description: Identifier for a table or view, either a UUID or its name and namespace + description: |- + Identifier for a tabular (table, view, or generic table) — either a UUID + or its name and namespace. Wire format primary names are `table-id` and + `table`; `view_id` / `view` and `generic_table_id` / `generic_table` are + accepted as input aliases for client ergonomics. TabularIdentUuid: oneOf: - type: object @@ -7475,12 +8118,25 @@ components: type: string enum: - view + - type: object + required: + - id + - type + properties: + id: + type: string + format: uuid + type: + type: string + enum: + - generic-table TabularType: type: string description: Type of tabular enum: - table - view + - generic-table TaskAttempt: type: object required: @@ -7616,6 +8272,17 @@ components: items: $ref: '#/components/schemas/TabularIdentUuid' description: Tabulars to undrop + UpdateGenericTableAssignmentsRequest: + type: object + properties: + deletes: + type: array + items: + $ref: '#/components/schemas/GenericTableAssignment' + writes: + type: array + items: + $ref: '#/components/schemas/GenericTableAssignment' UpdateNamespaceAssignmentsRequest: type: object properties: @@ -7749,6 +8416,28 @@ components: properties: delete-profile: $ref: '#/components/schemas/TabularDeleteProfile' + UpdateWarehouseFormatVersionPolicyRequest: + type: object + required: + - allowed-format-versions + properties: + allowed-format-versions: + type: array + items: + type: integer + format: int32 + description: |- + Iceberg table format versions that may be created in, or upgraded to, + within this warehouse. Must be a non-empty subset of `[1, 2, 3]`. + default-format-version: + type: + - integer + - 'null' + format: int32 + description: |- + Default Iceberg table format version applied when a create-table request + does not specify one. Must be a member of `allowed-format-versions`. When + omitted, resolves to v2 if allowed, otherwise the highest allowed version. UpdateWarehouseStorageRequest: type: object required: @@ -7965,6 +8654,7 @@ components: - get_all_tasks - control_all_tasks - set_protection + - set_format_version_policy - get_endpoint_statistics WarehouseAssignment: oneOf: @@ -8171,6 +8861,19 @@ components: view-id: type: string format: uuid + - type: object + description: Get tasks for a specific generic table + required: + - generic-table-id + - type + properties: + generic-table-id: + type: string + format: uuid + type: + type: string + enum: + - generic-table - type: object description: |- Get Warehouse-level tasks which are not associated with a specific entity @@ -8208,6 +8911,18 @@ components: view-id: type: string format: uuid + - type: object + required: + - generic-table-id + - type + properties: + generic-table-id: + type: string + format: uuid + type: + type: string + enum: + - generic-table WarehouseTaskInfo: type: object required: diff --git a/openapi/rest-catalog-open-api.yaml b/openapi/rest-catalog-open-api.yaml index 66285a99..e10ff55e 100644 --- a/openapi/rest-catalog-open-api.yaml +++ b/openapi/rest-catalog-open-api.yaml @@ -25,7 +25,7 @@ info: name: Apache 2.0 url: https://www.apache.org/licenses/LICENSE-2.0.html version: 0.0.1 - description: Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. + description: Defines the specification for the first version of the REST Catalog API. Implementations should ideally support all Iceberg table spec versions. servers: - url: "{scheme}://{host}/{basePath}" description: Server URL when the port can be inferred from the scheme @@ -80,7 +80,7 @@ paths: application/json: schema: $ref: '#/components/schemas/CatalogConfig' - example: {"overrides": {"warehouse": "s3://bucket/warehouse/"}, "defaults": {"clients": "4"}, "endpoints": ["GET /v1/{prefix}/namespaces/{namespace}", "GET /v1/{prefix}/namespaces", "POST /v1/{prefix}/namespaces", "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}", "GET /v1/{prefix}/namespaces/{namespace}/views/{view}"]} + example: {"overrides": {"warehouse": "s3://bucket/warehouse/"}, "defaults": {"clients": "4"}, "idempotency-key-lifetime": "PT30M", "endpoints": ["GET /v1/{prefix}/namespaces/{namespace}", "GET /v1/{prefix}/namespaces", "POST /v1/{prefix}/namespaces", "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}", "GET /v1/{prefix}/namespaces/{namespace}/views/{view}"]} 400: $ref: '#/components/responses/BadRequestErrorResponse' 401: @@ -147,9 +147,8 @@ paths: - $ref: '#/components/parameters/page-size' - name: parent in: query - description: An optional namespace, underneath which to list namespaces. If not provided or empty, all top-level namespaces should be listed. If parent is a multipart namespace, the parts must be separated by the unit separator (`0x1F`) byte. + description: An optional namespace, underneath which to list namespaces. If not provided, all top-level namespaces should be listed. For backward compatibility, empty string is treated as absent for now. If parent is a multipart namespace, the parts must be separated by the namespace separator as indicated via the /config override `namespace-separator`, which defaults to the unit separator `0x1F` byte (url encoded `%1F`). To be compatible with older clients, servers must use both the advertised separator and `0x1F` as valid separators when decoding namespaces. The `namespace-separator` should be provided in a url encoded form. required: false - allowEmptyValue: true schema: type: string example: "accounting%1Ftax" @@ -195,6 +194,8 @@ paths: tags: - Catalog API summary: Create a namespace + parameters: + - $ref: '#/components/parameters/idempotency-key' description: Create a namespace, with an optional set of properties. The server might also add properties, such as `last_modified_time` etc. operationId: createNamespace requestBody: @@ -306,6 +307,29 @@ paths: - Catalog API summary: Drop a namespace from the catalog. operationId: dropNamespace + parameters: + - $ref: '#/components/parameters/idempotency-key' + - name: force + in: query + description: If force and recursive are set to true, immediately delete all contents of the namespace without considering soft-delete policies. Force has no effect without recursive=true. + required: false + schema: + type: boolean + default: false + - name: recursive + in: query + description: Delete a namespace and its contents. This means all tables, views, and namespaces under this namespace will be deleted. The namespace itself will also be deleted. If the warehouse containing the namespace is configured with a soft-deletion profile, the `force` flag has to be provided. The deletion will not be a soft-deletion. Every table, view and namespace will be gone as soon as this call returns. Depending on whether the `purge` flag was set to true, the data will be queued for deletion too. Any pending `tabular_expiration` will be cancelled. If there is a running `tabular_expiration`, this call will fail with a `409 Conflict` error. + required: false + schema: + type: boolean + default: false + - name: purge + in: query + description: If recursive is true, also deletes table and view data. If false, only metadata is dropped from the catalog, table location remains untouched. Defaults to true for all tables managed by Lakekeeper. + required: false + schema: + type: boolean + default: true responses: 204: description: Success, no content @@ -339,28 +363,6 @@ paths: $ref: '#/components/responses/ServiceUnavailableResponse' 5XX: $ref: '#/components/responses/ServerErrorResponse' - parameters: - - name: force - in: query - description: If force and recursive are set to true, immediately delete all contents of the namespace without considering soft-delete policies. Force has no effect without recursive=true. - required: false - schema: - type: boolean - default: false - - name: recursive - in: query - description: Delete a namespace and its contents. This means all tables, views, and namespaces under this namespace will be deleted. The namespace itself will also be deleted. If the warehouse containing the namespace is configured with a soft-deletion profile, the `force` flag has to be provided. The deletion will not be a soft-deletion. Every table, view and namespace will be gone as soon as this call returns. Depending on whether the `purge` flag was set to true, the data will be queued for deletion too. Any pending `tabular_expiration` will be cancelled. If there is a running `tabular_expiration`, this call will fail with a `409 Conflict` error. - required: false - schema: - type: boolean - default: false - - name: purge - in: query - description: If recursive is true, also deletes table and view data. If false, only metadata is dropped from the catalog, table location remains untouched. Defaults to true for all tables managed by Lakekeeper. - required: false - schema: - type: boolean - default: true description: Drop a namespace from the catalog. By default, the namespace needs to be empty. You can however set `recursive=true` which will delete all tables, views and namespaces under this namespace. The namespace itself will also be deleted. If the warehouse containing the namespace is configured with a soft-deletion profile, the `force` flag has to be provided. The deletion will not be a soft-deletion. Every table, view and namespace will be gone as soon as this call returns. Depending on whether the `purge` flag was set to true, the data will be queued for deletion too. Any pending `tabular_expiration` will be cancelled. If there is a running `tabular_expiration`, this call will fail with a `409 Conflict` error. /v1/{prefix}/namespaces/{namespace}/properties: parameters: @@ -371,6 +373,8 @@ paths: - Catalog API summary: Set or remove properties on a namespace operationId: updateProperties + parameters: + - $ref: '#/components/parameters/idempotency-key' description: |- Set and/or remove properties on a namespace. The request body specifies a list of properties to remove and a map of key value pairs to update. Properties that are not in the request are not modified or removed by this call. @@ -483,6 +487,7 @@ paths: operationId: createTable parameters: - $ref: '#/components/parameters/data-access' + - $ref: '#/components/parameters/idempotency-key' requestBody: required: true content: @@ -514,7 +519,7 @@ paths: schema: $ref: '#/components/schemas/IcebergErrorResponse' examples: - NamespaceAlreadyExists: + TableAlreadyExists: $ref: '#/components/examples/TableAlreadyExistsError' 419: $ref: '#/components/responses/AuthenticationTimeoutResponse' @@ -527,12 +532,44 @@ paths: - $ref: '#/components/parameters/prefix' - $ref: '#/components/parameters/namespace' - $ref: '#/components/parameters/table' + - $ref: '#/components/parameters/data-access' post: tags: - Catalog API summary: Submit a scan for planning - description: "Submits a scan for server-side planning.\n\nPoint-in-time scans are planned by passing snapshot-id to identify the table snapshot to scan. Incremental scans are planned by passing both start-snapshot-id and end-snapshot-id. Requests that include both point in time config properties and incremental config properties are invalid. If the request does not include either incremental or point-in-time config properties, scan planning should produce a point-in-time scan of the latest snapshot in the table's main branch.\n\nResponses must include a valid status listed below. A \"cancelled\" status is considered invalid for this endpoint. \n- When \"completed\" the planning operation has produced plan tasks and\n file scan tasks that must be returned in the response (not fetched\n later by calling fetchPlanningResult)\n\n- When \"submitted\" the response must include a plan-id used to poll\n fetchPlanningResult to fetch the planning result when it is ready\n\n- When \"failed\" the response must be a valid error response\nThe response for a \"completed\" planning operation includes two types of tasks (file scan tasks and plan tasks) and both may be included in the response. Tasks must not be included for any other response status.\n\nResponses that include a plan-id indicate that the service is holding state or performing work for the client.\n\n- Clients should use the plan-id to fetch results from\n fetchPlanningResult when the response status is \"submitted\"\n\n- Clients should inform the service if planning results are no longer\n needed by calling cancelPlanning. Cancellation is not necessary after\n fetchScanTasks has been used to fetch scan tasks for each plan task.\n" + description: > + Submits a scan for server-side planning. + + Point-in-time scans are planned by passing snapshot-id to identify the table snapshot to scan. Incremental scans are planned by passing both start-snapshot-id and end-snapshot-id. Requests that include both point in time config properties and incremental config properties are invalid. If the request does not include either incremental or point-in-time config properties, scan planning should produce a point-in-time scan of the latest snapshot in the table's main branch. + + Responses must include a valid status listed below. A "cancelled" status is considered invalid for this endpoint. + + - When "completed" the planning operation has produced plan tasks and + + file scan tasks that must be returned in the response (not fetched + later by calling fetchPlanningResult) + + - When "submitted" the response must include a plan-id used to poll + + fetchPlanningResult to fetch the planning result when it is ready + + - When "failed" the response must be a valid error response + + The response for a "completed" planning operation includes two types of tasks (file scan tasks and plan tasks) and both may be included in the response. Tasks must not be included for any other response status. + + Responses that include a plan-id indicate that the service is holding state or performing work for the client. + + - Clients should use the plan-id to fetch results from + + fetchPlanningResult when the response status is "submitted" + + - Clients should inform the service if planning results are no longer + + needed by calling cancelPlanning. Cancellation is not necessary after + fetchScanTasks has been used to fetch scan tasks for each plan task. operationId: planTableScan + parameters: + - $ref: '#/components/parameters/idempotency-key' requestBody: content: application/json: @@ -572,6 +609,7 @@ paths: - $ref: '#/components/parameters/namespace' - $ref: '#/components/parameters/table' - $ref: '#/components/parameters/plan-id' + - $ref: '#/components/parameters/data-access' get: tags: - Catalog API @@ -629,6 +667,8 @@ paths: - Catalog API summary: Cancels scan planning for a plan-id operationId: cancelPlanning + parameters: + - $ref: '#/components/parameters/idempotency-key' description: > Cancels scan planning for a plan-id. @@ -651,12 +691,14 @@ paths: 403: $ref: '#/components/responses/ForbiddenResponse' 404: - description: Not Found - NoSuchTableException, the table does not exist - NoSuchNamespaceException, the namespace does not exist + description: Not Found - NoSuchPlanIdException, the plan-id does not exist - NoSuchTableException, the table does not exist - NoSuchNamespaceException, the namespace does not exist content: application/json: schema: $ref: '#/components/schemas/IcebergErrorResponse' examples: + PlanIdDoesNotExist: + $ref: '#/components/examples/NoSuchPlanIdError' TableDoesNotExist: $ref: '#/components/examples/NoSuchTableError' NamespaceDoesNotExist: @@ -677,6 +719,8 @@ paths: - Catalog API summary: Fetches result tasks for a plan task operationId: fetchScanTasks + parameters: + - $ref: '#/components/parameters/idempotency-key' description: Fetches result tasks for a plan task. requestBody: content: @@ -719,6 +763,9 @@ paths: tags: - Catalog API summary: Register a table in the given namespace using given metadata file location + parameters: + - $ref: '#/components/parameters/data-access' + - $ref: '#/components/parameters/idempotency-key' description: Register a table using given metadata file location. operationId: registerTable requestBody: @@ -752,7 +799,7 @@ paths: schema: $ref: '#/components/schemas/IcebergErrorResponse' examples: - NamespaceAlreadyExists: + TableAlreadyExists: $ref: '#/components/examples/TableAlreadyExistsError' 419: $ref: '#/components/responses/AuthenticationTimeoutResponse' @@ -782,7 +829,7 @@ paths: - $ref: '#/components/parameters/data-access' - name: If-None-Match in: header - description: An optional header that allows the server to return 304 (Not Modified) if the metadata is current. The content is the value of the ETag received in a CreateTableResponse or LoadTableResponse. + description: An optional header that allows the server to return 304 (Not Modified) if the metadata is current. The content is the value of the ETag received in a CreateTableResponse, LoadTableResponse or CommitTableResponse. required: false schema: type: string @@ -795,6 +842,7 @@ paths: schema: type: string enum: [all, refs] + - $ref: '#/components/parameters/referenced-by' responses: 200: $ref: '#/components/responses/LoadTableResponse' @@ -826,6 +874,8 @@ paths: - Catalog API summary: Commit updates to a table operationId: updateTable + parameters: + - $ref: '#/components/parameters/idempotency-key' description: |- Commit updates to a table. @@ -903,6 +953,7 @@ paths: operationId: dropTable description: Remove a table from the catalog parameters: + - $ref: '#/components/parameters/idempotency-key' - name: purgeRequested in: query required: false @@ -981,6 +1032,14 @@ paths: - Catalog API summary: Load vended credentials for a table from the catalog operationId: loadCredentials + parameters: + - name: planId + in: query + required: false + schema: + type: string + description: The plan ID that has been used for server-side scan planning + - $ref: '#/components/parameters/referenced-by' description: Load vended credentials for a table from the catalog. responses: 200: @@ -1013,6 +1072,8 @@ paths: tags: - Catalog API summary: Rename a table from its current name to a new name + parameters: + - $ref: '#/components/parameters/idempotency-key' description: Rename a table from one identifier to another. It's valid to move a table across namespaces, but the server implementation is not required to support it. operationId: renameTable requestBody: @@ -1110,6 +1171,8 @@ paths: - Catalog API summary: Commit updates to multiple tables in an atomic operation operationId: commitTransaction + parameters: + - $ref: '#/components/parameters/idempotency-key' requestBody: description: |- Commit updates to multiple tables in an atomic operation @@ -1265,7 +1328,7 @@ paths: schema: $ref: '#/components/schemas/ErrorModel' examples: - NamespaceAlreadyExists: + ViewAlreadyExists: $ref: '#/components/examples/ViewAlreadyExistsError' 419: $ref: '#/components/responses/AuthenticationTimeoutResponse' @@ -1291,6 +1354,8 @@ paths: The response also contains the view's full metadata, matching the view metadata JSON file. The catalog configuration may contain credentials that should be used for subsequent requests for the view. The configuration key "token" is used to pass an access token to be used as a bearer token for view requests. Otherwise, a token may be passed using a RFC 8693 token type as a configuration key. For example, "urn:ietf:params:oauth:token-type:jwt=". + parameters: + - $ref: '#/components/parameters/referenced-by' responses: 200: $ref: '#/components/responses/LoadViewResponse' @@ -1320,6 +1385,8 @@ paths: - Catalog API summary: Replace a view operationId: replaceView + parameters: + - $ref: '#/components/parameters/idempotency-key' description: Commit updates to a view. requestBody: required: true @@ -1389,6 +1456,15 @@ paths: summary: Drop a view from the catalog operationId: dropView description: Remove a view from the catalog + parameters: + - $ref: '#/components/parameters/idempotency-key' + - name: force + in: query + description: If true, ignore `protection-status` when dropping. + required: false + schema: + type: boolean + default: false responses: 204: description: Success, no content @@ -1413,14 +1489,6 @@ paths: $ref: '#/components/responses/ServiceUnavailableResponse' 5XX: $ref: '#/components/responses/ServerErrorResponse' - parameters: - - name: force - in: query - description: If true, ignore `protection-status` when dropping. - required: false - schema: - type: boolean - default: false head: tags: - Catalog API @@ -1451,6 +1519,8 @@ paths: summary: Rename a view from its current name to a new name description: Rename a view from one identifier to another. It's valid to move a view across namespaces, but the server implementation is not required to support it. operationId: renameView + parameters: + - $ref: '#/components/parameters/idempotency-key' requestBody: description: Current view identifier to rename and new view identifier to rename to content: @@ -1506,7 +1576,7 @@ components: name: namespace in: path required: true - description: A namespace identifier as a single string. Multipart namespace parts should be separated by the unit separator (`0x1F`) byte. + description: A namespace identifier as a single string. Multipart namespace parts must be separated by the namespace separator as indicated via the /config override `namespace-separator`, which defaults to the unit separator `0x1F` byte (url encoded `%1F`). To be compatible with older clients, servers must use both the advertised separator and `0x1F` as valid separators when decoding namespaces. The `namespace-separator` should be provided in a url encoded form. schema: type: string examples: @@ -1552,7 +1622,7 @@ components: Specific properties and handling for `vended-credentials` is documented in the `LoadTableResult` schema section of this spec document. - The protocol and specification for `remote-signing` is documented in the `s3-signer-open-api.yaml` OpenApi spec in the `aws` module. + The protocol and specification for `remote-signing` is documented in the `s3-signer-open-api.yaml` OpenApi spec in the `aws` module. required: false schema: @@ -1567,7 +1637,6 @@ components: name: pageToken in: query required: false - allowEmptyValue: true schema: $ref: '#/components/schemas/PageToken' page-size: @@ -1581,10 +1650,52 @@ components: etag: name: ETag in: header - description: Identifies a unique version of the table metadata. + description: |- + Identifies a unique version of the table metadata. + Implementations that support ETags should produce unique tags for responses that return different metadata content but represent the same version of table metadata. For example, the `snapshots` query parameter may result in different metadata representations depending on whether `refs` or `all` is provided, therefore should have distinct ETags. + required: false + schema: + type: string + referenced-by: + name: referenced-by + in: query + description: |- + A comma-separated list of fully qualified view names (namespace and view name) representing the view reference chain when an entity (table or view) is loaded via a view. The list should be ordered with the outermost view first, followed by any intermediate views it references, down to the view that directly references the entity. For a simple case where a view directly references the entity, the list contains a single view identifier. For nested views (a view referencing another view which references the entity), the list contains multiple view identifiers representing the complete dependency chain. + The view identifier is a composite string of the format {namespace}{separator}{viewName}. The namespace-separator (defined in /config) acts as the delimiter. When parsing, the last occurrence of this separator identifies the boundary between the namespace and the view name. Multipart namespaces must follow the encoding rules of the parent parameter in the list namespaces endpoint. + Multiple view identifiers are separated by commas. Servers should split the parameter value on comma characters to parse individual view identifiers. If view names contain commas, they must be url-encoded as %2C. + Example with multiple views (where prod%1Fanalytics is a nested namespace which has a quarterly_view which references to monthly_view which then references the entity being loaded) - prod%1Fanalytics%1Fquarterly_view,prod%1Fanalytics%1Fmonthly_view + required: false + schema: + type: string + idempotency-key: + name: Idempotency-Key + in: header required: false schema: type: string + format: uuid + minLength: 36 + maxLength: 36 + example: "017F22E2-79B0-7CC3-98C4-DC0C0C07398F" + description: | + Optional client-provided idempotency key for safe request retries. + + When present, the server ensures no additional effects for requests that carry the same + Idempotency-Key. If a prior request with this key has been finalized, the server returns + an equivalent final response without re-running the operation. The response body may + reflect a newer state of the catalog than existed at the time of the commit. + + Finalization rules: + - Finalize & replay: 200, 201, 204, and deterministic terminal 4xx (including 409 + such as AlreadyExists, NamespaceNotEmpty, etc.) + - Do not finalize (not stored/replayed): 5xx + + Key Requirements: + - Key format: UUIDv7 in string form (RFC 9562). + - The idempotency key must be globally unique (no reuse across different operations). + - Catalogs SHOULD NOT expire keys before the end of the advertised token lifetime. + - If Idempotency-Key is used, clients MUST reuse the same key when retrying the same + logical operation and MUST generate a new key for a different operation. ############################## # Application Schema Objects # ############################## @@ -1637,6 +1748,11 @@ components: type: string description: A list of endpoints that the server supports. The format of each endpoint must be " ". The HTTP verb and the resource path must be separated by a space character. example: ["GET /v1/{prefix}/namespaces/{namespace}", "GET /v1/{prefix}/namespaces", "POST /v1/{prefix}/namespaces", "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}", "GET /v1/{prefix}/namespaces/{namespace}/views/{view}"] + idempotency-key-lifetime: + type: string + format: duration + description: Client reuse window for an Idempotency-Key (ISO-8601 duration, e.g., PT30M, PT24H). Interpreted as the maximum time from the first submission using a key to the last retry during which a client may reuse that key. Servers SHOULD accept retries for at least this duration and MAY include a grace period to account for delays/clock skew. Clients SHOULD NOT reuse an Idempotency-Key after this window elapses; they SHOULD generate a new key for any subsequent attempt. Presence of this field indicates the server supports Idempotency-Key semantics for mutation endpoints. If absent, clients MUST assume idempotency is not supported. + example: PT30M CreateNamespaceRequest: type: object required: @@ -1878,15 +1994,12 @@ components: required: - type - term - - value properties: type: $ref: '#/components/schemas/ExpressionType' enum: ["is-null", "not-null", "is-nan", "not-nan"] term: $ref: '#/components/schemas/Term' - value: - type: object LiteralExpression: type: object required: @@ -1900,7 +2013,7 @@ components: term: $ref: '#/components/schemas/Term' value: - type: object + $ref: '#/components/schemas/PrimitiveTypeValue' SetExpression: type: object required: @@ -1916,7 +2029,7 @@ components: values: type: array items: - type: object + $ref: '#/components/schemas/PrimitiveTypeValue' Term: oneOf: - $ref: '#/components/schemas/Reference' @@ -2836,7 +2949,37 @@ components: items: $ref: '#/components/schemas/StorageCredential' LoadTableResult: - description: "Result used when a table is successfully loaded.\n\n\nThe table metadata JSON is returned in the `metadata` field. The corresponding file location of table metadata should be returned in the `metadata-location` field, unless the metadata is not yet committed. For example, a create transaction may return metadata that is staged but not committed.\nClients can check whether metadata has changed by comparing metadata locations after the table has been created.\n\n\nThe `config` map returns table-specific configuration for the table's resources, including its HTTP client and FileIO. For example, config may contain a specific FileIO implementation class for the table depending on its underlying storage.\n\n\nThe following configurations should be respected by clients:\n\n## General Configurations\n\n- `token`: Authorization bearer token to use for table requests if OAuth2 security is enabled \n\n## AWS Configurations\n\nThe following configurations should be respected when working with tables stored in AWS S3\n - `client.region`: region to configure client for making requests to AWS\n - `s3.access-key-id`: id for credentials that provide access to the data in S3\n - `s3.secret-access-key`: secret for credentials that provide access to data in S3 \n - `s3.session-token`: if present, this value should be used for as the session token \n - `s3.remote-signing-enabled`: if `true` remote signing should be performed as described in the `s3-signer-open-api.yaml` specification\n - `s3.cross-region-access-enabled`: if `true`, S3 Cross-Region bucket access is enabled\n\n## Storage Credentials\n\nCredentials for ADLS / GCS / S3 / ... are provided through the `storage-credentials` field.\nClients must first check whether the respective credentials exist in the `storage-credentials` field before checking the `config` for credentials.\n" + description: | + Result used when a table is successfully loaded. + + The table metadata JSON is returned in the `metadata` field. The corresponding file location of table metadata should be returned in the `metadata-location` field, unless the metadata is not yet committed. For example, a create transaction may return metadata that is staged but not committed. + Clients can check whether metadata has changed by comparing metadata locations after the table has been created. + + The `config` map returns table-specific configuration for the table's resources, including its HTTP client and FileIO. For example, config may contain a specific FileIO implementation class for the table depending on its underlying storage. + + The following configurations should be respected by clients: + + ## General Configurations + + - `token`: Authorization bearer token to use for table requests if OAuth2 security is enabled + - `scan-planning-mode`: Communicates to clients the supported planning mode. Clients should use this value to fail fast if the supported scanning mode is not available on the client. Valid values: + - `client`: Clients MUST use client-side scan planning + - `server`: Clients MUST use server-side scan planning via the `planTableScan` endpoint + + ## AWS Configurations + + The following configurations should be respected when working with tables stored in AWS S3 + - `client.region`: region to configure client for making requests to AWS + - `s3.access-key-id`: id for credentials that provide access to the data in S3 + - `s3.secret-access-key`: secret for credentials that provide access to data in S3 + - `s3.session-token`: if present, this value should be used for as the session token + - `s3.remote-signing-enabled`: if `true` remote signing should be performed as described in the `s3-signer-open-api.yaml` specification + - `s3.cross-region-access-enabled`: if `true`, S3 Cross-Region bucket access is enabled + + ## Storage Credentials + + Credentials for ADLS / GCS / S3 / ... are provided through the `storage-credentials` field. + Clients must first check whether the respective credentials exist in the `storage-credentials` field before checking the `config` for credentials. type: object required: - metadata @@ -2896,11 +3039,20 @@ components: status: $ref: '#/components/schemas/PlanStatus' const: "completed" + storage-credentials: + type: array + description: |- + Storage credentials for accessing the files returned in the scan result. + If the server returns storage credentials as part of the completed scan planning response, the expectation is for the client to use these credentials to read the files returned in the FileScanTasks as part of the scan result. + items: + $ref: '#/components/schemas/StorageCredential' CompletedPlanningWithIDResult: type: object allOf: - $ref: '#/components/schemas/CompletedPlanningResult' - type: object + required: + - plan-id properties: plan-id: description: ID used to track a planning request @@ -2921,6 +3073,7 @@ components: type: object required: - status + - plan-id properties: status: $ref: '#/components/schemas/PlanStatus' @@ -3783,6 +3936,10 @@ components: filter: description: Expression used to filter the table data $ref: '#/components/schemas/Expression' + min-rows-requested: + description: The minimum number of rows requested for the scan. This is used as a hint to the server to not have to return more rows than necessary. It is not required for the server to return that many rows since the scan may not produce that many rows. The server can also return more rows than requested. + type: integer + format: int64 case-sensitive: description: Enables case sensitive field matching for filter and select type: boolean @@ -4013,6 +4170,9 @@ components: application/json: schema: $ref: '#/components/schemas/CommitTableResponse' + headers: + etag: + $ref: '#/components/parameters/etag' LoadCredentialsResponse: description: Table credentials result when loading credentials for a table content: diff --git a/src/assets/delta.svg b/src/assets/delta.svg new file mode 100644 index 00000000..596b0ec1 --- /dev/null +++ b/src/assets/delta.svg @@ -0,0 +1,60 @@ + + + + + delta-logo-rev + + + + + + + delta-logo-rev + + + + diff --git a/src/assets/iceberg.svg b/src/assets/iceberg.svg new file mode 100644 index 00000000..ec1619e2 --- /dev/null +++ b/src/assets/iceberg.svg @@ -0,0 +1,119 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/assets/lance.png b/src/assets/lance.png new file mode 100644 index 00000000..d336418c Binary files /dev/null and b/src/assets/lance.png differ diff --git a/src/assets/vortex_logo.svg b/src/assets/vortex_logo.svg new file mode 100644 index 00000000..63fbd585 --- /dev/null +++ b/src/assets/vortex_logo.svg @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/src/assets/vortex_logo_dark_theme.svg b/src/assets/vortex_logo_dark_theme.svg new file mode 100644 index 00000000..258cc4ad --- /dev/null +++ b/src/assets/vortex_logo_dark_theme.svg @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/src/common/enums.ts b/src/common/enums.ts index b5c707af..194e8c40 100644 --- a/src/common/enums.ts +++ b/src/common/enums.ts @@ -17,7 +17,7 @@ export enum ObjectType { WAREHOUSE = 'Warehouse', STORAGE_PROFILE = 'StorageProfile', STORAGE_CREDENTIAL = 'StorageCredential', - DELETION_PROFILE = 'DeletionProfile', + CATALOG_SETTINGS = 'CatalogSettings', } export enum TokenType { diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index ce9c995c..3870e371 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -12,6 +12,7 @@ import { ServerAssignment, TableAssignment, ViewAssignment, + GenericTableAssignment, WarehouseAssignment, } from '../gen/management/types.gen'; import { Type } from './enums'; @@ -54,6 +55,7 @@ export enum RelationType { Namespace = 'namespace', View = 'view', Table = 'table', + GenericTable = 'generic-table', } export type AssignmentCollection = @@ -63,7 +65,8 @@ export type AssignmentCollection = | WarehouseAssignment[] | NamespaceAssignment[] | TableAssignment[] - | ViewAssignment[]; + | ViewAssignment[] + | GenericTableAssignment[]; export interface Item { name: string; diff --git a/src/common/permissionActions.ts b/src/common/permissionActions.ts index 1bfa7259..adc50691 100644 --- a/src/common/permissionActions.ts +++ b/src/common/permissionActions.ts @@ -5,6 +5,7 @@ import { LakekeeperNamespaceAction, LakekeeperTableAction, LakekeeperViewAction, + LakekeeperGenericTableAction, LakekeeperRoleAction, LakekeeperUserAction, OpenFgaServerAction, @@ -13,6 +14,7 @@ import { OpenFgaNamespaceAction, OpenFgaTableAction, OpenFgaViewAction, + OpenFgaGenericTableAction, OpenFgaRoleAction, } from '@/gen/management/types.gen'; @@ -60,18 +62,21 @@ const catalogWarehouseActions: LakekeeperWarehouseAction[] = [ { action: 'get_all_tasks' }, { action: 'control_all_tasks' }, { action: 'set_protection' }, + { action: 'set_format_version_policy' }, { action: 'get_endpoint_statistics' }, ]; const catalogNamespaceActions: LakekeeperNamespaceAction[] = [ { action: 'create_table' }, { action: 'create_view' }, + { action: 'create_generic_table' }, { action: 'create_namespace' }, { action: 'delete' }, { action: 'update_properties' }, { action: 'get_metadata' }, { action: 'list_tables' }, { action: 'list_views' }, + { action: 'list_generic_tables' }, { action: 'list_namespaces' }, { action: 'list_everything' }, { action: 'set_protection' }, @@ -105,6 +110,19 @@ const catalogViewActions: LakekeeperViewAction[] = [ { action: 'set_protection' }, ]; +const catalogGenericTableActions: LakekeeperGenericTableAction[] = [ + { action: 'drop' }, + { action: 'read_data' }, + { action: 'write_data' }, + { action: 'get_metadata' }, + { action: 'rename' }, + { action: 'include_in_list' }, + { action: 'undrop' }, + { action: 'get_tasks' }, + { action: 'control_tasks' }, + { action: 'set_protection' }, +]; + const catalogRoleActions: LakekeeperRoleAction[] = [ { action: 'read' }, { action: 'delete' }, @@ -173,6 +191,16 @@ const authorizerViewActions: OpenFgaViewAction[] = [ 'change_ownership', ]; +const authorizerGenericTableActions: OpenFgaGenericTableAction[] = [ + 'read_assignments', + 'grant_pass_grants', + 'grant_manage_grants', + 'grant_describe', + 'grant_select', + 'grant_modify', + 'change_ownership', +]; + const authorizerRoleActions: OpenFgaRoleAction[] = [ 'assume', 'can_grant_assignee', @@ -188,6 +216,7 @@ export const permissionActions = { catalogNamespaceActions, catalogTableActions, catalogViewActions, + catalogGenericTableActions, catalogRoleActions, catalogUserActions, // Authorizer actions (permission delegation) @@ -197,6 +226,7 @@ export const permissionActions = { authorizerNamespaceActions, authorizerTableActions, authorizerViewActions, + authorizerGenericTableActions, authorizerRoleActions, }; diff --git a/src/components/GenericTableHeader.vue b/src/components/GenericTableHeader.vue new file mode 100644 index 00000000..8a9066db --- /dev/null +++ b/src/components/GenericTableHeader.vue @@ -0,0 +1,53 @@ + + + diff --git a/src/components/GenericTableOverview.vue b/src/components/GenericTableOverview.vue new file mode 100644 index 00000000..9f09c0ca --- /dev/null +++ b/src/components/GenericTableOverview.vue @@ -0,0 +1,169 @@ + + + diff --git a/src/components/NamespaceGenericTables.vue b/src/components/NamespaceGenericTables.vue new file mode 100644 index 00000000..2b55dfdc --- /dev/null +++ b/src/components/NamespaceGenericTables.vue @@ -0,0 +1,284 @@ + + + diff --git a/src/components/NamespaceTables.vue b/src/components/NamespaceTables.vue index 619235d7..f30b3ca1 100644 --- a/src/components/NamespaceTables.vue +++ b/src/components/NamespaceTables.vue @@ -15,10 +15,21 @@ @update:options="paginationCheck($event)" :headers="headers" hover - :items="loadedTables" + :items="filteredRows" :sort-by="[{ key: 'name', order: 'asc' }]"> @@ -215,6 +247,7 @@