From 5380d450ef786535e8728eb600719f01587759b6 Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Sun, 25 Jan 2026 02:24:40 +0000 Subject: [PATCH 01/11] remove website, typescript, etc pubsublite uses --- it/google-cloud-platform/build.gradle | 2 - .../pubsublite/PubsubliteResourceManager.java | 215 ------------------ .../beam/it/gcp/pubsublite/package-info.java | 20 -- .../assets/symbols/java.g.yaml | 16 -- .../assets/symbols/python.g.yaml | 6 - sdks/standard_expansion_services.yaml | 3 - sdks/typescript/src/apache_beam/io/index.ts | 1 - .../src/apache_beam/io/pubsublite.ts | 55 ----- .../sql/extensions/create-external-table.md | 54 ----- .../content/en/documentation/io/connectors.md | 21 -- 10 files changed, 393 deletions(-) delete mode 100644 it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsublite/PubsubliteResourceManager.java delete mode 100644 it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsublite/package-info.java delete mode 100644 sdks/typescript/src/apache_beam/io/pubsublite.ts diff --git a/it/google-cloud-platform/build.gradle b/it/google-cloud-platform/build.gradle index 56a1d7307348..3a46f2b94d83 100644 --- a/it/google-cloud-platform/build.gradle +++ b/it/google-cloud-platform/build.gradle @@ -74,8 +74,6 @@ dependencies { implementation 'com.google.cloud:google-cloud-spanner' implementation 'com.google.cloud:google-cloud-pubsub' provided 'com.google.api.grpc:proto-google-cloud-pubsub-v1' - implementation 'com.google.cloud:google-cloud-pubsublite' - provided 'com.google.api.grpc:proto-google-cloud-pubsublite-v1' implementation 'com.google.cloud:google-cloud-datastore' implementation 'com.google.cloud:google-cloud-datastream' provided 'com.google.api.grpc:proto-google-cloud-datastream-v1' diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsublite/PubsubliteResourceManager.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsublite/PubsubliteResourceManager.java deleted file mode 100644 index 6f6e21bfd327..000000000000 --- a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsublite/PubsubliteResourceManager.java +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.it.gcp.pubsublite; - -import com.google.cloud.pubsublite.AdminClient; -import com.google.cloud.pubsublite.AdminClientSettings; -import com.google.cloud.pubsublite.CloudRegion; -import com.google.cloud.pubsublite.ProjectId; -import com.google.cloud.pubsublite.ReservationName; -import com.google.cloud.pubsublite.ReservationPath; -import com.google.cloud.pubsublite.SubscriptionName; -import com.google.cloud.pubsublite.SubscriptionPath; -import com.google.cloud.pubsublite.TopicName; -import com.google.cloud.pubsublite.TopicPath; -import com.google.cloud.pubsublite.proto.Reservation; -import com.google.cloud.pubsublite.proto.Subscription; -import com.google.cloud.pubsublite.proto.Topic; -import com.google.protobuf.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ExecutionException; -import org.apache.beam.it.common.ResourceManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Client for managing Pub/Sub Lite resources. */ -public class PubsubliteResourceManager implements ResourceManager { - private static final Logger LOG = LoggerFactory.getLogger(PubsubliteResourceManager.class); - - private final List cleanupReservations = new ArrayList<>(); - private final List cleanupTopics = new ArrayList<>(); - - public static final Integer DEFAULT_NUM_PARTITIONS = 100; - - // 5 days is the default retention period for the PSLite topic - public static final Duration DEFAULT_RETENTION_PERIOD = - Duration.newBuilder().setSeconds(3600 * 24 * 5).build(); - - // 30 GB per partition - public static final Long DEFAULT_PARTITION_SIZE = 30 * 1024 * 1024 * 1024L; - - /** - * Creates a new PubsubLite reservation with the specified number of capacity units. Capacity - * units represent 0.25 MiBps on a regional reservation, and 1 MiBps on a zonal reservation. - * - * @param reservationName the name of the reservation to create. - * @param cloudRegion the region in which the reservation will be created. - * @param projectId the project id associated with the reservation. - * @param capacity the number of capacity units for the reservation. - * @return the path of the created reservation. - */ - public ReservationPath createReservation( - String reservationName, String cloudRegion, String projectId, Long capacity) { - try (AdminClient client = - AdminClient.create( - AdminClientSettings.newBuilder().setRegion(CloudRegion.of(cloudRegion)).build())) { - ReservationPath reservationPath = - ReservationPath.newBuilder() - .setProject(ProjectId.of(projectId)) - .setLocation(CloudRegion.of(cloudRegion)) - .setName(ReservationName.of(reservationName)) - .build(); - client - .createReservation( - Reservation.newBuilder() - .setName(reservationPath.toString()) - .setThroughputCapacity(capacity) - .build()) - .get(); - cleanupReservations.add(reservationPath); - LOG.info("Created reservation {}", reservationPath); - return reservationPath; - } catch (ExecutionException | InterruptedException e) { - throw new RuntimeException( - String.format( - "Unable to create reservation %s in region %s with capacity %d", - reservationName, cloudRegion, capacity), - e); - } - } - - /** - * Creates a topic with the given name on Pub/Sub. - * - *

https://cloud.google.com/pubsub/lite/docs/reservations - * - * @param topicName Topic name to create. The underlying implementation may not use the topic name - * directly, and can add a prefix or a suffix to identify specific executions. - * @param reservationPath the path of the reservation under which to create the topic. - * @return The instance of the TopicName that was just created. - */ - public TopicName createTopic(String topicName, ReservationPath reservationPath) { - try (AdminClient client = - AdminClient.create( - AdminClientSettings.newBuilder().setRegion(reservationPath.location()).build())) { - TopicPath topicPath = - TopicPath.newBuilder() - .setName(TopicName.of(topicName)) - .setLocation(reservationPath.location()) - .setProject(reservationPath.project()) - .build(); - Topic topic = - client - .createTopic( - Topic.newBuilder() - .setName(topicPath.toString()) - .setPartitionConfig( - Topic.PartitionConfig.newBuilder() - .setCount(DEFAULT_NUM_PARTITIONS) - .build()) - .setRetentionConfig( - Topic.RetentionConfig.newBuilder() - .setPeriod(DEFAULT_RETENTION_PERIOD) - .setPerPartitionBytes(DEFAULT_PARTITION_SIZE) - .build()) - .setReservationConfig( - Topic.ReservationConfig.newBuilder() - .setThroughputReservation(reservationPath.toString()) - .build()) - .build()) - .get(); - cleanupTopics.add(topicPath); - LOG.info("Created topic {}", topicPath); - return TopicName.of(topic.getName()); - } catch (ExecutionException | InterruptedException e) { - throw new RuntimeException( - String.format("Unable to create topic %s in reservation %s", topicName, reservationPath), - e); - } - } - - /** - * Creates a new Pub/Sub Lite subscription for a specified topic. - * - * @param reservationPath the path of the reservation to add the subscription. - * @param topicName the name of the topic to add the subscription to. - * @param subscriptionName the name to use for the subscription. - * @return the created {@link SubscriptionName} instance. - */ - public SubscriptionName createSubscription( - ReservationPath reservationPath, TopicName topicName, String subscriptionName) { - try (AdminClient client = - AdminClient.create( - AdminClientSettings.newBuilder().setRegion(reservationPath.location()).build())) { - Subscription subscription = - client - .createSubscription( - Subscription.newBuilder() - .setTopic(topicName.toString()) - .setName( - SubscriptionPath.newBuilder() - .setLocation(reservationPath.location()) - .setName(SubscriptionName.of(subscriptionName)) - .setProject(reservationPath.project()) - .build() - .toString()) - .setDeliveryConfig( - Subscription.DeliveryConfig.newBuilder() - .setDeliveryRequirement( - Subscription.DeliveryConfig.DeliveryRequirement - .DELIVER_IMMEDIATELY) - .build()) - .build()) - .get(); - LOG.info("Created subscription {}", subscription.getName()); - return SubscriptionName.of(subscription.getName()); - } catch (ExecutionException | InterruptedException e) { - throw new RuntimeException( - String.format( - "Unable to create subscription %s for topic %s", subscriptionName, topicName), - e); - } - } - - /** Delete any topics or subscriptions created by this manager. */ - @Override - public void cleanupAll() { - for (TopicPath t : cleanupTopics) { - try (AdminClient client = - AdminClient.create( - AdminClientSettings.newBuilder().setRegion(t.location().region()).build())) { - client.deleteTopic(t).get(); - LOG.info("Deleted topic {}", t); - } catch (InterruptedException | ExecutionException e) { - System.out.println("Unable to delete topic " + t); - e.printStackTrace(); - } - } - for (ReservationPath r : cleanupReservations) { - try (AdminClient client = - AdminClient.create(AdminClientSettings.newBuilder().setRegion(r.location()).build())) { - client.deleteReservation(r).get(); - LOG.info("Deleted reservation {}", r); - } catch (InterruptedException | ExecutionException e) { - System.out.println("Unable to delete reservation " + r); - e.printStackTrace(); - } - } - } -} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsublite/package-info.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsublite/package-info.java deleted file mode 100644 index c135952d6cdf..000000000000 --- a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsublite/package-info.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** Package for managing Pub/Sub lite resources within integration tests. */ -package org.apache.beam.it.gcp.pubsublite; diff --git a/playground/frontend/playground_components/assets/symbols/java.g.yaml b/playground/frontend/playground_components/assets/symbols/java.g.yaml index e0f2b1d15d14..20431eff26f8 100644 --- a/playground/frontend/playground_components/assets/symbols/java.g.yaml +++ b/playground/frontend/playground_components/assets/symbols/java.g.yaml @@ -9024,22 +9024,6 @@ PubsubJsonClient: - pull properties: - FACTORY -PubsubLiteIO: - methods: - - addUuids - - deduplicate - - expand - - read - - write -PubsubLiteSink: - methods: - - finishBundle - - processElement - - startBundle -PubsubLiteTableProvider: - methods: - - buildBeamSqlTable - - getTableType PubsubMessage: methods: - equals diff --git a/playground/frontend/playground_components/assets/symbols/python.g.yaml b/playground/frontend/playground_components/assets/symbols/python.g.yaml index 0b9e5e142ded..ae3dc882fe82 100644 --- a/playground/frontend/playground_components/assets/symbols/python.g.yaml +++ b/playground/frontend/playground_components/assets/symbols/python.g.yaml @@ -8155,9 +8155,6 @@ ReadFromPubSub: - expand properties: - URN -ReadFromPubSubLite: - methods: - - expand ReadFromSnowflake: methods: - expand @@ -12050,9 +12047,6 @@ WriteToPubSub: - expand properties: - URN -WriteToPubSubLite: - methods: - - expand WriteToSnowflake: methods: - expand diff --git a/sdks/standard_expansion_services.yaml b/sdks/standard_expansion_services.yaml index 4a5afdfb6c38..531caca5a376 100644 --- a/sdks/standard_expansion_services.yaml +++ b/sdks/standard_expansion_services.yaml @@ -63,9 +63,6 @@ # skip_transforms: # # generate_sequence is already included in the Java IO expansion service # - 'beam:schematransform:org.apache.beam:generate_sequence:v1' -# # Handwritten wrappers exist in apache_beam/io/gcp/pubsublite/ -# - 'beam:schematransform:org.apache.beam:pubsublite_read:v1' -# - 'beam:schematransform:org.apache.beam:pubsublite_write:v1' # # Handwritten wrapper exists in apache_beam/io/gcp/spanner.py # - 'beam:schematransform:org.apache.beam:spanner_write:v1' # # Native IO exists in apache_beam/io/gcp/pubsub.py diff --git a/sdks/typescript/src/apache_beam/io/index.ts b/sdks/typescript/src/apache_beam/io/index.ts index 046f38b8c4d4..b3a28253ce5d 100644 --- a/sdks/typescript/src/apache_beam/io/index.ts +++ b/sdks/typescript/src/apache_beam/io/index.ts @@ -22,7 +22,6 @@ export * from "./bigqueryio"; export * from "./kafka"; export * from "./parquetio"; export * from "./pubsub"; -export * from "./pubsublite"; export * from "./schemaio"; import { requireForSerialization } from "../serialization"; diff --git a/sdks/typescript/src/apache_beam/io/pubsublite.ts b/sdks/typescript/src/apache_beam/io/pubsublite.ts deleted file mode 100644 index 510b39b9bf9a..000000000000 --- a/sdks/typescript/src/apache_beam/io/pubsublite.ts +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as beam from "../../apache_beam"; -import * as external from "../transforms/external"; -import { RowCoder } from "../coders/row_coder"; -import { serviceProviderFromJavaGradleTarget } from "../utils/service"; -import { camelToSnakeOptions } from "../utils/utils"; - -const PUBSUBLITE_EXPANSION_GRADLE_TARGET = - "sdks:java:io:google-cloud-platform:expansion-service:shadowJar"; - -// TODO: Schema-producing variants. -export function readFromPubSubLiteRaw( - subscriptionPath: string, - options: { minBundleTimeout?: number; deduplicate?: boolean } = {}, -): beam.AsyncPTransform> { - return beam.withName( - "readFromPubSubLiteRaw", - external.rawExternalTransform>( - "beam:transform:org.apache.beam:pubsublite_read:v1", - { subscription_path: subscriptionPath, ...camelToSnakeOptions(options) }, - serviceProviderFromJavaGradleTarget(PUBSUBLITE_EXPANSION_GRADLE_TARGET), - ), - ); -} - -export function writeToPubSubLiteRaw( - topicPath: string, - options: { addUuids?: boolean } = {}, -): beam.AsyncPTransform, {}> { - return beam.withName( - "writeToPubSubLiteRaw", - external.rawExternalTransform, {}>( - "beam:transform:org.apache.beam:pubsublite_write:v1", - { topic_path: topicPath, ...camelToSnakeOptions(options) }, - serviceProviderFromJavaGradleTarget(PUBSUBLITE_EXPANSION_GRADLE_TARGET), - ), - ); -} diff --git a/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md b/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md index ad6ba66beb20..eeac3bbec034 100644 --- a/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md +++ b/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md @@ -429,58 +429,6 @@ TYPE pubsub LOCATION 'projects/testing-integration/topics/user-location' ``` -## Pub/Sub Lite - -### Syntax -``` -CREATE EXTERNAL TABLE [ IF NOT EXISTS ] tableName( - publish_timestamp DATETIME, - event_timestamp DATETIME, - message_key BYTES, - attributes ARRAY>>, - payload [BYTES, ROW] -) -TYPE pubsublite -// For writing -LOCATION 'projects/[PROJECT]/locations/[GCP-LOCATION]/topics/[TOPIC]' -// For reading -LOCATION 'projects/[PROJECT]/locations/[GCP-LOCATION]/subscriptions/[SUBSCRIPTION]' -``` - -* `LOCATION`: - * `PROJECT`: ID of the Google Cloud Project - * `TOPIC`: The Pub/Sub Lite topic name. - * `SUBSCRIPTION`: The Pub/Sub Lite subscription name. - * `GCP-LOCATION`: The location for this Pub/Sub Lite topic os subscription. -* `TBLPROPERTIES`: - * `timestampAttributeKey`: Optional. The key which contains the event - timestamp associated with the Pub/Sub message. If not specified, the - message publish timestamp is used as an event timestamp for - windowing/watermarking. - * `deadLetterQueue`: Optional, supports - [Generic DLQ Handling](#generic-dlq-handling) - * `format`: Optional. Allows you to specify the payload format. - -### Read Mode - -PubsubLiteIO supports reading from subscriptions. - -### Write Mode - -PubsubLiteIO supports writing to topics. - -### Supported Payload - -* Pub/Sub Lite supports [Generic Payload Handling](#generic-payload-handling). - -### Example - -``` -CREATE EXTERNAL TABLE locations (event_timestamp TIMESTAMP, attributes ARRAY>>, payload ROW) -TYPE pubsublite -LOCATION 'projects/testing-integration/locations/us-central1-a/topics/user-location' -``` - ## Kafka KafkaIO is experimental in Beam SQL. @@ -934,5 +882,3 @@ DLQ handling are supported: field and "payload" byte array field. * `pubsub`: Pub/Sub Topic * DLQ_ID is the full path of the Pub/Sub Topic. -* `pubsublite`: Pub/Sub Lite Topic - * DLQ_ID is the full path of the Pub/Sub Lite Topic. diff --git a/website/www/site/content/en/documentation/io/connectors.md b/website/www/site/content/en/documentation/io/connectors.md index 9797195518e9..fb04e381cf84 100644 --- a/website/www/site/content/en/documentation/io/connectors.md +++ b/website/www/site/content/en/documentation/io/connectors.md @@ -939,27 +939,6 @@ This table provides a consolidated, at-a-glance overview of the available built- ✘ ✘ - - Pub/Sub Lite - ✔ - ✔ - - ✔ - native - - - ✔ - via X-language - - Not available - - ✔ - via X-language - - ✔ - ✔ - ✘ - InfluxDB ✔ From 8ad72ea73e7ed33ba7d7eada5658234b39d6f414 Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Tue, 27 Jan 2026 14:09:49 +0000 Subject: [PATCH 02/11] rebase --- website/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/website/Dockerfile b/website/Dockerfile index e09d06f22ead..10a17d7110f8 100644 --- a/website/Dockerfile +++ b/website/Dockerfile @@ -40,6 +40,7 @@ RUN apt-get update \ gnupg2 \ gosu \ lynx \ + && pip3 install --upgrade pip --break-system-packages || true \ && apt-get autoremove -yqq --purge \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -58,10 +59,9 @@ RUN npm update -g npm RUN npm install postcss postcss-cli autoprefixer # Install yarn -ENV COREPACK_HOME=/usr/local/share/corepack -RUN mkdir -p "$COREPACK_HOME" \ - && corepack enable \ - && corepack prepare yarn@1.22.22 --activate +RUN set -eux; \ + corepack enable; \ + corepack prepare yarn@stable --activate # Install hugo extended version v0.117.0 RUN HUGOHOME="$(mktemp -d)" \ From 0fd82a0b35e0ada9b889d061b7f1e1eb27294336 Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Tue, 27 Jan 2026 14:12:06 +0000 Subject: [PATCH 03/11] rebase conflict --- website/Dockerfile | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/website/Dockerfile b/website/Dockerfile index 10a17d7110f8..65c4f74a03e7 100644 --- a/website/Dockerfile +++ b/website/Dockerfile @@ -27,7 +27,8 @@ ENV DEBIAN_FRONTEND=noninteractive \ LANG=C.UTF-8 \ LC_ALL=C.UTF-8 \ LC_CTYPE=C.UTF-8 \ - LC_MESSAGES=C.UTF-8 + LC_MESSAGES=C.UTF-8 \ + COREPACK_HOME=/opt/.corepack WORKDIR /opt/ @@ -54,14 +55,12 @@ RUN curl -sL https://deb.nodesource.com/setup_lts.x | bash - \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -RUN npm update -g npm - -RUN npm install postcss postcss-cli autoprefixer - -# Install yarn -RUN set -eux; \ - corepack enable; \ - corepack prepare yarn@stable --activate +# Update npm and enable corepack to install yarn +RUN mkdir -p $COREPACK_HOME && chmod 777 $COREPACK_HOME \ + && npm update -g npm \ + && npm install -g postcss postcss-cli autoprefixer \ + && corepack enable \ + && corepack prepare yarn@stable --activate # Install hugo extended version v0.117.0 RUN HUGOHOME="$(mktemp -d)" \ From 9116211a18c460fbead4624ce759698b73cf727b Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Sun, 25 Jan 2026 03:07:52 +0000 Subject: [PATCH 04/11] try to fix dependency flutter issues --- playground/frontend/pubspec.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/playground/frontend/pubspec.yaml b/playground/frontend/pubspec.yaml index 066ab4aa6ad7..d8d222a0450f 100644 --- a/playground/frontend/pubspec.yaml +++ b/playground/frontend/pubspec.yaml @@ -30,7 +30,7 @@ dependencies: app_state: ^0.9.4 collection: ^1.15.0 easy_localization: ^3.0.1 - easy_localization_ext: ^0.1.1 + easy_localization_ext: ^0.1.0 # Pin to 0.1.0 to avoid addAllRecursive conflict easy_localization_loader: ^1.0.0 equatable: ^2.0.5 expansion_widget: ^0.0.2 @@ -49,6 +49,10 @@ dependencies: url_launcher: ^6.0.12 url_strategy: ^0.2.0 +dependency_overrides: + # This fixes the 'WidgetStateProperty' error by forcing the old version + keyed_collection_widgets: 0.4.2 + dev_dependencies: build_runner: ^2.1.4 fake_async: ^1.3.0 From 2b9fe58689a018b9c0be08145d2ff3009cbdaecf Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Sun, 25 Jan 2026 03:18:58 +0000 Subject: [PATCH 05/11] remove pins, needs to be a separate exercise --- playground/frontend/pubspec.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/playground/frontend/pubspec.yaml b/playground/frontend/pubspec.yaml index d8d222a0450f..066ab4aa6ad7 100644 --- a/playground/frontend/pubspec.yaml +++ b/playground/frontend/pubspec.yaml @@ -30,7 +30,7 @@ dependencies: app_state: ^0.9.4 collection: ^1.15.0 easy_localization: ^3.0.1 - easy_localization_ext: ^0.1.0 # Pin to 0.1.0 to avoid addAllRecursive conflict + easy_localization_ext: ^0.1.1 easy_localization_loader: ^1.0.0 equatable: ^2.0.5 expansion_widget: ^0.0.2 @@ -49,10 +49,6 @@ dependencies: url_launcher: ^6.0.12 url_strategy: ^0.2.0 -dependency_overrides: - # This fixes the 'WidgetStateProperty' error by forcing the old version - keyed_collection_widgets: 0.4.2 - dev_dependencies: build_runner: ^2.1.4 fake_async: ^1.3.0 From b29beaf4673fa3886ecefa07dbaba9daadd87f4e Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Tue, 27 Jan 2026 14:15:41 +0000 Subject: [PATCH 06/11] revert previous fix to website certificate issue since already fixed, but add a slight improvement --- website/Dockerfile | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/website/Dockerfile b/website/Dockerfile index 65c4f74a03e7..b5258f702ae8 100644 --- a/website/Dockerfile +++ b/website/Dockerfile @@ -27,8 +27,7 @@ ENV DEBIAN_FRONTEND=noninteractive \ LANG=C.UTF-8 \ LC_ALL=C.UTF-8 \ LC_CTYPE=C.UTF-8 \ - LC_MESSAGES=C.UTF-8 \ - COREPACK_HOME=/opt/.corepack + LC_MESSAGES=C.UTF-8 WORKDIR /opt/ @@ -55,12 +54,15 @@ RUN curl -sL https://deb.nodesource.com/setup_lts.x | bash - \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -# Update npm and enable corepack to install yarn -RUN mkdir -p $COREPACK_HOME && chmod 777 $COREPACK_HOME \ - && npm update -g npm \ - && npm install -g postcss postcss-cli autoprefixer \ - && corepack enable \ - && corepack prepare yarn@stable --activate +# Install npm +RUN npm update -g npm +RUN npm install postcss postcss-cli autoprefixer + +# Install yarn +ENV COREPACK_HOME=/usr/local/share/corepack +RUN mkdir -p "$COREPACK_HOME" \ + && corepack enable \ + && corepack prepare yarn@stable --activate # Install hugo extended version v0.117.0 RUN HUGOHOME="$(mktemp -d)" \ From 1928ed560ccbe6f015d25e0bd4413d592d78c56e Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Wed, 28 Jan 2026 18:58:05 +0000 Subject: [PATCH 07/11] revert changes on generated files and separate improvements --- .../assets/symbols/java.g.yaml | 16 ++++++++++++++++ .../assets/symbols/python.g.yaml | 6 ++++++ website/Dockerfile | 5 ++--- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/playground/frontend/playground_components/assets/symbols/java.g.yaml b/playground/frontend/playground_components/assets/symbols/java.g.yaml index 20431eff26f8..e0f2b1d15d14 100644 --- a/playground/frontend/playground_components/assets/symbols/java.g.yaml +++ b/playground/frontend/playground_components/assets/symbols/java.g.yaml @@ -9024,6 +9024,22 @@ PubsubJsonClient: - pull properties: - FACTORY +PubsubLiteIO: + methods: + - addUuids + - deduplicate + - expand + - read + - write +PubsubLiteSink: + methods: + - finishBundle + - processElement + - startBundle +PubsubLiteTableProvider: + methods: + - buildBeamSqlTable + - getTableType PubsubMessage: methods: - equals diff --git a/playground/frontend/playground_components/assets/symbols/python.g.yaml b/playground/frontend/playground_components/assets/symbols/python.g.yaml index ae3dc882fe82..0b9e5e142ded 100644 --- a/playground/frontend/playground_components/assets/symbols/python.g.yaml +++ b/playground/frontend/playground_components/assets/symbols/python.g.yaml @@ -8155,6 +8155,9 @@ ReadFromPubSub: - expand properties: - URN +ReadFromPubSubLite: + methods: + - expand ReadFromSnowflake: methods: - expand @@ -12047,6 +12050,9 @@ WriteToPubSub: - expand properties: - URN +WriteToPubSubLite: + methods: + - expand WriteToSnowflake: methods: - expand diff --git a/website/Dockerfile b/website/Dockerfile index b5258f702ae8..e09d06f22ead 100644 --- a/website/Dockerfile +++ b/website/Dockerfile @@ -40,7 +40,6 @@ RUN apt-get update \ gnupg2 \ gosu \ lynx \ - && pip3 install --upgrade pip --break-system-packages || true \ && apt-get autoremove -yqq --purge \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -54,15 +53,15 @@ RUN curl -sL https://deb.nodesource.com/setup_lts.x | bash - \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -# Install npm RUN npm update -g npm + RUN npm install postcss postcss-cli autoprefixer # Install yarn ENV COREPACK_HOME=/usr/local/share/corepack RUN mkdir -p "$COREPACK_HOME" \ && corepack enable \ - && corepack prepare yarn@stable --activate + && corepack prepare yarn@1.22.22 --activate # Install hugo extended version v0.117.0 RUN HUGOHOME="$(mktemp -d)" \ From 33eac6ac2a0745490c9a637f9a2268900c20d4d8 Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Thu, 29 Jan 2026 04:17:23 +0000 Subject: [PATCH 08/11] revert website changes until later --- .../sql/extensions/create-external-table.md | 52 +++++++++++++++++++ .../content/en/documentation/io/connectors.md | 21 ++++++++ 2 files changed, 73 insertions(+) diff --git a/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md b/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md index eeac3bbec034..95f23b489594 100644 --- a/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md +++ b/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md @@ -429,6 +429,58 @@ TYPE pubsub LOCATION 'projects/testing-integration/topics/user-location' ``` +## Pub/Sub Lite + +### Syntax +``` +CREATE EXTERNAL TABLE [ IF NOT EXISTS ] tableName( + publish_timestamp DATETIME, + event_timestamp DATETIME, + message_key BYTES, + attributes ARRAY>>, + payload [BYTES, ROW] +) +TYPE pubsublite +// For writing +LOCATION 'projects/[PROJECT]/locations/[GCP-LOCATION]/topics/[TOPIC]' +// For reading +LOCATION 'projects/[PROJECT]/locations/[GCP-LOCATION]/subscriptions/[SUBSCRIPTION]' +``` + +* `LOCATION`: + * `PROJECT`: ID of the Google Cloud Project + * `TOPIC`: The Pub/Sub Lite topic name. + * `SUBSCRIPTION`: The Pub/Sub Lite subscription name. + * `GCP-LOCATION`: The location for this Pub/Sub Lite topic os subscription. +* `TBLPROPERTIES`: + * `timestampAttributeKey`: Optional. The key which contains the event + timestamp associated with the Pub/Sub message. If not specified, the + message publish timestamp is used as an event timestamp for + windowing/watermarking. + * `deadLetterQueue`: Optional, supports + [Generic DLQ Handling](#generic-dlq-handling) + * `format`: Optional. Allows you to specify the payload format. + +### Read Mode + +PubsubLiteIO supports reading from subscriptions. + +### Write Mode + +PubsubLiteIO supports writing to topics. + +### Supported Payload + +* Pub/Sub Lite supports [Generic Payload Handling](#generic-payload-handling). + +### Example + +``` +CREATE EXTERNAL TABLE locations (event_timestamp TIMESTAMP, attributes ARRAY>>, payload ROW) +TYPE pubsublite +LOCATION 'projects/testing-integration/locations/us-central1-a/topics/user-location' +``` + ## Kafka KafkaIO is experimental in Beam SQL. diff --git a/website/www/site/content/en/documentation/io/connectors.md b/website/www/site/content/en/documentation/io/connectors.md index fb04e381cf84..f00f81878575 100644 --- a/website/www/site/content/en/documentation/io/connectors.md +++ b/website/www/site/content/en/documentation/io/connectors.md @@ -939,6 +939,27 @@ This table provides a consolidated, at-a-glance overview of the available built- ✘ ✘ + + Pub/Sub Lite + ✔ + ✔ + + ✔ + native + + + ✔ + via X-language + + Not available + + ✔ + via X-language + + ✔ + ✔ + ✘ + InfluxDB ✔ From af6052a70fb3671831388d8dde9fbc384d508b4c Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Thu, 29 Jan 2026 04:21:05 +0000 Subject: [PATCH 09/11] one more revert --- .../documentation/dsls/sql/extensions/create-external-table.md | 2 ++ website/www/site/content/en/documentation/io/connectors.md | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md b/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md index 95f23b489594..42f34076e2b6 100644 --- a/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md +++ b/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md @@ -934,3 +934,5 @@ DLQ handling are supported: field and "payload" byte array field. * `pubsub`: Pub/Sub Topic * DLQ_ID is the full path of the Pub/Sub Topic. +* `pubsublite`: Pub/Sub Lite Topic + * DLQ_ID is the full path of the Pub/Sub Lite Topic. \ No newline at end of file diff --git a/website/www/site/content/en/documentation/io/connectors.md b/website/www/site/content/en/documentation/io/connectors.md index f00f81878575..9797195518e9 100644 --- a/website/www/site/content/en/documentation/io/connectors.md +++ b/website/www/site/content/en/documentation/io/connectors.md @@ -939,7 +939,7 @@ This table provides a consolidated, at-a-glance overview of the available built- ✘ ✘ - + Pub/Sub Lite ✔ ✔ From 52e058f86db930597c619768f664d4d7c25bc7de Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Thu, 29 Jan 2026 04:23:31 +0000 Subject: [PATCH 10/11] one more change :) --- .../documentation/dsls/sql/extensions/create-external-table.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md b/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md index 42f34076e2b6..4730defaa8fd 100644 --- a/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md +++ b/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md @@ -935,4 +935,4 @@ DLQ handling are supported: * `pubsub`: Pub/Sub Topic * DLQ_ID is the full path of the Pub/Sub Topic. * `pubsublite`: Pub/Sub Lite Topic - * DLQ_ID is the full path of the Pub/Sub Lite Topic. \ No newline at end of file + * DLQ_ID is the full path of the Pub/Sub Lite Topic. \ No newline at end of file From 62a8dd015eec2523a7dadb315fe2e18c07937d95 Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Thu, 29 Jan 2026 04:26:06 +0000 Subject: [PATCH 11/11] checkout file instead --- .../documentation/dsls/sql/extensions/create-external-table.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md b/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md index 4730defaa8fd..ad6ba66beb20 100644 --- a/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md +++ b/website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md @@ -935,4 +935,4 @@ DLQ handling are supported: * `pubsub`: Pub/Sub Topic * DLQ_ID is the full path of the Pub/Sub Topic. * `pubsublite`: Pub/Sub Lite Topic - * DLQ_ID is the full path of the Pub/Sub Lite Topic. \ No newline at end of file + * DLQ_ID is the full path of the Pub/Sub Lite Topic.