Revert the last cpp protobuf update and GCS reader support.

The required riegeli version depends on protobug that is incompatible with TF. Ported from #175

PiperOrigin-RevId: 829128090

Author: ArrayRecord Team

MODULE.bazel

@@ -13,41 +13,34 @@
 # limitations under the License.
 
 # TODO(fchern): automate version string alignment with setup.py
-VERSION = "0.8.2"
+VERSION = "0.8.0"
 
 module(
     name = "array_record",
     version = VERSION,
     repo_name = "com_google_array_record",
 )
 
-bazel_dep(name = "rules_proto", version = "7.0.2")
-bazel_dep(name = "rules_python", version = "1.6.0")
+bazel_dep(name = "rules_proto", version = "7.1.0")
+bazel_dep(name = "rules_python", version = "1.4.1")
 bazel_dep(name = "platforms", version = "0.0.11")
-
-# We force the protobuf version to be 28.3 because 29.x (that is required by rules_python) causes
-# segfault when importing TensorFlow due to a conflict with TF's protobuf version.
-bazel_dep(name = "protobuf", version = "29.0-rc3")
-single_version_override(
-    module_name = "protobuf",
-    version = "28.3",
-)
-
+bazel_dep(name = "protobuf", version = "31.1")
 bazel_dep(name = "googletest", version = "1.15.2")
-bazel_dep(name = "abseil-cpp", version = "20240722.0")
+bazel_dep(name = "abseil-cpp", version = "20250127.1")
 bazel_dep(name = "abseil-py", version = "2.1.0")
 bazel_dep(name = "eigen", version = "3.4.0.bcr.3")
-bazel_dep(name = "riegeli", version = "0.0.0-20241218-3385e3c")
+bazel_dep(name = "riegeli", version = "0.0.0-20250717-5b2e77e")
 bazel_dep(name = "pybind11_bazel", version = "2.12.0")
+bazel_dep(name = "google_cloud_cpp", version = "3.0.0-rc0")
 
 SUPPORTED_PYTHON_VERSIONS = [
+    "3.10",
     "3.11",
     "3.12",
     "3.13",
-    "3.14",
 ]
 
-DEFAULT_PYTHON_VERSION = "3.13"
+DEFAULT_PYTHON_VERSION = "3.10"
 
 python_configure = use_extension("@pybind11_bazel//:python_configure.bzl", "extension")
 use_repo(python_configure, "local_config_python")

oss/build_whl.sh

@@ -28,11 +28,6 @@ function main() {
   write_to_bazelrc "build --host_cxxopt=-std=c++17"
   write_to_bazelrc "build --experimental_repo_remote_exec"
   write_to_bazelrc "common --check_direct_dependencies=error"
-  # Reduce noise during build.
-  write_to_bazelrc "build --cxxopt=-Wno-deprecated-declarations --host_cxxopt=-Wno-deprecated-declarations"
-  write_to_bazelrc "build --cxxopt=-Wno-parentheses --host_cxxopt=-Wno-parentheses"
-  write_to_bazelrc "build --cxxopt=-Wno-sign-compare --host_cxxopt=-Wno-sign-compare"
-
   PLATFORM="$(uname)"
 
   if [ -n "${CROSSTOOL_TOP}" ]; then
@@ -95,11 +90,11 @@ function main() {
   $PYTHON_BIN -m pip install ${OUTPUT_DIR}/all_dist/array_record*.whl
   $PYTHON_BIN -c 'import array_record'
   $PYTHON_BIN -c 'from array_record.python import array_record_data_source'
-  # TF does not have a Python 3.14 wheel yet.
-  if (( "${PYTHON_MINOR_VERSION}" < 14 )); then
-    $PYTHON_BIN -m pip install jax tensorflow>=2.20.0 grain --only-binary h5py
-    $PYTHON_BIN oss/test_import_tensorflow.py
+  # TF is not available on Python 3.13 and above.
+  if [ "$(uname)" != "Darwin" ] && (( "${PYTHON_MINOR_VERSION}" < 13 )); then
+    $PYTHON_BIN -m pip install jax tensorflow>=2.20.0 grain
     $PYTHON_BIN oss/test_import_grain.py
+    $PYTHON_BIN oss/test_import_tensorflow.py
   fi
 }
 

python/BUILD

@@ -20,6 +20,8 @@ pybind_extension(
         "@riegeli//riegeli/base:initializer",
         "@riegeli//riegeli/bytes:fd_reader",
         "@riegeli//riegeli/bytes:fd_writer",
+        "@riegeli//riegeli/gcs:gcs_object",
+        "@riegeli//riegeli/gcs:gcs_reader",
     ],
 )
 

python/array_record_module.cc

@@ -22,6 +22,7 @@ limitations under the License.
 #include <vector>
 
 #include "absl/status/status.h"
+#include "absl/strings/match.h"
 #include "absl/strings/str_format.h"
 #include "absl/strings/string_view.h"
 #include "cpp/array_record_reader.h"
@@ -34,6 +35,8 @@ limitations under the License.
 #include "riegeli/base/maker.h"
 #include "riegeli/bytes/fd_reader.h"
 #include "riegeli/bytes/fd_writer.h"
+#include "riegeli/gcs/gcs_object.h"
+#include "riegeli/gcs/gcs_reader.h"
 
 namespace py = pybind11;
 
@@ -50,10 +53,13 @@ PYBIND11_MODULE(array_record_module, m) {
                throw py::value_error(
                    std::string(status_or_option.status().message()));
              }
+             riegeli::FdWriterBase::Options file_writer_options;
+             file_writer_options.set_buffer_size(size_t{16} << 20);
              // Release the GIL because IO is time consuming.
              py::gil_scoped_release scoped_release;
              return new array_record::ArrayRecordWriter(
-                 riegeli::Maker<riegeli::FdWriter>(path),
+                 riegeli::Maker<riegeli::FdWriter>(
+                     path, std::move(file_writer_options)),
                  status_or_option.value());
            }),
            py::arg("path"), py::arg("options") = "")
@@ -84,18 +90,29 @@ PYBIND11_MODULE(array_record_module, m) {
                    std::string(status_or_option.status().message()));
              }
              riegeli::FdReaderBase::Options file_reader_options;
+             riegeli::GcsReader::Options gcs_reader_options;
              if (kwargs.contains("file_reader_buffer_size")) {
                auto file_reader_buffer_size =
                    kwargs["file_reader_buffer_size"].cast<int64_t>();
                file_reader_options.set_buffer_size(file_reader_buffer_size);
+               gcs_reader_options.set_buffer_size(file_reader_buffer_size);
              }
              // Release the GIL because IO is time consuming.
              py::gil_scoped_release scoped_release;
-             return new array_record::ArrayRecordReader(
-                 riegeli::Maker<riegeli::FdReader>(
-                     path, std::move(file_reader_options)),
-                 status_or_option.value(),
-                 array_record::ArrayRecordGlobalPool());
+             if (absl::StartsWith(path, "gs://")) {
+               return new array_record::ArrayRecordReader(
+                   riegeli::Maker<riegeli::GcsReader>(
+                       google::cloud::storage::Client(),
+                       riegeli::GcsObject(path), std::move(gcs_reader_options)),
+                   status_or_option.value(),
+                   array_record::ArrayRecordGlobalPool());
+             } else {
+               return new array_record::ArrayRecordReader(
+                   riegeli::Maker<riegeli::FdReader>(
+                       path, std::move(file_reader_options)),
+                   status_or_option.value(),
+                   array_record::ArrayRecordGlobalPool());
+             }
            }),
            py::arg("path"), py::arg("options") = "", R"(
            ArrayRecordReader for fast sequential or random access.

setup.py

@@ -31,23 +31,23 @@ def has_ext_modules(self):
 
 setup(
     name='array_record',
-    version='0.8.2',
+    version='0.8.0',
     description='A file format that achieves a new frontier of IO efficiency',
     author='ArrayRecord team',
     author_email='no-reply@google.com',
     packages=find_packages(),
     include_package_data=True,
     package_data={'': ['*.so']},
-    python_requires='>=3.11',
+    python_requires='>=3.10',
     install_requires=REQUIRED_PACKAGES,
     extras_require={'beam': BEAM_EXTRAS, 'test': TEST_EXTRAS},
     url='https://github.com/google/array_record',
     license='Apache-2.0',
     classifiers=[
+        'Programming Language :: Python :: 3.10',
         'Programming Language :: Python :: 3.11',
         'Programming Language :: Python :: 3.12',
         'Programming Language :: Python :: 3.13',
-        'Programming Language :: Python :: 3.14',
     ],
     zip_safe=False,
     distclass=BinaryDistribution,