CAS: answer directory probes inside a part from the part manifest, no LIST per part file

docs/en/antalya/cas/architecture/read-path.md

@@ -12,7 +12,12 @@ doc_type: 'reference'
 A `CAS` read never touches a classical local-metadata path: there is no local directory listing to
 consult, only a ref resolve followed by object-store reads. This page covers the three ways a file
 access is served, the full chain for the common case, the two caches that sit on that chain, and
-how a part still open inside a write transaction serves its own reads.
+how a part still open inside a write transaction serves its own reads. A directory probe on a path
+inside a part (`<table>/<part>/<file>`, which `MergeTree` issues for every checksum entry at load)
+is answered from the part's retained folder manifest: a plain file is not a directory, a nested
+directory is and lists its children. No object-store `LIST` is involved; only a probe whose part
+does not resolve falls back to a listing: the table-level file listing on an `Atomic` table, the
+mirrored live-tree listing on a non-`Atomic` table.
 
 ## How a file access is served {#access-kinds}
 

src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedMetadataStorage.cpp

@@ -1394,6 +1394,30 @@ bool ContentAddressedMetadataStorage::liveTreeDirHasChildren(const std::string &
     return !store()->listMirroredChildren(scope).empty();
 }
 
+bool ContentAddressedMetadataStorage::tableSubdirExists(const Cas::TableFilePath & tf) const
+{
+    /// At least one verbatim file under it.
+    const auto life = readableNamespaceFilesLife(liveNamespace(tf.table_uuid));
+    if (!life)
+        return false;
+    const std::string prefix = tf.tail + "/";
+    for (const auto & name : store()->listNamespaceFiles(*life))
+        if (name.starts_with(prefix))
+            return true;
+    return false;
+}
+
+std::vector<std::string> ContentAddressedMetadataStorage::tableSubdirChildren(const Cas::TableFilePath & tf) const
+{
+    /// Verbatim files under <subdir>/, first-component collapsed.
+    std::unordered_set<std::string> result;
+    if (const auto life = readableNamespaceFilesLife(liveNamespace(tf.table_uuid)))
+        for (const auto & name : store()->listNamespaceFiles(*life))
+            if (name.starts_with(tf.tail + "/"))
+                addFirstComponent(result, name.substr(tf.tail.size() + 1));
+    return toVector(std::move(result));
+}
+
 Cas::RootNamespace ContentAddressedMetadataStorage::liveNamespace(const std::string & table_uuid) const
 {
     /// Path mirroring: the namespace is the table's canonical disk path with the
@@ -1605,6 +1629,23 @@ ContentAddressedMetadataStorage::DirRoute ContentAddressedMetadataStorage::class
                 return dr;
             }
         }
+        /// A path with a part-shaped component followed by more components: a file or nested
+        /// directory of a live, detached or moving part IF that ref resolves (shadow is routed
+        /// above). For an Atomic table path, the parser calls every first component after the
+        /// table root except `deduplication_logs` the part component; a non-Atomic table path
+        /// anchors on the rightmost part-shaped component instead. Either way, whether this really
+        /// is a part is decided by the ref at answer time, not by the path:
+        /// `existsDirectory`/`listDirectory` take the old table-subdirectory branch when it does
+        /// not resolve. Classification stays pure path computation, so the parse that branch needs
+        /// travels with the shape.
+        if (r && !r->ref.empty() && !r->file.empty())
+        {
+            dr.shape = DirShape::PartFile;
+            dr.p = std::move(p);
+            dr.r = std::move(r);
+            dr.tf = Cas::parseTableFilePath(path);
+            return dr;
+        }
         /// No sub-shape matched: fall through, identical to today's post-`if (p)` continuation.
     }
 
@@ -1700,16 +1741,18 @@ bool ContentAddressedMetadataStorage::existsDirectory(const std::string & path)
             return view && view->hasDirectory(*dr.projection_prefix);
         }
         case DirShape::TableSubdir:
+            return tableSubdirExists(*dr.tf);
+        case DirShape::PartFile:
         {
-            /// At least one verbatim file under it.
-            const auto life = readableNamespaceFilesLife(liveNamespace(dr.tf->table_uuid));
-            if (!life)
-                return false;
-            const std::string prefix = dr.tf->tail + "/";
-            for (const auto & name : store()->listNamespaceFiles(*life))
-                if (name.starts_with(prefix))
-                    return true;
-            return false;
+            /// A resolved part answers from its folder view: a plain file has no entries under
+            /// its own name with a trailing slash, a nested directory has. An unresolved ref is not
+            /// a part we know (a table subdirectory path, or a non-Atomic part-shaped table
+            /// component), so it answers as the table subdirectory or generic directory that the
+            /// same path denotes. A failed resolution or manifest read propagates.
+            auto view = partAccess()->getView(dr.r->refKey(), Cas::Freshness::CachedForLoad);
+            if (view)
+                return view->hasDirectory(dr.r->file + "/");
+            return dr.tf ? tableSubdirExists(*dr.tf) : liveTreeDirHasChildren(path);
         }
         case DirShape::GenericIntermediate:
             /// Exists iff a server-root-scoped mirrored LIST finds any object. Keeps `cd`/existence
@@ -1894,14 +1937,13 @@ std::vector<std::string> ContentAddressedMetadataStorage::listDirectory(const st
             return view ? view->listChildren(*dr.projection_prefix) : std::vector<std::string>{};
         }
         case DirShape::TableSubdir:
+            return tableSubdirChildren(*dr.tf);
+        case DirShape::PartFile:
         {
-            /// Verbatim files under <subdir>/, first-component collapsed.
-            std::unordered_set<std::string> result;
-            if (const auto life = readableNamespaceFilesLife(liveNamespace(dr.tf->table_uuid)))
-                for (const auto & name : store()->listNamespaceFiles(*life))
-                    if (name.starts_with(dr.tf->tail + "/"))
-                        addFirstComponent(result, name.substr(dr.tf->tail.size() + 1));
-            return toVector(std::move(result));
+            auto view = partAccess()->getView(dr.r->refKey(), Cas::Freshness::CachedForLoad);
+            if (view)
+                return view->listChildren(dr.r->file + "/");
+            return dr.tf ? tableSubdirChildren(*dr.tf) : listLiveTreeChildren(path);
         }
         case DirShape::GenericIntermediate:
             /// The disk root "", `store`, or any loose-file container above a table dir: a

src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedMetadataStorage.h

@@ -474,6 +474,13 @@ class ContentAddressedMetadataStorage final : public IMetadataStorage, public IC
     /// always considered present.
     bool liveTreeDirHasChildren(const std::string & path) const;
 
+    /// The table-level-subdirectory answers (a LIST of the life's `_files/` prefix filtered by
+    /// `tf.tail + "/"`), shared by the `TableSubdir` shape and by a `PartFile` whose ref does not
+    /// resolve: an unresolved ref answers as the table subdirectory or generic directory that the
+    /// same path denotes.
+    bool tableSubdirExists(const Cas::TableFilePath & tf) const;
+    std::vector<std::string> tableSubdirChildren(const Cas::TableFilePath & tf) const;
+
     /// Resolves one parsed path to its namespace, reference, and in-tree file. Detached paths are
     /// re-split here so their references remain in the table namespace with a `detached/` prefix;
     /// shadow paths map to a namespace derived from the literal shadow directory.
@@ -516,6 +523,7 @@ class ContentAddressedMetadataStorage final : public IMetadataStorage, public IC
         MovingContainer,
         PartDir,
         ProjectionDir,
+        PartFile,
         TableSubdir,
         GenericIntermediate,
     };

src/Disks/tests/gtest_ca_wiring.cpp

@@ -870,6 +870,23 @@ TEST(CASWiringRoute, DirShapeDispatchOrderIsStable)
     EXPECT_EQ(storage->classifyDirectoryForTest("shadow/bk1").shape,            DS::ShadowIntermediate);
     EXPECT_EQ(storage->classifyDirectoryForTest("a11/a11a11a1-1111-4111-8111-111111111111/deduplication_logs").shape, DS::TableSubdir);
     EXPECT_EQ(storage->classifyDirectoryForTest("store").shape,                 DS::GenericIntermediate);
+
+    /// A path INSIDE a part (file or nested directory): its own shape, decided by the ref at answer
+    /// time. Atomic, detached, moving, non-Atomic and a temporary restore part all route here; a
+    /// projection dir, a table-level subdir and a shadow part file keep their shapes.
+    const std::string tbl = "a11/a11a11a1-1111-4111-8111-111111111111";
+    EXPECT_EQ(storage->classifyDirectoryForTest(tbl + "/all_1_1_0/columns.txt").shape,            DS::PartFile);
+    EXPECT_EQ(storage->classifyDirectoryForTest(tbl + "/all_1_1_0/sub").shape,                    DS::PartFile);
+    EXPECT_EQ(storage->classifyDirectoryForTest(tbl + "/detached/all_1_1_0/columns.txt").shape,   DS::PartFile);
+    EXPECT_EQ(storage->classifyDirectoryForTest(tbl + "/moving/all_1_1_0/columns.txt").shape,     DS::PartFile);
+    EXPECT_EQ(storage->classifyDirectoryForTest(tbl + "/tmp_restore_all_1_1_0-abcdefgh/columns.txt").shape, DS::PartFile);
+    EXPECT_EQ(storage->classifyDirectoryForTest("data/db/tbl/all_1_1_0/columns.txt").shape,        DS::PartFile);
+    EXPECT_EQ(storage->classifyDirectoryForTest(tbl + "/all_1_1_0/p.proj").shape,                 DS::ProjectionDir);
+    EXPECT_EQ(storage->classifyDirectoryForTest(tbl + "/deduplication_logs").shape,               DS::TableSubdir);
+    EXPECT_EQ(storage->classifyDirectoryForTest("shadow/bk1/store/" + tbl + "/all_1_1_0/columns.txt").shape, DS::ShadowIntermediate);
+    /// The old branch's parse travels with the shape: present on an Atomic path, absent on non-Atomic.
+    EXPECT_TRUE(storage->classifyDirectoryForTest(tbl + "/all_1_1_0/columns.txt").tf.has_value());
+    EXPECT_FALSE(storage->classifyDirectoryForTest("data/db/tbl/all_1_1_0/columns.txt").tf.has_value());
 }
 
 /// ==== M-W Task 3: the write path through IMetadataTransaction ====