Spr 865 fs snapshot validation

include/storage/vacuumer.hh

@@ -92,6 +92,14 @@ public:
         _global_file_size_threshold = size;
     }
 
+    /**
+     * @brief Return whether the vacuumer is enabled
+     *
+     * @return true - enabled
+     * @return false - disabled
+     */
+    bool is_enabled() const { return _vacuum_start_enabled; }
+
     /**
      * @brief Cleanup DB's entries from vacuum storage
      *        - memory, global vacuum file and partials

include/test/file_system_check.hh

@@ -10,7 +10,7 @@ namespace springtail::test {
     class FSCheck
     {
     public:
-        FSCheck();
+        explicit FSCheck(uint64_t max_xid = constant::LATEST_XID, bool all_xids = false);
         ~FSCheck() = default;
 
         /**
@@ -33,14 +33,23 @@ namespace springtail::test {
          *
          * @param db_id - database id
          * @param table_id - table id
+         * @param all_xids - check all available xids
          */
         void
-        check_db_table(uint64_t db_id, uint64_t table_id);
+        check_db_table(uint64_t db_id, uint64_t table_id, bool all_xids = false);
 
     private:
         std::map<uint64_t, std::string> _databases;     ///< map of database id to database name
+        std::map<uint64_t, uint64_t> _db_id_to_cutoff_xid;  ///< cuttoff xid per database
         std::filesystem::path _table_base;              ///< directory where all the tables are stored
+        uint64_t _max_xid;                              ///< maximum xid
+        uint64_t _max_recorded_xid{0};                  ///< maximum xid found in system tables
+        bool _all_xids;                                 ///< iterate over all xids
 
+        /**
+         * @brief Storage for namespace data
+         *
+         */
         struct FSNamespace
         {
             uint64_t ns_id;
@@ -50,28 +59,43 @@ namespace springtail::test {
             bool exists;
         };
 
+        /**
+         * @brief Storage for root data
+         *
+         */
         struct FSRoot
         {
             uint64_t xid;
             uint64_t extent_id;
             uint64_t snapshot_xid;
         };
 
+        /**
+         * @brief Storage for stats data
+         *
+         */
         struct FSStats
         {
             uint64_t xid;
             uint64_t row_count;
             uint64_t end_offset;
         };
 
+        /**
+         * @brief Storage for index data
+         *
+         */
         struct FSIndex
         {
             uint64_t xid;
             uint64_t lsn;
             Index index;
         };
 
-
+        /**
+         * @brief Storage for table data
+         *
+         */
         struct FSTable
         {
             uint64_t ns_id;
@@ -86,24 +110,44 @@ namespace springtail::test {
             std::map<uint64_t, FSStats> xid_to_stats;
         };
 
+        /**
+         * @brief Map of database id and namespace id namespace data
+         *
+         */
         std::map<std::pair<uint64_t, uint64_t>, FSNamespace> _db_ns_id_map;
+
+        /**
+         * @brief Map of database id and table id to table data
+         *
+         */
         std::map<std::pair<uint64_t, uint64_t>, FSTable> _db_tbl_id_map;
 
         /**
          * @brief Read all namespaces for the database
          *
          * @param db_id - database id
+         * @param max_xid - maximum xid that limits data scan
          */
         void
-        _read_namespaces(uint64_t db_id);
+        _read_namespaces(uint64_t db_id, uint64_t max_xid);
 
         /**
          * @brief Read all table data for the given database
          *
-         * @param db_id
+         * @param db_id - database id
+         * @param max_xid - maximum xid that limits data scan
+         */
+        void
+        _read_tables(uint64_t db_id, uint64_t max_xid);
+
+        /**
+         * @brief Read all information for the given database from the system tables.
+         *
+         * @param db_id - database id
+         * @param max_xid - maximum xid that limits data scan
          */
         void
-        _read_tables(uint64_t db_id);
+        _read_database_info(uint64_t db_id, uint64_t max_xid);
 
         /**
          * @brief Validate primary key index
@@ -127,10 +171,11 @@ namespace springtail::test {
          * @brief Internal function for checking specific database.
          *
          * @param db_id - database id
-         * @param db_name - database name
+         * @param first_xid - first xid
+         * @param cuttoff_xid - cutoff xid
          */
         void
-        _check_db(uint64_t db_id, const std::string &db_name);
+        _check_db(uint64_t db_id, uint64_t first_xid, uint64_t cutoff_xid);
 
         /**
          * @brief Internal function to reading specific database table
@@ -154,6 +199,14 @@ namespace springtail::test {
         template<typename Tbl>
         std::pair<TablePtr, std::shared_ptr<std::vector<FieldPtr>>>
         _get_table_and_fields(uint64_t db_id);
+
+        /**
+         * @brief Save the next max recorded xid if applicable
+         *
+         * @param xid - xid value
+         */
+        void
+        _record_max_xid(uint64_t xid);
     };
 
 } // springtail::test

src/sys_tbl_mgr/CMakeLists.txt

@@ -37,6 +37,7 @@ add_executable(file_system_check test/file_system_check.cc)
 target_include_directories(file_system_check PRIVATE ${CMAKE_SOURCE_DIR}/include ${VCPKG_INCLUDE_PATH})
 target_link_libraries(file_system_check
   PUBLIC springtail_test_check
+  PRIVATE Boost::program_options
 )
 add_dsymutil_command(file_system_check)
 

src/sys_tbl_mgr/test/file_system_check.cc

@@ -1,15 +1,30 @@
+#include <boost/program_options.hpp>
+
 #include <common/init.hh>
 
 #include <test/file_system_check.hh>
+
 using namespace springtail;
+namespace po = boost::program_options;
 
 int
 main(int argc, char *argv[])
 {
+    uint64_t max_xid = 0;
+    bool all_xids = false;
+
+    po::options_description desc("Options");
+    desc.add_options()("max_xid,mx", po::value<uint64_t>(&max_xid)->default_value(constant::LATEST_XID), "Maximum xid, default is the latest");
+    desc.add_options()("all_xids,ax", po::value<bool>(&all_xids)->default_value(false), "Flag to check all xids, default is the false");
+
+    po::variables_map vm;
+    po::store(po::parse_command_line(argc, argv, desc), vm);
+    po::notify(vm);
+
     // no logging
     springtail_init(false, std::nullopt, LOG_NONE);
 
-    std::shared_ptr<test::FSCheck> fs_check = std::make_shared<test::FSCheck>();
+    auto fs_check = std::make_shared<test::FSCheck>(max_xid, all_xids);
     fs_check->check_dbs();
     fs_check.reset();