From 4b8a99af18840f7b79a013a9fb8442275921ef9f Mon Sep 17 00:00:00 2001 From: "wangyi.ywq" Date: Mon, 21 Mar 2022 15:10:15 +0800 Subject: [PATCH 1/6] support file min_max userkey in EventListener --- db/db_impl_compaction_flush.cc | 7 +++++++ include/rocksdb/listener.h | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/db/db_impl_compaction_flush.cc b/db/db_impl_compaction_flush.cc index cc544b6fc6..1fd42cd3d6 100644 --- a/db/db_impl_compaction_flush.cc +++ b/db/db_impl_compaction_flush.cc @@ -638,6 +638,8 @@ void DBImpl::NotifyOnFlushCompleted( info.largest_seqno = file_meta->fd.largest_seqno; info.table_properties = prop; info.flush_reason = cfd->GetFlushReason(); + info.min_userkey = file_meta->smallest.user_key().ToString(); + info.max_userkey = file_meta->largest.user_key().ToString(); for (auto listener : immutable_db_options_.listeners) { listener->OnFlushCompleted(this, info); } @@ -1202,6 +1204,8 @@ void DBImpl::NotifyOnCompactionCompleted( auto fn = TableFileName(c->immutable_cf_options()->cf_paths, fmd->fd.GetNumber(), fmd->fd.GetPathId()); info.input_files.push_back(fn); + info.input_min_max.push_back({fmd->smallest.user_key().ToString(), + fmd->largest.user_key().ToString()}); if (info.table_properties.count(fn) == 0) { std::shared_ptr tp; auto s = current->GetTableProperties(&tp, fmd, &fn); @@ -1218,6 +1222,9 @@ void DBImpl::NotifyOnCompactionCompleted( info.output_files.push_back(TableFileName( c->immutable_cf_options()->cf_paths, newf.second.fd.GetNumber(), newf.second.fd.GetPathId())); + info.output_min_max.push_back( + {newf.second.smallest.user_key().ToString(), + newf.second.largest.user_key().ToString()}); } for (auto listener : immutable_db_options_.listeners) { listener->OnCompactionCompleted(this, info); diff --git a/include/rocksdb/listener.h b/include/rocksdb/listener.h index 3ff0c0b241..5007e22ca6 100644 --- a/include/rocksdb/listener.h +++ b/include/rocksdb/listener.h @@ -239,6 +239,10 @@ struct FlushJobInfo { TableProperties table_properties; FlushReason flush_reason; + + std::string min_userkey; + + std::string max_userkey; }; struct TableTransientStat { @@ -277,9 +281,11 @@ struct CompactionJobInfo { int output_level; // the names of the compaction input files. std::vector input_files; + std::vector> input_min_max; // the names of the compaction output files. std::vector output_files; + std::vector> output_min_max; // Table properties for input and output tables. // The map is keyed by values from input_files and output_files. TablePropertiesCollection table_properties; From db761e72fe54e617a889bdc009387aceb8b66968 Mon Sep 17 00:00:00 2001 From: "wangyi.ywq" Date: Mon, 21 Mar 2022 15:40:24 +0800 Subject: [PATCH 2/6] variable rename --- db/db_impl_compaction_flush.cc | 7 ++++--- include/rocksdb/listener.h | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/db/db_impl_compaction_flush.cc b/db/db_impl_compaction_flush.cc index 1fd42cd3d6..fb386f2940 100644 --- a/db/db_impl_compaction_flush.cc +++ b/db/db_impl_compaction_flush.cc @@ -1204,8 +1204,9 @@ void DBImpl::NotifyOnCompactionCompleted( auto fn = TableFileName(c->immutable_cf_options()->cf_paths, fmd->fd.GetNumber(), fmd->fd.GetPathId()); info.input_files.push_back(fn); - info.input_min_max.push_back({fmd->smallest.user_key().ToString(), - fmd->largest.user_key().ToString()}); + info.input_files_min_max_userkey.push_back( + {fmd->smallest.user_key().ToString(), + fmd->largest.user_key().ToString()}); if (info.table_properties.count(fn) == 0) { std::shared_ptr tp; auto s = current->GetTableProperties(&tp, fmd, &fn); @@ -1222,7 +1223,7 @@ void DBImpl::NotifyOnCompactionCompleted( info.output_files.push_back(TableFileName( c->immutable_cf_options()->cf_paths, newf.second.fd.GetNumber(), newf.second.fd.GetPathId())); - info.output_min_max.push_back( + info.output_files_min_max_userkey.push_back( {newf.second.smallest.user_key().ToString(), newf.second.largest.user_key().ToString()}); } diff --git a/include/rocksdb/listener.h b/include/rocksdb/listener.h index 5007e22ca6..9df6ce0842 100644 --- a/include/rocksdb/listener.h +++ b/include/rocksdb/listener.h @@ -281,11 +281,11 @@ struct CompactionJobInfo { int output_level; // the names of the compaction input files. std::vector input_files; - std::vector> input_min_max; + std::vector> input_files_min_max_userkey; // the names of the compaction output files. std::vector output_files; - std::vector> output_min_max; + std::vector> output_files_min_max_userkey; // Table properties for input and output tables. // The map is keyed by values from input_files and output_files. TablePropertiesCollection table_properties; From f5910a35aee52551bffab176ff5769dec25143c1 Mon Sep 17 00:00:00 2001 From: "wangyi.ywq" Date: Mon, 21 Mar 2022 15:47:52 +0800 Subject: [PATCH 3/6] variable rename --- db/db_impl_compaction_flush.cc | 4 ++-- include/rocksdb/listener.h | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/db/db_impl_compaction_flush.cc b/db/db_impl_compaction_flush.cc index fb386f2940..a10155d3d7 100644 --- a/db/db_impl_compaction_flush.cc +++ b/db/db_impl_compaction_flush.cc @@ -1204,7 +1204,7 @@ void DBImpl::NotifyOnCompactionCompleted( auto fn = TableFileName(c->immutable_cf_options()->cf_paths, fmd->fd.GetNumber(), fmd->fd.GetPathId()); info.input_files.push_back(fn); - info.input_files_min_max_userkey.push_back( + info.input_files_boundries_user_key.push_back( {fmd->smallest.user_key().ToString(), fmd->largest.user_key().ToString()}); if (info.table_properties.count(fn) == 0) { @@ -1223,7 +1223,7 @@ void DBImpl::NotifyOnCompactionCompleted( info.output_files.push_back(TableFileName( c->immutable_cf_options()->cf_paths, newf.second.fd.GetNumber(), newf.second.fd.GetPathId())); - info.output_files_min_max_userkey.push_back( + info.output_files_boundries_user_key.push_back( {newf.second.smallest.user_key().ToString(), newf.second.largest.user_key().ToString()}); } diff --git a/include/rocksdb/listener.h b/include/rocksdb/listener.h index 9df6ce0842..e8e856a02c 100644 --- a/include/rocksdb/listener.h +++ b/include/rocksdb/listener.h @@ -240,9 +240,9 @@ struct FlushJobInfo { FlushReason flush_reason; - std::string min_userkey; + std::string smallest_userkey; - std::string max_userkey; + std::string largest_userkey; }; struct TableTransientStat { @@ -281,11 +281,11 @@ struct CompactionJobInfo { int output_level; // the names of the compaction input files. std::vector input_files; - std::vector> input_files_min_max_userkey; + std::vector> input_files_boundries_user_key; // the names of the compaction output files. std::vector output_files; - std::vector> output_files_min_max_userkey; + std::vector> output_files_boundries_user_key; // Table properties for input and output tables. // The map is keyed by values from input_files and output_files. TablePropertiesCollection table_properties; From f81f99e6cdae8583341a2c9c0223ab7e4d2c1277 Mon Sep 17 00:00:00 2001 From: "wangyi.ywq" Date: Tue, 22 Mar 2022 16:25:56 +0800 Subject: [PATCH 4/6] add fileRange interface of TablePropertiesCollectionIterator --- db/db_impl.cc | 8 ++++++++ include/rocksdb/listener.h | 1 + 2 files changed, 9 insertions(+) diff --git a/db/db_impl.cc b/db/db_impl.cc index d9652d831f..5cef75ea1d 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -170,6 +170,14 @@ class TablePropertiesCollectionIteratorImpl assert(Valid()); return filename_; } + + std::pair fileRange() const override { + assert(Valid()); + FileMetaData* f = *iter_; + return {f->smallest.user_key().ToString(), + f->largest.user_key().ToString()}; + } + const std::shared_ptr& properties() const override { assert(Valid()); return properties_; diff --git a/include/rocksdb/listener.h b/include/rocksdb/listener.h index e8e856a02c..6f2066d51a 100644 --- a/include/rocksdb/listener.h +++ b/include/rocksdb/listener.h @@ -37,6 +37,7 @@ class TablePropertiesCollectionIterator { virtual bool Valid() const = 0; virtual Status status() const = 0; + virtual std::pair fileRange() const = 0; public: // No copying allowed From d4a09cbc3200e7d92fe8ee61cd7ea58bb90ea317 Mon Sep 17 00:00:00 2001 From: "wangyi.ywq" Date: Tue, 22 Mar 2022 16:34:15 +0800 Subject: [PATCH 5/6] fix typo --- db/db_impl_compaction_flush.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/db_impl_compaction_flush.cc b/db/db_impl_compaction_flush.cc index a10155d3d7..0e74a337de 100644 --- a/db/db_impl_compaction_flush.cc +++ b/db/db_impl_compaction_flush.cc @@ -638,8 +638,8 @@ void DBImpl::NotifyOnFlushCompleted( info.largest_seqno = file_meta->fd.largest_seqno; info.table_properties = prop; info.flush_reason = cfd->GetFlushReason(); - info.min_userkey = file_meta->smallest.user_key().ToString(); - info.max_userkey = file_meta->largest.user_key().ToString(); + info.smallest_userkey = file_meta->smallest.user_key().ToString(); + info.largest_userkey = file_meta->largest.user_key().ToString(); for (auto listener : immutable_db_options_.listeners) { listener->OnFlushCompleted(this, info); } From c1847e4837f5f7ee2d0a60c984447e5360421c31 Mon Sep 17 00:00:00 2001 From: "wangyi.ywq" Date: Fri, 25 Mar 2022 15:12:45 +0800 Subject: [PATCH 6/6] rename bounderies --- db/db_impl.cc | 5 ++--- db/db_impl_compaction_flush.cc | 10 ++++------ include/rocksdb/listener.h | 6 +++--- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/db/db_impl.cc b/db/db_impl.cc index 5cef75ea1d..daba9e8f91 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -171,11 +171,10 @@ class TablePropertiesCollectionIteratorImpl return filename_; } - std::pair fileRange() const override { + std::pair boundaries() const override { assert(Valid()); FileMetaData* f = *iter_; - return {f->smallest.user_key().ToString(), - f->largest.user_key().ToString()}; + return {f->smallest.user_key(), f->largest.user_key()}; } const std::shared_ptr& properties() const override { diff --git a/db/db_impl_compaction_flush.cc b/db/db_impl_compaction_flush.cc index 0e74a337de..3b47b4a2f8 100644 --- a/db/db_impl_compaction_flush.cc +++ b/db/db_impl_compaction_flush.cc @@ -1204,9 +1204,8 @@ void DBImpl::NotifyOnCompactionCompleted( auto fn = TableFileName(c->immutable_cf_options()->cf_paths, fmd->fd.GetNumber(), fmd->fd.GetPathId()); info.input_files.push_back(fn); - info.input_files_boundries_user_key.push_back( - {fmd->smallest.user_key().ToString(), - fmd->largest.user_key().ToString()}); + info.input_boundries.push_back( + {fmd->smallest.user_key(), fmd->largest.user_key()}); if (info.table_properties.count(fn) == 0) { std::shared_ptr tp; auto s = current->GetTableProperties(&tp, fmd, &fn); @@ -1223,9 +1222,8 @@ void DBImpl::NotifyOnCompactionCompleted( info.output_files.push_back(TableFileName( c->immutable_cf_options()->cf_paths, newf.second.fd.GetNumber(), newf.second.fd.GetPathId())); - info.output_files_boundries_user_key.push_back( - {newf.second.smallest.user_key().ToString(), - newf.second.largest.user_key().ToString()}); + info.output_boundries.push_back( + {newf.second.smallest.user_key(), newf.second.largest.user_key()}); } for (auto listener : immutable_db_options_.listeners) { listener->OnCompactionCompleted(this, info); diff --git a/include/rocksdb/listener.h b/include/rocksdb/listener.h index 6f2066d51a..3bffff1c34 100644 --- a/include/rocksdb/listener.h +++ b/include/rocksdb/listener.h @@ -37,7 +37,7 @@ class TablePropertiesCollectionIterator { virtual bool Valid() const = 0; virtual Status status() const = 0; - virtual std::pair fileRange() const = 0; + virtual std::pair boundaries() const = 0; public: // No copying allowed @@ -282,11 +282,11 @@ struct CompactionJobInfo { int output_level; // the names of the compaction input files. std::vector input_files; - std::vector> input_files_boundries_user_key; + std::vector input_boundries; // the names of the compaction output files. std::vector output_files; - std::vector> output_files_boundries_user_key; + std::vector> output_boundries; // Table properties for input and output tables. // The map is keyed by values from input_files and output_files. TablePropertiesCollection table_properties;