From b899a40b7cc46154de2f26afb17ca1ba201a2c55 Mon Sep 17 00:00:00 2001 From: Pierre Alexandre Tremblay Date: Wed, 16 Apr 2025 08:47:16 +0200 Subject: [PATCH 01/12] first working KDTreeQuery refactor with questions --- include/flucoma/clients/nrt/KDTreeClient.hpp | 86 +++++++++++--------- 1 file changed, 46 insertions(+), 40 deletions(-) diff --git a/include/flucoma/clients/nrt/KDTreeClient.hpp b/include/flucoma/clients/nrt/KDTreeClient.hpp index 49d3f5388..a15f7c468 100644 --- a/include/flucoma/clients/nrt/KDTreeClient.hpp +++ b/include/flucoma/clients/nrt/KDTreeClient.hpp @@ -149,13 +149,20 @@ constexpr auto KDTreeQueryParams = defineParameters( KDTreeRef::makeParam("tree", "KDTree"), LongParam("numNeighbours", "Number of Nearest Neighbours", 1), FloatParam("radius", "Maximum distance", 0, Min(0)), - InputDataSetClientRef::makeParam("dataSet", "DataSet Name"), + InputDataSetClientRef::makeParam("lookupDataSet", "Lookup DataSet Name"), InputBufferParam("inputPointBuffer", "Input Point Buffer"), BufferParam("predictionBuffer", "Prediction Buffer")); class KDTreeQuery : public FluidBaseClient, ControlIn, ControlOut { - enum { kTree, kNumNeighbors, kRadius, kDataSet, kInputBuffer, kOutputBuffer }; + enum { + kTree, + kNumNeighbors, + kRadius, + kLookupDataSet, + kInputBuffer, + kOutputBuffer + }; public: using ParamDescType = decltype(KDTreeQueryParams); @@ -173,8 +180,7 @@ class KDTreeQuery : public FluidBaseClient, ControlIn, ControlOut static constexpr auto& getParameterDescriptors() { return KDTreeQueryParams; } - KDTreeQuery(ParamSetViewType& p, FluidContext& c) - : mParams(p), mRTBuffer(c.allocator()) + KDTreeQuery(ParamSetViewType& p, FluidContext& c) : mParams(p) { controlChannelsIn(1); controlChannelsOut({1, 1}); @@ -188,74 +194,74 @@ class KDTreeQuery : public FluidBaseClient, ControlIn, ControlOut { if (input[0](0) > 0) { + output[0](0) = 0; + auto kdtreeptr = get().get().lock(); if (!kdtreeptr) - { - // c.reportError("FluidKDTree RT Query: No FluidKDTree found"); - return; - } + return; // c.reportError("FluidKDTree RT Query: No FluidKDTree found"); if (!kdtreeptr->initialized()) - { - // c.reportError("FluidKDTree RT Query: tree not fitted"); - return; - } + return; // c.reportError("FluidKDTree RT Query: tree not fitted"); index k = get(); if (k > kdtreeptr->size() || k < 0) return; // c.reportError("FluidKDTree RT Query has wrong k size"); - index dims = kdtreeptr->dims(); + + index dims = kdtreeptr->dims(); + InOutBuffersCheck bufCheck(dims); if (!bufCheck.checkInputs(get().get(), get().get())) return; // c.reportError("FluidKDTree RT Query i/o buffers are // unavailable"); - auto datasetClientPtr = get().get().lock(); - if (!datasetClientPtr) - datasetClientPtr = kdtreeptr->getDataSet().get().lock(); - if (!datasetClientPtr) - { - // c.reportError("Could not obtain reference FluidDataSet"); - return; - } + auto lookupDSpointer = get().get().lock(); - auto dataset = datasetClientPtr->getDataSet(); - index pointSize = dataset.pointSize(); - auto outBuf = BufferAdaptor::Access(get().get()); - index maxK = outBuf.samps(0).size() / pointSize; - if (maxK <= 0) return; - index outputSize = maxK * pointSize; + index pointSize = lookupDSpointer ? lookupDSpointer->dims().value() : 1; + + auto outBuf = BufferAdaptor::Access(get().get()); + auto outSamps = outBuf.samps(0); + + index numPoints = outSamps.size() / pointSize; + if (numPoints <= 0) + return; // c.reportError("FluidKDTree RT Query output buffer is too + // small for one point") RealVector point(dims, c.allocator()); point <<= BufferAdaptor::ReadAccess(get().get()) .samps(0, dims, 0); - if (mRTBuffer.size() != outputSize) - { - mRTBuffer = RealVector(outputSize, c.allocator()); - mRTBuffer.fill(0); - } auto [dists, ids] = kdtreeptr->algorithm().kNearest( point, k, get(), c.allocator()); - mNumValidKs = std::min(asSigned(ids.size()), maxK); + if (lookupDSpointer) + { + auto lookupDS = lookupDSpointer->getDataSet(); + + auto lookupFn = [&lookupDS, outSamps, pointSize, + n = 0](auto id) mutable { + if (auto point = lookupDS.get(*id); point.data() != nullptr) + outSamps(Slice(n, pointSize)) <<= point; + n += pointSize; + }; - for (index i = 0; i < mNumValidKs; i++) + std::for_each_n(ids.begin(), numPoints, lookupFn); + } + else { - dataset.get(*ids[asUnsigned(i)], - mRTBuffer(Slice(i * pointSize, pointSize))); + std::transform(dists.begin(), dists.begin() + numPoints, + outSamps.begin(), [](auto p) { return std::sqrt(p); }); } - outBuf.samps(0, outputSize, 0) <<= mRTBuffer; + + mLastNumPoints = ids.size(); } - output[0](0) = mNumValidKs; + output[0](0) = mLastNumPoints; } private: - RealVector mRTBuffer; - index mNumValidKs = 0; + index mLastNumPoints{0}; InputDataSetClientRef mDataSetClient; }; From 12eb08239fca21e6eaffd62c7638bfb238a7870b Mon Sep 17 00:00:00 2001 From: Pierre Alexandre Tremblay Date: Wed, 16 Apr 2025 08:48:43 +0200 Subject: [PATCH 02/12] corrected the error for non-triggered state --- include/flucoma/clients/nrt/KDTreeClient.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/flucoma/clients/nrt/KDTreeClient.hpp b/include/flucoma/clients/nrt/KDTreeClient.hpp index a15f7c468..8386b9892 100644 --- a/include/flucoma/clients/nrt/KDTreeClient.hpp +++ b/include/flucoma/clients/nrt/KDTreeClient.hpp @@ -194,7 +194,7 @@ class KDTreeQuery : public FluidBaseClient, ControlIn, ControlOut { if (input[0](0) > 0) { - output[0](0) = 0; + output[0](0) = mLastNumPoints = 0; auto kdtreeptr = get().get().lock(); if (!kdtreeptr) @@ -256,7 +256,7 @@ class KDTreeQuery : public FluidBaseClient, ControlIn, ControlOut mLastNumPoints = ids.size(); } - output[0](0) = mLastNumPoints; + output[0](0) = mLastNumPoints; // updates the output if successful or if not triggered } From 9a97b53b167eba8e4c41b99e6efb9e8a57be9ad8 Mon Sep 17 00:00:00 2001 From: Pierre Alexandre Tremblay Date: Wed, 16 Apr 2025 11:40:14 +0200 Subject: [PATCH 03/12] wise catch from a wise man --- include/flucoma/clients/nrt/KDTreeClient.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/flucoma/clients/nrt/KDTreeClient.hpp b/include/flucoma/clients/nrt/KDTreeClient.hpp index 8386b9892..24c84f61d 100644 --- a/include/flucoma/clients/nrt/KDTreeClient.hpp +++ b/include/flucoma/clients/nrt/KDTreeClient.hpp @@ -253,7 +253,7 @@ class KDTreeQuery : public FluidBaseClient, ControlIn, ControlOut outSamps.begin(), [](auto p) { return std::sqrt(p); }); } - mLastNumPoints = ids.size(); + mLastNumPoints = min(ids.size(), numPoints); } output[0](0) = mLastNumPoints; // updates the output if successful or if not triggered From 1f44637023371272f43eb3f6138f5bca487a8dc5 Mon Sep 17 00:00:00 2001 From: Pierre Alexandre Tremblay Date: Wed, 16 Apr 2025 14:35:41 +0200 Subject: [PATCH 04/12] even better when it compiles --- include/flucoma/clients/nrt/KDTreeClient.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/flucoma/clients/nrt/KDTreeClient.hpp b/include/flucoma/clients/nrt/KDTreeClient.hpp index 24c84f61d..a52000a82 100644 --- a/include/flucoma/clients/nrt/KDTreeClient.hpp +++ b/include/flucoma/clients/nrt/KDTreeClient.hpp @@ -253,7 +253,7 @@ class KDTreeQuery : public FluidBaseClient, ControlIn, ControlOut outSamps.begin(), [](auto p) { return std::sqrt(p); }); } - mLastNumPoints = min(ids.size(), numPoints); + mLastNumPoints = std::min(ids.size(), numPoints); } output[0](0) = mLastNumPoints; // updates the output if successful or if not triggered From 2c7ce2bab1aaa0bc30038c13091c32c8596e6e26 Mon Sep 17 00:00:00 2001 From: Pierre Alexandre Tremblay Date: Wed, 16 Apr 2025 14:47:04 +0200 Subject: [PATCH 05/12] formated --- include/flucoma/clients/nrt/KDTreeClient.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/flucoma/clients/nrt/KDTreeClient.hpp b/include/flucoma/clients/nrt/KDTreeClient.hpp index a52000a82..a930b7d39 100644 --- a/include/flucoma/clients/nrt/KDTreeClient.hpp +++ b/include/flucoma/clients/nrt/KDTreeClient.hpp @@ -256,7 +256,8 @@ class KDTreeQuery : public FluidBaseClient, ControlIn, ControlOut mLastNumPoints = std::min(ids.size(), numPoints); } - output[0](0) = mLastNumPoints; // updates the output if successful or if not triggered + output[0](0) = + mLastNumPoints; // updates the output if successful or if not triggered } From 712ac4b199118d2bc3316cda95d488d60ed833dc Mon Sep 17 00:00:00 2001 From: Pierre Alexandre Tremblay Date: Wed, 16 Apr 2025 14:54:08 +0200 Subject: [PATCH 06/12] 3rd time lucky sorry for the noise --- include/flucoma/clients/nrt/KDTreeClient.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/flucoma/clients/nrt/KDTreeClient.hpp b/include/flucoma/clients/nrt/KDTreeClient.hpp index a930b7d39..c77a97acf 100644 --- a/include/flucoma/clients/nrt/KDTreeClient.hpp +++ b/include/flucoma/clients/nrt/KDTreeClient.hpp @@ -253,7 +253,7 @@ class KDTreeQuery : public FluidBaseClient, ControlIn, ControlOut outSamps.begin(), [](auto p) { return std::sqrt(p); }); } - mLastNumPoints = std::min(ids.size(), numPoints); + mLastNumPoints = std::min(asSigned(ids.size()), numPoints); } output[0](0) = From 2774a4e0c3f1e46afe404457a775c9a53dcf9b56 Mon Sep 17 00:00:00 2001 From: Pierre Alexandre Tremblay Date: Wed, 11 Jun 2025 15:42:29 +0200 Subject: [PATCH 07/12] now with a shared function --- include/flucoma/clients/nrt/KDTreeClient.hpp | 52 ++++++++++---------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/include/flucoma/clients/nrt/KDTreeClient.hpp b/include/flucoma/clients/nrt/KDTreeClient.hpp index c77a97acf..b0cc5ba0a 100644 --- a/include/flucoma/clients/nrt/KDTreeClient.hpp +++ b/include/flucoma/clients/nrt/KDTreeClient.hpp @@ -78,22 +78,12 @@ class KDTreeClient : public FluidBaseClient, MessageResult kNearest(InputBufferPtr data, Optional nNeighbours) const { - // we can deprecate ancillary parameters in favour of optional args by - // falling back to using parameters when arg not present index k = nNeighbours ? nNeighbours.value() : get(); - // alternatively we could just be hardcore and ignore parameters and have - // message handlers fallback to a default when arg missing (which would be - // eventual behaviour, I guess) index k = nNeighbours.value_or(1); - if (k > mAlgorithm.size()) return Error(SmallDataSet); - // if (k <= 0 && get() <= 0) return Error(SmallK); - if (!mAlgorithm.initialized()) return Error(NoDataFitted); - InBufferCheck bufCheck(mAlgorithm.dims()); - if (!bufCheck.checkInputs(data.get())) - return Error(bufCheck.error()); - RealVector point(mAlgorithm.dims()); - point <<= - BufferAdaptor::ReadAccess(data.get()).samps(0, mAlgorithm.dims(), 0); - auto [dists, ids] = mAlgorithm.kNearest(point, k, get()); + + auto reply = computeKnearest(data, k); + auto dists = reply.value().first; + auto ids = reply.value().second; + StringVector result(asSigned(ids.size())); std::transform(ids.cbegin(), ids.cend(), result.begin(), [](const std::string* x) { @@ -105,19 +95,9 @@ class KDTreeClient : public FluidBaseClient, MessageResult kNearestDist(InputBufferPtr data, Optional nNeighbours) const { - // TODO: refactor with kNearest index k = nNeighbours ? nNeighbours.value() : get(); - if (k > mAlgorithm.size()) return Error(SmallDataSet); - // if (k <= 0 && get() <= 0) return Error(SmallK); - if (!mAlgorithm.initialized()) return Error(NoDataFitted); - InBufferCheck bufCheck(mAlgorithm.dims()); - if (!bufCheck.checkInputs(data.get())) - return Error(bufCheck.error()); - RealVector point(mAlgorithm.dims()); - point <<= - BufferAdaptor::ReadAccess(data.get()).samps(0, mAlgorithm.dims(), 0); - auto [dist, ids] = mAlgorithm.kNearest(point, k, get()); - return {dist}; + auto reply = computeKnearest(data, k); + return {reply.value().first}; } static auto getMessageDescriptors() @@ -141,6 +121,24 @@ class KDTreeClient : public FluidBaseClient, private: InputDataSetClientRef mDataSetClient; + + MessageResult + computeKnearest(InputBufferPtr data, index k) const + { + if (k > mAlgorithm.size()) + return Error(SmallDataSet); + if (k <= 0 && get() <= 0) + return Error(SmallK); + if (!mAlgorithm.initialized()) + return Error(NoDataFitted); + InBufferCheck bufCheck(mAlgorithm.dims()); + if (!bufCheck.checkInputs(data.get())) + return Error(bufCheck.error()); + RealVector point(mAlgorithm.dims()); + point <<= + BufferAdaptor::ReadAccess(data.get()).samps(0, mAlgorithm.dims(), 0); + return {mAlgorithm.kNearest(point, k, get())}; + } }; using KDTreeRef = SharedClientRef; From c0d024582392b8dc5ad66b24c08a8faba1c3f989 Mon Sep 17 00:00:00 2001 From: Pierre Alexandre Tremblay Date: Thu, 12 Jun 2025 10:13:59 +0200 Subject: [PATCH 08/12] fix error (no squaring needed here as the algo layer does that for us) --- include/flucoma/clients/nrt/KDTreeClient.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/flucoma/clients/nrt/KDTreeClient.hpp b/include/flucoma/clients/nrt/KDTreeClient.hpp index b0cc5ba0a..909d69297 100644 --- a/include/flucoma/clients/nrt/KDTreeClient.hpp +++ b/include/flucoma/clients/nrt/KDTreeClient.hpp @@ -248,7 +248,7 @@ class KDTreeQuery : public FluidBaseClient, ControlIn, ControlOut else { std::transform(dists.begin(), dists.begin() + numPoints, - outSamps.begin(), [](auto p) { return std::sqrt(p); }); + outSamps.begin(), [](auto p) { return p; }); } mLastNumPoints = std::min(asSigned(ids.size()), numPoints); From c10b59692ead8d02f45685b2ce4a98dbf9f49627 Mon Sep 17 00:00:00 2001 From: Pierre Alexandre Tremblay Date: Thu, 12 Jun 2025 13:25:38 +0200 Subject: [PATCH 09/12] much better / much terser copy --- include/flucoma/clients/nrt/KDTreeClient.hpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/include/flucoma/clients/nrt/KDTreeClient.hpp b/include/flucoma/clients/nrt/KDTreeClient.hpp index 909d69297..339f499da 100644 --- a/include/flucoma/clients/nrt/KDTreeClient.hpp +++ b/include/flucoma/clients/nrt/KDTreeClient.hpp @@ -246,10 +246,7 @@ class KDTreeQuery : public FluidBaseClient, ControlIn, ControlOut std::for_each_n(ids.begin(), numPoints, lookupFn); } else - { - std::transform(dists.begin(), dists.begin() + numPoints, - outSamps.begin(), [](auto p) { return p; }); - } + std::copy_n(dists.begin(), numPoints, outSamps.begin()); mLastNumPoints = std::min(asSigned(ids.size()), numPoints); } From 58e2a0619d394238483495bc51307fc246484b8b Mon Sep 17 00:00:00 2001 From: Pierre Alexandre Tremblay Date: Thu, 12 Jun 2025 17:47:35 +0200 Subject: [PATCH 10/12] adds the replies --- include/flucoma/clients/common/Result.hpp | 12 +++++++++++- include/flucoma/clients/nrt/KDTreeClient.hpp | 2 ++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/include/flucoma/clients/common/Result.hpp b/include/flucoma/clients/common/Result.hpp index 33d66016b..9db70abd8 100644 --- a/include/flucoma/clients/common/Result.hpp +++ b/include/flucoma/clients/common/Result.hpp @@ -57,7 +57,7 @@ class Result bool ok() const noexcept { return (mStatus == Status::kOk); } - Status status() { return mStatus; } + Status status() const noexcept { return mStatus; } void set(Status r) noexcept { mStatus = r; } @@ -92,6 +92,16 @@ class MessageResult : public Result operator T() const { return mData; } T& value() { return mData; } const T& value() const { return mData; } + + template + MessageResult(MessageResult const& x) : Result(x.status(), x.message()) + { + if constexpr (std::is_convertible_v) + { + if (x.ok()) mData = x.mData; + } + } + private: T mData; bool hasData; diff --git a/include/flucoma/clients/nrt/KDTreeClient.hpp b/include/flucoma/clients/nrt/KDTreeClient.hpp index 339f499da..690c727b8 100644 --- a/include/flucoma/clients/nrt/KDTreeClient.hpp +++ b/include/flucoma/clients/nrt/KDTreeClient.hpp @@ -81,6 +81,8 @@ class KDTreeClient : public FluidBaseClient, index k = nNeighbours ? nNeighbours.value() : get(); auto reply = computeKnearest(data, k); + if (!reply.ok()) return reply; + auto dists = reply.value().first; auto ids = reply.value().second; From 338ec040574390cb8088f2ae54a271084c688c2e Mon Sep 17 00:00:00 2001 From: Pierre Alexandre Tremblay Date: Fri, 13 Jun 2025 09:42:19 +0200 Subject: [PATCH 11/12] forgot the test there... --- include/flucoma/clients/nrt/KDTreeClient.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/flucoma/clients/nrt/KDTreeClient.hpp b/include/flucoma/clients/nrt/KDTreeClient.hpp index 690c727b8..f886a1384 100644 --- a/include/flucoma/clients/nrt/KDTreeClient.hpp +++ b/include/flucoma/clients/nrt/KDTreeClient.hpp @@ -99,6 +99,8 @@ class KDTreeClient : public FluidBaseClient, { index k = nNeighbours ? nNeighbours.value() : get(); auto reply = computeKnearest(data, k); + if (!reply.ok()) return reply; + return {reply.value().first}; } From a6cc8bcf4accd9c6a24a940f20d9dfae4da6fb6c Mon Sep 17 00:00:00 2001 From: Pierre Alexandre Tremblay Date: Fri, 13 Jun 2025 10:39:54 +0200 Subject: [PATCH 12/12] this condition used to be commented (with <=) but in effect we use 0 and 0 to get all ranked, so we just need to check neither are negative. --- include/flucoma/clients/nrt/KDTreeClient.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/flucoma/clients/nrt/KDTreeClient.hpp b/include/flucoma/clients/nrt/KDTreeClient.hpp index f886a1384..1a0dff0d9 100644 --- a/include/flucoma/clients/nrt/KDTreeClient.hpp +++ b/include/flucoma/clients/nrt/KDTreeClient.hpp @@ -131,7 +131,7 @@ class KDTreeClient : public FluidBaseClient, { if (k > mAlgorithm.size()) return Error(SmallDataSet); - if (k <= 0 && get() <= 0) + if (k < 0 && get() < 0) return Error(SmallK); if (!mAlgorithm.initialized()) return Error(NoDataFitted);