MLP: add random seed

include/flucoma/algorithms/public/MLP.hpp

@@ -10,14 +10,15 @@ under the European Union’s Horizon 2020 research and innovation programme
 
 #pragma once
 
+#include "../util/EigenRandom.hpp"
 #include "../util/FluidEigenMappings.hpp"
 #include "../util/NNFuncs.hpp"
 #include "../util/NNLayer.hpp"
 #include "../../data/FluidDataSet.hpp"
 #include "../../data/FluidIndex.hpp"
+#include "../../data/FluidMemory.hpp"
 #include "../../data/FluidTensor.hpp"
 #include "../../data/TensorTypes.hpp"
-#include "../../data/FluidMemory.hpp"
 #include <Eigen/Core>
 #include <random>
 
@@ -30,9 +31,8 @@ class MLP
   using ArrayXXd = Eigen::ArrayXXd;
 
 public:
-
   void init(index inputSize, index outputSize,
-            FluidTensor<index, 1> hiddenSizes, index hiddenAct, index outputAct)
+            FluidTensor<index, 1> hiddenSizes, index hiddenAct, index outputAct, index seed)
   {
     mLayers.clear();
     std::vector<index> sizes = {inputSize};
@@ -50,7 +50,7 @@ class MLP
       mLayers.push_back(NNLayer(sizes[asUnsigned(i)], sizes[asUnsigned(i + 1)],
                                 activations[asUnsigned(i)]));
     }
-    for (auto&& l : mLayers) l.init();
+    for (auto&& l : mLayers) l.init(seed);
     mInitialized = true;
     mTrained = false;
   }
@@ -77,7 +77,6 @@ class MLP
 
   void clear()
   {
-    for (auto&& l : mLayers) l.init();
     mInitialized = false;
     mTrained = false;
   }
@@ -218,6 +217,7 @@ class MLP
   bool                 mInitialized{false};
   bool                 mTrained{false};
   index mMaxLayerSize;
+  RandomSeed mSeed; 
 };
 } // namespace algorithm
 } // namespace fluid

include/flucoma/algorithms/public/SGD.hpp

@@ -32,10 +32,10 @@ class SGD
 
   double train(MLP& model, InputRealMatrixView in, RealMatrixView out,
                index nIter, index batchSize, double learningRate,
-               double momentum, double valFrac)
+               double momentum, double valFrac, index seed)
   {
     return train(model, in, out,
-                 SimpleDataSampler(in.rows(), batchSize, valFrac, true), nIter,
+                 SimpleDataSampler(in.rows(), batchSize, valFrac, true, seed), nIter,
                  learningRate, momentum);
   }
 
@@ -47,6 +47,7 @@ class SGD
     using namespace _impl;
     using namespace std;
     using namespace Eigen;
+    MLP originalModel(model); 
     index nExamples = in.rows();
     // index inputSize = in.cols();
     index outputSize = out.cols();
@@ -104,7 +105,9 @@ class SGD
     bool isNan = !((finalPred == finalPred)).all();
     if (isNan)
     {
-      model.clear();
+      using std::swap; 
+      //just return model to exactly its pre-call state
+      swap(model,originalModel); 
       return -1;
     }
     error = model.loss(finalPred, output);

include/flucoma/algorithms/util/NNLayer.hpp

@@ -11,6 +11,7 @@ under the European Union’s Horizon 2020 research and innovation programme
 #pragma once
 
 #include "NNFuncs.hpp"
+#include "../util/EigenRandom.hpp"
 #include "../../data/FluidIndex.hpp"
 #include "../../data/FluidMemory.hpp"
 #include "../../data/TensorTypes.hpp"
@@ -46,10 +47,11 @@ class NNLayer
     initGrads();
   }
 
-  void init()
+  void init(RandomSeed seed)
   {
     double dev = std::sqrt(6.0 / (mWeights.rows() + mWeights.cols()));
-    mWeights = dev * MatrixXd::Random(mWeights.rows(), mWeights.cols()).array();
+    mWeights = EigenRandom<MatrixXd>(mWeights.rows(), mWeights.cols(), seed,
+                                     Range{-dev, dev});
     mBiases = VectorXd::Zero(mWeights.cols());
     initGrads();
   }

include/flucoma/clients/nrt/MLPClassifierClient.hpp

@@ -31,7 +31,7 @@ struct MLPClassifierData
   index                      dims() const { return mlp.dims(); }
   void                       clear()
   {
-    mlp.clear();
+    mlp.clear(); 
     encoder.clear();
   }
   bool initialized() const { return mlp.initialized(); }
@@ -66,7 +66,8 @@ constexpr auto MLPClassifierParams = defineParameters(
     FloatParam("learnRate", "Learning Rate", 0.01, Min(0.0), Max(1.0)),
     FloatParam("momentum", "Momentum", 0.5, Min(0.0), Max(0.99)),
     LongParam("batchSize", "Batch Size", 50, Min(1)),
-    FloatParam("validation", "Validation Amount", 0.2, Min(0), Max(0.9)));
+    FloatParam("validation", "Validation Amount", 0.2, Min(0), Max(0.9)),
+    LongParam("seed", "Random Seed", -1));
 
 
 class MLPClassifierClient : public FluidBaseClient,
@@ -83,7 +84,8 @@ class MLPClassifierClient : public FluidBaseClient,
     kRate,
     kMomentum,
     kBatchSize,
-    kVal
+    kVal,
+    kRandomSeed
   };
 
 public:
@@ -161,9 +163,10 @@ class MLPClassifierClient : public FluidBaseClient,
     {
       mAlgorithm.mlp.init(sourceDataSet.pointSize(),
                           mAlgorithm.encoder.numLabels(), get<kHidden>(),
-                          get<kActivation>(), 1); // sigmoid output
+                          get<kActivation>(), 1,
+                          get<kRandomSeed>()); // sigmoid output
     }
-    
+
     if (auto missingIDs = sourceDataSet.checkIDs(targetDataSet);
         missingIDs.size() == 0)
     {

include/flucoma/clients/nrt/MLPRegressorClient.hpp

@@ -37,7 +37,8 @@ constexpr auto MLPRegressorParams = defineParameters(
     FloatParam("learnRate", "Learning Rate", 0.01, Min(0.0), Max(1.0)),
     FloatParam("momentum", "Momentum", 0.9, Min(0.0), Max(0.99)),
     LongParam("batchSize", "Batch Size", 50, Min(1)),
-    FloatParam("validation", "Validation Amount", 0.2, Min(0), Max(0.9)));
+    FloatParam("validation", "Validation Amount", 0.2, Min(0), Max(0.9)),
+    LongParam("seed","Random Seed", -1));
 
 class MLPRegressorClient : public FluidBaseClient,
                            OfflineIn,
@@ -57,7 +58,8 @@ class MLPRegressorClient : public FluidBaseClient,
     kRate,
     kMomentum,
     kBatchSize,
-    kVal
+    kVal, 
+    kRandomSeed
   };
 
 public:
@@ -120,7 +122,8 @@ class MLPRegressorClient : public FluidBaseClient,
     {
 
       mAlgorithm.init(sourceDataSet.pointSize(), targetDataSet.pointSize(),
-                      get<kHidden>(), get<kActivation>(), outputAct);
+                      get<kHidden>(), get<kActivation>(), outputAct,
+                      get<kRandomSeed>());
     }
 
     if (auto missingIDs = sourceDataSet.checkIDs(targetDataSet);

include/flucoma/data/FluidDataSetSampler.hpp

@@ -46,9 +46,9 @@ class FluidDataSetSampler : public detail::DataSampler<FluidDataSetSampler>
 public:
   template <typename DataSetA, typename DataSetB>
   FluidDataSetSampler(DataSetA const& in, DataSetB const& out, index batchSize,
-                      double validationFraction, bool shuffle = true)
+                      double validationFraction, bool shuffle = true, index seed = -1)
       : detail::DataSampler<FluidDataSetSampler>(in.size(), batchSize,
-                                                 validationFraction, shuffle),
+                                                 validationFraction, shuffle, seed),
         mIdxMaps{in.indexMap(out)}
   {}
 };

include/flucoma/data/FluidJSON.hpp

@@ -425,7 +425,7 @@ void from_json(const nlohmann::json &j, MLP &mlp) {
       hiddenSizes(i) =  j["layers"][asUnsigned(i)]["cols"].get<index>();
     }
   }
-  mlp.init(inputSize,outputSize, hiddenSizes, activation, finalActivation);
+  mlp.init(inputSize,outputSize, hiddenSizes, activation, finalActivation, -1);//FIXME why do we this line?
   for (index i = 0; i < nLayers; i++){
     auto l = j["layers"][asUnsigned(i)];
     index rows = l["rows"].get<index>();

include/flucoma/data/SimpleDataSampler.hpp

@@ -37,9 +37,9 @@ class SimpleDataSampler : public detail::DataSampler<SimpleDataSampler>
 
 public:
   SimpleDataSampler(index size, index batchSize, double validationFraction,
-                    bool shuffle)
+                    bool shuffle, index seed)
       : detail::DataSampler<SimpleDataSampler>(size, batchSize,
-                                               validationFraction, shuffle)
+                                               validationFraction, shuffle, seed)
   {}
 };
 

include/flucoma/data/detail/DataSampler.hpp

@@ -57,9 +57,9 @@ class DataSampler
   };
 
   bool                  mShuffle;
-  index                 mTrainCount;  
-  std::random_device    mRnd;
-  std::mt19937          mGen{mRnd()};
+  index                 mSeed;
+  index                 mTrainCount;    
+  std::mt19937          mGen;
   std::vector<index>    mIdx;
   index                 mBatchSize;
   FluidTensor<index, 2> mBatch;
@@ -78,10 +78,11 @@ class DataSampler
 
 protected:
   DataSampler(index size, index batchSize, double validationFraction,
-              bool shuffle)
-      : mShuffle{shuffle},
+              bool shuffle, index seed)
+      : mShuffle{shuffle}, mSeed{seed},
         mTrainCount{
             std::lrint((1 - std::clamp(validationFraction, 0.0, 1.0)) * size)},
+        mGen(static_cast<size_t>(seed > 0 ? seed : std::random_device()())),
         mIdx(makeIndex(size, mShuffle)),
         mBatchSize{std::min(mTrainCount, batchSize)},
         mBatch(batchSize + (mTrainCount % mBatchSize), 2),
@@ -90,12 +91,9 @@ class DataSampler
 public:
   void reset()
   {
-    using std::begin, std::end;
-
+    if (mSeed > 0) mGen.seed(asUnsigned(mSeed));
     mBatchCount = 0;
-    if (mShuffle)
-      std::shuffle(begin(mIdx), begin(mIdx) + mTrainCount,
-                   mGen); // preserve validation set
+    mIdx = makeIndex(mIdx.size(), mShuffle);
   }
 
   // Returns in / out indices for this batch (not the data)

tests/CMakeLists.txt

@@ -117,6 +117,9 @@ add_test_executable(TestTransientSlice algorithms/public/TestTransientSlice.cpp)
 add_test_executable(TestMLP algorithms/public/TestMLP.cpp)
 add_test_executable(TestKMeans algorithms/public/TestKMeans.cpp)
 
+add_test_executable(TestDataSampler data/detail/TestDataSampler.cpp)
+add_test_executable(TestSGD algorithms/public/TestSGD.cpp)
+
 target_link_libraries(TestNoveltySeg PRIVATE TestSignals)
 target_link_libraries(TestOnsetSeg PRIVATE TestSignals)
 target_link_libraries(TestEnvelopeSeg PRIVATE TestSignals)
@@ -154,4 +157,7 @@ catch_discover_tests(TestMLP WORKING_DIRECTORY "${CMAKE_BINARY_DIR}")
 catch_discover_tests(TestKMeans WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 catch_discover_tests(TestEigenRandom WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 
+catch_discover_tests(TestDataSampler)
+catch_discover_tests(TestSGD)
+
 add_compile_tests("FluidTensor Compilation Tests" data/compile_tests/TestFluidTensor_Compile.cpp) 

tests/algorithms/public/TestMLP.cpp

@@ -9,6 +9,7 @@
 #include <flucoma/data/FluidJSON.hpp>
 #include <flucoma/data/FluidTensor.hpp>
 #include <flucoma/data/FluidDataSetSampler.hpp>
+#include <vector> 
 #include <iostream>
 
 namespace fluid::algorithm {
@@ -106,7 +107,7 @@ TEST_CASE("MLP works on precomputed example")
   // Make a network and set initial conditions
   MLP   mlp = MLP();
   index act = static_cast<index>(NNActivations::Activation::kSigmoid);
-  mlp.init(3, 1, {2}, act, act);
+  mlp.init(3, 1, {2}, act, act,-1);
   FluidTensor<double, 2> layer0Coeffs = {{0.1, 0.2}, {0.3, 0.1}, {0.5, 0}};
   FluidTensor<double, 2> layer1Coeffs = {{0.1}, {0.2}};
   FluidTensor<double, 1> layer0Bias = {0.1, 0.1};
@@ -117,7 +118,7 @@ TEST_CASE("MLP works on precomputed example")
 
   // train for a single iteration 
   SGD sgd;
-  sgd.train(mlp, x, y, 1, 1, 0.1, 0.0, 0.0);
+  sgd.train(mlp, x, y, 1, 1, 0.1, 0.0, 0.0,-1);
 
   // get our hand computed data 
   auto [W1, W2, b1, b2] = manual(0.0, 0.1);
@@ -221,7 +222,6 @@ TEST_CASE("Test batch loader for mismatched fluid datasets")
   index i = 0;
   for (auto batch : ds)
   {
-    std::cout << "ping\n";
     index expectedSize = i++ == 0 ? batchSize + (N % batchSize) : batchSize;
     CHECK(batch->rows() == expectedSize);
     auto inputidx = batch->col(0);
@@ -233,5 +233,37 @@ TEST_CASE("Test batch loader for mismatched fluid datasets")
   }
 }
 
+TEST_CASE("MLP does repeatable things with manually set seed")
+{
+  using Tensor = FluidTensor<double, 2>; 
+  using Vector = FluidTensor<double, 1>; 
+
+  std::vector weights(5, Tensor(3,3)); 
+  Vector biases(3); 
+  index dummyActivation; 
+
+  MLP model; 
+
+  //same seed should give same result, different seed should give differet result
+  model.init(3, 3,FluidTensor<index,1>{3},0,0,42); 
+  model.getParameters(0, weights[0],biases,dummyActivation); 
+  model.init(3, 3,FluidTensor<index,1>{3},0,0,42); 
+  model.getParameters(0, weights[1],biases,dummyActivation); 
+  model.init(3, 3,FluidTensor<index,1>{3},0,0,2875); 
+  model.getParameters(0, weights[2],biases,dummyActivation); 
+
+  //automatic seeding should give different succcesive results
+  model.init(3, 3,FluidTensor<index,1>{3},0,0,-1); 
+  model.getParameters(0, weights[3],biases,dummyActivation); 
+  model.init(3, 3,FluidTensor<index,1>{3},0,0,-1); 
+  model.getParameters(0, weights[4],biases,dummyActivation); 
+
+  //only weights are stochastic
+  using Catch::Matchers::RangeEquals; 
+
+  REQUIRE_THAT(weights[1], RangeEquals(weights[0])); 
+  REQUIRE_THAT(weights[1], !RangeEquals(weights[2])); 
+  REQUIRE_THAT(weights[3], !RangeEquals(weights[4])); 
+}
 
 } // namespace fluid::algorithm