From 23258a19b06fb02ea429edbd1977c5425fd58177 Mon Sep 17 00:00:00 2001 From: "liyuchenmike@gmail.com" Date: Thu, 11 Aug 2016 17:29:47 +0800 Subject: [PATCH 01/16] / SINGA-236 Memory Pool \n / Added the memory pool feature to management Block data allocated on CPU side. \n / Added relevant test cases. --- include/singa/core/common.h | 3 + include/singa/core/memory.h | 52 +++++++++++ src/core/memory/memory.cc | 151 ++++++++++++++++++++++++++++++- test/singa/test_memory.cc | 174 ++++++++++++++++++++++++++++++++++++ 4 files changed, 379 insertions(+), 1 deletion(-) diff --git a/include/singa/core/common.h b/include/singa/core/common.h index caa7c679b0..77210144e7 100644 --- a/include/singa/core/common.h +++ b/include/singa/core/common.h @@ -67,6 +67,9 @@ class Block { // ref) : data_(ptr), size_(size), offset_(offset), ref_count_(ref) {} void* mutable_data() const { return static_cast(data_) + offset_; } const void* data() const { return static_cast(data_) + offset_; } + + // TODO(wangwei) check if the set is correct and add lock if shared sturcture is allowed + void set_data(void* ptr) { data_ = ptr; } size_t size() const { return size_; } size_t offset() const { return offset_; } int IncRefCount() { diff --git a/include/singa/core/memory.h b/include/singa/core/memory.h index f664f95ced..2d2e78b191 100644 --- a/include/singa/core/memory.h +++ b/include/singa/core/memory.h @@ -23,6 +23,7 @@ #include #include "singa/proto/core.pb.h" #include "singa/singa_config.h" +#include "singa/core/common.h" #ifdef USE_CUDA #include "cnmem.h" @@ -50,6 +51,57 @@ class DeviceMemPool { // size_t init_size_ = 0, max_size_ = 0; }; +class CppMemPool { + public: + // initial pool size (MB), and the size of each memory uint in the memory pool (KB) + CppMemPool(size_t init_size_mb = 256, size_t uint_size_kb = 1); + + // return a new pool based on the current pool + // once returned, the old pool will be invalid + // re-initial with pool size (MB), and set the size of each memory uint in the memory pool (KB) + void RsetMemPool(size_t init_size_mb = 256, size_t uint_size_kb = 1); + + // create the memory requested, if size is larger than memUintSize, malloc from system call + // is_ptr_null indicate whether the pointer is null and if so we will initialize it in the malloc function, + // otherwise we will use the ptr directly and access its data and functions. + // after the malloc, the data pointer of the block will be changed and the orginal data pointer will be lost. + void Malloc(Block** ptr, const size_t size, bool is_ptr_null = true); + void Free(Block* ptr); + + std::pair GetMemUsage(); + size_t GetNumFreeUints(){return numUints - numAllocatedUintsInPool;}; + + // release all memory. + // all pointers allocated in the pool must be freed before calling the descturctor. + ~CppMemPool(); + + protected: + // each structure define a memory uint in the memory pool + // the structure is a static double linked list + struct _Uint { + struct _Uint *pPrev, *pNext; + Block* pBlk; + }; + + // pointer to the memory pool + void* pMemPool; + + // head pointer to allocated memory uint + struct _Uint* pAllocatedMemUint; + // head pointer to free memory uint + struct _Uint* pFreeMemUint; + + // the size of each memory uint with/out the meta data of the uint + size_t memUintSize, memUintSizeNoMeta; + + // the number of memory uints in the pool + size_t numUints; + // the number of allocated uints which are resided in the memory pool + size_t numAllocatedUintsInPool; + // the number of allocated uints including the ones resided outside the memory pool + size_t numAllocatedUints; +}; + #ifdef USE_CUDA class CnMemPool : public DeviceMemPool { public: diff --git a/src/core/memory/memory.cc b/src/core/memory/memory.cc index cb33a48cdd..943574f628 100644 --- a/src/core/memory/memory.cc +++ b/src/core/memory/memory.cc @@ -21,8 +21,157 @@ #include "singa/proto/core.pb.h" #include -#ifdef USE_CUDA namespace singa { + +std::pair CppMemPool::GetMemUsage() { + size_t total,free; + total = memUintSize * numUints; + free = total - memUintSize * numAllocatedUintsInPool; + return std::make_pair(free,total); +} + +CppMemPool::CppMemPool(size_t init_size_mb, size_t uint_size_kb) { + pMemPool = NULL ; + pAllocatedMemUint = pFreeMemUint = NULL; + memUintSize = memUintSizeNoMeta = 0; + numUints = numAllocatedUintsInPool = numAllocatedUints = 0; + RsetMemPool(init_size_mb,uint_size_kb); +} + + +void CppMemPool::RsetMemPool(size_t init_size_mb, size_t uint_size_kb) { + + if(numAllocatedUintsInPool == 0) { // in the case the pool is empty + // setting up the parameters in the memory pool + const size_t kNBytesPerKB = (1u << 10); + const size_t kNBytesPerMB = (1u << 20); + memUintSize = uint_size_kb * kNBytesPerKB; + memUintSizeNoMeta = memUintSize - sizeof(struct _Uint); + size_t poolSize = init_size_mb * kNBytesPerMB; + bool memAligned = poolSize % memUintSize == 0; + numUints = memAligned ? (poolSize / memUintSize) : (poolSize / memUintSize + 1); + CHECK_GE(numUints,1); + poolSize = memUintSize * numUints; + + // intialize the memory pool + pMemPool = malloc(poolSize); + CHECK(pMemPool != NULL); + for(size_t idx = 0; idx < numUints; idx++) { + struct _Uint *pCurUint = (struct _Uint*)((char *)pMemPool + idx * memUintSize); + pCurUint->pPrev = NULL; + pCurUint->pNext = pFreeMemUint; + if(pFreeMemUint != NULL) { + pFreeMemUint->pPrev = pCurUint; + } + pFreeMemUint = pCurUint; + pCurUint->pBlk = NULL; + } + } else { // the pool is not empty, create a new one and copy the old to the new one + CppMemPool* pNewPool = new CppMemPool(init_size_mb, uint_size_kb); + struct _Uint* pCurUint = pAllocatedMemUint; + for(size_t idx = 0; idx < numAllocatedUintsInPool; idx++) { + Block* pOldBlk = pCurUint->pBlk; + const void* pData = pOldBlk->data(); + pNewPool->Malloc(&pOldBlk, pOldBlk->size(), false); + size_t copySize = pOldBlk->size() - pOldBlk->offset(); + memcpy(pOldBlk->mutable_data(),pData,copySize); + pCurUint = pCurUint->pNext; + } + // swap the new pool with the current + std::swap(pNewPool->pMemPool,pMemPool); + std::swap(pNewPool->pAllocatedMemUint,pAllocatedMemUint); + std::swap(pNewPool->pFreeMemUint,pFreeMemUint); + std::swap(pNewPool->memUintSize,memUintSize); + std::swap(pNewPool->memUintSizeNoMeta,memUintSizeNoMeta); + std::swap(pNewPool->numUints,numUints); + std::swap(pNewPool->numAllocatedUintsInPool,numAllocatedUintsInPool); + pNewPool->numAllocatedUints = 0; + delete pNewPool; + } +} + +void CppMemPool::Malloc(Block** ptr, const size_t size, bool is_ptr_null) { + numAllocatedUints++; + // the size is larger than the memory uint size + if(size > memUintSizeNoMeta || pFreeMemUint == NULL) { + void* pData = malloc(size); + if(is_ptr_null) { + *ptr = new Block(pData,size); + } else { + CHECK_EQ((*ptr)->size(),size); + (*ptr)->set_data(pData); + } + return; + } + + // otherwise retrieve from one of the memory uint + numAllocatedUintsInPool++; + struct _Uint *pCurUint = pFreeMemUint; + pFreeMemUint = pCurUint->pNext; + if(pFreeMemUint != NULL) { + pFreeMemUint->pPrev = NULL; + } + + pCurUint->pNext = pAllocatedMemUint; + if(pAllocatedMemUint != NULL) { + pAllocatedMemUint->pPrev = pCurUint; + } + + pAllocatedMemUint = pCurUint; + void* pData = (void*)((char *)pCurUint + sizeof(struct _Uint)); + if(is_ptr_null) { + *ptr = new Block(pData,size); + } else { + CHECK_EQ((*ptr)->size(),size); + (*ptr)->set_data(pData); + } + CHECK(pCurUint->pBlk == NULL); + pCurUint->pBlk = *ptr; +} + +void CppMemPool::Free(Block* ptr) { + void* pData = ptr->mutable_data(); + if(pMemPool < pData && pData < (void*)((char*)pMemPool + numUints * memUintSize)) { + struct _Uint *pCurUint = (struct _Uint*)((char*)pData-sizeof(struct _Uint)); + CHECK(ptr == pCurUint->pBlk); + + if(pCurUint == pAllocatedMemUint) { + pAllocatedMemUint = pCurUint->pNext; + if(pAllocatedMemUint != NULL) { + pAllocatedMemUint->pPrev = NULL; + } + } else { + struct _Uint *pCurPrevUint = pCurUint->pPrev; + pCurUint->pPrev = NULL; + pCurPrevUint->pNext = pCurUint->pNext; + if(pCurUint->pNext != NULL) { + pCurUint->pNext->pPrev = pCurPrevUint; + } + } + + pCurUint->pNext = pFreeMemUint; + if(pFreeMemUint != NULL) { + pFreeMemUint->pPrev = pCurUint; + } + + pFreeMemUint = pCurUint; + pCurUint->pBlk = NULL; + numAllocatedUintsInPool--; + } + else { + free(pData); + } + numAllocatedUints--; + delete ptr; +} + +CppMemPool::~CppMemPool() { + CHECK_EQ(numAllocatedUints,0); + free(pMemPool); +} + + +#ifdef USE_CUDA std::atomic CnMemPool::pool_count(0); std::pair CnMemPool::GetMemUsage() { size_t free, total; diff --git a/test/singa/test_memory.cc b/test/singa/test_memory.cc index 33a374724e..4e0dfff065 100644 --- a/test/singa/test_memory.cc +++ b/test/singa/test_memory.cc @@ -25,6 +25,180 @@ #include "singa/singa_config.h" #include "singa/utils/timer.h" #include "singa/utils/cuda_utils.h" +#include + +// this tests allocated a number of memory blocks in the memory pool +// the pool consists of 1024 uints and each uint has a size of 1000 bytes +// we malloc 1024 blocks where half of the block will reside outside the pool, +// and the other half will be inside the pool +TEST(CppMemPool, Malloc) { + singa::CppMemPool pool(1,1); + const int numOfTests = 1024; + const size_t dataSizeSmall = 1000; + const size_t dataSizeLarge = 2000; + singa::Block** pptr = new singa::Block*[numOfTests]; + + for(int i = 0; i < numOfTests; i++) { + const size_t dataSize = (i%2) ? dataSizeSmall : dataSizeLarge; + pool.Malloc(&(pptr[i]),dataSize); + int* data = static_cast(pptr[i]->mutable_data()); + for(int idx = 0; idx < (int)dataSize/4; idx++) { + data[idx] = i; + } + data = static_cast(pptr[i]->mutable_data()); + int sum = 0; + for(int idx = 0; idx < (int)dataSize/4; idx++) { + sum += data[idx]; + } + CHECK_EQ(sum,i*dataSize/4); + } + CHECK_EQ(512,pool.GetNumFreeUints()); + + for(int i = 0; i < numOfTests; i++) { + pool.Free(pptr[i]); + } + CHECK_EQ(1024,pool.GetNumFreeUints()); + + delete[] pptr; +} + +// this tests intialize a pool with size 2M bytes and each memory unit has a size of 2048 bytes +// we then allocated 1024 memory block with half of the blocks with size 2000 and the other half with size 1000 +// then we reset the pool to size 1M bytes and memory uint size to 1000 bytes to test the reset function +TEST(CppMemPool, MallocAndRest) { + singa::CppMemPool pool(2,2); + const int numOfTests = 1024; + const size_t dataSizeSmall = 1000; + const size_t dataSizeLarge = 2000; + singa::Block** pptr = new singa::Block*[numOfTests]; + + for(int i = 0; i < numOfTests; i++) { + const size_t dataSize = (i%2) ? dataSizeSmall : dataSizeLarge; + pool.Malloc(&(pptr[i]),dataSize); + int* data = static_cast(pptr[i]->mutable_data()); + for(int idx = 0; idx < (int)dataSize/4; idx++) { + data[idx] = i; + } + data = static_cast(pptr[i]->mutable_data()); + int sum = 0; + for(int idx = 0; idx < (int)dataSize/4; idx++) { + sum += data[idx]; + } + CHECK_EQ(sum,i*dataSize/4); + } + CHECK_EQ(0,pool.GetNumFreeUints()); + + pool.RsetMemPool(1,1); + CHECK_EQ(512,pool.GetNumFreeUints()); + for(int i = 0; i < numOfTests; i++) { + const size_t dataSize = (i%2) ? dataSizeSmall : dataSizeLarge; + int* data = static_cast(pptr[i]->mutable_data()); + for(int idx = 0; idx < (int)dataSize/4; idx++) { + data[idx] = i; + } + data = static_cast(pptr[i]->mutable_data()); + int sum = 0; + for(int idx = 0; idx < (int)dataSize/4; idx++) { + sum += data[idx]; + } + CHECK_EQ(sum,i*dataSize/4); + } + + for(int i = 0; i < numOfTests; i++) { + pool.Free(pptr[i]); + } + CHECK_EQ(1024,pool.GetNumFreeUints()); + + delete[] pptr; +} + +// this tests initialize a pool with size 1M bytes and uint size of 1024 bytes +// then 1024 memory blocks are allocated, half of them in the pool and the other half outside the pool +// subsequently, we randomly free 512 blocks and after that allocate them back to the pool +// after reset the pool to a size of 2M bytes and uint size of 2048 bytes, +// we free all memory blocks allocated. +TEST(CppMemPool, RandomFree) { + singa::CppMemPool pool(1,1); + const int numOfTests = 1024; + const size_t dataSizeSmall = 1000; + const size_t dataSizeLarge = 2000; + singa::Block** pptr = new singa::Block*[numOfTests]; + + for(int i = 0; i < numOfTests; i++) { + const size_t dataSize = (i%2) ? dataSizeSmall : dataSizeLarge; + pool.Malloc(&(pptr[i]),dataSize); + int* data = static_cast(pptr[i]->mutable_data()); + for(int idx = 0; idx < (int)dataSize/4; idx++) { + data[idx] = i; + } + data = static_cast(pptr[i]->mutable_data()); + int sum = 0; + for(int idx = 0; idx < (int)dataSize/4; idx++) { + sum += data[idx]; + } + CHECK_EQ(sum,i*dataSize/4); + } + CHECK_EQ(512,pool.GetNumFreeUints()); + + // randomized free pointers + int* randomPool = new int[numOfTests]; + for(int i = 0; i < numOfTests; i++) { + randomPool[i] = i; + } + int iter = 0; + while(iter != numOfTests/2) { // random free half of the memory blocks + int pos = std::rand() % (numOfTests-iter); + int i = randomPool[pos]; + std::swap(randomPool[pos],randomPool[numOfTests-1-iter]); + + // check value before deletion + const size_t dataSize = (i%2) ? dataSizeSmall : dataSizeLarge; + int* data = static_cast(pptr[i]->mutable_data()); + for(int idx = 0; idx < (int)dataSize/4; idx++) { + data[idx] = i; + } + data = static_cast(pptr[i]->mutable_data()); + int sum = 0; + for(int idx = 0; idx < (int)dataSize/4; idx++) { + sum += data[idx]; + } + CHECK_EQ(sum,i*dataSize/4); + + pool.Free(pptr[i]); + iter++; + } + + // test the unfreed memory block value + for(int pos = 0; pos < numOfTests/2; pos++) { + int i = randomPool[pos]; + const size_t dataSize = (i%2) ? dataSizeSmall : dataSizeLarge; + int* data = static_cast(pptr[i]->mutable_data()); + for(int idx = 0; idx < (int)dataSize/4; idx++) { + data[idx] = i; + } + data = static_cast(pptr[i]->mutable_data()); + int sum = 0; + for(int idx = 0; idx < (int)dataSize/4; idx++) { + sum += data[idx]; + } + CHECK_EQ(sum,i*dataSize/4); + } + + for(int pos = numOfTests/2; pos < numOfTests; pos++) { + int i = randomPool[pos]; + const size_t dataSize = (i%2) ? dataSizeSmall : dataSizeLarge; + pool.Malloc(&(pptr[i]),dataSize); + } + + pool.RsetMemPool(2,2); + for(int i = 0; i < numOfTests; i++) { + pool.Free(pptr[i]); + } + CHECK_EQ(1024,pool.GetNumFreeUints()); + + delete[] randomPool; + delete[] pptr; +} #ifdef USE_CUDA /* From b9cc5ae7454d28291e9b8b6c2b13f5adc6399d32 Mon Sep 17 00:00:00 2001 From: WANG Ji Date: Sat, 6 Aug 2016 17:36:28 +0800 Subject: [PATCH 02/16] SINGA-231 Batchnormlized VGG model for cifar-10 In this ticket, we implemented a batch normalized VGG model for cifar10 dataset (refer to http://torch.ch/blog/2015/07/30/cifar.html). * +vgg-parallel.cc for parallel training * +vgg.py using python language * fix a bug in ResetLike() method in tensor.h, which before did not reset shape. * fix a bug in local_updater.cc, which may cause race condition when multi-threads try to initialize mutexes concurrently. * revise batch nomalization layer to support 2D tensor input --- examples/cifar10/CMakeLists.txt | 5 + examples/cifar10/train_vgg_cifar10.py | 162 +++++++++++++ examples/cifar10/vgg-parallel.cc | 333 ++++++++++++++++++++++++++ examples/cifar10/vgg.py | 52 ++++ src/core/tensor/tensor.cc | 2 +- src/model/layer/batchnorm.cc | 25 +- src/model/layer/batchnorm.h | 3 +- src/model/layer/cudnn_batchnorm.cc | 31 ++- src/model/updater/local_updater.cc | 1 + src/python/singa/layer.py | 10 +- src/python/singa/net.py | 6 +- 11 files changed, 613 insertions(+), 17 deletions(-) create mode 100644 examples/cifar10/train_vgg_cifar10.py create mode 100644 examples/cifar10/vgg-parallel.cc create mode 100644 examples/cifar10/vgg.py diff --git a/examples/cifar10/CMakeLists.txt b/examples/cifar10/CMakeLists.txt index 92f884ccc5..76c0b73ea2 100644 --- a/examples/cifar10/CMakeLists.txt +++ b/examples/cifar10/CMakeLists.txt @@ -10,4 +10,9 @@ ADD_EXECUTABLE(alexnet-parallel alexnet-parallel.cc) ADD_DEPENDENCIES(alexnet-parallel singa_core singa_model singa_utils) TARGET_LINK_LIBRARIES(alexnet-parallel singa_core singa_utils singa_model protobuf ${SINGA_LIBKER_LIBS}) SET_TARGET_PROPERTIES(alexnet-parallel PROPERTIES LINK_FLAGS "${LINK_FLAGS} -pthread") + +ADD_EXECUTABLE(vgg-parallel vgg-parallel.cc) +ADD_DEPENDENCIES(vgg-parallel singa_core singa_model singa_utils) +TARGET_LINK_LIBRARIES(vgg-parallel singa_core singa_utils singa_model protobuf ${SINGA_LIBKER_LIBS}) +SET_TARGET_PROPERTIES(vgg-parallel PROPERTIES LINK_FLAGS "${LINK_FLAGS} -pthread") ENDIF(USE_CUDNN) diff --git a/examples/cifar10/train_vgg_cifar10.py b/examples/cifar10/train_vgg_cifar10.py new file mode 100644 index 0000000000..e9df04ee6e --- /dev/null +++ b/examples/cifar10/train_vgg_cifar10.py @@ -0,0 +1,162 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +""" CIFAR10 dataset is at https://www.cs.toronto.edu/~kriz/cifar.html. +It includes 5 binary dataset, each contains 10000 images. 1 row (1 image) +includes 1 label & 3072 pixels. 3072 pixels are 3 channels of a 32x32 image +""" + +import cPickle +import numpy as np +import os +import sys +import math + +sys.path.append(os.path.join(os.path.dirname(__file__), '../../build/python')) +from singa import initializer +from singa import utils +from singa import optimizer +from singa import device +from singa import tensor +from singa.proto import core_pb2 + +import vgg + + +def load_dataset(filepath): + print 'Loading data file %s' % filepath + with open(filepath, 'rb') as fd: + cifar10 = cPickle.load(fd) + image = cifar10['data'].astype(dtype=np.uint8) + image = image.reshape((-1, 3, 32, 32)) + label = np.asarray(cifar10['labels'], dtype=np.uint8) + label = label.reshape(label.size, 1) + return image, label + + +def load_train_data(dir_path, num_batches=5): + labels = [] + batchsize = 10000 + images = np.empty((num_batches * batchsize, 3, 32, 32), dtype=np.uint8) + for did in range(1, num_batches + 1): + fname_train_data = dir_path + "/data_batch_{}".format(did) + image, label = load_dataset(fname_train_data) + images[(did - 1) * batchsize:did * batchsize] = image + labels.extend(label) + images = np.array(images, dtype=np.float32) + labels = np.array(labels, dtype=np.int32) + return images, labels + + +def load_test_data(dir_path): + images, labels = load_dataset(dir_path + "/test_batch") + return np.array(images, dtype=np.float32), np.array(labels, dtype=np.int32) + + +def get_lr(epoch): + return 0.01 / float(1 << ((epoch / 30))) + #if epoch < 100: + # return 0.01 + #elif epoch < 150: + # return 0.005 + #elif epoch < 200: + # return 0.001 + #elif epoch < 250: + # return 0.0001 + + +def train(data_dir, net, num_epoch=250, batch_size=128): + print 'Creating Device............' + cuda = device.create_cuda_gpus(2)[1] + net.to_device(cuda) + print 'Start intialization............' + opt = optimizer.SGD(momentum=0.9, weight_decay=0.0005) + for (p, name) in zip(net.param_values(), net.param_names()): + print name, p.shape + if len(p.shape) > 1: + if 'mean' in name or 'beta' in name: + p.set_value(0.0) + elif 'var' in name: + p.set_value(1.0) + elif 'gamma' in name: + initializer.uniform(p, 0, 1) + elif 'conv' in name: + initializer.gaussian(p, 0, math.sqrt(2.0/(9.0 * p.shape[0]))) + else: + initializer.gaussian(p, 0, 0.02) + + #stdv = 1.0/math.sqrt(p.shape[1]) + #initializer.uniform(p, -stdv, stdv) + else: + p.set_value(0) + #print specs.name, filler.type, p.l1() + print name, p.l1() + print 'Loading data ..................' + train_x, train_y = load_train_data(data_dir) + test_x, test_y = load_test_data(data_dir) + mean = train_x.mean() + std = train_x.std() + train_x -= mean + test_x -= mean + train_x /= std + test_x /= std + + tx = tensor.Tensor((batch_size, 3, 32, 32), cuda) + ty = tensor.Tensor((batch_size,), cuda, core_pb2.kInt) + num_train_batch = train_x.shape[0] / batch_size + num_test_batch = test_x.shape[0] / batch_size + idx = np.arange(train_x.shape[0], dtype=np.int32) + for epoch in range(num_epoch): + np.random.shuffle(idx) + loss, acc = 0.0, 0.0 + print 'Epoch %d' % epoch + for b in range(num_train_batch): + x = train_x[idx[b * batch_size: (b + 1) * batch_size]] + y = train_y[idx[b * batch_size: (b + 1) * batch_size]] + tx.copy_from_numpy(x) + ty.copy_from_numpy(y) + grads, (l, a) = net.train(tx, ty) + loss += l + acc += a + for (s, p, g) in zip(net.param_specs(), net.param_values(), grads): + opt.apply_with_lr(epoch, get_lr(epoch), g, p, str(s.name)) + # update progress bar + utils.update_progress(b * 1.0 / num_train_batch, + 'training loss = %f, accuracy = %f' % (l, a)) + info = '\ntraining loss = %f, training accuracy = %f' \ + % (loss / num_train_batch, acc / num_train_batch) + print info + + loss, acc = 0.0, 0.0 + for b in range(num_test_batch): + x = test_x[b * batch_size: (b + 1) * batch_size] + y = test_y[b * batch_size: (b + 1) * batch_size] + tx.copy_from_numpy(x) + ty.copy_from_numpy(y) + l, a = net.evaluate(tx, ty) + loss += l + acc += a + + print 'test loss = %f, test accuracy = %f' \ + % (loss / num_test_batch, acc / num_test_batch) + net.save('model.bin') # save model params into checkpoint file + +if __name__ == '__main__': + data_dir = 'cifar-10-batches-py' + assert os.path.exists(data_dir), \ + 'Pls download the cifar10 dataset via "download_data.py py"' + net = vgg.create_net() + train(data_dir, net) diff --git a/examples/cifar10/vgg-parallel.cc b/examples/cifar10/vgg-parallel.cc new file mode 100644 index 0000000000..ba308e9d32 --- /dev/null +++ b/examples/cifar10/vgg-parallel.cc @@ -0,0 +1,333 @@ +/************************************************************ +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*************************************************************/ + +#include "cifar10.h" +#include "singa/model/feed_forward_net.h" +#include "singa/model/optimizer.h" +#include "singa/model/updater.h" +#include "singa/model/initializer.h" +#include "singa/model/metric.h" +#include "singa/utils/channel.h" +#include "singa/utils/string.h" +#include "singa/core/memory.h" +#include "../../src/model/layer/cudnn_convolution.h" +#include "../../src/model/layer/cudnn_activation.h" +#include "../../src/model/layer/cudnn_pooling.h" +#include "../../src/model/layer/cudnn_lrn.h" +#include "../../src/model/layer/cudnn_dropout.h" +#include "../../src/model/layer/cudnn_batchnorm.h" +#include "../../src/model/layer/dense.h" +#include "../../src/model/layer/flatten.h" +#include +#include +#include + +namespace singa { + +const float default_wd = 0.0005f; + +LayerConf GenConvConf(string name, int nb_filter, int kernel, int stride, + int pad, float std = .02f, float bias = .0f) { + LayerConf conf; + conf.set_name(name); + conf.set_type("CudnnConvolution"); + ConvolutionConf *conv = conf.mutable_convolution_conf(); + conv->set_num_output(nb_filter); + conv->add_kernel_size(kernel); + conv->add_stride(stride); + conv->add_pad(pad); + conv->set_bias_term(true); + + ParamSpec *wspec = conf.add_param(); + wspec->set_name(name + "_weight"); + auto wfill = wspec->mutable_filler(); + wfill->set_type("Gaussian"); + wfill->set_std(sqrt(2.0f/(nb_filter*9.0f))); + + ParamSpec *bspec = conf.add_param(); + bspec->set_name(name + "_bias"); + auto bfill = bspec->mutable_filler(); + bfill->set_value(bias); + // bspec->set_lr_mult(2); + // bspec->set_decay_mult(0); + return conf; +} + +LayerConf GenPoolingConf(string name, bool max_pool, int kernel, int stride, + int pad) { + LayerConf conf; + conf.set_name(name); + conf.set_type("CudnnPooling"); + PoolingConf *pool = conf.mutable_pooling_conf(); + pool->set_kernel_size(kernel); + pool->set_stride(stride); + pool->set_pad(pad); + if (!max_pool) pool->set_pool(PoolingConf_PoolMethod_AVE); + return conf; +} + +LayerConf GenReLUConf(string name) { + LayerConf conf; + conf.set_name(name); + conf.set_type("RELU"); + return conf; +} + +LayerConf GenDenseConf(string name, int num_output, float std, float wd = default_wd) { + LayerConf conf; + conf.set_name(name); + conf.set_type("Dense"); + DenseConf *dense = conf.mutable_dense_conf(); + dense->set_num_output(num_output); + + ParamSpec *wspec = conf.add_param(); + wspec->set_name(name + "_weight"); + wspec->set_decay_mult(wd); + auto wfill = wspec->mutable_filler(); + wfill->set_type("Gaussian"); + wfill->set_std(std); + + ParamSpec *bspec = conf.add_param(); + bspec->set_name(name + "_bias"); + bspec->set_lr_mult(2); + bspec->set_decay_mult(0); + + return conf; +} + +LayerConf GenFlattenConf(string name) { + LayerConf conf; + conf.set_name(name); + conf.set_type("Flatten"); + return conf; +} + +LayerConf GenBatchNormConf(string name) { + LayerConf conf; + conf.set_name(name); + conf.set_type("CudnnBatchNorm"); + ParamSpec *gammaspec = conf.add_param(); + gammaspec->set_name(name + "_gamma"); + auto gammafill = gammaspec->mutable_filler(); + gammafill->set_type("uniform"); + gammafill->set_min(0); + gammafill->set_max(1); + + ParamSpec *betaspec = conf.add_param(); + betaspec->set_name(name + "_beta"); + auto betafill = betaspec->mutable_filler(); + betafill->set_type("constant"); + betafill->set_value(0); + + ParamSpec *meanspec = conf.add_param(); + meanspec->set_name(name + "_mean"); + auto meanfill = meanspec->mutable_filler(); + meanfill->set_type("constant"); + meanfill->set_value(0); + + ParamSpec *varspec = conf.add_param(); + varspec->set_name(name + "_var"); + auto varfill = varspec->mutable_filler(); + varfill->set_type("constant"); + varfill->set_value(1); + + return conf; +} + +LayerConf GenDropoutConf(string name, float dropout_ratio) { + LayerConf conf; + conf.set_name(name); + conf.set_type("CudnnDropout"); + DropoutConf *dropout = conf.mutable_dropout_conf(); + dropout->set_dropout_ratio(dropout_ratio); + + return conf; +} + +void ConvBNReLU(FeedForwardNet& net, string name, int nb_filter, Shape* shape = nullptr) { + net.Add(new CudnnConvolution(), GenConvConf(name+"_conv", nb_filter, 3, 1, 1), shape); + net.Add(new CudnnBatchNorm(), GenBatchNormConf(name+"_bn")); + net.Add(new CudnnActivation(), GenReLUConf(name+"_relu")); +} + +FeedForwardNet CreateNet() { + FeedForwardNet net; + Shape s{3, 32, 32}; + ConvBNReLU(net, "conv1_1", 64, &s); + net.Add(new CudnnDropout(), GenDropoutConf("drop1", 0.3)); + ConvBNReLU(net, "conv1_2", 64); + net.Add(new CudnnPooling(), GenPoolingConf("pool1", true, 2, 2, 0)); + ConvBNReLU(net, "conv2_1", 128); + net.Add(new CudnnDropout(), GenDropoutConf("drop2", 0.4)); + ConvBNReLU(net, "conv2_2", 128); + net.Add(new CudnnPooling(), GenPoolingConf("pool2", true, 2, 2, 0)); + ConvBNReLU(net, "conv3_1", 256); + net.Add(new CudnnDropout(), GenDropoutConf("drop3_1", 0.4)); + ConvBNReLU(net, "conv3_2", 256); + net.Add(new CudnnDropout(), GenDropoutConf("drop3_2", 0.4)); + ConvBNReLU(net, "conv3_3", 256); + net.Add(new CudnnPooling(), GenPoolingConf("pool3", true, 2, 2, 0)); + ConvBNReLU(net, "conv4_1", 512); + net.Add(new CudnnDropout(), GenDropoutConf("drop4_1", 0.4)); + ConvBNReLU(net, "conv4_2", 512); + net.Add(new CudnnDropout(), GenDropoutConf("drop4_2", 0.4)); + ConvBNReLU(net, "conv4_3", 512); + net.Add(new CudnnPooling(), GenPoolingConf("pool4", true, 2, 2, 0)); + ConvBNReLU(net, "conv5_1", 512); + net.Add(new CudnnDropout(), GenDropoutConf("drop5_1", 0.4)); + ConvBNReLU(net, "conv5_2", 512); + net.Add(new CudnnDropout(), GenDropoutConf("drop5_2", 0.4)); + ConvBNReLU(net, "conv5_3", 512); + net.Add(new CudnnPooling(), GenPoolingConf("pool5", true, 2, 2, 0)); + net.Add(new Flatten(), GenFlattenConf("flat")); + net.Add(new CudnnDropout(), GenDropoutConf("flat_drop", 0.5)); + net.Add(new Dense(), GenDenseConf("ip1", 512, 0.02)); + net.Add(new CudnnBatchNorm(), GenBatchNormConf("ip1_bn")); + net.Add(new CudnnActivation(), GenReLUConf("ip1_relu")); + net.Add(new CudnnDropout(), GenDropoutConf("ip1_drop", 0.5)); + net.Add(new Dense(), GenDenseConf("ip2", 10, 0.02)); + + return net; +} + +void Train(float lr, int num_epoch, string data_dir) { + Cifar10 data(data_dir); + Tensor train_x, train_y, test_x, test_y; + Tensor train_x_1, train_x_2, train_y_1, train_y_2; + { + auto train = data.ReadTrainData(); + size_t nsamples = train.first.shape(0); + auto mtrain = + Reshape(train.first, Shape{nsamples, train.first.Size() / nsamples}); + const Tensor &mean = Average(mtrain, 0); + SubRow(mean, &mtrain); + Tensor std = Square(mtrain); + std = Average(std, 0); + std = Sqrt(std);; + std += 1e-6f; + DivRow(std, &mtrain); + + train_x = Reshape(mtrain, train.first.shape()); + train_y = train.second; + + LOG(INFO) << "Slicing training data..."; + train_x_1.Reshape(Shape{nsamples / 2, train.first.shape(1), + train.first.shape(2), train.first.shape(3)}); + LOG(INFO) << "Copying first data slice..."; + CopyDataToFrom(&train_x_1, train_x, train_x.Size() / 2); + train_x_2.Reshape(Shape{nsamples / 2, train.first.shape(1), + train.first.shape(2), train.first.shape(3)}); + LOG(INFO) << "Copying second data slice..."; + CopyDataToFrom(&train_x_2, train_x, train_x.Size() / 2, 0, + train_x.Size() / 2); + train_y_1.Reshape(Shape{nsamples / 2}); + train_y_1.AsType(kInt); + LOG(INFO) << "Copying first label slice..."; + CopyDataToFrom(&train_y_1, train_y, train_y.Size() / 2); + train_y_2.Reshape(Shape{nsamples / 2}); + train_y_2.AsType(kInt); + LOG(INFO) << "Copying second label slice..."; + CopyDataToFrom(&train_y_2, train_y, train_y.Size() / 2, 0, + train_y.Size() / 2); + + auto test = data.ReadTestData(); + nsamples = test.first.shape(0); + auto mtest = + Reshape(test.first, Shape{nsamples, test.first.Size() / nsamples}); + SubRow(mean, &mtest); + DivRow(std, &mtest); + test_x = Reshape(mtest, test.first.shape()); + test_y = test.second; + } + + CHECK_EQ(train_x.shape(0), train_y.shape(0)); + CHECK_EQ(test_x.shape(0), test_y.shape(0)); + LOG(INFO) << "Total Training samples = " << train_y.shape(0) + << ", Total Test samples = " << test_y.shape(0); + CHECK_EQ(train_x_1.shape(0), train_y_1.shape(0)); + LOG(INFO) << "On net 1, Training samples = " << train_y_1.shape(0) + << ", Test samples = " << test_y.shape(0); + CHECK_EQ(train_x_2.shape(0), train_y_2.shape(0)); + LOG(INFO) << "On net 2, Training samples = " << train_y_2.shape(0); + + auto net_1 = CreateNet(); + auto net_2 = CreateNet(); + + SGD sgd; + OptimizerConf opt_conf; + opt_conf.set_momentum(0.9); + auto reg = opt_conf.mutable_regularizer(); + reg->set_coefficient(0.0005); + sgd.Setup(opt_conf); + sgd.SetLearningRateGenerator([lr](int epoch) { + return 0.01f / static_cast(1u << (epoch/30)); + }); + + SoftmaxCrossEntropy loss_1, loss_2; + Accuracy acc_1, acc_2; + /// Create updater aggregating gradient on CPU + std::shared_ptr updater = std::make_shared(2, &sgd); + + /// Only need to register parameter once. + net_1.Compile(true, true, updater, &loss_1, &acc_1); + net_2.Compile(true, false, updater, &loss_2, &acc_2); + + MemPoolConf mem_conf; + mem_conf.add_device(0); + mem_conf.add_device(1); + std::shared_ptr mem_pool(new CnMemPool(mem_conf)); + std::shared_ptr cuda_1(new CudaGPU(0, mem_pool)); + std::shared_ptr cuda_2(new CudaGPU(1, mem_pool)); + net_1.ToDevice(cuda_1); + net_2.ToDevice(cuda_2); + + train_x_1.ToDevice(cuda_1); + train_y_1.ToDevice(cuda_1); + test_x.ToDevice(cuda_1); + test_y.ToDevice(cuda_1); + train_x_2.ToDevice(cuda_2); + train_y_2.ToDevice(cuda_2); + + LOG(INFO) << "Launching thread..."; + std::thread t1 = + net_1.TrainThread(50, num_epoch, train_x_1, train_y_1, test_x, test_y); + std::thread t2 = net_2.TrainThread(50, num_epoch, train_x_2, train_y_2); + t1.join(); + t2.join(); +} +} + +int main(int argc, char **argv) { + singa::InitChannel(nullptr); + int pos = singa::ArgPos(argc, argv, "-epoch"); + int nEpoch = 1; + if (pos != -1) nEpoch = atoi(argv[pos + 1]); + pos = singa::ArgPos(argc, argv, "-lr"); + float lr = 0.001; + if (pos != -1) lr = atof(argv[pos + 1]); + pos = singa::ArgPos(argc, argv, "-data"); + string data = "cifar-10-batches-bin"; + if (pos != -1) data = argv[pos + 1]; + + LOG(INFO) << "Start training"; + singa::Train(lr, nEpoch, data); + LOG(INFO) << "End training"; +} diff --git a/examples/cifar10/vgg.py b/examples/cifar10/vgg.py new file mode 100644 index 0000000000..8063307f5b --- /dev/null +++ b/examples/cifar10/vgg.py @@ -0,0 +1,52 @@ +import sys +import os + +sys.path.append(os.path.join(os.path.dirname(__file__), '../../build/python')) +from singa import layer +from singa import metric +from singa import loss +from singa import net as ffnet +from singa.proto import core_pb2 + +def ConvBnReLU(net, name, nb_filers, sample_shape=None): + net.add(layer.Conv2D(name + '_1', nb_filers, 3, 1, pad=1, + input_sample_shape=sample_shape)) + net.add(layer.BatchNormalization(name + '_2')) + net.add(layer.Activation(name + '_3')) + +def create_net(): + net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy()) + ConvBnReLU(net, 'conv1_1', 64, (3, 32, 32)) + net.add(layer.Dropout('drop1', 0.3, engine='cudnn')) + ConvBnReLU(net, 'conv1_2', 64) + net.add(layer.MaxPooling2D('pool1', 2, 2, border_mode='valid')) + ConvBnReLU(net, 'conv2_1', 128) + net.add(layer.Dropout('drop2_1', 0.4, engine='cudnn')) + ConvBnReLU(net, 'conv2_2', 128) + net.add(layer.MaxPooling2D('pool2', 2, 2, border_mode='valid')) + ConvBnReLU(net, 'conv3_1', 256) + net.add(layer.Dropout('drop3_1', 0.4, engine='cudnn')) + ConvBnReLU(net, 'conv3_2', 256) + net.add(layer.Dropout('drop3_2', 0.4, engine='cudnn')) + ConvBnReLU(net, 'conv3_3', 256) + net.add(layer.MaxPooling2D('pool3', 2, 2, border_mode='valid')) + ConvBnReLU(net, 'conv4_1', 512) + net.add(layer.Dropout('drop4_1', 0.4, engine='cudnn')) + ConvBnReLU(net, 'conv4_2', 512) + net.add(layer.Dropout('drop4_2', 0.4, engine='cudnn')) + ConvBnReLU(net, 'conv4_3', 512) + net.add(layer.MaxPooling2D('pool4', 2, 2, border_mode='valid')) + ConvBnReLU(net, 'conv5_1', 512) + net.add(layer.Dropout('drop5_1', 0.4, engine='cudnn')) + ConvBnReLU(net, 'conv5_2', 512) + net.add(layer.Dropout('drop5_2', 0.4, engine='cudnn')) + ConvBnReLU(net, 'conv5_3', 512) + net.add(layer.MaxPooling2D('pool5', 2, 2, border_mode='valid')) + net.add(layer.Flatten('flat')) + net.add(layer.Dropout('drop_flat', 0.5, engine='cudnn')) + net.add(layer.Dense('ip1', 512)) + net.add(layer.BatchNormalization('batchnorm_ip1')) + net.add(layer.Activation('relu_ip1')) + net.add(layer.Dropout('drop_ip2', 0.5, engine='cudnn')) + net.add(layer.Dense('ip2', 10)) + return net diff --git a/src/core/tensor/tensor.cc b/src/core/tensor/tensor.cc index 4972a86a83..c16bd2908e 100644 --- a/src/core/tensor/tensor.cc +++ b/src/core/tensor/tensor.cc @@ -80,11 +80,11 @@ void Tensor::ResetLike(const Tensor &in) { if (block_ == nullptr || device_ != in.device_ || MemSize() != in.MemSize()) { if (block_ != nullptr && block_->DecRefCount() == 0) device_->FreeBlock(block_); - shape_ = in.shape_; device_ = in.device_; data_type_ = in.data_type_; block_ = device_->NewBlock(in.MemSize()); } + shape_ = in.shape_; } void Tensor::Reshape(const Shape &shape) { diff --git a/src/model/layer/batchnorm.cc b/src/model/layer/batchnorm.cc index b6edc9e173..6ea9f2af12 100644 --- a/src/model/layer/batchnorm.cc +++ b/src/model/layer/batchnorm.cc @@ -27,8 +27,18 @@ void BatchNorm::Setup(const Shape& in_sample, const LayerConf& conf) { out_sample_shape_ = in_sample; factor_ = conf.batchnorm_conf().factor(); channels_ = in_sample.at(0); - height_ = in_sample.at(1); - width_ = in_sample.at(2); + if (in_sample.size() == 3u) + height_ = in_sample.at(1); + else + height_ = 1; + if (in_sample.size() == 3u) + width_ = in_sample.at(2); + else + width_ = 1; + if (in_sample.size() == 1u) + is_2d_ = true; + else + is_2d_ = false; bnScale_.Reshape(Shape{channels_ * height_ * width_}); bnBias_.ResetLike(bnScale_); @@ -92,7 +102,8 @@ const Tensor BatchNorm::Forward(int flag, const Tensor& input) { AddRow(bnBias_, &output); } - output.Reshape(Shape{output.shape(0), channels_, height_, width_}); + if (!is_2d_) + output.Reshape(Shape{output.shape(0), channels_, height_, width_}); return output; } @@ -170,10 +181,16 @@ const std::pair> BatchNorm::Backward( SumRows(dy, &dbnBias_); param_grad.push_back(dbnScale_); param_grad.push_back(dbnBias_); + Tensor dummy; + dummy.ResetLike(runningMean_); + dummy.SetValue(.0f); + param_grad.push_back(dummy); + param_grad.push_back(dummy); } else { LOG(ERROR) << "Do not call backward for evaluation phase"; } - dx.Reshape(Shape{dx.shape(0), channels_, height_, width_}); + if (!is_2d_) + dx.Reshape(Shape{dx.shape(0), channels_, height_, width_}); return std::make_pair(dx, param_grad); } diff --git a/src/model/layer/batchnorm.h b/src/model/layer/batchnorm.h index 6ff818bc78..f3d83abd12 100644 --- a/src/model/layer/batchnorm.h +++ b/src/model/layer/batchnorm.h @@ -44,7 +44,7 @@ class BatchNorm : public Layer { /// \copydoc Layer::Backward(int, const Tensor&, const Tensor&); const std::pair> Backward( int flag, const Tensor& grad) override; - const std::vector param_values() override { + virtual const std::vector param_values() override { return std::vector { bnScale_, bnBias_, runningMean_, runningVariance_ }; } @@ -77,6 +77,7 @@ class BatchNorm : public Layer { protected: float factor_; size_t channels_, height_, width_; + bool is_2d_ = false; Tensor bnScale_, bnBias_; Tensor dbnScale_, dbnBias_; Tensor runningMean_, runningVariance_; diff --git a/src/model/layer/cudnn_batchnorm.cc b/src/model/layer/cudnn_batchnorm.cc index 9e1e8928c7..461f1b6da0 100644 --- a/src/model/layer/cudnn_batchnorm.cc +++ b/src/model/layer/cudnn_batchnorm.cc @@ -75,14 +75,20 @@ const Tensor CudnnBatchNorm::Forward(int flag, const Tensor& input) { auto shape = input.shape(); auto dtype = input.data_type(); Tensor output; + Tensor x; + if(is_2d_) + x = Reshape(input, Shape{shape.at(0), shape.at(1), 1, 1}); + else + x = input; + shape = x.shape(); if (!has_init_cudnn_) InitCudnn(shape, dtype); // TODO(wangji): check device id of input and params - output.ResetLike(input); + output.ResetLike(x); if ((flag & kTrain) == kTrain) { output.device()->Exec( [=](Context* ctx) { - Block *inBlock = input.block(), *outBlock = output.block(), + Block *inBlock = x.block(), *outBlock = output.block(), *saveMeanBlock = resultSaveMean_.block(), *saveVarBlock = resultSaveVariance_.block(), *runningMeanBlock = runningMean_.block(), @@ -110,7 +116,7 @@ const Tensor CudnnBatchNorm::Forward(int flag, const Tensor& input) { saveMeanBlock->mutable_data(), saveVarBlock->mutable_data())); }, - {input.block(), + {x.block(), bnScale_.block(), bnBias_.block()}, {output.block(), @@ -118,11 +124,11 @@ const Tensor CudnnBatchNorm::Forward(int flag, const Tensor& input) { runningVariance_.block(), resultSaveMean_.block(), resultSaveVariance_.block()}); - buf_.push(input); + buf_.push(x); } else { output.device()->Exec( [=](Context* ctx) { - Block *inBlock = input.block(), *outBlock = output.block(), + Block *inBlock = x.block(), *outBlock = output.block(), *runningMeanBlock = runningMean_.block(), *runningVarBlock = runningVariance_.block(), *bnScaleBlock = bnScale_.block(), @@ -145,13 +151,15 @@ const Tensor CudnnBatchNorm::Forward(int flag, const Tensor& input) { runningVarBlock->data(), epsilon)); }, - {input.block(), + {x.block(), bnScale_.block(), bnBias_.block(), runningMean_.block(), runningVariance_.block()}, {output.block()}); } + if (is_2d_) + output.Reshape(Shape{shape.at(0), shape.at(1)}); return output; } @@ -160,13 +168,13 @@ const std::pair> CudnnBatchNorm::Backward( vector param_grad; Tensor dx; if ((flag & kTrain) == kTrain) { - Tensor input = buf_.top(); + Tensor x = buf_.top(); buf_.pop(); dx.ResetLike(grad); dx.device()->Exec( [=](Context* ctx) { Block *dyblock = grad.block(), *dxblock = dx.block(), - *xblock = input.block(), + *xblock = x.block(), *bnScaleBlock = bnScale_.block(), *dbnScaleBlock = dbnScale_.block(), *dbnBiasBlock = dbnBias_.block(), @@ -208,6 +216,13 @@ const std::pair> CudnnBatchNorm::Backward( } param_grad.push_back(dbnScale_); param_grad.push_back(dbnBias_); + Tensor dummy; + dummy.ResetLike(dbnScale_); + dummy.SetValue(.0f); + param_grad.push_back(dummy); + param_grad.push_back(dummy); + if (is_2d_) + dx.Reshape(Shape{dx.shape().at(0), dx.shape().at(1)}); return std::make_pair(dx, param_grad); } } // namespace diff --git a/src/model/updater/local_updater.cc b/src/model/updater/local_updater.cc index eab4a7cb74..c3c67934d7 100644 --- a/src/model/updater/local_updater.cc +++ b/src/model/updater/local_updater.cc @@ -33,6 +33,7 @@ void LocalUpdater::Register(const string& name, const ParamSpec& specs) { } dev_index_[name] = 0; to_updater_finished_[name] = 0; + mtx_[name]; } void LocalUpdater::Apply(int step, const string& name, Tensor& grad, diff --git a/src/python/singa/layer.py b/src/python/singa/layer.py index 937a7e192d..a443e1ab97 100644 --- a/src/python/singa/layer.py +++ b/src/python/singa/layer.py @@ -327,10 +327,16 @@ def __init__(self, name, momentum=0.9, engine='cudnn', beta_specs['name'] = name + '_beta' if 'name' not in gamma_specs: gamma_specs['name'] = name + '_gamma' - self.conf.param.extend([_construct_param_specs_from_dict(beta_specs)]) + mean_specs = {'init': 'constant', 'value': 0, 'name': name+'_mean'} + var_specs = {'init': 'constant', 'value': 1, 'name': name+'_var'} self.conf.param.extend([_construct_param_specs_from_dict(gamma_specs)]) - self.param_specs.append(_construct_param_specs_from_dict(beta_specs)) + self.conf.param.extend([_construct_param_specs_from_dict(beta_specs)]) + self.conf.param.extend([_construct_param_specs_from_dict(mean_specs)]) + self.conf.param.extend([_construct_param_specs_from_dict(var_specs)]) self.param_specs.append(_construct_param_specs_from_dict(gamma_specs)) + self.param_specs.append(_construct_param_specs_from_dict(beta_specs)) + self.param_specs.append(_construct_param_specs_from_dict(mean_specs)) + self.param_specs.append(_construct_param_specs_from_dict(var_specs)) _check_engine(engine, ['cudnn']) self.layer = _create_layer(engine, 'BatchNorm') if input_sample_shape is not None: diff --git a/src/python/singa/net.py b/src/python/singa/net.py index 084db4bbca..c0ba61d2d9 100644 --- a/src/python/singa/net.py +++ b/src/python/singa/net.py @@ -64,6 +64,9 @@ def param_specs(self): specs.extend(lyr.param_specs) return specs + def param_names(self): + return [spec.name for spec in self.param_specs()] + def train(self, x, y): out = self.forward(kTrain, x) l = self.loss.forward(kTrain, out, y) @@ -89,9 +92,10 @@ def predict(self, x): return tensor.softmax(xx) def forward(self, flag, x): + #print x.l1() for lyr in self.layers: x = lyr.forward(flag, x) - # print lyr.name, x.l1() + # print lyr.name, x.l1() return x def backward(self, flag=kTrain): From a447ad02299fe163bd5b72a799cc42180a6fd2ba Mon Sep 17 00:00:00 2001 From: xiangrui Date: Tue, 2 Aug 2016 16:09:29 +0800 Subject: [PATCH 03/16] SINGA-232 Alexnet on Imagenet Implement Alexnet on Imagenet. 1. The model is following the imagenet paper. 2. The data is created offline, including multiple training files, one test file and one mean file. All of them are in binary format. This part is implemented via writer, encoder in SINGA. 3. Loading data in multiple threads. This part needs reader, decoder and transformer in SINGA. 4. This example need OpenCV support. 5. snapshot, jpgencoder, timer, binfile_reader are slightly modified. --- CMakeLists.txt | 2 +- examples/CMakeLists.txt | 1 + examples/imagenet/CMakeLists.txt | 16 ++ examples/imagenet/README.md | 43 ++++ examples/imagenet/alexnet.cc | 410 +++++++++++++++++++++++++++++++ examples/imagenet/create_data.sh | 3 + examples/imagenet/ilsvrc12.cc | 69 ++++++ examples/imagenet/ilsvrc12.h | 380 ++++++++++++++++++++++++++++ examples/imagenet/run.sh | 3 + include/singa/io/snapshot.h | 8 +- include/singa/utils/timer.h | 6 +- src/core/tensor/tensor.cc | 8 +- src/io/binfile_reader.cc | 6 +- src/io/jpg_encoder.cc | 2 +- src/io/snapshot.cc | 8 +- 15 files changed, 950 insertions(+), 15 deletions(-) create mode 100644 examples/imagenet/CMakeLists.txt create mode 100644 examples/imagenet/README.md create mode 100644 examples/imagenet/alexnet.cc create mode 100755 examples/imagenet/create_data.sh create mode 100644 examples/imagenet/ilsvrc12.cc create mode 100644 examples/imagenet/ilsvrc12.h create mode 100755 examples/imagenet/run.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 23f8ef6323..8c6afad6e8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.6) PROJECT(singa) -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -O2 ") LIST(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Thirdparty) #message(STATUS "module path: ${CMAKE_MODULE_PATH}") diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 3490c38e37..6014f27319 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1 +1,2 @@ ADD_SUBDIRECTORY(cifar10) +ADD_SUBDIRECTORY(imagenet) diff --git a/examples/imagenet/CMakeLists.txt b/examples/imagenet/CMakeLists.txt new file mode 100644 index 0000000000..71fbbb1ce3 --- /dev/null +++ b/examples/imagenet/CMakeLists.txt @@ -0,0 +1,16 @@ +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) +INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/include) + +IF(USE_CUDNN) + IF(USE_OPENCV) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp ") + ADD_EXECUTABLE(imagenet alexnet.cc) + ADD_DEPENDENCIES(imagenet singa_core singa_model singa_utils singa_io) + TARGET_LINK_LIBRARIES(imagenet singa_core singa_utils singa_model singa_io protobuf ${SINGA_LIBKER_LIBS}) + + ADD_EXECUTABLE(createdata ilsvrc12.cc) + ADD_DEPENDENCIES(createdata singa_core singa_io singa_model singa_utils) + TARGET_LINK_LIBRARIES(createdata singa_core singa_utils singa_io singa_model protobuf ${SINGA_LIBKER_LIBS}) + #SET_TARGET_PROPERTIES(createdata PROPERTIES LINK_FLAGS "${LINK_FLAGS}") + ENDIF(USE_OPENCV) +ENDIF(USE_CUDNN) diff --git a/examples/imagenet/README.md b/examples/imagenet/README.md new file mode 100644 index 0000000000..2e0389a2cf --- /dev/null +++ b/examples/imagenet/README.md @@ -0,0 +1,43 @@ +# Example of alexnet + +### Data download +* Please refer to step1-3 on [Instructions to create ImageNet 2012 data](https://github.com/amd/OpenCL-caffe/wiki/Instructions-to-create-ImageNet-2012-data) + to download and decompress the data. +* You can download the training and validation list by + [get_ilsvrc_aux.sh](https://github.com/BVLC/caffe/blob/master/data/ilsvrc12/get_ilsvrc_aux.sh) + or from [Imagenet](http://www.image-net.org/download-images). + +### Data preprocessing +* Assuming you have downloaded the data and the list. + Now we should transform the data into binary files. You can run: + + sh create_data.sh + + The script will generate a test file(`test.bin`), a mean file(`mean.bin`) and + several training files(`trainX.bin`) in the specified output folder. +* You can also change the parameters in `create_data.sh`. + + `-trainlist `: the file of training list; + + `-trainfolder `: the folder of training images; + + `-testlist `: the file of test list; + + `-testfolder `: the folder of test images; + + `-outdata `: the folder to save output files, including mean, training and test files. + The script will generate these files in the specified folder; + + `-filesize `: number of training images that stores in each binary file. + +### Training +* After preparing data, you can run the following command to train the Alexnet model. + + sh run.sh +* You may change the parameters in `run.sh`. + + `-epoch `: number of epoch to be trained, default is 90; + + `-lr `: base learning rate, the learning rate will decrease each 20 epochs, + more specifically, `lr = lr * exp(0.1 * (epoch / 20))`; + + `-batchsize `: batchsize, it should be changed regarding to your memory; + + `-filesize `: number of training images that stores in each binary file, it is the + same as the `filesize` in data preprocessing; + + `-ntrain `: number of training images; + + `-ntest `: number of test images; + + `-data `: the folder which stores the binary files, it is exactly the output + folder in data preprocessing step; + + `-pfreq `: the frequency(in batch) of printing current model status(loss and accuracy); + + `-nthreads `: the number of threads to load data which feed to the model. \ No newline at end of file diff --git a/examples/imagenet/alexnet.cc b/examples/imagenet/alexnet.cc new file mode 100644 index 0000000000..22cc88f34d --- /dev/null +++ b/examples/imagenet/alexnet.cc @@ -0,0 +1,410 @@ +/************************************************************ +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*************************************************************/ + +#include "singa/singa_config.h" +#ifdef USE_OPENCV +#include +#include "../../src/model/layer/cudnn_activation.h" +#include "../../src/model/layer/cudnn_convolution.h" +#include "../../src/model/layer/cudnn_dropout.h" +#include "../../src/model/layer/cudnn_lrn.h" +#include "../../src/model/layer/cudnn_pooling.h" +#include "../../src/model/layer/dense.h" +#include "../../src/model/layer/flatten.h" +#include "./ilsvrc12.h" +#include "singa/io/snapshot.h" +#include "singa/model/feed_forward_net.h" +#include "singa/model/initializer.h" +#include "singa/model/metric.h" +#include "singa/model/optimizer.h" +#include "singa/utils/channel.h" +#include "singa/utils/string.h" +#include "singa/utils/timer.h" +namespace singa { + +LayerConf GenConvConf(string name, int nb_filter, int kernel, int stride, + int pad, float std, float bias = .0f) { + LayerConf conf; + conf.set_name(name); + conf.set_type("CudnnConvolution"); + ConvolutionConf *conv = conf.mutable_convolution_conf(); + conv->set_num_output(nb_filter); + conv->add_kernel_size(kernel); + conv->add_stride(stride); + conv->add_pad(pad); + conv->set_bias_term(true); + + ParamSpec *wspec = conf.add_param(); + wspec->set_name(name + "_weight"); + auto wfill = wspec->mutable_filler(); + wfill->set_type("Gaussian"); + wfill->set_std(std); + + ParamSpec *bspec = conf.add_param(); + bspec->set_name(name + "_bias"); + bspec->set_lr_mult(2); + bspec->set_decay_mult(0); + auto bfill = bspec->mutable_filler(); + bfill->set_value(bias); + return conf; +} + +LayerConf GenPoolingConf(string name, bool max_pool, int kernel, int stride, + int pad) { + LayerConf conf; + conf.set_name(name); + conf.set_type("CudnnPooling"); + PoolingConf *pool = conf.mutable_pooling_conf(); + pool->set_kernel_size(kernel); + pool->set_stride(stride); + pool->set_pad(pad); + if (!max_pool) pool->set_pool(PoolingConf_PoolMethod_AVE); + return conf; +} + +LayerConf GenReLUConf(string name) { + LayerConf conf; + conf.set_name(name); + conf.set_type("RELU"); + return conf; +} + +LayerConf GenDenseConf(string name, int num_output, float std, float wd, + float bias = .0f) { + LayerConf conf; + conf.set_name(name); + conf.set_type("Dense"); + DenseConf *dense = conf.mutable_dense_conf(); + dense->set_num_output(num_output); + + ParamSpec *wspec = conf.add_param(); + wspec->set_name(name + "_weight"); + wspec->set_decay_mult(wd); + auto wfill = wspec->mutable_filler(); + wfill->set_type("Gaussian"); + wfill->set_std(std); + + ParamSpec *bspec = conf.add_param(); + bspec->set_name(name + "_bias"); + bspec->set_lr_mult(2); + bspec->set_decay_mult(0); + auto bfill = bspec->mutable_filler(); + bfill->set_value(bias); + + return conf; +} + +LayerConf GenLRNConf(string name) { + LayerConf conf; + conf.set_name(name); + conf.set_type("CudnnLRN"); + LRNConf *lrn = conf.mutable_lrn_conf(); + lrn->set_local_size(5); + lrn->set_alpha(1e-04); + lrn->set_beta(0.75); + return conf; +} + +LayerConf GenFlattenConf(string name) { + LayerConf conf; + conf.set_name(name); + conf.set_type("Flatten"); + return conf; +} + +LayerConf GenDropoutConf(string name, float dropout_ratio) { + LayerConf conf; + conf.set_name(name); + conf.set_type("CudnnDropout"); + DropoutConf *dropout = conf.mutable_dropout_conf(); + dropout->set_dropout_ratio(dropout_ratio); + return conf; +} + +FeedForwardNet CreateNet() { + FeedForwardNet net; + Shape s{3, 227, 227}; + + net.Add(new CudnnConvolution(), GenConvConf("conv1", 96, 11, 4, 0, 0.01), &s); + net.Add(new CudnnActivation(), GenReLUConf("relu1")); + net.Add(new CudnnPooling(), GenPoolingConf("pool1", true, 3, 2, 0)); + net.Add(new CudnnLRN(), GenLRNConf("lrn1")); + net.Add(new CudnnConvolution(), + GenConvConf("conv2", 256, 5, 1, 2, 0.01, 1.0)); + net.Add(new CudnnActivation(), GenReLUConf("relu2")); + net.Add(new CudnnPooling(), GenPoolingConf("pool2", true, 3, 2, 0)); + net.Add(new CudnnLRN(), GenLRNConf("lrn2")); + net.Add(new CudnnConvolution(), GenConvConf("conv3", 384, 3, 1, 1, 0.01)); + net.Add(new CudnnActivation(), GenReLUConf("relu3")); + net.Add(new CudnnConvolution(), + GenConvConf("conv4", 384, 3, 1, 1, 0.01, 1.0)); + net.Add(new CudnnActivation(), GenReLUConf("relu4")); + net.Add(new CudnnConvolution(), + GenConvConf("conv5", 256, 3, 1, 1, 0.01, 1.0)); + net.Add(new CudnnActivation(), GenReLUConf("relu5")); + net.Add(new CudnnPooling(), GenPoolingConf("pool5", true, 3, 2, 0)); + net.Add(new Flatten(), GenFlattenConf("flat")); + net.Add(new Dense(), GenDenseConf("ip6", 4096, 0.005, 1, 1.0)); + net.Add(new CudnnActivation(), GenReLUConf("relu6")); + net.Add(new CudnnDropout(), GenDropoutConf("drop6", 0.5)); + net.Add(new Dense(), GenDenseConf("ip7", 4096, 0.005, 1, 1.0)); + net.Add(new CudnnActivation(), GenReLUConf("relu7")); + net.Add(new CudnnDropout(), GenDropoutConf("drop7", 0.5)); + net.Add(new Dense(), GenDenseConf("ip8", 1000, 0.01, 1)); + + return net; +} + +void TrainOneEpoch(FeedForwardNet &net, ILSVRC &data, + std::shared_ptr device, int epoch, string bin_folder, + size_t num_train_files, size_t batchsize, float lr, + Channel *train_ch, size_t pfreq, int nthreads) { + float loss = 0.0f, metric = 0.0f; + float load_time = 0.0f, train_time = 0.0f; + size_t b = 0; + size_t n_read; + Timer timer, ttr; + Tensor prefetch_x, prefetch_y; + string binfile = bin_folder + "/train1.bin"; + timer.Tick(); + data.LoadData(kTrain, binfile, batchsize, &prefetch_x, &prefetch_y, &n_read, + nthreads); + load_time += timer.Elapsed(); + CHECK_EQ(n_read, batchsize); + Tensor train_x(prefetch_x.shape(), device); + Tensor train_y(prefetch_y.shape(), device, kInt); + std::thread th; + for (size_t fno = 1; fno <= num_train_files; fno++) { + binfile = bin_folder + "/train" + std::to_string(fno) + ".bin"; + while (true) { + if (th.joinable()) { + th.join(); + load_time += timer.Elapsed(); + // LOG(INFO) << "num of samples: " << n_read; + if (n_read < batchsize) { + if (n_read > 0) { + LOG(WARNING) << "Pls set batchsize to make num_total_samples " + << "% batchsize == 0. Otherwise, the last " << n_read + << " samples would not be used"; + } + break; + } + } + if (n_read == batchsize) { + train_x.CopyData(prefetch_x); + train_y.CopyData(prefetch_y); + } + timer.Tick(); + th = data.AsyncLoadData(kTrain, binfile, batchsize, &prefetch_x, + &prefetch_y, &n_read, nthreads); + if (n_read < batchsize) continue; + CHECK_EQ(train_x.shape(0), train_y.shape(0)); + ttr.Tick(); + auto ret = net.TrainOnBatch(epoch, train_x, train_y); + train_time += ttr.Elapsed(); + loss += ret.first; + metric += ret.second; + b++; + } + if (b % pfreq == 0) { + train_ch->Send( + "Epoch " + std::to_string(epoch) + ", training loss = " + + std::to_string(loss / b) + ", accuracy = " + + std::to_string(metric / b) + ", lr = " + std::to_string(lr) + + ", time of loading " + std::to_string(batchsize) + " images = " + + std::to_string(load_time / b) + + " ms, time of training (batchsize = " + std::to_string(batchsize) + + ") = " + std::to_string(train_time / b) + " ms."); + loss = 0.0f; + metric = 0.0f; + load_time = 0.0f; + train_time = 0.0f; + b = 0; + } + } +} + +void TestOneEpoch(FeedForwardNet &net, ILSVRC &data, + std::shared_ptr device, int epoch, string bin_folder, + size_t num_test_images, size_t batchsize, Channel *val_ch, + int nthreads) { + float loss = 0.0f, metric = 0.0f; + float load_time = 0.0f, eval_time = 0.0f; + size_t n_read; + string binfile = bin_folder + "/test.bin"; + Timer timer, tte; + Tensor prefetch_x, prefetch_y; + timer.Tick(); + data.LoadData(kEval, binfile, batchsize, &prefetch_x, &prefetch_y, &n_read, + nthreads); + load_time += timer.Elapsed(); + Tensor test_x(prefetch_x.shape(), device); + Tensor test_y(prefetch_y.shape(), device, kInt); + int remain = (int)num_test_images - n_read; + CHECK_EQ(n_read, batchsize); + std::thread th; + while (true) { + if (th.joinable()) { + th.join(); + load_time += timer.Elapsed(); + remain -= n_read; + if (remain < 0) break; + if (n_read < batchsize) break; + } + test_x.CopyData(prefetch_x); + test_y.CopyData(prefetch_y); + timer.Tick(); + th = data.AsyncLoadData(kEval, binfile, batchsize, &prefetch_x, &prefetch_y, + &n_read, nthreads); + + CHECK_EQ(test_x.shape(0), test_y.shape(0)); + tte.Tick(); + auto ret = net.EvaluateOnBatch(test_x, test_y); + eval_time += tte.Elapsed(); + ret.first.ToHost(); + ret.second.ToHost(); + loss += Sum(ret.first); + metric += Sum(ret.second); + } + loss /= num_test_images; + metric /= num_test_images; + val_ch->Send("Epoch " + std::to_string(epoch) + ", val loss = " + + std::to_string(loss) + ", accuracy = " + std::to_string(metric) + + ", time of loading " + std::to_string(num_test_images) + + " images = " + std::to_string(load_time) + + " ms, time of evaluating " + std::to_string(num_test_images) + + " images = " + std::to_string(eval_time) + " ms."); +} + +void Checkpoint(FeedForwardNet &net, string prefix) { + Snapshot snapshot(prefix, Snapshot::kWrite, 200); + auto names = net.GetParamNames(); + auto values = net.GetParamValues(); + for (size_t k = 0; k < names.size(); k++) { + values.at(k).ToHost(); + snapshot.Write(names.at(k), values.at(k)); + } + LOG(INFO) << "Write snapshot into " << prefix; +} + +void Train(int num_epoch, float lr, size_t batchsize, size_t train_file_size, + string bin_folder, size_t num_train_images, size_t num_test_images, + size_t pfreq, int nthreads) { + ILSVRC data; + data.ReadMean(bin_folder + "/mean.bin"); + auto net = CreateNet(); + auto cuda = std::make_shared(0); + net.ToDevice(cuda); + SGD sgd; + OptimizerConf opt_conf; + opt_conf.set_momentum(0.9); + auto reg = opt_conf.mutable_regularizer(); + reg->set_coefficient(0.0005); + sgd.Setup(opt_conf); + sgd.SetLearningRateGenerator( + [lr](int epoch) { return lr * std::pow(0.1, epoch / 20); }); + + SoftmaxCrossEntropy loss; + Accuracy acc; + net.Compile(true, &sgd, &loss, &acc); + + Channel *train_ch = GetChannel("train_perf"); + train_ch->EnableDestStderr(true); + Channel *val_ch = GetChannel("val_perf"); + val_ch->EnableDestStderr(true); + size_t num_train_files = num_train_images / train_file_size + + (num_train_images % train_file_size ? 1 : 0); + for (int epoch = 0; epoch < num_epoch; epoch++) { + float epoch_lr = sgd.GetLearningRate(epoch); + TrainOneEpoch(net, data, cuda, epoch, bin_folder, num_train_files, + batchsize, epoch_lr, train_ch, pfreq, nthreads); + if (epoch % 10 == 0 && epoch > 0) { + string prefix = "snapshot_epoch" + std::to_string(epoch); + Checkpoint(net, prefix); + } + TestOneEpoch(net, data, cuda, epoch, bin_folder, num_test_images, batchsize, + val_ch, nthreads); + } +} +} + +int main(int argc, char **argv) { + singa::InitChannel(nullptr); + + if (argc == 1) { + std::cout << "Usage:\n" + << "\t-epoch : number of epoch to be trained, default is 90;\n" + << "\t-lr : base learning rate;\n" + << "\t-batchsize : batchsize, it should be changed regarding " + "to your memory;\n" + << "\t-filesize : number of training images that stores in " + "each binary file;\n" + << "\t-ntrain : number of training images;\n" + << "\t-ntest : number of test images;\n" + << "\t-data : the folder which stores the binary files;\n" + << "\t-pfreq : the frequency(in batch) of printing current " + "model status(loss and accuracy);\n" + << "\t-nthreads `: the number of threads to load data which " + "feed to the model.\n"; + return 0; + } + int pos = singa::ArgPos(argc, argv, "-epoch"); + int nEpoch = 90; + if (pos != -1) nEpoch = atoi(argv[pos + 1]); + + pos = singa::ArgPos(argc, argv, "-lr"); + float lr = 0.01; + if (pos != -1) lr = atof(argv[pos + 1]); + + pos = singa::ArgPos(argc, argv, "-batchsize"); + int batchsize = 256; + if (pos != -1) batchsize = atof(argv[pos + 1]); + + pos = singa::ArgPos(argc, argv, "-filesize"); + size_t train_file_size = 1280; + if (pos != -1) train_file_size = atoi(argv[pos + 1]); + + pos = singa::ArgPos(argc, argv, "-ntrain"); + size_t num_train_images = 1281167; + if (pos != -1) num_train_images = atoi(argv[pos + 1]); + + pos = singa::ArgPos(argc, argv, "-ntest"); + size_t num_test_images = 50000; + if (pos != -1) num_test_images = atoi(argv[pos + 1]); + + pos = singa::ArgPos(argc, argv, "-data"); + string bin_folder = "imagenet_data"; + if (pos != -1) bin_folder = argv[pos + 1]; + + pos = singa::ArgPos(argc, argv, "-pfreq"); + size_t pfreq = 100; + if (pos != -1) pfreq = atoi(argv[pos + 1]); + + pos = singa::ArgPos(argc, argv, "-nthreads"); + int nthreads = 12; + if (pos != -1) nthreads = atoi(argv[pos + 1]); + + LOG(INFO) << "Start training"; + singa::Train(nEpoch, lr, batchsize, train_file_size, bin_folder, + num_train_images, num_test_images, pfreq, nthreads); + LOG(INFO) << "End training"; +} +#endif diff --git a/examples/imagenet/create_data.sh b/examples/imagenet/create_data.sh new file mode 100755 index 0000000000..dd3d9b8225 --- /dev/null +++ b/examples/imagenet/create_data.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env sh +../../build/bin/createdata -trainlist "imagenet/label/train.txt" -trainfolder "imagenet/ILSVRC2012_img_train" \ + -testlist "imagenet/label/val.txt" -testfolder "imagenet/ILSVRC2012_img_val" -outdata "imagenet_data" -filesize 1280 diff --git a/examples/imagenet/ilsvrc12.cc b/examples/imagenet/ilsvrc12.cc new file mode 100644 index 0000000000..2bc07f2aeb --- /dev/null +++ b/examples/imagenet/ilsvrc12.cc @@ -0,0 +1,69 @@ +/************************************************************ +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*************************************************************/ +#include "singa/singa_config.h" +#ifdef USE_OPENCV +#include "ilsvrc12.h" +#include "singa/utils/channel.h" +#include "singa/utils/string.h" +int main(int argc, char **argv) { + if (argc == 1) { + std::cout << "Usage:\n" + << "\t-trainlist : the file of training list;\n" + << "\t-trainfolder : the folder of training images;\n" + << "\t-testlist : the file of test list;\n" + << "\t-testfolder : the folder of test images;\n" + << "\t-outdata : the folder to save output files;\n" + << "\t-filesize : number of training images that stores in " + "each binary file.\n"; + return 0; + } + int pos = singa::ArgPos(argc, argv, "-trainlist"); + string train_image_list = "imagenet/label/train.txt"; + if (pos != -1) train_image_list = argv[pos + 1]; + + pos = singa::ArgPos(argc, argv, "-trainfolder"); + string train_image_folder = "imagenet/ILSVRC2012_img_train"; + if (pos != -1) train_image_folder = argv[pos + 1]; + + pos = singa::ArgPos(argc, argv, "-testlist"); + string test_image_list = "imagenet/label/val.txt"; + if (pos != -1) test_image_list = argv[pos + 1]; + + pos = singa::ArgPos(argc, argv, "-testfolder"); + string test_image_folder = "imagenet/ILSVRC2012_img_val"; + if (pos != -1) test_image_folder = argv[pos + 1]; + + pos = singa::ArgPos(argc, argv, "-outdata"); + string bin_folder = "imagenet_data"; + if (pos != -1) bin_folder = argv[pos + 1]; + + pos = singa::ArgPos(argc, argv, "-filesize"); + size_t train_file_size = 1280; + if (pos != -1) train_file_size = atoi(argv[pos + 1]); + singa::ILSVRC data; + LOG(INFO) << "Creating training and test data..."; + data.CreateTrainData(train_image_list, train_image_folder, bin_folder, + train_file_size); + data.CreateTestData(test_image_list, test_image_folder, bin_folder); + LOG(INFO) << "Data created!"; + return 0; +} +#endif // USE_OPENCV diff --git a/examples/imagenet/ilsvrc12.h b/examples/imagenet/ilsvrc12.h new file mode 100644 index 0000000000..a6d4238f2b --- /dev/null +++ b/examples/imagenet/ilsvrc12.h @@ -0,0 +1,380 @@ +/************************************************************ +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*************************************************************/ +#include "singa/singa_config.h" +#ifdef USE_OPENCV +#ifndef SINGA_EXAMPLES_IMAGENET_ILSVRC12_H_ +#define SINGA_EXAMPLES_IMAGENET_ILSVRC12_H_ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "singa/core/tensor.h" +#include "singa/io/decoder.h" +#include "singa/io/encoder.h" +#include "singa/io/reader.h" +#include "singa/io/transformer.h" +#include "singa/io/writer.h" +#include "singa/proto/io.pb.h" +#include "singa/utils/timer.h" + +using std::string; +using namespace singa::io; +namespace singa { +/// For reading ILSVRC2012 image data as tensors. +class ILSVRC { + public: + /// Setup encoder, decoder + ILSVRC(); + ~ILSVRC() { + if (encoder != nullptr) delete encoder; + if (decoder != nullptr) delete decoder; + if (transformer != nullptr) delete transformer; + if (reader != nullptr) { + reader->Close(); + delete reader; + } + if (writer != nullptr) { + writer->Close(); + delete writer; + } + } + /// Create binary files for training data + /// train_image_list: list file of training images + /// train_image_folder: folder where stores original training images + /// train_bin_folder: folder to store binary files + /// train_file_size: number of images that are contain in one binary file + void CreateTrainData(string train_image_list, string train_image_folder, + string train_bin_folder, size_t train_file_size); + /// Create binary files for test data + /// train_image_list: list file of test images + /// train_image_folder: folder where saves original test images + /// train_bin_folder: folder to save binary files + void CreateTestData(string test_image_list, string test_image_folder, + string test_bin_folder); + /// Load data from a binary file, return pair + /// suppose the data will be loaded file by file. + /// flag: kTrain or kTest + /// file: binary file which stores the images + /// read_size: number of images to be loaded + /// offset: offset in the file + /// n_read: number of images which are read + size_t LoadData(int flag, string file, size_t read_size, Tensor *x, Tensor *y, + size_t *n_read, int nthreads); + + std::thread AsyncLoadData(int flag, string file, size_t read_size, Tensor *x, + Tensor *y, size_t *n_read, int nthreads); + + void DecodeTransform(int flag, int thid, int nthreads, + vector images, Tensor *x, Tensor *y); + std::thread AsyncDecodeTransform(int flag, int thid, int nthreads, + vector images, Tensor *x, + Tensor *y); + + /// Read mean from path + void ReadMean(string path); + + protected: + /// Read one image at path, resize the image + Tensor ReadImage(string path); + /// Write buff to the file in kCreate/kAppend mode + void Write(string outfile, singa::io::Mode mode); + void WriteMean(Tensor &mean, string path); + + private: + /// size for resizing + const size_t kImageSize = 256; + const size_t kImageNBytes = 3 * kImageSize * kImageSize; + /// size for cropping + const size_t kCropSize = 227; + Tensor mean; + string last_read_file = ""; + + JPGEncoder *encoder = nullptr; + JPGDecoder *decoder = nullptr; + ImageTransformer *transformer = nullptr; + BinFileReader *reader = nullptr; + BinFileWriter *writer = nullptr; +}; + +ILSVRC::ILSVRC() { + EncoderConf en_conf; + en_conf.set_image_dim_order("CHW"); + encoder = new JPGEncoder(); + encoder->Setup(en_conf); + + DecoderConf de_conf; + de_conf.set_image_dim_order("CHW"); + decoder = new JPGDecoder(); + decoder->Setup(de_conf); + + TransformerConf trans_conf; + trans_conf.add_crop_shape(kCropSize); + trans_conf.add_crop_shape(kCropSize); + trans_conf.set_image_dim_order("CHW"); + trans_conf.set_horizontal_mirror(true); + transformer = new ImageTransformer(); + transformer->Setup(trans_conf); +} + +Tensor ILSVRC::ReadImage(string path) { + cv::Mat mat = cv::imread(path, CV_LOAD_IMAGE_COLOR); + CHECK(mat.data != NULL) << "OpenCV load image fail: " << path; + cv::Size size(kImageSize, kImageSize); + cv::Mat resized; + cv::resize(mat, resized, size); + CHECK_EQ((size_t)resized.size().height, kImageSize); + CHECK_EQ((size_t)resized.size().width, kImageSize); + // dimension_order: CHW + Shape shape{(size_t)resized.channels(), (size_t)resized.rows, + (size_t)resized.cols}; + Tensor image(shape, singa::kUChar); + unsigned char *data = new unsigned char[kImageNBytes]; + for (int i = 0; i < resized.rows; i++) + for (int j = 0; j < resized.cols; j++) + for (int k = 0; k < resized.channels(); k++) + data[k * kImageSize * kImageSize + i * kImageSize + j] = + resized.at(i, j)[k]; + image.CopyDataFromHostPtr(data, kImageNBytes); + delete[] data; + + return image; +} + +void ILSVRC::WriteMean(Tensor &mean, string path) { + Tensor mean_lb(Shape{1}, kInt); + std::vector input; + input.push_back(mean); + input.push_back(mean_lb); + BinFileWriter bfwriter; + bfwriter.Open(path, kCreate); + bfwriter.Write(path, encoder->Encode(input)); + bfwriter.Flush(); + bfwriter.Close(); +} + +void ILSVRC::CreateTrainData(string image_list, string input_folder, + string output_folder, size_t file_size = 12800) { + std::vector> file_list; + size_t *sum = new size_t[kImageNBytes]; + for (size_t i = 0; i < kImageNBytes; i++) sum[i] = 0u; + string image_file_name; + int label; + string outfile; + std::ifstream image_list_file(image_list.c_str(), std::ios::in); + while (image_list_file >> image_file_name >> label) + file_list.push_back(std::make_pair(image_file_name, label)); + LOG(INFO) << "Data Shuffling"; + std::shuffle(file_list.begin(), file_list.end(), + std::default_random_engine()); + LOG(INFO) << "Total number of training images is " << file_list.size(); + size_t num_train_images = file_list.size(); + num_train_images = 12900; + if (file_size == 0) file_size = num_train_images; + // todo: accelerate with omp + for (size_t imageid = 0; imageid < num_train_images; imageid++) { + string path = input_folder + "/" + file_list[imageid].first; + Tensor image = ReadImage(path); + auto image_data = image.data(); + for (size_t i = 0; i < kImageNBytes; i++) + sum[i] += static_cast(image_data[i]); + label = file_list[imageid].second; + Tensor lb(Shape{1}, kInt); + lb.CopyDataFromHostPtr(&label, 1); + std::vector input; + input.push_back(image); + input.push_back(lb); + // LOG(INFO) << path << "\t" << label; + string encoded_str = encoder->Encode(input); + if (writer == nullptr) { + writer = new BinFileWriter(); + outfile = output_folder + "/train" + + std::to_string(imageid / file_size + 1) + ".bin"; + writer->Open(outfile, kCreate); + } + writer->Write(path, encoded_str); + if ((imageid + 1) % file_size == 0) { + writer->Flush(); + writer->Close(); + LOG(INFO) << "Write " << file_size << " images into " << outfile; + delete writer; + writer = nullptr; + } + } + if (writer != nullptr) { + writer->Flush(); + writer->Close(); + LOG(INFO) << "Write " << num_train_images % file_size << " images into " + << outfile; + delete writer; + writer = nullptr; + } + size_t num_file = + num_train_images / file_size + ((num_train_images % file_size) ? 1 : 0); + LOG(INFO) << "Write " << num_train_images << " images into " << num_file + << " binary files"; + Tensor mean = Tensor(Shape{3, kImageSize, kImageSize}, kUChar); + unsigned char *mean_data = new unsigned char[kImageNBytes]; + for (size_t i = 0; i < kImageNBytes; i++) + mean_data[i] = static_cast(sum[i] / num_train_images); + mean.CopyDataFromHostPtr(mean_data, kImageNBytes); + string mean_path = output_folder + "/mean.bin"; + WriteMean(mean, mean_path); + delete[] mean_data; + delete[] sum; +} + +void ILSVRC::CreateTestData(string image_list, string input_folder, + string output_folder) { + std::vector> file_list; + string image_file_name; + string outfile = output_folder + "/test.bin"; + int label; + std::ifstream image_list_file(image_list.c_str(), std::ios::in); + while (image_list_file >> image_file_name >> label) + file_list.push_back(std::make_pair(image_file_name, label)); + LOG(INFO) << "Total number of test images is " << file_list.size(); + size_t num_test_images = file_list.size(); + num_test_images = 500; + for (size_t imageid = 0; imageid < num_test_images; imageid++) { + string path = input_folder + "/" + file_list[imageid].first; + Tensor image = ReadImage(path); + label = file_list[imageid].second; + Tensor lb(Shape{1}, singa::kInt); + lb.CopyDataFromHostPtr(&label, 1); + std::vector input; + input.push_back(image); + input.push_back(lb); + string encoded_str = encoder->Encode(input); + if (writer == nullptr) { + writer = new BinFileWriter(); + writer->Open(outfile, kCreate); + } + writer->Write(path, encoded_str); + } + if (writer != nullptr) { + writer->Flush(); + writer->Close(); + delete writer; + writer = nullptr; + } + LOG(INFO) << "Write " << num_test_images << " images into " << outfile; +} + +void ILSVRC::ReadMean(string path) { + BinFileReader bfreader; + string key, value; + bfreader.Open(path); + bfreader.Read(&key, &value); + auto ret = decoder->Decode(value); + bfreader.Close(); + mean = ret[0]; +} +/// A wrapper method to spawn a thread to execute LoadData() method. +std::thread ILSVRC::AsyncLoadData(int flag, string file, size_t read_size, + Tensor *x, Tensor *y, size_t *n_read, + int nthreads) { + return std::thread( + [=]() { LoadData(flag, file, read_size, x, y, n_read, nthreads); }); +} + +size_t ILSVRC::LoadData(int flag, string file, size_t read_size, Tensor *x, + Tensor *y, size_t *n_read, int nthreads) { + x->Reshape(Shape{read_size, 3, kCropSize, kCropSize}); + y->AsType(kInt); + y->Reshape(Shape{read_size}); + if (file != last_read_file) { + if (reader != nullptr) { + reader->Close(); + delete reader; + reader = nullptr; + } + reader = new BinFileReader(); + reader->Open(file, 100 << 20); + last_read_file = file; + } else if (reader == nullptr) { + reader = new BinFileReader(); + reader->Open(file, 100 << 20); + } + vector images; + for (size_t i = 0; i < read_size; i++) { + string image_path; + string *image = new string(); + bool ret = reader->Read(&image_path, image); + if (ret == false) { + reader->Close(); + delete reader; + reader = nullptr; + break; + } + images.push_back(image); + } + int nimg = images.size(); + *n_read = nimg; + + vector threads; + for (int i = 1; i < nthreads; i++) { + threads.push_back(AsyncDecodeTransform(flag, i, nthreads, images, x, y)); + } + DecodeTransform(flag, 0, nthreads, images, x, y); + for (size_t i = 0; i < threads.size(); i++) threads[i].join(); + for (int k = 0; k < nimg; k++) delete images.at(k); + return nimg; +} + +/// A wrapper method to spawn a thread to execute Decodetransform() method. +std::thread ILSVRC::AsyncDecodeTransform(int flag, int thid, int nthreads, + vector images, Tensor *x, + Tensor *y) { + return std::thread( + [=]() { DecodeTransform(flag, thid, nthreads, images, x, y); }); +} + +void ILSVRC::DecodeTransform(int flag, int thid, int nthreads, + vector images, Tensor *x, Tensor *y) { + int nimg = images.size(); + int start = nimg / nthreads * thid; + int end = start + nimg / nthreads; + for (int k = start; k < end; k++) { + std::vector pair = decoder->Decode(*images.at(k)); + auto tmp_image = pair[0] - mean; + Tensor aug_image = transformer->Apply(flag, tmp_image); + CopyDataToFrom(x, aug_image, aug_image.Size(), k * aug_image.Size()); + CopyDataToFrom(y, pair[1], 1, k); + } + if (thid == 0) { + for (int k = nimg / nthreads * nthreads; k < nimg; k++) { + std::vector pair = decoder->Decode(*images.at(k)); + auto tmp_image = pair[0] - mean; + Tensor aug_image = transformer->Apply(flag, tmp_image); + CopyDataToFrom(x, aug_image, aug_image.Size(), k * aug_image.Size()); + CopyDataToFrom(y, pair[1], 1, k); + } + } +} +} // namespace singa + +#endif // SINGA_EXAMPLES_IMAGENET_ILSVRC12_H_ +#endif // USE_OPENCV diff --git a/examples/imagenet/run.sh b/examples/imagenet/run.sh new file mode 100755 index 0000000000..5c27b5c6e5 --- /dev/null +++ b/examples/imagenet/run.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env sh +../../build/bin/imagenet -epoch 90 -lr 0.01 -batchsize 256 -filesize 1280 -ntrain 1281167 -ntest 50000 \ + -data "imagenet_data" -pfreq 100 -nthreads 12 diff --git a/include/singa/io/snapshot.h b/include/singa/io/snapshot.h index 75455722ae..0d5aa66faf 100644 --- a/include/singa/io/snapshot.h +++ b/include/singa/io/snapshot.h @@ -49,7 +49,8 @@ class Snapshot { /// i.e. /// name and shape, one line per parameter. /// kRead for reading snapshot, whereas kWrite for dumping out snapshot. - Snapshot(const std::string& prefix, Mode mode); + /// max_param_size: in MB + Snapshot(const std::string& prefix, Mode mode, int max_param_size = 10); ~Snapshot() {} /// Read parameters saved as tensors from checkpoint file. std::vector> Read(); @@ -67,8 +68,9 @@ class Snapshot { private: std::string prefix_; Mode mode_; - std::unique_ptr bin_writer_ptr_, text_writer_ptr_; - std::unique_ptr bin_reader_ptr_; + std::unique_ptr bin_writer_ptr_; + std::unique_ptr text_writer_ptr_; + std::unique_ptr bin_reader_ptr_; /// Check whether parameter name is unique. std::unordered_set param_names_; /// Preload key-parameter tensor pairs for seeking a specified key. diff --git a/include/singa/utils/timer.h b/include/singa/utils/timer.h index bdd6c5c5b9..1372d3c8fd 100644 --- a/include/singa/utils/timer.h +++ b/include/singa/utils/timer.h @@ -11,6 +11,7 @@ class Timer { typedef std::chrono::duration Seconds; typedef std::chrono::duration Milliseconds; typedef std::chrono::duration> Hours; + typedef std::chrono::duration Microseconds; /// Init the internal time point to the current time Timer() { Tick(); } @@ -23,8 +24,9 @@ class Timer { int Elapsed() const { static_assert(std::is_same::value || std::is_same::value || - std::is_same::value, - "Template arg must be Seconds | Milliseconds | Hours"); + std::is_same::value || + std::is_same::value, + "Template arg must be Seconds | Milliseconds | Hours | Microseconds"); auto now = std::chrono::high_resolution_clock::now(); return std::chrono::duration_cast(now - last_).count(); } diff --git a/src/core/tensor/tensor.cc b/src/core/tensor/tensor.cc index c16bd2908e..a580bd2834 100644 --- a/src/core/tensor/tensor.cc +++ b/src/core/tensor/tensor.cc @@ -34,12 +34,12 @@ Tensor::Tensor() { device_ = defaultDevice; } Tensor::Tensor(const Shape &shape, DataType dtype) : data_type_(dtype), device_(defaultDevice), shape_(shape) { - device_ = defaultDevice; + //device_ = defaultDevice; block_ = device_->NewBlock(Product(shape_) * SizeOf(data_type_)); } Tensor::Tensor(Shape &&shape, DataType dtype) : data_type_(dtype), device_(defaultDevice), shape_(shape) { - device_ = defaultDevice; + //device_ = defaultDevice; block_ = device_->NewBlock(Product(shape_) * SizeOf(data_type_)); } Tensor::Tensor(const Shape &shape, std::shared_ptr device, @@ -127,7 +127,9 @@ void Tensor::ToDevice(std::shared_ptr dst) { } } -void Tensor::ToHost() { ToDevice(device_->host()); } +void Tensor::ToHost() { + if (device_ != defaultDevice) ToDevice(device_->host()); +} template void Tensor::CopyDataFromHostPtr(const DType *src, const size_t num, diff --git a/src/io/binfile_reader.cc b/src/io/binfile_reader.cc index 77e34d824a..9b52a5d495 100644 --- a/src/io/binfile_reader.cc +++ b/src/io/binfile_reader.cc @@ -98,7 +98,7 @@ bool BinFileReader::OpenFile() { buf_ = new char[capacity_]; fdat_.open(path_, std::ios::in | std::ios::binary); CHECK(fdat_.is_open()) << "Cannot open file " << path_; - return fdat_.is_open(); + return fdat_.is_open(); } bool BinFileReader::ReadField(std::string* content) { @@ -108,7 +108,9 @@ bool BinFileReader::ReadField(std::string* content) { int len = *reinterpret_cast(buf_ + offset_); offset_ += ssize; if (!PrepareNextField(len)) return false; - for (int i = 0; i < len; ++i) content->push_back(buf_[offset_ + i]); + content->reserve(len); + content->insert(0, buf_ + offset_, len); + //for (int i = 0; i < len; ++i) content->push_back(buf_[offset_ + i]); offset_ += len; return true; } diff --git a/src/io/jpg_encoder.cc b/src/io/jpg_encoder.cc index 9db799df7f..8335a91211 100644 --- a/src/io/jpg_encoder.cc +++ b/src/io/jpg_encoder.cc @@ -72,7 +72,7 @@ std::string JPGEncoder::Encode(vector& data) { // suppose each image is attached with at most one label if (data.size() == 2) { const int* label = data[1].data(); - CHECK_EQ(label[0], 2); + //CHECK_EQ(label[0], 2); record.add_label(label[0]); } diff --git a/src/io/snapshot.cc b/src/io/snapshot.cc index 3b9b8cedd3..58c7044222 100644 --- a/src/io/snapshot.cc +++ b/src/io/snapshot.cc @@ -29,17 +29,17 @@ #include namespace singa { -Snapshot::Snapshot(const std::string& prefix, Mode mode) +Snapshot::Snapshot(const std::string& prefix, Mode mode, int max_param_size /*in MB*/) : prefix_(prefix), mode_(mode), bin_writer_ptr_(mode_ == kWrite ? (new io::BinFileWriter) : nullptr), text_writer_ptr_(mode_ == kWrite ? (new io::TextFileWriter) : nullptr), bin_reader_ptr_(mode_ == kRead ? (new io::BinFileReader) : nullptr) { if (mode_ == kWrite) { - bin_writer_ptr_->Open(prefix + ".model", io::kCreate); + bin_writer_ptr_->Open(prefix + ".model", io::kCreate, max_param_size << 20); text_writer_ptr_->Open(prefix + ".desc", io::kCreate); } else if (mode == kRead) { - bin_reader_ptr_->Open(prefix + ".model"); + bin_reader_ptr_->Open(prefix + ".model", max_param_size << 20); std::string key, serialized_str; singa::TensorProto tp; while (bin_reader_ptr_->Read(&key, &serialized_str)) { @@ -63,6 +63,7 @@ void Snapshot::Write(const std::string& key, const Tensor& param) { std::string serialized_str; CHECK(tp.SerializeToString(&serialized_str)); bin_writer_ptr_->Write(key, serialized_str); +// bin_writer_ptr_->Flush(); std::string desc_str = "parameter name: " + key; Shape shape = param.shape(); @@ -71,6 +72,7 @@ void Snapshot::Write(const std::string& key, const Tensor& param) { desc_str += "\tshape:"; for (size_t s : shape) desc_str += " " + std::to_string(s); text_writer_ptr_->Write(key, desc_str); + // text_writer_ptr_->Flush(); } std::vector> Snapshot::Read() { From e0e28d9ab3d157184c8833607d109240079b39b9 Mon Sep 17 00:00:00 2001 From: Xiangrui Date: Tue, 9 Aug 2016 21:04:24 +0800 Subject: [PATCH 04/16] SINGA-232 Alexnet on Imagenet Replace CudnnDropout with Dropout Add argument "-h" --- examples/imagenet/alexnet.cc | 14 +++++++------- examples/imagenet/ilsvrc12.cc | 5 +++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/examples/imagenet/alexnet.cc b/examples/imagenet/alexnet.cc index 22cc88f34d..270312c5d9 100644 --- a/examples/imagenet/alexnet.cc +++ b/examples/imagenet/alexnet.cc @@ -24,7 +24,7 @@ #include #include "../../src/model/layer/cudnn_activation.h" #include "../../src/model/layer/cudnn_convolution.h" -#include "../../src/model/layer/cudnn_dropout.h" +#include "../../src/model/layer/dropout.h" #include "../../src/model/layer/cudnn_lrn.h" #include "../../src/model/layer/cudnn_pooling.h" #include "../../src/model/layer/dense.h" @@ -133,7 +133,7 @@ LayerConf GenFlattenConf(string name) { LayerConf GenDropoutConf(string name, float dropout_ratio) { LayerConf conf; conf.set_name(name); - conf.set_type("CudnnDropout"); + conf.set_type("Dropout"); DropoutConf *dropout = conf.mutable_dropout_conf(); dropout->set_dropout_ratio(dropout_ratio); return conf; @@ -164,10 +164,10 @@ FeedForwardNet CreateNet() { net.Add(new Flatten(), GenFlattenConf("flat")); net.Add(new Dense(), GenDenseConf("ip6", 4096, 0.005, 1, 1.0)); net.Add(new CudnnActivation(), GenReLUConf("relu6")); - net.Add(new CudnnDropout(), GenDropoutConf("drop6", 0.5)); + net.Add(new Dropout(), GenDropoutConf("drop6", 0.5)); net.Add(new Dense(), GenDenseConf("ip7", 4096, 0.005, 1, 1.0)); net.Add(new CudnnActivation(), GenReLUConf("relu7")); - net.Add(new CudnnDropout(), GenDropoutConf("drop7", 0.5)); + net.Add(new Dropout(), GenDropoutConf("drop7", 0.5)); net.Add(new Dense(), GenDenseConf("ip8", 1000, 0.01, 1)); return net; @@ -348,8 +348,8 @@ void Train(int num_epoch, float lr, size_t batchsize, size_t train_file_size, int main(int argc, char **argv) { singa::InitChannel(nullptr); - - if (argc == 1) { + int pos = singa::ArgPos(argc, argv, "-h"); + if (pos != -1) { std::cout << "Usage:\n" << "\t-epoch : number of epoch to be trained, default is 90;\n" << "\t-lr : base learning rate;\n" @@ -366,7 +366,7 @@ int main(int argc, char **argv) { "feed to the model.\n"; return 0; } - int pos = singa::ArgPos(argc, argv, "-epoch"); + pos = singa::ArgPos(argc, argv, "-epoch"); int nEpoch = 90; if (pos != -1) nEpoch = atoi(argv[pos + 1]); diff --git a/examples/imagenet/ilsvrc12.cc b/examples/imagenet/ilsvrc12.cc index 2bc07f2aeb..c9e6d2fb88 100644 --- a/examples/imagenet/ilsvrc12.cc +++ b/examples/imagenet/ilsvrc12.cc @@ -24,7 +24,8 @@ #include "singa/utils/channel.h" #include "singa/utils/string.h" int main(int argc, char **argv) { - if (argc == 1) { + int pos = singa::ArgPos(argc, argv, "-h"); + if (pos != -1) { std::cout << "Usage:\n" << "\t-trainlist : the file of training list;\n" << "\t-trainfolder : the folder of training images;\n" @@ -35,7 +36,7 @@ int main(int argc, char **argv) { "each binary file.\n"; return 0; } - int pos = singa::ArgPos(argc, argv, "-trainlist"); + pos = singa::ArgPos(argc, argv, "-trainlist"); string train_image_list = "imagenet/label/train.txt"; if (pos != -1) train_image_list = argv[pos + 1]; From 0f30b11dd3068c95a31b344854aaff3a10107b7f Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Tue, 9 Aug 2016 20:06:29 +0800 Subject: [PATCH 05/16] SINGA-231 Batchnormlized VGG model for cifar-10 Merge the training of vgg and alexnet into train.py The validation accuracy of vgg could reach 0.89 --- examples/cifar10/alexnet.py | 16 ++- examples/cifar10/predict.py | 14 ++- examples/cifar10/run-parallel.sh | 1 + examples/cifar10/train.py | 63 ++++++---- examples/cifar10/train_vgg_cifar10.py | 162 -------------------------- examples/cifar10/vgg-parallel.cc | 24 ++-- examples/cifar10/vgg.py | 66 +++++++++-- 7 files changed, 138 insertions(+), 208 deletions(-) delete mode 100644 examples/cifar10/train_vgg_cifar10.py diff --git a/examples/cifar10/alexnet.py b/examples/cifar10/alexnet.py index 4b3daec8f4..96c339a261 100644 --- a/examples/cifar10/alexnet.py +++ b/examples/cifar10/alexnet.py @@ -14,15 +14,21 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================= +''' This model is created following the structure from +https://code.google.com/p/cuda-convnet/source/browse/trunk/example-layers/layers-18pct.cfg +Following the same setting for hyper-parameters and data pre-processing, the final +validation accuracy would be about 82%. +''' + import sys import os sys.path.append(os.path.join(os.path.dirname(__file__), '../../build/python')) from singa import layer +from singa import initializer from singa import metric from singa import loss from singa import net as ffnet -from singa.proto import core_pb2 def create_net(): @@ -44,4 +50,12 @@ def create_net(): net.add(layer.MaxPooling2D('pool3', 3, 2, pad=1)) net.add(layer.Flatten('flat')) net.add(layer.Dense('dense', 10, W_specs=W2_specs.copy(), b_specs=b_specs.copy())) + for (p, specs) in zip(net.param_values(), net.param_specs()): + filler = specs.filler + if filler.type == 'gaussian': + initializer.gaussian(p, filler.mean, filler.std) + else: + p.set_value(0) + print specs.name, filler.type, p.l1() + return net diff --git a/examples/cifar10/predict.py b/examples/cifar10/predict.py index d083d0b319..07b114562a 100644 --- a/examples/cifar10/predict.py +++ b/examples/cifar10/predict.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================= - +import cPickle as pickle import numpy as np import sys import os @@ -27,6 +27,15 @@ def predict(net, images, cuda, topk=5): + '''Predict the label of each image. + + Args: + net, a pretrained neural net + images, a batch of images [batch_size, 3, 32, 32], which have been + pre-processed + cuda, the cuda device + topk, return the topk labels for each image. + ''' x = tensor.from_numpy(images.astype(np.float32)) x.to_device(cuda) y = net.predict(x) @@ -40,7 +49,7 @@ def predict(net, images, cuda, topk=5): def load_dataset(filepath): print 'Loading data file %s' % filepath with open(filepath, 'rb') as fd: - cifar10 = cPickle.load(fd) + cifar10 = pickle.load(fd) image = cifar10['data'].astype(dtype=np.uint8) image = image.reshape((-1, 3, 32, 32)) label = np.asarray(cifar10['labels'], dtype=np.uint8) @@ -79,4 +88,5 @@ def compute_image_mean(train_dir): mean = compute_image_mean('cifar-10-batches-py') test_images, _ = load_test_data('cifar-10-batches-py') + # minus mean is for alexnet; vgg uses a different pre-processing strategy print predict(model, test_images - mean, cuda) diff --git a/examples/cifar10/run-parallel.sh b/examples/cifar10/run-parallel.sh index 6a9109a777..18193db54a 100755 --- a/examples/cifar10/run-parallel.sh +++ b/examples/cifar10/run-parallel.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh ../../build/bin/alexnet-parallel -epoch 4 +#../../build/bin/vgg-parallel -epoch 4 diff --git a/examples/cifar10/train.py b/examples/cifar10/train.py index f4caca4c3a..cb4110db34 100644 --- a/examples/cifar10/train.py +++ b/examples/cifar10/train.py @@ -23,9 +23,9 @@ import numpy as np import os import sys +import argparse sys.path.append(os.path.join(os.path.dirname(__file__), '../../build/python')) -from singa import initializer from singa import utils from singa import optimizer from singa import device @@ -33,6 +33,7 @@ from singa.proto import core_pb2 import alexnet +import vgg def load_dataset(filepath): @@ -65,7 +66,28 @@ def load_test_data(dir_path): return np.array(images, dtype=np.float32), np.array(labels, dtype=np.int32) -def get_lr(epoch): +def normalize_for_vgg(train_x, test_x): + mean = train_x.mean() + std = train_x.std() + train_x -= mean + test_x -= mean + train_x /= std + test_x /= std + return train_x, test_x + + +def normalize_for_alexnet(train_x, test_x): + mean = np.average(train_x, axis=0) + train_x -= mean + test_x -= mean + return train_x, test_x + + +def vgg_lr(epoch): + return 0.01 / float(1 << ((epoch / 30))) + + +def alexnet_lr(epoch): if epoch < 120: return 0.001 elif epoch < 130: @@ -74,32 +96,21 @@ def get_lr(epoch): return 0.00001 -def train(data_dir, net, num_epoch=140, batch_size=100): +def train(data, net, max_epoch, get_lr, weight_decay, batch_size=100): print 'Start intialization............' cuda = device.create_cuda_gpu() net.to_device(cuda) opt = optimizer.SGD(momentum=0.9, weight_decay=0.004) for (p, specs) in zip(net.param_values(), net.param_specs()): - filler = specs.filler - if filler.type == 'gaussian': - initializer.gaussian(p, filler.mean, filler.std) - else: - p.set_value(0) opt.register(p, specs) - print specs.name, filler.type, p.l1() - print 'Loading data ..................' - train_x, train_y = load_train_data(data_dir) - test_x, test_y = load_test_data(data_dir) - mean = np.average(train_x, axis=0) - train_x -= mean - test_x -= mean tx = tensor.Tensor((batch_size, 3, 32, 32), cuda) ty = tensor.Tensor((batch_size,), cuda, core_pb2.kInt) + train_x, train_y, test_x, test_y = data num_train_batch = train_x.shape[0] / batch_size num_test_batch = test_x.shape[0] / batch_size idx = np.arange(train_x.shape[0], dtype=np.int32) - for epoch in range(num_epoch): + for epoch in range(max_epoch): np.random.shuffle(idx) loss, acc = 0.0, 0.0 print 'Epoch %d' % epoch @@ -135,8 +146,20 @@ def train(data_dir, net, num_epoch=140, batch_size=100): net.save('model.bin') # save model params into checkpoint file if __name__ == '__main__': - data_dir = 'cifar-10-batches-py' - assert os.path.exists(data_dir), \ + parser = argparse.ArgumentParser(description='Train vgg/alexnet for cifar10') + parser.add_argument('model', choices=['vgg', 'alexnet'], default='alexnet') + parser.add_argument('data', default='cifar-10-batches-py') + args = parser.parse_args() + assert os.path.exists(args.data), \ 'Pls download the cifar10 dataset via "download_data.py py"' - net = alexnet.create_net() - train(data_dir, net) + print 'Loading data ..................' + train_x, train_y = load_train_data(args.data) + test_x, test_y = load_test_data(args.data) + if args.model == 'alexnet': + train_x, test_x = normalize_for_alexnet(train_x, test_x) + net = alexnet.create_net() + train((train_x, train_y, test_x, test_y), net, 140, alexnet_lr, 0.004) + else: + train_x, test_x = normalize_for_vgg(train_x, test_x) + net = vgg.create_net() + train((train_x, train_y, test_x, test_y), net, 250, vgg_lr, 0.0005) diff --git a/examples/cifar10/train_vgg_cifar10.py b/examples/cifar10/train_vgg_cifar10.py deleted file mode 100644 index e9df04ee6e..0000000000 --- a/examples/cifar10/train_vgg_cifar10.py +++ /dev/null @@ -1,162 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= -""" CIFAR10 dataset is at https://www.cs.toronto.edu/~kriz/cifar.html. -It includes 5 binary dataset, each contains 10000 images. 1 row (1 image) -includes 1 label & 3072 pixels. 3072 pixels are 3 channels of a 32x32 image -""" - -import cPickle -import numpy as np -import os -import sys -import math - -sys.path.append(os.path.join(os.path.dirname(__file__), '../../build/python')) -from singa import initializer -from singa import utils -from singa import optimizer -from singa import device -from singa import tensor -from singa.proto import core_pb2 - -import vgg - - -def load_dataset(filepath): - print 'Loading data file %s' % filepath - with open(filepath, 'rb') as fd: - cifar10 = cPickle.load(fd) - image = cifar10['data'].astype(dtype=np.uint8) - image = image.reshape((-1, 3, 32, 32)) - label = np.asarray(cifar10['labels'], dtype=np.uint8) - label = label.reshape(label.size, 1) - return image, label - - -def load_train_data(dir_path, num_batches=5): - labels = [] - batchsize = 10000 - images = np.empty((num_batches * batchsize, 3, 32, 32), dtype=np.uint8) - for did in range(1, num_batches + 1): - fname_train_data = dir_path + "/data_batch_{}".format(did) - image, label = load_dataset(fname_train_data) - images[(did - 1) * batchsize:did * batchsize] = image - labels.extend(label) - images = np.array(images, dtype=np.float32) - labels = np.array(labels, dtype=np.int32) - return images, labels - - -def load_test_data(dir_path): - images, labels = load_dataset(dir_path + "/test_batch") - return np.array(images, dtype=np.float32), np.array(labels, dtype=np.int32) - - -def get_lr(epoch): - return 0.01 / float(1 << ((epoch / 30))) - #if epoch < 100: - # return 0.01 - #elif epoch < 150: - # return 0.005 - #elif epoch < 200: - # return 0.001 - #elif epoch < 250: - # return 0.0001 - - -def train(data_dir, net, num_epoch=250, batch_size=128): - print 'Creating Device............' - cuda = device.create_cuda_gpus(2)[1] - net.to_device(cuda) - print 'Start intialization............' - opt = optimizer.SGD(momentum=0.9, weight_decay=0.0005) - for (p, name) in zip(net.param_values(), net.param_names()): - print name, p.shape - if len(p.shape) > 1: - if 'mean' in name or 'beta' in name: - p.set_value(0.0) - elif 'var' in name: - p.set_value(1.0) - elif 'gamma' in name: - initializer.uniform(p, 0, 1) - elif 'conv' in name: - initializer.gaussian(p, 0, math.sqrt(2.0/(9.0 * p.shape[0]))) - else: - initializer.gaussian(p, 0, 0.02) - - #stdv = 1.0/math.sqrt(p.shape[1]) - #initializer.uniform(p, -stdv, stdv) - else: - p.set_value(0) - #print specs.name, filler.type, p.l1() - print name, p.l1() - print 'Loading data ..................' - train_x, train_y = load_train_data(data_dir) - test_x, test_y = load_test_data(data_dir) - mean = train_x.mean() - std = train_x.std() - train_x -= mean - test_x -= mean - train_x /= std - test_x /= std - - tx = tensor.Tensor((batch_size, 3, 32, 32), cuda) - ty = tensor.Tensor((batch_size,), cuda, core_pb2.kInt) - num_train_batch = train_x.shape[0] / batch_size - num_test_batch = test_x.shape[0] / batch_size - idx = np.arange(train_x.shape[0], dtype=np.int32) - for epoch in range(num_epoch): - np.random.shuffle(idx) - loss, acc = 0.0, 0.0 - print 'Epoch %d' % epoch - for b in range(num_train_batch): - x = train_x[idx[b * batch_size: (b + 1) * batch_size]] - y = train_y[idx[b * batch_size: (b + 1) * batch_size]] - tx.copy_from_numpy(x) - ty.copy_from_numpy(y) - grads, (l, a) = net.train(tx, ty) - loss += l - acc += a - for (s, p, g) in zip(net.param_specs(), net.param_values(), grads): - opt.apply_with_lr(epoch, get_lr(epoch), g, p, str(s.name)) - # update progress bar - utils.update_progress(b * 1.0 / num_train_batch, - 'training loss = %f, accuracy = %f' % (l, a)) - info = '\ntraining loss = %f, training accuracy = %f' \ - % (loss / num_train_batch, acc / num_train_batch) - print info - - loss, acc = 0.0, 0.0 - for b in range(num_test_batch): - x = test_x[b * batch_size: (b + 1) * batch_size] - y = test_y[b * batch_size: (b + 1) * batch_size] - tx.copy_from_numpy(x) - ty.copy_from_numpy(y) - l, a = net.evaluate(tx, ty) - loss += l - acc += a - - print 'test loss = %f, test accuracy = %f' \ - % (loss / num_test_batch, acc / num_test_batch) - net.save('model.bin') # save model params into checkpoint file - -if __name__ == '__main__': - data_dir = 'cifar-10-batches-py' - assert os.path.exists(data_dir), \ - 'Pls download the cifar10 dataset via "download_data.py py"' - net = vgg.create_net() - train(data_dir, net) diff --git a/examples/cifar10/vgg-parallel.cc b/examples/cifar10/vgg-parallel.cc index ba308e9d32..c6b7fa1a8d 100644 --- a/examples/cifar10/vgg-parallel.cc +++ b/examples/cifar10/vgg-parallel.cc @@ -32,7 +32,7 @@ #include "../../src/model/layer/cudnn_activation.h" #include "../../src/model/layer/cudnn_pooling.h" #include "../../src/model/layer/cudnn_lrn.h" -#include "../../src/model/layer/cudnn_dropout.h" +#include "../../src/model/layer/dropout.h" #include "../../src/model/layer/cudnn_batchnorm.h" #include "../../src/model/layer/dense.h" #include "../../src/model/layer/flatten.h" @@ -155,7 +155,7 @@ LayerConf GenBatchNormConf(string name) { LayerConf GenDropoutConf(string name, float dropout_ratio) { LayerConf conf; conf.set_name(name); - conf.set_type("CudnnDropout"); + conf.set_type("Dropout"); DropoutConf *dropout = conf.mutable_dropout_conf(); dropout->set_dropout_ratio(dropout_ratio); @@ -172,37 +172,37 @@ FeedForwardNet CreateNet() { FeedForwardNet net; Shape s{3, 32, 32}; ConvBNReLU(net, "conv1_1", 64, &s); - net.Add(new CudnnDropout(), GenDropoutConf("drop1", 0.3)); + net.Add(new Dropout(), GenDropoutConf("drop1", 0.3)); ConvBNReLU(net, "conv1_2", 64); net.Add(new CudnnPooling(), GenPoolingConf("pool1", true, 2, 2, 0)); ConvBNReLU(net, "conv2_1", 128); - net.Add(new CudnnDropout(), GenDropoutConf("drop2", 0.4)); + net.Add(new Dropout(), GenDropoutConf("drop2", 0.4)); ConvBNReLU(net, "conv2_2", 128); net.Add(new CudnnPooling(), GenPoolingConf("pool2", true, 2, 2, 0)); ConvBNReLU(net, "conv3_1", 256); - net.Add(new CudnnDropout(), GenDropoutConf("drop3_1", 0.4)); + net.Add(new Dropout(), GenDropoutConf("drop3_1", 0.4)); ConvBNReLU(net, "conv3_2", 256); - net.Add(new CudnnDropout(), GenDropoutConf("drop3_2", 0.4)); + net.Add(new Dropout(), GenDropoutConf("drop3_2", 0.4)); ConvBNReLU(net, "conv3_3", 256); net.Add(new CudnnPooling(), GenPoolingConf("pool3", true, 2, 2, 0)); ConvBNReLU(net, "conv4_1", 512); - net.Add(new CudnnDropout(), GenDropoutConf("drop4_1", 0.4)); + net.Add(new Dropout(), GenDropoutConf("drop4_1", 0.4)); ConvBNReLU(net, "conv4_2", 512); - net.Add(new CudnnDropout(), GenDropoutConf("drop4_2", 0.4)); + net.Add(new Dropout(), GenDropoutConf("drop4_2", 0.4)); ConvBNReLU(net, "conv4_3", 512); net.Add(new CudnnPooling(), GenPoolingConf("pool4", true, 2, 2, 0)); ConvBNReLU(net, "conv5_1", 512); - net.Add(new CudnnDropout(), GenDropoutConf("drop5_1", 0.4)); + net.Add(new Dropout(), GenDropoutConf("drop5_1", 0.4)); ConvBNReLU(net, "conv5_2", 512); - net.Add(new CudnnDropout(), GenDropoutConf("drop5_2", 0.4)); + net.Add(new Dropout(), GenDropoutConf("drop5_2", 0.4)); ConvBNReLU(net, "conv5_3", 512); net.Add(new CudnnPooling(), GenPoolingConf("pool5", true, 2, 2, 0)); net.Add(new Flatten(), GenFlattenConf("flat")); - net.Add(new CudnnDropout(), GenDropoutConf("flat_drop", 0.5)); + net.Add(new Dropout(), GenDropoutConf("flat_drop", 0.5)); net.Add(new Dense(), GenDenseConf("ip1", 512, 0.02)); net.Add(new CudnnBatchNorm(), GenBatchNormConf("ip1_bn")); net.Add(new CudnnActivation(), GenReLUConf("ip1_relu")); - net.Add(new CudnnDropout(), GenDropoutConf("ip1_drop", 0.5)); + net.Add(new Dropout(), GenDropoutConf("ip1_drop", 0.5)); net.Add(new Dense(), GenDenseConf("ip2", 10, 0.02)); return net; diff --git a/examples/cifar10/vgg.py b/examples/cifar10/vgg.py index 8063307f5b..0b9bb56717 100644 --- a/examples/cifar10/vgg.py +++ b/examples/cifar10/vgg.py @@ -1,12 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +""" The VGG model is adapted from http://torch.ch/blog/2015/07/30/cifar.html. +The best validation accuracy we achieved is about 89% without data augmentation. +The performance could be improved by tuning some hyper-parameters, including +learning rate, weight decay, max_epoch, parameter initialization, etc. +""" + import sys import os +import math sys.path.append(os.path.join(os.path.dirname(__file__), '../../build/python')) + from singa import layer +from singa import initializer from singa import metric from singa import loss from singa import net as ffnet -from singa.proto import core_pb2 + def ConvBnReLU(net, name, nb_filers, sample_shape=None): net.add(layer.Conv2D(name + '_1', nb_filers, 3, 1, pad=1, @@ -14,39 +39,58 @@ def ConvBnReLU(net, name, nb_filers, sample_shape=None): net.add(layer.BatchNormalization(name + '_2')) net.add(layer.Activation(name + '_3')) + def create_net(): net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy()) ConvBnReLU(net, 'conv1_1', 64, (3, 32, 32)) - net.add(layer.Dropout('drop1', 0.3, engine='cudnn')) + net.add(layer.Dropout('drop1', 0.3, engine='cuda')) ConvBnReLU(net, 'conv1_2', 64) net.add(layer.MaxPooling2D('pool1', 2, 2, border_mode='valid')) ConvBnReLU(net, 'conv2_1', 128) - net.add(layer.Dropout('drop2_1', 0.4, engine='cudnn')) + net.add(layer.Dropout('drop2_1', 0.4, engine='cuda')) ConvBnReLU(net, 'conv2_2', 128) net.add(layer.MaxPooling2D('pool2', 2, 2, border_mode='valid')) ConvBnReLU(net, 'conv3_1', 256) - net.add(layer.Dropout('drop3_1', 0.4, engine='cudnn')) + net.add(layer.Dropout('drop3_1', 0.4, engine='cuda')) ConvBnReLU(net, 'conv3_2', 256) - net.add(layer.Dropout('drop3_2', 0.4, engine='cudnn')) + net.add(layer.Dropout('drop3_2', 0.4, engine='cuda')) ConvBnReLU(net, 'conv3_3', 256) net.add(layer.MaxPooling2D('pool3', 2, 2, border_mode='valid')) ConvBnReLU(net, 'conv4_1', 512) - net.add(layer.Dropout('drop4_1', 0.4, engine='cudnn')) + net.add(layer.Dropout('drop4_1', 0.4, engine='cuda')) ConvBnReLU(net, 'conv4_2', 512) - net.add(layer.Dropout('drop4_2', 0.4, engine='cudnn')) + net.add(layer.Dropout('drop4_2', 0.4, engine='cuda')) ConvBnReLU(net, 'conv4_3', 512) net.add(layer.MaxPooling2D('pool4', 2, 2, border_mode='valid')) ConvBnReLU(net, 'conv5_1', 512) - net.add(layer.Dropout('drop5_1', 0.4, engine='cudnn')) + net.add(layer.Dropout('drop5_1', 0.4, engine='cuda')) ConvBnReLU(net, 'conv5_2', 512) - net.add(layer.Dropout('drop5_2', 0.4, engine='cudnn')) + net.add(layer.Dropout('drop5_2', 0.4, engine='cuda')) ConvBnReLU(net, 'conv5_3', 512) net.add(layer.MaxPooling2D('pool5', 2, 2, border_mode='valid')) net.add(layer.Flatten('flat')) - net.add(layer.Dropout('drop_flat', 0.5, engine='cudnn')) + net.add(layer.Dropout('drop_flat', 0.5, engine='cuda')) net.add(layer.Dense('ip1', 512)) net.add(layer.BatchNormalization('batchnorm_ip1')) net.add(layer.Activation('relu_ip1')) - net.add(layer.Dropout('drop_ip2', 0.5, engine='cudnn')) + net.add(layer.Dropout('drop_ip2', 0.5, engine='cuda')) net.add(layer.Dense('ip2', 10)) + print 'Start intialization............' + for (p, name) in zip(net.param_values(), net.param_names()): + print name, p.shape + if len(p.shape) > 1: + if 'mean' in name or 'beta' in name: + p.set_value(0.0) + elif 'var' in name: + p.set_value(1.0) + elif 'gamma' in name: + initializer.uniform(p, 0, 1) + elif 'conv' in name: + initializer.gaussian(p, 0, math.sqrt(2.0/(9.0 * p.shape[0]))) + else: + initializer.gaussian(p, 0, 0.02) + else: + p.set_value(0) + print name, p.l1() + return net From 246c37516ae8f5caed5a1e86a92857ded22f9bf1 Mon Sep 17 00:00:00 2001 From: zhaojing Date: Sat, 25 Jun 2016 00:15:06 +0800 Subject: [PATCH 06/16] SINGA-218 Implementation for RNN CUDNN version - cudnn rnn implementation (cudnn_rnn,h, cudnn_rnn.cc, rnn.cc, rnn.h, test_cudnn_rnn.cc). - The weight shape now are manually calculated instead of using API provided by CUDNN. - Test for RNN_cudnn_Tanh (unidirectional, 1 hidden layer). --- src/model/layer/cudnn_rnn.cc | 328 +++++++++++++++++++++++++++++++++++ src/model/layer/cudnn_rnn.h | 85 +++++++++ src/model/layer/rnn.cc | 53 ++++++ src/model/layer/rnn.h | 31 +++- src/proto/model.proto | 17 ++ test/singa/test_cudnn_rnn.cc | 212 ++++++++++++++++++++++ 6 files changed, 720 insertions(+), 6 deletions(-) create mode 100644 src/model/layer/cudnn_rnn.cc create mode 100644 src/model/layer/cudnn_rnn.h create mode 100644 src/model/layer/rnn.cc create mode 100644 test/singa/test_cudnn_rnn.cc diff --git a/src/model/layer/cudnn_rnn.cc b/src/model/layer/cudnn_rnn.cc new file mode 100644 index 0000000000..6f04e5ca70 --- /dev/null +++ b/src/model/layer/cudnn_rnn.cc @@ -0,0 +1,328 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "./cudnn_rnn.h" +#ifdef USE_CUDNN +#include +#include +#include "./cudnn_utils.h" +#include "singa/utils/logging.h" + +namespace singa { +CudnnRNN::~CudnnRNN() { + if (weight_desc_ != nullptr) + CUDNN_CHECK(cudnnDestroyFilterDescriptor(weight_desc_)); + if (dropout_desc_ != nullptr) + CUDNN_CHECK(cudnnDestroyDropoutDescriptor(dropout_desc_)); + if (rnn_desc_ != nullptr) + CUDNN_CHECK(cudnnDestroyRNNDescriptor(rnn_desc_)); + if (x_descs_ != nullptr) + for (size_t i = 0; i < seqLength_; i++) + CUDNN_CHECK(cudnnDestroyTensorDescriptor(x_descs_[i])); + if (y_descs_ != nullptr) + for (size_t i = 0; i < seqLength_; i++) + CUDNN_CHECK(cudnnDestroyTensorDescriptor(y_descs_[i])); + if (hx_desc_ != nullptr) + CUDNN_CHECK(cudnnDestroyTensorDescriptor(hx_desc_)); + if (hy_desc_ != nullptr) + CUDNN_CHECK(cudnnDestroyTensorDescriptor(hy_desc_)); + if (cx_desc_ != nullptr) + CUDNN_CHECK(cudnnDestroyTensorDescriptor(cx_desc_)); + if (cy_desc_ != nullptr) + CUDNN_CHECK(cudnnDestroyTensorDescriptor(cy_desc_)); +} + +void CudnnRNN::Setup(const Shape& in_sample, const LayerConf &conf) { + RNN::Setup(in_sample, conf); + RNNConf rnn_conf = conf.rnn_conf(); + // convert MB to bytes + workspace_byte_limit_ = rnn_conf.workspace_byte_limit() << 20; + inputMode_ = ToLowerCase(rnn_conf.inputmode()); + direction_ = ToLowerCase(rnn_conf.direction()); + mode_ = ToLowerCase(rnn_conf.mode()); + CHECK(inputMode_ == "cudnn_linear_input" || inputMode_ == "cudnn_skip_input") + << "CudnnRNN only supports two inputmodes: cudnn_linear_input, " + "cudnn_skip_input"; + CHECK(direction_ == "cudnn_undirectional" || direction_ == "cudnn_bidirectional") + << "CudnnRNN only supports two directions: cudnn_undirectional, " + "cudnn_bidirectional"; + CHECK(mode_ == "cudnn_rnn_relu" || mode_ == "cudnn_rnn_tanh" || + mode_ == "cudnn_lstm" || mode_ == "cudnn_gru") + << "CudnnRNN only supports four modes: cudnn_rnn_relu, " + "cudnn_rnn_tanh, cudnn_lstm and cudnn_gru"; + // the first constant (4) is the size of float + // the second constant (2, 8, 6) is the number of sets of params + if (mode_ == "cudnn_rnn_relu" || mode_ == "cudnn_rnn_tanh") + weightSize_ = 4 * 2 * (hiddenSize_ * in_sample[2] + hiddenSize_); + else if (mode_ == "cudnn_lstm") + weightSize_ = 4 * 8 * (hiddenSize_ * in_sample[2] + hiddenSize_); + else if (mode_ == "cudnn_gru") + weightSize_ = 4 * 6 * (hiddenSize_ * in_sample[2] + hiddenSize_); + if (direction_ == "cudnn_bidirectional") + weightSize_ = weightSize_ * 2; +} + +void CudnnRNN::ToDevice(std::shared_ptr device) { + weight_.ToDevice(device); + workspace_.ToDevice(device); +} + +void CudnnRNN::InitCudnn(const Tensor &input) { + CHECK(!has_init_cudnn_); + DataType dtype = input.data_type(); + auto dev = input.device(); + Context *ctx = dev->context(0); + seqLength_ = input.shape(0); + size_t batchsize = input.shape(1); /*(seqLength, minibatch, inputSize) !!! */ + size_t inputSize = input.shape(2); + size_t numDirections; + if (direction_ == "cudnn_undirectional") + numDirections = 1; + else + numDirections = 2; + x_descs_ = new cudnnTensorDescriptor_t[seqLength_]; + y_descs_ = new cudnnTensorDescriptor_t[seqLength_]; + for (size_t i = 0; i < seqLength_; i++) + CUDNN_CHECK(cudnnCreateTensorDescriptor(&x_descs_[i])); + for (size_t i = 0; i < seqLength_; i++) + CUDNN_CHECK(cudnnCreateTensorDescriptor(&y_descs_[i])); + CUDNN_CHECK(cudnnCreateTensorDescriptor(&hx_desc_)); + CUDNN_CHECK(cudnnCreateTensorDescriptor(&cx_desc_)); + CUDNN_CHECK(cudnnCreateTensorDescriptor(&hy_desc_)); + CUDNN_CHECK(cudnnCreateTensorDescriptor(&cy_desc_)); + CUDNN_CHECK(cudnnCreateFilterDescriptor(&weight_desc_)); + CUDNN_CHECK(cudnnCreateDropoutDescriptor(&dropout_desc_)); + CUDNN_CHECK(cudnnCreateRNNDescriptor(&rnn_desc_)); + + + int dimA[3] = {batchsize, inputSize, 1}; + int strideA[3] = {dimA[2] * dimA[1], dimA[2], 1}; + for (size_t i = 0; i < seqLength_; i++){ + dimA[0] = batchsize; + dimA[1] = inputSize; + dimA[2] = 1; + strideA[0] = dimA[2] * dimA[1]; + strideA[1] = dimA[2]; + strideA[2] = 1; + CUDNN_CHECK(cudnnSetTensorNdDescriptor(x_descs_[i], GetCudnnDataType(dtype), 3, + dimA, strideA)); + dimA[0] = batchsize; + dimA[1] = hiddenSize_ * numDirections; + dimA[2] = 1; + strideA[0] = dimA[2] * dimA[1]; + strideA[1] = dimA[2]; + strideA[2] = 1; + CUDNN_CHECK(cudnnSetTensorNdDescriptor(y_descs_[i], GetCudnnDataType(dtype), 3, + dimA, strideA)); + } + + dimA[0] = numLayers_; + dimA[1] = batchsize; + dimA[2] = hiddenSize_ * numDirections; + strideA[0] = dimA[2] * dimA[1]; + strideA[1] = dimA[2]; + strideA[2] = 1; + CUDNN_CHECK(cudnnSetTensorNdDescriptor(hx_desc_, GetCudnnDataType(dtype), 3, + dimA, strideA)); + CUDNN_CHECK(cudnnSetTensorNdDescriptor(cx_desc_, GetCudnnDataType(dtype), 3, + dimA, strideA)); + CUDNN_CHECK(cudnnSetTensorNdDescriptor(hy_desc_, GetCudnnDataType(dtype), 3, + dimA, strideA)); + CUDNN_CHECK(cudnnSetTensorNdDescriptor(cy_desc_, GetCudnnDataType(dtype), 3, + dimA, strideA)); + + size_t dropoutStatesSize; + CUDNN_CHECK(cudnnDropoutGetStatesSize(ctx->cudnn_handle, &dropoutStatesSize)); + dropoutStates_ = Tensor(Shape{dropoutStatesSize}, dev, dtype); + CUDNN_CHECK(cudnnSetDropoutDescriptor(dropout_desc_, ctx->cudnn_handle, dropout_, this->dropoutStates_.block()->mutable_data(), dropoutStatesSize, 0x01234567)); + + cudnnRNNInputMode_t inputMode; + cudnnDirectionMode_t direction; + cudnnRNNMode_t mode; + + if (inputMode_ == "cudnn_linear_input" || inputMode_ == "cudnn_skip_input"){ + if (inputMode_ == "cudnn_linear_input") + inputMode = CUDNN_LINEAR_INPUT; + else if (inputMode_ == "cudnn_skip_input") + inputMode = CUDNN_SKIP_INPUT; + } + if (direction_ == "cudnn_undirectional" || direction_ == "cudnn_bidirectional"){ + if (direction_ == "cudnn_undirectional") + direction = CUDNN_UNIDIRECTIONAL; + else if (direction_ == "cudnn_bidirectional") + direction = CUDNN_BIDIRECTIONAL; + } + if (mode_ == "cudnn_rnn_relu" || mode_ == "cudnn_rnn_tanh" || + mode_ == "cudnn_lstm" || mode_ == "cudnn_gru"){ + if (mode_ == "cudnn_rnn_relu") + mode = CUDNN_RNN_RELU; + else if (mode_ == "cudnn_rnn_tanh") + mode = CUDNN_RNN_TANH; + else if (mode_ == "cudnn_lstm") + mode = CUDNN_LSTM; + else if (mode_ == "cudnn_gru") + mode = CUDNN_GRU; + } + CUDNN_CHECK(cudnnSetRNNDescriptor(rnn_desc_, hiddenSize_, numLayers_, dropout_desc_, inputMode, direction, mode, GetCudnnDataType(dtype))); + + size_t weightSize; + CUDNN_CHECK(cudnnGetRNNParamsSize(ctx->cudnn_handle, rnn_desc_, x_descs_[0], &weightSize, GetCudnnDataType(dtype))); + CHECK_EQ(weightSize, weightSize_); + + int filterDimA[3] = {weightSize_, 1, 1}; + CUDNN_CHECK(cudnnSetFilterNdDescriptor(weight_desc_, GetCudnnDataType(dtype), CUDNN_TENSOR_NCHW, 3, filterDimA)); + + + CUDNN_CHECK(cudnnGetRNNWorkspaceSize(ctx->cudnn_handle, rnn_desc_, seqLength_, x_descs_, &workspace_count_)); + workspace_ = Tensor(Shape{workspace_count_}, dev, dtype); + + CUDNN_CHECK(cudnnGetRNNTrainingReserveSize(ctx->cudnn_handle, rnn_desc_, seqLength_, x_descs_, &ReserveSize_)); + reserve_ = Tensor(Shape{ReserveSize_}, dev, dtype); + has_init_cudnn_ = true; +} + +const vector CudnnRNN::Forward(int flag, const vector& inputs) { + /*(seqLength, minibatch, inputSize) !!! */ + singa::Tensor input = inputs[0]; + singa::Tensor hx = inputs[1]; + singa:: Tensor cx = inputs[2]; + CHECK_EQ(input.device()->lang(), kCuda); + CHECK_EQ(input.device()->lang(), this->weight_.device()->lang()); + CHECK_EQ(input.nDim(), 3u); + vector data_output; + if (flag & kTrain) buf_.push(input); // buffer the input for backward + size_t batchsize = input.shape(1); /*(seqLength, minibatch, inputSize) !!! */ + DataType dtype = input.data_type(); + auto dev = input.device(); + + if (!has_init_cudnn_) InitCudnn(input); + + + size_t numDirections; + if (direction_ == "cudnn_undirectional") + numDirections = 1; + else + numDirections = 2; + + Shape shape{seqLength_, batchsize, hiddenSize_ * numDirections}; + Tensor output(shape, dev, dtype); + Shape shape1{numLayers_, batchsize, hiddenSize_ * numDirections}; + Tensor hy(shape1, dev, dtype); + Tensor cy(shape1, dev, dtype); + + output.device()->Exec([input, output, hx, hy, cx, cy, this](Context *ctx) { + Block *inblock = input.block(), *outblock = output.block(), + *wblock = this->weight_.block(), *hxblock = hx.block(), + *hyblock = hy.block(), *cxblock = cx.block(), + *cyblock = cy.block(); + cudnnRNNForwardTraining( + ctx->cudnn_handle, this->rnn_desc_, seqLength_, this->x_descs_, + inblock->data(), this->hx_desc_, hxblock->data(), this->cx_desc_, + cxblock->data(), this->weight_desc_, wblock->data(), this->y_descs_, + outblock->mutable_data(), this->hy_desc_, hyblock->mutable_data(), + cy_desc_, cyblock->mutable_data(), this->workspace_.block()->mutable_data(), + this->workspace_count_ * sizeof(float), this->reserve_.block()->mutable_data(), + this->ReserveSize_ * sizeof(float)); +}, {input.block(), weight_.block(), hx.block(), cx.block()}, + {output.block(), hy.block(), cy.block()}, workspace_.block()); + buf_.push(output); + buf_.push(hx); + buf_.push(hy); // in order to assign shape to dhy + buf_.push(cx); + buf_.push(cy); // in order to assign shape to dcy + data_output.push_back(output); + data_output.push_back(hy); + data_output.push_back(cy); + return data_output; +} + +const std::pair, vector> CudnnRNN::Backward( + int flag, const vector& grads) { + CHECK(has_init_cudnn_); + singa::Tensor grad = grads[0]; + singa::Tensor dhy = grads[1]; + singa::Tensor dcy = grads[2]; + CHECK_EQ(grad.device()->lang(), kCuda); + CHECK_EQ(grad.nDim(), 3u); + CHECK(!buf_.empty()); + Tensor cy = buf_.top(); + buf_.pop(); + CHECK(!buf_.empty()); + Tensor cx = buf_.top(); + buf_.pop(); + CHECK(!buf_.empty()); + Tensor hy = buf_.top(); + buf_.pop(); + CHECK(!buf_.empty()); + Tensor hx = buf_.top(); + buf_.pop(); + CHECK(!buf_.empty()); + Tensor src_output = buf_.top(); + buf_.pop(); + CHECK(!buf_.empty()); + Tensor src_data = buf_.top(); + buf_.pop(); + vector param_grad; + vector data_grad; + Tensor dx; + dx.ResetLike(src_data); + Tensor dw; + dw.ResetLike(weight_); + Tensor dhx; + dhx.ResetLike(hx); + Tensor dcx; + dcx.ResetLike(cx); + + + dx.device()->Exec([grad, dw, src_data, src_output, hx, this](Context *ctx) { + Block *inblock = src_data.block(), *srcoutblock = src_output.block(), + *dwblock = dw.block(), *hxblock = hx.block(); + cudnnRNNBackwardWeights( + ctx->cudnn_handle, this->rnn_desc_, seqLength_, this->x_descs_, + inblock->data(), this->hx_desc_, hxblock->data(), this->y_descs_, + srcoutblock->data(), this->workspace_.block()->mutable_data(), + this->workspace_count_ * sizeof(float), this->weight_desc_, + dwblock->mutable_data(), this->reserve_.block()->mutable_data(), + this->ReserveSize_ * sizeof(float)); + }, {src_data.block(), hx.block(), src_output.block()}, {dw.block(), workspace_.block()}); + + // LOG(ERROR) << "backward src"; + dx.device()->Exec([grad, dw, src_output, dx, cy, cx, hy, hx, dhy, dcy, dhx, dcx, this](Context *ctx) { + Block *srcoutblock = src_output.block(), *wblock = this->weight_.block(), *dxblock = dx.block(), + *dyblock = grad.block(), *cxblock = cx.block(), *hxblock = hx.block(), *dhyblock = dhy.block(), + *dcyblock = dcy.block(), *dhxblock = dhx.block(), *dcxblock = dcx.block(); + cudnnRNNBackwardData( + ctx->cudnn_handle, this->rnn_desc_, seqLength_, this->y_descs_, srcoutblock->data(), + this->y_descs_, dyblock->data(), this->hy_desc_, dhyblock->data(), + this->cy_desc_, dcyblock->data(), this->weight_desc_, wblock->data(), + this->hx_desc_, hxblock->data(), this->cx_desc_, cxblock->data(), + this->x_descs_, dxblock->mutable_data(), this->hx_desc_, dhxblock->mutable_data(), + this->cx_desc_, dcxblock->mutable_data(), this->workspace_.block()->mutable_data(), + this->workspace_count_ * sizeof(float), this->reserve_.block()->mutable_data(), + this->ReserveSize_ * sizeof(float)); + }, {hx.block(), src_output.block(), grad.block(), grad.block(), dhy.block(), dcy.block(), + this->weight_.block(), hx.block(), cx.block()}, + {dx.block(), dhx.block(), dcx.block(), reserve_.block(), workspace_.block()}); + param_grad.push_back(dw); + data_grad.push_back(dx); + data_grad.push_back(dhx); + data_grad.push_back(dcx); + return std::make_pair(data_grad, param_grad); +} + +} // namespace singa +#endif // USE_CUDNN diff --git a/src/model/layer/cudnn_rnn.h b/src/model/layer/cudnn_rnn.h new file mode 100644 index 0000000000..b1e9f43e59 --- /dev/null +++ b/src/model/layer/cudnn_rnn.h @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_MODEL_LAYER_CUDNN_RNN_H_ +#define SRC_MODEL_LAYER_CUDNN_RNN_H_ +#include "singa/singa_config.h" +#ifdef USE_CUDNN +#include +#include +#include +#include "./rnn.h" +#include "singa/core/common.h" +#include "singa/model/layer.h" +#include "singa/proto/core.pb.h" +#include "singa/utils/string.h" +#include +#include +#include "./cudnn_utils.h" +#include "singa/utils/logging.h" + +namespace singa { +class CudnnRNN : public RNN { + public: + ~CudnnRNN(); + /// \copydoc Layer::layer_type() + const std::string layer_type() const override { return "CudnnRNN"; } + + const vector Forward(int flag, const vector& inputs) override; + const std::pair, vector> Backward(int flag, const vector& grads) override; + + /// \copydoc Layer::Setup(const LayerConf&); + void Setup(const Shape& in_sample, const LayerConf &conf) override; + + void ToDevice(std::shared_ptr device) override; + + size_t workspace_byte_limit() { return workspace_byte_limit_; } + // string prefer() { return prefer_; } + string inputMode() const { return inputMode_; } + string direction() const { return direction_; } + string mode() const { return mode_; } + + protected: + /// Init cudnn related data structures. + void InitCudnn(const Tensor& input); + + protected: + bool has_init_cudnn_ = false; + cudnnTensorDescriptor_t* x_descs_ = nullptr; + cudnnTensorDescriptor_t* y_descs_ = nullptr; + cudnnTensorDescriptor_t hx_desc_ = nullptr; + cudnnTensorDescriptor_t cx_desc_ = nullptr; + cudnnTensorDescriptor_t hy_desc_ = nullptr; + cudnnTensorDescriptor_t cy_desc_ = nullptr; + cudnnFilterDescriptor_t weight_desc_ = nullptr; + cudnnRNNDescriptor_t rnn_desc_ = nullptr; + cudnnDropoutDescriptor_t dropout_desc_ = nullptr; + size_t workspace_byte_limit_, workspace_count_; + size_t ReserveSize_; + Tensor workspace_; + string inputMode_; + string direction_; + string mode_; + Tensor reserve_; + Tensor dropoutStates_; +}; + +} // namespace singa + +#endif // USE_CUDNN +#endif // SRC_MODEL_LAYER_CUDNN_RNN_H_ diff --git a/src/model/layer/rnn.cc b/src/model/layer/rnn.cc new file mode 100644 index 0000000000..493a5e42b5 --- /dev/null +++ b/src/model/layer/rnn.cc @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "./rnn.h" +#include +#include "singa/model/layer.h" + +namespace singa { + +void RNN::Setup(const Shape& in_sample, const LayerConf &conf) { + Layer::Setup(in_sample, conf); + RNNConf rnn_conf = conf.rnn_conf(); + hiddenSize_ = rnn_conf.hiddensize(); + CHECK_GT(hiddenSize_, 0u); + + numLayers_ = rnn_conf.numlayers(); + CHECK_GT(numLayers_, 0u); + + dropout_ = rnn_conf.dropout(); + CHECK_GE(dropout_, 0u); +} + +const vector RNN::Forward(int flag, const vector& inputs) { + vector data_output; + return data_output; +} + +const std::pair, vector> RNN::Backward(int flag, const vector& grads) { + vector param_grad; + vector data_grad; + return std::make_pair(data_grad, param_grad); +} + +void RNN::ToDevice(std::shared_ptr device) { + Layer::ToDevice(device); + weight_.ToDevice(device); +} +} /* singa */ diff --git a/src/model/layer/rnn.h b/src/model/layer/rnn.h index 35c86bd5b5..ec5a35d71b 100644 --- a/src/model/layer/rnn.h +++ b/src/model/layer/rnn.h @@ -38,21 +38,32 @@ class RNN : public Layer { const std::string layer_type() const override { return "RNN"; } /// \copydoc Layer::Setup(const LayerConf&); - void Setup(const LayerConf& conf) override; + void Setup(const vector& in_shape, const LayerConf& conf) override; /// \copydoc Layer::Forward(int flag, const vector&) - const vector Forward(int flag, const vector& input) override; + const vector Forward(int flag, const vector& inputs) override; /// \copydoc Layer::Backward(int, const vector&); const std::pair, vector> Backward( - int flag, const vector& grad) override; + int flag, const vector& grads) override; - void ToDevice(Device* device) override; + size_t hiddenSize() const { return hiddenSize_; } + size_t numLayers() const { return numLayers_; } + size_t weightSize() const { return weightSize_; } + float dropout() const { return dropout_; } + + void set_weight(Tensor w) { + weight_.ResetLike(w); + weight_.CopyData(w); + } + + + void ToDevice(std::shared_ptr device) override; /// Return the internal state stack, which should be empty at the beginning /// of /// one iteration. - std::stack states() const { return states_; } + // std::stack states() const { return states_; } protected: /// Storing input or output from Forward(), which are used in Backward(). @@ -60,7 +71,15 @@ class RNN : public Layer { /// 1. push the 'input' or 'output' into states_ if the flag of Forward() is /// for kTrain and 'input' or 'output' is necessary for Backward(). /// 2. pop data out in Backward(). - std::stack states_; + // std::stack states_; + std::stack buf_; + size_t hiddenSize_; + size_t numLayers_; + size_t numLinearLayer_; + size_t seqLength_; + size_t weightSize_; /*all the weights and biases*/ + float dropout_; + Tensor weight_; }; } // namespace singa #endif // SRC_MODEL_LAYER_RNN_H_ diff --git a/src/proto/model.proto b/src/proto/model.proto index b1318d9c0f..d8193f17fa 100644 --- a/src/proto/model.proto +++ b/src/proto/model.proto @@ -203,6 +203,7 @@ message LayerConf { optional ConcatConf concat_conf = 104; optional ContrastiveLossConf contrastive_loss_conf = 105; optional ConvolutionConf convolution_conf = 106; + optional RNNConf rnn_conf = 140; // optional DataConf data_conf = 107; optional DropoutConf dropout_conf = 108; // optional DummyDataConf dummy_data_conf = 109; @@ -391,6 +392,22 @@ message ConvolutionConf { optional string prefer = 51 [default = "fastest"]; } +message RNNConf { + optional uint32 hiddensize = 1; // The number of hiddensize + optional uint32 numlayers = 2; // The number of stacked RNN layers + optional float dropout = 3 [default = 0]; + optional int32 workspace_byte_limit = 50 [default = 512]; + // cudnn inputmode + // options: "cudnn_linear_input", "cudnn_skip_input" + optional string inputmode = 51 [default = "cudnn_linear_input"]; + // cudnn direction + // options: "cudnn_undirectional", "cudnn_bidirectional" + optional string direction = 52 [default = "cudnn_undirectional"]; + // cudnn RNN mode + // options: "cudnn_rnn_relu", "cudnn_rnn_tanh", "cudnn_lstm", "cudnn_gru" + optional string mode = 53 [default = "cudnn_rnn_relu"]; +} + /* message DataConf { enum DB { diff --git a/test/singa/test_cudnn_rnn.cc b/test/singa/test_cudnn_rnn.cc new file mode 100644 index 0000000000..1a79d7b0fa --- /dev/null +++ b/test/singa/test_cudnn_rnn.cc @@ -0,0 +1,212 @@ +/************************************************************ +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*************************************************************/ + +#include "../src/model/layer/cudnn_rnn.h" +#ifdef USE_CUDNN + +#include "gtest/gtest.h" + +using singa::CudnnRNN; +using singa::Shape; +TEST(CudnnRNN, Setup) { + CudnnRNN rnn; + EXPECT_EQ("CudnnRNN", rnn.layer_type()); + + singa::LayerConf conf; + singa::RNNConf *rnnconf = conf.mutable_rnn_conf(); + rnnconf->set_hiddensize(2); + rnnconf->set_numlayers(1); + rnnconf->set_dropout(0); + rnnconf->set_inputmode("cudnn_linear_input"); + rnnconf->set_direction("cudnn_undirectional"); + rnnconf->set_mode("cudnn_rnn_tanh"); + // MB + rnnconf->set_workspace_byte_limit(256); + rnn.Setup(Shape{4, 1, 2}, conf); + + EXPECT_EQ(2u, rnn.hiddenSize()); + EXPECT_EQ(1u, rnn.numLayers()); + EXPECT_EQ(0u, rnn.dropout()); + EXPECT_EQ("cudnn_linear_input", rnn.inputMode()); + EXPECT_EQ("cudnn_undirectional", rnn.direction()); + EXPECT_EQ("cudnn_rnn_tanh", rnn.mode()); + EXPECT_EQ(256u << 20, rnn.workspace_byte_limit()); +} + +TEST(CudnnRNN, Forward) { + auto cuda = std::make_shared(); + const size_t seqLength = 4, batchsize = 1, dim = 2; + const size_t numLayers = 1, hiddensize = 2, numDirections = 1; + const float x[seqLength * batchsize * dim] = {1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f}; + singa::Tensor in(singa::Shape{seqLength, batchsize, dim}, cuda); + in.CopyDataFromHostPtr(x, seqLength * batchsize * dim); + + + + const float hx_data[numLayers * batchsize * hiddensize * numDirections] = {1.0f, 1.0f}; + singa::Tensor hx(singa::Shape{numLayers, batchsize, hiddensize * numDirections}, cuda); + hx.CopyDataFromHostPtr(hx_data, numLayers * batchsize * hiddensize * numDirections); + + const float cx_data[numLayers * batchsize * hiddensize * numDirections] = {1.0f, 1.0f}; + singa::Tensor cx(singa::Shape{numLayers, batchsize, hiddensize * numDirections}, cuda); + cx.CopyDataFromHostPtr(cx_data, numLayers * batchsize * hiddensize * numDirections); + + CudnnRNN rnn; + + singa::LayerConf conf; + singa::RNNConf *rnnconf = conf.mutable_rnn_conf(); + rnnconf->set_hiddensize(2); + rnnconf->set_numlayers(1); + rnnconf->set_dropout(0); + rnnconf->set_inputmode("cudnn_linear_input"); + rnnconf->set_direction("cudnn_undirectional"); + rnnconf->set_mode("cudnn_rnn_tanh"); + // MB + rnnconf->set_workspace_byte_limit(256); + rnn.Setup(Shape{4, 1, 2}, conf); + + + size_t weightSize = rnn.weightSize(); + float we[weightSize]; + for (size_t i = 0; i < weightSize; i++) + we[i] = 1.0f; + singa::Tensor weight(singa::Shape{weightSize, 1, 1}, cuda); + weight.CopyDataFromHostPtr(we, weightSize); + rnn.set_weight(weight); + + vector input_array; + input_array.push_back(in); + input_array.push_back(hx); + input_array.push_back(cx); + const auto ret = rnn.Forward(singa::kTrain, input_array); + // singa::CppCPU host(0, 1); + singa::Tensor out1 = ret[0]; + out1.ToHost(); + const float *outptr1 = out1.data(); + EXPECT_EQ(8u, out1.Size()); + EXPECT_NEAR(1.0f, outptr1[0], 0.0001); // tanh 6 + EXPECT_NEAR(1.0f, outptr1[1], 0.0001); + EXPECT_NEAR(1.0f, outptr1[2], 0.0001); + EXPECT_NEAR(1.0f, outptr1[3], 0.0001); + EXPECT_NEAR(1.0f, outptr1[4], 0.0001); + EXPECT_NEAR(1.0f, outptr1[5], 0.0001); + EXPECT_NEAR(1.0f, outptr1[6], 0.0001); + EXPECT_NEAR(1.0f, outptr1[7], 0.0001); + + singa::Tensor hy1 = ret[1]; + hy1.ToHost(); + const float *hyptr1 = hy1.data(); + EXPECT_EQ(2u, hy1.Size()); + EXPECT_NEAR(1.0f, hyptr1[0], 0.0001); + EXPECT_NEAR(1.0f, hyptr1[1], 0.0001); +} + +TEST(CudnnRNN, Backward) { + // src_data + auto cuda = std::make_shared(); + const size_t seqLength = 4, batchsize = 1, dim = 2; + const size_t numLayers = 1, hiddensize = 2, numDirections = 1; + const float x[seqLength * batchsize * dim] = {1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f}; + singa::Tensor in(singa::Shape{seqLength, batchsize, dim}, cuda); + in.CopyDataFromHostPtr(x, seqLength * batchsize * dim); + + const float hx_data[numLayers * batchsize * hiddensize * numDirections] = {1.0f, 1.0f}; + singa::Tensor hx(singa::Shape{numLayers, batchsize, hiddensize * numDirections}, cuda); + hx.CopyDataFromHostPtr(hx_data, numLayers * batchsize * hiddensize * numDirections); + + const float cx_data[numLayers * batchsize * hiddensize * numDirections] = {1.0f, 1.0f}; + singa::Tensor cx(singa::Shape{numLayers, batchsize, hiddensize * numDirections}, cuda); + cx.CopyDataFromHostPtr(cx_data, numLayers * batchsize * hiddensize * numDirections); + + CudnnRNN rnn; + + singa::LayerConf conf; + singa::RNNConf *rnnconf = conf.mutable_rnn_conf(); + rnnconf->set_hiddensize(2); + rnnconf->set_numlayers(1); + rnnconf->set_dropout(0); + rnnconf->set_inputmode("cudnn_linear_input"); + rnnconf->set_direction("cudnn_undirectional"); + rnnconf->set_mode("cudnn_rnn_tanh"); + // MB + rnnconf->set_workspace_byte_limit(256); + rnn.Setup(Shape{4, 1, 2}, conf); + + size_t weightSize = rnn.weightSize(); + float we[weightSize]; + for (size_t i = 0; i < weightSize; i++) + we[i] = 1.0f; + singa::Tensor weight(singa::Shape{weightSize, 1, 1}, cuda); + weight.CopyDataFromHostPtr(we, weightSize); + rnn.set_weight(weight); + + + vector input_array; + input_array.push_back(in); + input_array.push_back(hx); + input_array.push_back(cx); + const auto ret = rnn.Forward(singa::kTrain, input_array); + + // grad + const float dy[seqLength * batchsize * hiddensize * numDirections] = {1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f}; + singa::Tensor grad(singa::Shape{seqLength, batchsize, hiddensize * numDirections}, + cuda); + grad.CopyDataFromHostPtr(dy, seqLength * batchsize * hiddensize * numDirections); + + const float dhy_data[numLayers * batchsize * hiddensize * numDirections] = {1.0f, 1.0f}; + singa::Tensor dhy(singa::Shape{numLayers, batchsize, hiddensize * numDirections}, + cuda); + dhy.CopyDataFromHostPtr(dhy_data, numLayers * batchsize * hiddensize * numDirections); + + const float dcy_data[numLayers * batchsize * hiddensize * numDirections] = {1.0f, 1.0f}; + singa::Tensor dcy(singa::Shape{numLayers, batchsize, hiddensize * numDirections}, + cuda); + dcy.CopyDataFromHostPtr(dcy_data, numLayers * batchsize * hiddensize * numDirections); + + vector grad_array; + grad_array.push_back(grad); + grad_array.push_back(dhy); + grad_array.push_back(dcy); + const auto ret_back = rnn.Backward(singa::kTrain, grad_array); + // singa::CppCPU host(0, 1); + singa::Tensor in_grad = ret_back.first[0]; + in_grad.ToHost(); + const float *dx = in_grad.data(); + EXPECT_EQ(8u, in_grad.Size()); + EXPECT_NEAR(0.14, dx[0], 0.0001); + EXPECT_NEAR(0.14, dx[1], 0.0001); + EXPECT_NEAR(0.1596, dx[2], 0.0001); + EXPECT_NEAR(0.1596, dx[3], 0.0001); + EXPECT_NEAR(0.1623, dx[4], 0.0001); + EXPECT_NEAR(0.1623, dx[5], 0.0001); + EXPECT_NEAR(0.1627, dx[6], 0.0001); + EXPECT_NEAR(0.1627, dx[7], 0.0001); + + singa::Tensor dhx_grad = ret_back.first[1]; + dhx_grad.ToHost(); + const float *dhx = dhx_grad.data(); + EXPECT_EQ(2u, dhx_grad.Size()); + EXPECT_NEAR(0.1627, dhx[0], 0.0001); + EXPECT_NEAR(0.1627, dhx[1], 0.0001); +} +#endif // USE_CUDNN From f551092cf3f2018af47e9fe29205237dc24251e4 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Fri, 5 Aug 2016 16:43:40 +0800 Subject: [PATCH 07/16] SINGA-218 Implementation for RNN CUDNN version Finish the CudnnRNN layer. Pass test for tanh rnn. RNN forward accepts a vector of input tensors: x(i) is the i-th input tensor, hx is the init hidden tensor which could be a dummy tensor. A dummy tensor is a tensor created without shape/device/data_type, during compuation, cudnnRNN would use 0s for this tensor. cx is not necessary for relu/tanh/gru rnn. For lstm, it could also be a dummy tensor like hx. The output is: . relu/tanh/gru rnns does not have cy. lstm have both hy and cy. RNN backward accepts a vector of input gradient tensors: . dhy is necessry for all rnns, but could be a dummy tensor, in which case a tensor with 0s would be used for dhy during computation. dcy is used only for lstm, which could also be a dummy tensor. The output is: >, where dhx is a tensor for the gradient of hx. dcx is only used for lstm. The CudnnRNN must be moved onto cuda, otherwise memory error would happen (the weight is on cpu). --- CMakeLists.txt | 2 +- cmake/Thirdparty/FindCUDNN.cmake | 2 +- include/singa/core/common.h | 15 +- include/singa/core/device.h | 9 +- include/singa/model/layer.h | 9 - include/singa/utils/context.h | 291 -------------- src/CMakeLists.txt | 1 + src/core/memory/memory.cc | 2 +- src/core/tensor/tensor.cc | 26 +- src/core/tensor/tensor_math.h | 2 +- src/core/tensor/tensor_math_cpp.h | 2 +- src/core/tensor/tensor_math_cuda.h | 2 +- src/model/layer/cudnn_rnn.cc | 610 +++++++++++++++++------------ src/model/layer/cudnn_rnn.h | 44 ++- src/model/layer/rnn.cc | 59 ++- src/model/layer/rnn.h | 31 +- src/model/optimizer/adagrad.cc | 4 +- src/model/optimizer/nesterov.cc | 4 +- src/model/optimizer/rmsprop.cc | 1 + src/model/optimizer/sgd.cc | 4 +- src/proto/model.proto | 18 +- test/singa/test_cudnn_rnn.cc | 273 ++++++------- 22 files changed, 628 insertions(+), 783 deletions(-) delete mode 100644 include/singa/utils/context.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c6afad6e8..38014ce097 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.6) PROJECT(singa) -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -O2 ") +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -g") LIST(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Thirdparty) #message(STATUS "module path: ${CMAKE_MODULE_PATH}") diff --git a/cmake/Thirdparty/FindCUDNN.cmake b/cmake/Thirdparty/FindCUDNN.cmake index eefab9d25d..cefc4fe1d1 100644 --- a/cmake/Thirdparty/FindCUDNN.cmake +++ b/cmake/Thirdparty/FindCUDNN.cmake @@ -27,7 +27,7 @@ IF(CUDNN_FOUND) ELSE() SET(CUDNN_VERSION "${CUDNN_VERSION_MAJOR}.${CUDNN_VERSION_MINOR}.${CUDNN_VERSION_PATCH}") ENDIF() - MESSAGE(STATUS "Found Cudnn_v${CUDNN_VERSION} at ${CUDNN_INCLUDE_DIR}") + MESSAGE(STATUS "Found Cudnn_v${CUDNN_VERSION} at ${CUDNN_INCLUDE_DIR} ${CUDNN_LIBRARIES}") MARK_AS_ADVANCED(CUDNN_INCLUDE_DIR CUDNN_LIBRARIES) ENDIF() diff --git a/include/singa/core/common.h b/include/singa/core/common.h index 77210144e7..e7c7ea268f 100644 --- a/include/singa/core/common.h +++ b/include/singa/core/common.h @@ -65,11 +65,17 @@ class Block { // Disabled as it is not used currently. // Block(void* ptr, size_t size, size_t offset, std::shared_ptr> // ref) : data_(ptr), size_(size), offset_(offset), ref_count_(ref) {} - void* mutable_data() const { return static_cast(data_) + offset_; } - const void* data() const { return static_cast(data_) + offset_; } // TODO(wangwei) check if the set is correct and add lock if shared sturcture is allowed void set_data(void* ptr) { data_ = ptr; } + void* mutable_data() { + initialized_ = true; + return static_cast(data_) + offset_; + } + const void* data() const { + CHECK(initialized_) << "Must initialize data before reading it"; + return static_cast(data_) + offset_; + } size_t size() const { return size_; } size_t offset() const { return offset_; } int IncRefCount() { @@ -80,11 +86,16 @@ class Block { } int ref_count() const { return ref_count_.load(); } + bool initialized() const { + return initialized_; + } + private: Block() {} void* data_ = nullptr; size_t size_ = 0; size_t offset_ = 0; + bool initialized_ = false; // Disabled as it is not used currently. // std::shared_ptr> ref_count_ = nullptr; std::atomic ref_count_; diff --git a/include/singa/core/device.h b/include/singa/core/device.h index cd9a811907..778a13034e 100644 --- a/include/singa/core/device.h +++ b/include/singa/core/device.h @@ -100,7 +100,7 @@ class Device { return lang_; } - std::shared_ptr host() const { return host_;} + virtual std::shared_ptr host() const { return host_;} Context* context(int k) { return &ctx_; @@ -140,6 +140,9 @@ class Device { Context ctx_; }; +/// a singleton CppDevice as the host for all devices. +extern std::shared_ptr defaultDevice; + /// Represent a CPU device which may have multiple threads/executors. /// It runs cpp code. class CppCPU : public Device { @@ -147,6 +150,7 @@ class CppCPU : public Device { ~CppCPU() {}; CppCPU(); + std::shared_ptr host() const override { return defaultDevice;} void SetRandSeed(unsigned seed) override; protected: void DoExec(function&& fn, int executor) override; @@ -161,9 +165,6 @@ class CppCPU : public Device { void Free(void* ptr) override; }; -/// a singleton CppDevice as the host for all devices. -extern std::shared_ptr defaultDevice; - // Implement Device using OpenCL libs. // class OpenclDevice : public Device { }; diff --git a/include/singa/model/layer.h b/include/singa/model/layer.h index c35f9b84c6..d31bd955f4 100644 --- a/include/singa/model/layer.h +++ b/include/singa/model/layer.h @@ -158,12 +158,10 @@ class Layer { /// Move the layer (including its parameters and other internal Tensor) onto /// the given device virtual void ToDevice(std::shared_ptr device) { - //for (auto p : param_values_) p->ToDevice(device); } /// Set the data type of Tensor in this layer. virtual void AsType(DataType dtype) { - //for (auto p : param_values_) p->AsType(dtype); } /// Serialize the layer info (including params) into a LayerConf proto message @@ -202,12 +200,6 @@ class Layer { return vector{}; } - /// Return a pointer to the 'i'-th parameter Tensor. - Tensor param_value(size_t i) { - CHECK_LT(i, param_values_.size()); - return param_values().at(i); - } - /// Return names of all parmaeters. const vector param_names() { vector pname; @@ -227,7 +219,6 @@ class Layer { protected: std::string name_; - vector param_values_; vector param_specs_; }; diff --git a/include/singa/utils/context.h b/include/singa/utils/context.h deleted file mode 100644 index 6e897e8db2..0000000000 --- a/include/singa/utils/context.h +++ /dev/null @@ -1,291 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#ifndef SINGA_UTILS_CONTEXT_H_ -#define SINGA_UTILS_CONTEXT_H_ - -#include -#include -#include -#include -#include - -#include "singa/utils/logging.h" - -#ifdef USE_GPU -#include -#include -#include -#include -// CUDA: various checks for different function calls. -#define CUDA_CHECK(condition) \ -/* Code block avoids redefinition of cudaError_t error */ \ -do { \ -cudaError_t error = condition; \ -CHECK_EQ(error, cudaSuccess) << " " << cudaGetErrorString(error); \ -} while (0) - -#ifdef USE_CUDNN -#include -#endif - -#endif // USE_GPU - -namespace singa { - -/** - * Context is used as a global singleton, which stores the mapping from CPU - * thread id to GPU device id. If a thread has no GPU, then its associated - * device id is -1. It manages (e.g., creating) the handlers for GPU - * devices. It also manages the GPU and CPU random generators, which are created - * when accessed. One CPU thread has a CPU random generator. A GPU device - * has a GPU random generator, which is accessible after assigning the GPU - * device with a CPU thread via SetupDevice. - */ -class Context { - public: - /** - * Destructor, release random generators and handlers. - */ - ~Context() { -#ifdef USE_GPU - for (auto& entry : device_id_) { - if (entry.second != -1) { - cudaSetDevice(entry.second); - if (cublas_handle_[entry.second] != nullptr) { - cublasDestroy(cublas_handle_[entry.second]); - cublas_handle_[entry.second] = nullptr; - } - if (curand_generator_[entry.second] != nullptr) { - curandDestroyGenerator(curand_generator_[entry.second]); - curand_generator_[entry.second] = nullptr; - } - } - } -#ifdef USE_CUDNN - for (auto& handle : cudnn_handle_) { - if (handle != nullptr) - CHECK_EQ(cudnnDestroy(handle), CUDNN_STATUS_SUCCESS); - handle = nullptr; - } -#endif -#endif - for (auto& entry : rand_generator_) { - if (entry.second != nullptr) { - delete entry.second; - entry.second = nullptr; - } - } - } - /** - * Constructor, init handlers and GPU rand generators to nullptr. - */ - Context() { - for (int i = 0; i < kMaxNumGPU; i++) { -#ifdef USE_GPU - cublas_handle_.push_back(nullptr); - curand_generator_.push_back(nullptr); -#ifdef USE_CUDNN - cudnn_handle_.push_back(nullptr); -#endif -#endif - } - } - - /** - * @return the device ID of the current thread. - */ - int device_id() { - return device_id(std::this_thread::get_id()); - } - /** - * @return the ID of the device attached to a given CPU thread, or -1 if this - * thread has not been attached GPU device. - */ - int device_id(const std::thread::id& tid) { - if (device_id_.find(tid) != device_id_.end()) - return device_id_[tid]; - else - return -2; - } - /** - * Setup the CPU thread, which may be assigned a GPU device. - * If there is no GPU device, then set did to -1. - * Set the random seed to -1. - * @param[in] thread::id CPU thread ID - * @param[in] device_id GPU device ID - */ - void SetupDevice(const std::thread::id& tid, const int did) { - SetupDevice(tid, did, -1); - } - /** - * @copy SetupDevice(const int, const int); - * @param[in] seed random seed - */ - void SetupDevice(const std::thread::id& tid, const int did, const int seed) { - device_id_[tid] = did; - seed_[tid] = seed; - } - - /** - * Activate the GPU device by calling cudaSetDevice. - */ - void ActivateDevice(const int device_id) { - CHECK_GE(device_id, 0); -#ifdef USE_GPU - cudaSetDevice(device_id); -#endif - } - - /** - * \copybreif rand_generator(const std::thread::id&); - * @return the CPU random generator for the calling thread. - */ - std::mt19937* rand_generator() { - return rand_generator(std::this_thread::get_id()); - } - /** - * Get the CPU random generator. - * If the generator does not exist, then create it now. - * If the seed is not set, i.e., seed=-1, then get a seed from system time. - * @param[in] thread::id CPU thread ID - * @return the CPU random generator - */ - std::mt19937* rand_generator(const std::thread::id& tid) { - if (rand_generator_.find(tid) == rand_generator_.end()) { - // CHECK(seed_.find(tid) != seed_.end()); - auto seed = static_cast(seed_[tid]); - if (seed_.find(tid) == seed_.end() || seed_.at(tid) == -1) - seed = std::chrono::system_clock::now().time_since_epoch().count(); - rand_generator_[tid] = new std::mt19937(seed); - } - return rand_generator_[tid]; - } -#ifdef USE_GPU - /** - * \copybreif cublas_handle_(const std::thread::id&); - * @return cublas handle for the calling thread. - */ - cublasHandle_t cublas_handle() { - return cublas_handle(std::this_thread::get_id()); - } - /** - * Get the handler of the GPU which is assigned to the given thread. - * Calls cublas_handle(const int); - */ - cublasHandle_t cublas_handle(const std::thread::id thread_id) { - return cublas_handle(device_id(thread_id)); - } - /** - * Get the handler of the GPU device given its device ID. The device - * must be set up via SetupDevice(const std::thread::id, const int) before - * calling this function. - * @param[in] device_id GPU device ID - * @return the GPU handler - */ - cublasHandle_t cublas_handle(const int device_id) { - CHECK_GE(device_id, 0); - if (cublas_handle_.at(device_id) == nullptr) { - cudaSetDevice(device_id); - cublasCreate(&cublas_handle_[device_id]); - } - return cublas_handle_[device_id]; - } - /** - * Get the rand generator of the GPU device assigned to the given thread. - */ - curandGenerator_t curand_generator(const std::thread::id thread_id) { - return curand_generator(device_id(thread_id)); - } - /** - * Get the random generator of the GPU device given the device id. - * @param[in] device_id GPU device ID - * @return random generator. If it does not exist, then create one. - * The random seed will be set to CURAND_RNG_PSEUDO_DEFAULT if it is not set. - */ - curandGenerator_t curand_generator(const int device_id) { - CHECK_GE(device_id, 0); - CHECK_LT(device_id, cudnn_handle_.size()); - if (curand_generator_.at(device_id) == nullptr) { - // TODO(wangwei) handle user set seed - /* - CHECK(seed_.find(tid) != seed_.end()); - auto seed = seed_[tid]; - */ - ActivateDevice(device_id); - curandCreateGenerator(&curand_generator_[device_id], - CURAND_RNG_PSEUDO_DEFAULT); - } - return curand_generator_[device_id]; - } - -#ifdef USE_CUDNN - cudnnHandle_t cudnn_handle() { - return cudnn_handle(std::this_thread::get_id()); - } - - cudnnHandle_t cudnn_handle(const std::thread::id thread_id) { - return cudnn_handle(device_id(thread_id)); - } - - cudnnHandle_t cudnn_handle(const int device_id) { - CHECK_GE(device_id, 0); - CHECK_LT(device_id, cudnn_handle_.size()); - } -#endif // USE_CUDNN - - protected: - //!< max num of GPUs per process - const int kMaxNumGPU = 64; - //!< map from thread id to device id - std::unordered_map device_id_; - //!< map from thread id to cpu rand generator - std::unordered_map rand_generator_; - //!< map from thread id to cpu rand generator seed - std::unordered_map seed_; -#ifdef USE_GPU - //!< cublas handler indexed by GPU device ID - std::vector cublas_handle_; - //!< cublas rand generator indexed by GPU device ID - std::vector curand_generator_; - -#ifdef USE_CUDNN - std::vector cudnn_handle_; -#endif -#endif // USE_GPU -}; - -} // namespace singa - -#endif // SINGA_UTILS_CONTEXT_H_ - if (cudnn_handle_.at(device_id) == nullptr) { - ActivateDevice(device_id); - // LOG(ERROR) << "create cudnn handle for device " << device_id; - CHECK_EQ(cudnnCreate(&cudnn_handle_[device_id]), CUDNN_STATUS_SUCCESS); - } - // LOG(ERROR) << "use cudnn handle from device " << device_id; - return cudnn_handle_[device_id]; - } -#endif - -#endif // USE_GPU - -#ifdef USE_OPENCL diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 65a81fc190..38e6aa36d1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -64,6 +64,7 @@ AUX_SOURCE_DIRECTORY(model/metric model_source) AUX_SOURCE_DIRECTORY(model/updater model_source) #MESSAGE(STATUS "MODEL ${model_source}") ADD_LIBRARY(singa_model SHARED ${model_source}) +MESSAGE(STATUS "model linker libs ${SINGA_LINKER_LIBS}") TARGET_LINK_LIBRARIES(singa_model ${SINGA_LINKER_LIBS}) LIST(APPEND SINGA_LINKER_LIBS singa_model) diff --git a/src/core/memory/memory.cc b/src/core/memory/memory.cc index 943574f628..6932b7aa33 100644 --- a/src/core/memory/memory.cc +++ b/src/core/memory/memory.cc @@ -71,7 +71,7 @@ void CppMemPool::RsetMemPool(size_t init_size_mb, size_t uint_size_kb) { struct _Uint* pCurUint = pAllocatedMemUint; for(size_t idx = 0; idx < numAllocatedUintsInPool; idx++) { Block* pOldBlk = pCurUint->pBlk; - const void* pData = pOldBlk->data(); + void* pData = pOldBlk->mutable_data(); pNewPool->Malloc(&pOldBlk, pOldBlk->size(), false); size_t copySize = pOldBlk->size() - pOldBlk->offset(); memcpy(pOldBlk->mutable_data(),pData,copySize); diff --git a/src/core/tensor/tensor.cc b/src/core/tensor/tensor.cc index a580bd2834..846bda715c 100644 --- a/src/core/tensor/tensor.cc +++ b/src/core/tensor/tensor.cc @@ -34,22 +34,30 @@ Tensor::Tensor() { device_ = defaultDevice; } Tensor::Tensor(const Shape &shape, DataType dtype) : data_type_(dtype), device_(defaultDevice), shape_(shape) { - //device_ = defaultDevice; - block_ = device_->NewBlock(Product(shape_) * SizeOf(data_type_)); + device_ = defaultDevice; + size_t size = Product(shape_) * SizeOf(data_type_); + if (size) + block_ = device_->NewBlock(size); } Tensor::Tensor(Shape &&shape, DataType dtype) : data_type_(dtype), device_(defaultDevice), shape_(shape) { - //device_ = defaultDevice; - block_ = device_->NewBlock(Product(shape_) * SizeOf(data_type_)); + device_ = defaultDevice; + size_t size = Product(shape_) * SizeOf(data_type_); + if (size) + block_ = device_->NewBlock(size); } Tensor::Tensor(const Shape &shape, std::shared_ptr device, DataType dtype) : data_type_(dtype), device_(device), shape_(shape) { - block_ = device_->NewBlock(Product(shape_) * SizeOf(data_type_)); + size_t size = Product(shape_) * SizeOf(data_type_); + if (size) + block_ = device_->NewBlock(size); } Tensor::Tensor(Shape &&shape, std::shared_ptr device, DataType dtype) : data_type_(dtype), device_(device), shape_(shape) { - block_ = device_->NewBlock(Product(shape_) * SizeOf(data_type_)); + size_t size = Product(shape_) * SizeOf(data_type_); + if (size) + block_ = device_->NewBlock(size); } Tensor::Tensor(const Tensor &in) : transpose_(in.transpose_), @@ -57,7 +65,8 @@ Tensor::Tensor(const Tensor &in) device_(in.device_), block_(in.block()), shape_(in.shape_) { - block_->IncRefCount(); + if (block_ != nullptr) + block_->IncRefCount(); } Tensor::Tensor(Tensor &&in) @@ -118,7 +127,8 @@ void Tensor::ToDevice(std::shared_ptr dst) { // TODO(wangwei) the comparison is very strict. May compare against device ID? if (device_ != dst) { Tensor tmp(shape_, dst, data_type_); - if (block_ != nullptr && Size()) tmp.CopyData(*this); + if (block_ != nullptr && Size() && block_->initialized()) + tmp.CopyData(*this); if (block_ != nullptr && block_->DecRefCount() == 0) device_->FreeBlock(block_); block_ = tmp.block_; diff --git a/src/core/tensor/tensor_math.h b/src/core/tensor/tensor_math.h index 7732dd29ae..1914ca6148 100644 --- a/src/core/tensor/tensor_math.h +++ b/src/core/tensor/tensor_math.h @@ -341,7 +341,7 @@ void SoftmaxCrossEntropyBwd(const size_t batchsize, const size_t dim, template void RowMax(const size_t nrow, const size_t ncol, const Block *in, - const Block *ret, Context* ctx) { + Block *ret, Context* ctx) { LOG(FATAL) << "Not Implemented"; } // ************************************** diff --git a/src/core/tensor/tensor_math_cpp.h b/src/core/tensor/tensor_math_cpp.h index 3e0c8ad4ae..941931d19f 100644 --- a/src/core/tensor/tensor_math_cpp.h +++ b/src/core/tensor/tensor_math_cpp.h @@ -551,7 +551,7 @@ void SoftmaxCrossEntropyBwd(const size_t batchsize, template <> void RowMax(const size_t nrow, const size_t ncol, - const Block *in, const Block *out, Context *ctx) { + const Block *in, Block *out, Context *ctx) { const float *inPtr = static_cast(in->data()); float *outPtr = static_cast(out->mutable_data()); for (size_t r = 0; r < nrow; r++) { diff --git a/src/core/tensor/tensor_math_cuda.h b/src/core/tensor/tensor_math_cuda.h index 43bfa1b42b..8b6e939db6 100644 --- a/src/core/tensor/tensor_math_cuda.h +++ b/src/core/tensor/tensor_math_cuda.h @@ -424,7 +424,7 @@ void SoftmaxCrossEntropyBwd(const size_t batchsize, template <> void RowMax(const size_t nrow, const size_t ncol, - const Block* in, const Block* out, + const Block* in, Block* out, Context* ctx) { const float* inPtr = static_cast(in->data()); float* outPtr = static_cast(out->mutable_data()); diff --git a/src/model/layer/cudnn_rnn.cc b/src/model/layer/cudnn_rnn.cc index 6f04e5ca70..242a342e92 100644 --- a/src/model/layer/cudnn_rnn.cc +++ b/src/model/layer/cudnn_rnn.cc @@ -30,12 +30,6 @@ CudnnRNN::~CudnnRNN() { CUDNN_CHECK(cudnnDestroyDropoutDescriptor(dropout_desc_)); if (rnn_desc_ != nullptr) CUDNN_CHECK(cudnnDestroyRNNDescriptor(rnn_desc_)); - if (x_descs_ != nullptr) - for (size_t i = 0; i < seqLength_; i++) - CUDNN_CHECK(cudnnDestroyTensorDescriptor(x_descs_[i])); - if (y_descs_ != nullptr) - for (size_t i = 0; i < seqLength_; i++) - CUDNN_CHECK(cudnnDestroyTensorDescriptor(y_descs_[i])); if (hx_desc_ != nullptr) CUDNN_CHECK(cudnnDestroyTensorDescriptor(hx_desc_)); if (hy_desc_ != nullptr) @@ -44,284 +38,392 @@ CudnnRNN::~CudnnRNN() { CUDNN_CHECK(cudnnDestroyTensorDescriptor(cx_desc_)); if (cy_desc_ != nullptr) CUDNN_CHECK(cudnnDestroyTensorDescriptor(cy_desc_)); -} - -void CudnnRNN::Setup(const Shape& in_sample, const LayerConf &conf) { - RNN::Setup(in_sample, conf); - RNNConf rnn_conf = conf.rnn_conf(); - // convert MB to bytes - workspace_byte_limit_ = rnn_conf.workspace_byte_limit() << 20; - inputMode_ = ToLowerCase(rnn_conf.inputmode()); - direction_ = ToLowerCase(rnn_conf.direction()); - mode_ = ToLowerCase(rnn_conf.mode()); - CHECK(inputMode_ == "cudnn_linear_input" || inputMode_ == "cudnn_skip_input") - << "CudnnRNN only supports two inputmodes: cudnn_linear_input, " - "cudnn_skip_input"; - CHECK(direction_ == "cudnn_undirectional" || direction_ == "cudnn_bidirectional") - << "CudnnRNN only supports two directions: cudnn_undirectional, " - "cudnn_bidirectional"; - CHECK(mode_ == "cudnn_rnn_relu" || mode_ == "cudnn_rnn_tanh" || - mode_ == "cudnn_lstm" || mode_ == "cudnn_gru") - << "CudnnRNN only supports four modes: cudnn_rnn_relu, " - "cudnn_rnn_tanh, cudnn_lstm and cudnn_gru"; - // the first constant (4) is the size of float - // the second constant (2, 8, 6) is the number of sets of params - if (mode_ == "cudnn_rnn_relu" || mode_ == "cudnn_rnn_tanh") - weightSize_ = 4 * 2 * (hiddenSize_ * in_sample[2] + hiddenSize_); - else if (mode_ == "cudnn_lstm") - weightSize_ = 4 * 8 * (hiddenSize_ * in_sample[2] + hiddenSize_); - else if (mode_ == "cudnn_gru") - weightSize_ = 4 * 6 * (hiddenSize_ * in_sample[2] + hiddenSize_); - if (direction_ == "cudnn_bidirectional") - weightSize_ = weightSize_ * 2; + if (dhx_desc_ != nullptr) + CUDNN_CHECK(cudnnDestroyTensorDescriptor(dhx_desc_)); + if (dhy_desc_ != nullptr) + CUDNN_CHECK(cudnnDestroyTensorDescriptor(dhy_desc_)); + if (dcx_desc_ != nullptr) + CUDNN_CHECK(cudnnDestroyTensorDescriptor(dcx_desc_)); + if (dcy_desc_ != nullptr) + CUDNN_CHECK(cudnnDestroyTensorDescriptor(dcy_desc_)); + DestroyIODescriptors(); } void CudnnRNN::ToDevice(std::shared_ptr device) { - weight_.ToDevice(device); + RNN::ToDevice(device); workspace_.ToDevice(device); + reserve_space_.ToDevice(device); } -void CudnnRNN::InitCudnn(const Tensor &input) { - CHECK(!has_init_cudnn_); - DataType dtype = input.data_type(); - auto dev = input.device(); - Context *ctx = dev->context(0); - seqLength_ = input.shape(0); - size_t batchsize = input.shape(1); /*(seqLength, minibatch, inputSize) !!! */ - size_t inputSize = input.shape(2); - size_t numDirections; - if (direction_ == "cudnn_undirectional") - numDirections = 1; - else - numDirections = 2; - x_descs_ = new cudnnTensorDescriptor_t[seqLength_]; - y_descs_ = new cudnnTensorDescriptor_t[seqLength_]; - for (size_t i = 0; i < seqLength_; i++) - CUDNN_CHECK(cudnnCreateTensorDescriptor(&x_descs_[i])); - for (size_t i = 0; i < seqLength_; i++) - CUDNN_CHECK(cudnnCreateTensorDescriptor(&y_descs_[i])); - CUDNN_CHECK(cudnnCreateTensorDescriptor(&hx_desc_)); - CUDNN_CHECK(cudnnCreateTensorDescriptor(&cx_desc_)); - CUDNN_CHECK(cudnnCreateTensorDescriptor(&hy_desc_)); - CUDNN_CHECK(cudnnCreateTensorDescriptor(&cy_desc_)); - CUDNN_CHECK(cudnnCreateFilterDescriptor(&weight_desc_)); +void CudnnRNN::DestroyIODescriptors() { + if (x_descs_ != nullptr) { + for (size_t i = 0; i < seq_length_; i++) { + CUDNN_CHECK(cudnnDestroyTensorDescriptor(x_descs_[i])); + CUDNN_CHECK(cudnnDestroyTensorDescriptor(dx_descs_[i])); + } + delete [] x_descs_; + delete [] dx_descs_; + } + if (y_descs_ != nullptr) { + for (size_t i = 0; i < seq_length_; i++) { + CUDNN_CHECK(cudnnDestroyTensorDescriptor(y_descs_[i])); + CUDNN_CHECK(cudnnDestroyTensorDescriptor(dy_descs_[i])); + } + delete [] y_descs_; + delete [] dy_descs_; + } +} + +void CudnnRNN::UpdateIODescriptors(size_t len, const vector &inputs) { + bool reset = false; + if (seq_length_ < len) { + DestroyIODescriptors(); + x_descs_ = new cudnnTensorDescriptor_t[len]; + dx_descs_ = new cudnnTensorDescriptor_t[len]; + y_descs_ = new cudnnTensorDescriptor_t[len]; + dy_descs_ = new cudnnTensorDescriptor_t[len]; + for (size_t i = 0; i < len; i++) { + CUDNN_CHECK(cudnnCreateTensorDescriptor(&x_descs_[i])); + CUDNN_CHECK(cudnnCreateTensorDescriptor(&dx_descs_[i])); + CUDNN_CHECK(cudnnCreateTensorDescriptor(&y_descs_[i])); + CUDNN_CHECK(cudnnCreateTensorDescriptor(&dy_descs_[i])); + } + reset = true; + } + + for (size_t i = 0; i < len; i++) { + CHECK_EQ(inputs[i].shape(1), input_dim_); + if (inputs[i].shape(0) != batch_size_ || reset) { + int d[3] = {1, 1, 1}, s[3] = {1, 1, 1}; + d[0] = static_cast(inputs[i].shape(0)); + CHECK_GT(d[0], 0); + d[1] = static_cast(inputs[i].shape(1)); + s[0] = d[1] * d[2]; + s[1] = d[2]; + CUDNN_CHECK(cudnnSetTensorNdDescriptor(x_descs_[i], dtype_, 3, d, s)); + CUDNN_CHECK(cudnnSetTensorNdDescriptor(dx_descs_[i], dtype_, 3, d, s)); + + d[0] = static_cast(inputs[i].shape(0)); + d[1] = static_cast(hidden_dim_ * num_directions_); + s[0] = d[1] * d[2]; + s[1] = d[2]; + CUDNN_CHECK(cudnnSetTensorNdDescriptor(y_descs_[i], dtype_, 3, d, s)); + CUDNN_CHECK(cudnnSetTensorNdDescriptor(dy_descs_[i], dtype_, 3, d, s)); + } + } +} + +// must be called after setting IO descriptors +void CudnnRNN::SetRNNDescriptor(shared_ptr dev) { + auto ctx = dev->context(0); CUDNN_CHECK(cudnnCreateDropoutDescriptor(&dropout_desc_)); + size_t state_size; + CUDNN_CHECK(cudnnDropoutGetStatesSize(ctx->cudnn_handle, &state_size)); + dropout_state_ = Tensor(Shape{state_size}, dev, kChar); + CUDNN_CHECK(cudnnSetDropoutDescriptor( + dropout_desc_, ctx->cudnn_handle, dropout_, + dropout_state_.block()->mutable_data(), state_size, seed_)); + CUDNN_CHECK(cudnnCreateRNNDescriptor(&rnn_desc_)); + cudnnRNNInputMode_t input_mode; + if (input_mode_ == "linear") + input_mode = CUDNN_LINEAR_INPUT; + else if (input_mode_ == "skip") + input_mode = CUDNN_SKIP_INPUT; + cudnnDirectionMode_t direction; + if (direction_ == "unidirectional") + direction = CUDNN_UNIDIRECTIONAL; + else if (direction_ == "bidirectional") + direction = CUDNN_BIDIRECTIONAL; + + cudnnRNNMode_t rnn_mode; + if (rnn_mode_ == "relu") + rnn_mode = CUDNN_RNN_RELU; + else if (rnn_mode_ == "tanh") + rnn_mode = CUDNN_RNN_TANH; + else if (rnn_mode_ == "lstm") + rnn_mode = CUDNN_LSTM; + else if (rnn_mode_ == "gru") + rnn_mode = CUDNN_GRU; + CUDNN_CHECK(cudnnSetRNNDescriptor(rnn_desc_, hidden_dim_, num_stacks_, + dropout_desc_, input_mode, direction, + rnn_mode, dtype_)); + + size_t weight_size; + CUDNN_CHECK(cudnnGetRNNParamsSize(ctx->cudnn_handle, rnn_desc_, x_descs_[0], + &weight_size, dtype_)); + // check the size manually calculated + CHECK_EQ(weight_size, weight_.Size() * sizeof(float)); + int filter_dim[3] = {static_cast(weight_size), 1, 1}; + CUDNN_CHECK(cudnnCreateFilterDescriptor(&weight_desc_)); + CUDNN_CHECK(cudnnSetFilterNdDescriptor(weight_desc_, dtype_, + CUDNN_TENSOR_NCHW, 3, filter_dim)); +} - int dimA[3] = {batchsize, inputSize, 1}; - int strideA[3] = {dimA[2] * dimA[1], dimA[2], 1}; - for (size_t i = 0; i < seqLength_; i++){ - dimA[0] = batchsize; - dimA[1] = inputSize; - dimA[2] = 1; - strideA[0] = dimA[2] * dimA[1]; - strideA[1] = dimA[2]; - strideA[2] = 1; - CUDNN_CHECK(cudnnSetTensorNdDescriptor(x_descs_[i], GetCudnnDataType(dtype), 3, - dimA, strideA)); - dimA[0] = batchsize; - dimA[1] = hiddenSize_ * numDirections; - dimA[2] = 1; - strideA[0] = dimA[2] * dimA[1]; - strideA[1] = dimA[2]; - strideA[2] = 1; - CUDNN_CHECK(cudnnSetTensorNdDescriptor(y_descs_[i], GetCudnnDataType(dtype), 3, - dimA, strideA)); +void CudnnRNN::ResetHiddenAndCellDescriptors(size_t batch_size) { + if (batch_size_ == 0) { + CUDNN_CHECK(cudnnCreateTensorDescriptor(&cx_desc_)); + CUDNN_CHECK(cudnnCreateTensorDescriptor(&dcx_desc_)); + CUDNN_CHECK(cudnnCreateTensorDescriptor(&cy_desc_)); + CUDNN_CHECK(cudnnCreateTensorDescriptor(&dcy_desc_)); + CUDNN_CHECK(cudnnCreateTensorDescriptor(&hx_desc_)); + CUDNN_CHECK(cudnnCreateTensorDescriptor(&dhx_desc_)); + CUDNN_CHECK(cudnnCreateTensorDescriptor(&hy_desc_)); + CUDNN_CHECK(cudnnCreateTensorDescriptor(&dhy_desc_)); } - - dimA[0] = numLayers_; - dimA[1] = batchsize; - dimA[2] = hiddenSize_ * numDirections; - strideA[0] = dimA[2] * dimA[1]; - strideA[1] = dimA[2]; - strideA[2] = 1; - CUDNN_CHECK(cudnnSetTensorNdDescriptor(hx_desc_, GetCudnnDataType(dtype), 3, - dimA, strideA)); - CUDNN_CHECK(cudnnSetTensorNdDescriptor(cx_desc_, GetCudnnDataType(dtype), 3, - dimA, strideA)); - CUDNN_CHECK(cudnnSetTensorNdDescriptor(hy_desc_, GetCudnnDataType(dtype), 3, - dimA, strideA)); - CUDNN_CHECK(cudnnSetTensorNdDescriptor(cy_desc_, GetCudnnDataType(dtype), 3, - dimA, strideA)); - size_t dropoutStatesSize; - CUDNN_CHECK(cudnnDropoutGetStatesSize(ctx->cudnn_handle, &dropoutStatesSize)); - dropoutStates_ = Tensor(Shape{dropoutStatesSize}, dev, dtype); - CUDNN_CHECK(cudnnSetDropoutDescriptor(dropout_desc_, ctx->cudnn_handle, dropout_, this->dropoutStates_.block()->mutable_data(), dropoutStatesSize, 0x01234567)); - - cudnnRNNInputMode_t inputMode; - cudnnDirectionMode_t direction; - cudnnRNNMode_t mode; - - if (inputMode_ == "cudnn_linear_input" || inputMode_ == "cudnn_skip_input"){ - if (inputMode_ == "cudnn_linear_input") - inputMode = CUDNN_LINEAR_INPUT; - else if (inputMode_ == "cudnn_skip_input") - inputMode = CUDNN_SKIP_INPUT; + int dim[3] = {1, 1, 1}; + dim[0] = static_cast(num_stacks_ * num_directions_); + dim[1] = static_cast(batch_size); + dim[2] = static_cast(hidden_dim_); + int stride[3] = {1, 1, 1}; + stride[0] = dim[1] * dim[2]; + stride[1] = dim[2]; + CUDNN_CHECK(cudnnSetTensorNdDescriptor(hx_desc_, dtype_, 3, dim, stride)); + CUDNN_CHECK(cudnnSetTensorNdDescriptor(dhx_desc_, dtype_, 3, dim, stride)); + CUDNN_CHECK(cudnnSetTensorNdDescriptor(hy_desc_, dtype_, 3, dim, stride)); + CUDNN_CHECK(cudnnSetTensorNdDescriptor(dhy_desc_, dtype_, 3, dim, stride)); + CUDNN_CHECK(cudnnSetTensorNdDescriptor(cx_desc_, dtype_, 3, dim, stride)); + CUDNN_CHECK(cudnnSetTensorNdDescriptor(dcx_desc_, dtype_, 3, dim, stride)); + CUDNN_CHECK(cudnnSetTensorNdDescriptor(cy_desc_, dtype_, 3, dim, stride)); + CUDNN_CHECK(cudnnSetTensorNdDescriptor(dcy_desc_, dtype_, 3, dim, stride)); +} + +void CudnnRNN::UpdateSpaces(size_t seq_length, shared_ptr dev) { + size_t count; + auto ctx = dev->context(0); + CUDNN_CHECK(cudnnGetRNNWorkspaceSize(ctx->cudnn_handle, rnn_desc_, + seq_length, x_descs_, &count)); + if (workspace_.Size() != count) { + workspace_ = Tensor(Shape{count}, dev, kChar); + // workspace_.SetValue(0); } - if (direction_ == "cudnn_undirectional" || direction_ == "cudnn_bidirectional"){ - if (direction_ == "cudnn_undirectional") - direction = CUDNN_UNIDIRECTIONAL; - else if (direction_ == "cudnn_bidirectional") - direction = CUDNN_BIDIRECTIONAL; + + CUDNN_CHECK(cudnnGetRNNTrainingReserveSize(ctx->cudnn_handle, rnn_desc_, + seq_length, x_descs_, &count)); + if (reserve_space_.Size() != count) { + reserve_space_ = Tensor(Shape{count}, dev, kChar); + // reserve_space_.SetValue(0); } - if (mode_ == "cudnn_rnn_relu" || mode_ == "cudnn_rnn_tanh" || - mode_ == "cudnn_lstm" || mode_ == "cudnn_gru"){ - if (mode_ == "cudnn_rnn_relu") - mode = CUDNN_RNN_RELU; - else if (mode_ == "cudnn_rnn_tanh") - mode = CUDNN_RNN_TANH; - else if (mode_ == "cudnn_lstm") - mode = CUDNN_LSTM; - else if (mode_ == "cudnn_gru") - mode = CUDNN_GRU; +} + +void CudnnRNN::UpdateStates(size_t num_x, const vector &inputs) { + UpdateIODescriptors(num_x, inputs); + size_t new_batch_size = inputs.at(0).shape(0); + if (batch_size_ != new_batch_size) + ResetHiddenAndCellDescriptors(new_batch_size); + if (rnn_desc_ == nullptr) + SetRNNDescriptor(inputs.at(0).device()); + UpdateSpaces(num_x, inputs.at(0).device()); + batch_size_ = new_batch_size; + seq_length_ = num_x; +} + +Tensor CudnnRNN::MergeInputs(size_t num, const vector &in) { + if (num == 1) + return in.at(0); + size_t size = 0; + for (size_t i = 0; i < num; i++) size += in.at(i).Size(); + Tensor out(Shape{size}, in.at(0).device(), in.at(0).data_type()); + for (size_t i = 0, offset = 0; i < num; i++) { + CopyDataToFrom(&out, in.at(i), in.at(i).Size(), offset); + offset += in.at(i).Size(); + } + return out; +} + +vector CudnnRNN::SplitOutput(size_t num, size_t dim, + const vector &in, + const Tensor output) { + vector outputs; + if (num == 1) { + outputs.push_back(output); + } else { + for (size_t i = 0, offset = 0; offset < output.Size(); i++) { + Shape s{in.at(i).shape(0), dim}; + Tensor out(s, output.device(), output.data_type()); + CopyDataToFrom(&out, output, out.Size(), 0, offset); + outputs.push_back(out); + offset += out.Size(); + } + CHECK_EQ(num, outputs.size()); } - CUDNN_CHECK(cudnnSetRNNDescriptor(rnn_desc_, hiddenSize_, numLayers_, dropout_desc_, inputMode, direction, mode, GetCudnnDataType(dtype))); + return outputs; +} - size_t weightSize; - CUDNN_CHECK(cudnnGetRNNParamsSize(ctx->cudnn_handle, rnn_desc_, x_descs_[0], &weightSize, GetCudnnDataType(dtype))); - CHECK_EQ(weightSize, weightSize_); +const vector CudnnRNN::Forward(int flag, const vector &inputs) { + DataType dtype = inputs.at(0).data_type(); + auto dev = inputs.at(0).device(); - int filterDimA[3] = {weightSize_, 1, 1}; - CUDNN_CHECK(cudnnSetFilterNdDescriptor(weight_desc_, GetCudnnDataType(dtype), CUDNN_TENSOR_NCHW, 3, filterDimA)); + // copy input data into a block of contiguous memory + // hx (and cx) is at the end of inputs + CHECK_GT(inputs.size(), 1u + has_cell_); + size_t num_x = inputs.size() - has_cell_ - 1; + Tensor input = MergeInputs(num_x, inputs); + LOG(INFO) << "input size " << input.Size() << " value " << input.L1(); - - CUDNN_CHECK(cudnnGetRNNWorkspaceSize(ctx->cudnn_handle, rnn_desc_, seqLength_, x_descs_, &workspace_count_)); - workspace_ = Tensor(Shape{workspace_count_}, dev, dtype); + if (rnn_desc_ != nullptr) + CHECK_EQ(dtype_, GetCudnnDataType(dtype)) + << "Cannot change cudnn data type during training from " << dtype_ + << " to " << GetCudnnDataType(dtype); + else + dtype_ = GetCudnnDataType(dtype); - CUDNN_CHECK(cudnnGetRNNTrainingReserveSize(ctx->cudnn_handle, rnn_desc_, seqLength_, x_descs_, &ReserveSize_)); - reserve_ = Tensor(Shape{ReserveSize_}, dev, dtype); - has_init_cudnn_ = true; -} + UpdateStates(num_x, inputs); + // CheckFowardShapes(); -const vector CudnnRNN::Forward(int flag, const vector& inputs) { - /*(seqLength, minibatch, inputSize) !!! */ - singa::Tensor input = inputs[0]; - singa::Tensor hx = inputs[1]; - singa:: Tensor cx = inputs[2]; - CHECK_EQ(input.device()->lang(), kCuda); - CHECK_EQ(input.device()->lang(), this->weight_.device()->lang()); - CHECK_EQ(input.nDim(), 3u); - vector data_output; - if (flag & kTrain) buf_.push(input); // buffer the input for backward - size_t batchsize = input.shape(1); /*(seqLength, minibatch, inputSize) !!! */ - DataType dtype = input.data_type(); - auto dev = input.device(); - - if (!has_init_cudnn_) InitCudnn(input); - - - size_t numDirections; - if (direction_ == "cudnn_undirectional") - numDirections = 1; - else - numDirections = 2; - - Shape shape{seqLength_, batchsize, hiddenSize_ * numDirections}; - Tensor output(shape, dev, dtype); - Shape shape1{numLayers_, batchsize, hiddenSize_ * numDirections}; - Tensor hy(shape1, dev, dtype); - Tensor cy(shape1, dev, dtype); - - output.device()->Exec([input, output, hx, hy, cx, cy, this](Context *ctx) { - Block *inblock = input.block(), *outblock = output.block(), - *wblock = this->weight_.block(), *hxblock = hx.block(), - *hyblock = hy.block(), *cxblock = cx.block(), - *cyblock = cy.block(); - cudnnRNNForwardTraining( - ctx->cudnn_handle, this->rnn_desc_, seqLength_, this->x_descs_, - inblock->data(), this->hx_desc_, hxblock->data(), this->cx_desc_, - cxblock->data(), this->weight_desc_, wblock->data(), this->y_descs_, - outblock->mutable_data(), this->hy_desc_, hyblock->mutable_data(), - cy_desc_, cyblock->mutable_data(), this->workspace_.block()->mutable_data(), - this->workspace_count_ * sizeof(float), this->reserve_.block()->mutable_data(), - this->ReserveSize_ * sizeof(float)); -}, {input.block(), weight_.block(), hx.block(), cx.block()}, - {output.block(), hy.block(), cy.block()}, workspace_.block()); - buf_.push(output); - buf_.push(hx); - buf_.push(hy); // in order to assign shape to dhy - buf_.push(cx); - buf_.push(cy); // in order to assign shape to dcy - data_output.push_back(output); - data_output.push_back(hy); - data_output.push_back(cy); - return data_output; + Shape outshape{input.Size() * hidden_dim_ / input_dim_ * num_directions_}; + Tensor output(outshape, dev, dtype); + LOG(INFO) << "output size " << output.Size(); + Tensor hx = inputs.at(num_x); + Shape state_shape{num_stacks_ * num_directions_, batch_size_, hidden_dim_}; + Tensor hy(state_shape, dev, dtype); + Tensor cy, cx; + if (has_cell_) { + cx = inputs.at(num_x + 1); + cy.ResetLike(hy); + } + + LOG(INFO) << "hidden size " << hy.Size(); + LOG(INFO) << "weight size " << weight_.Size() << " value " << weight_.L1(); + Block *inb = input.block(), *outb = output.block(), + *wb = this->weight_.block(), *hxb = hx.block(), *cxb = cx.block(), + *hyb = hy.block(), *cyb = cy.block(), + *wspace = this->workspace_.block(), + *rspace = this->reserve_space_.block(); + if (flag & kTrain) { + dev->Exec( + [inb, outb, wb, hxb, cxb, hyb, cyb, wspace, rspace, this](Context *ctx) { + // clang-format off + cudnnRNNForwardTraining( + ctx->cudnn_handle, + this->rnn_desc_, + this->seq_length_, + this->x_descs_, inb->data(), + this->hx_desc_, hxb == nullptr ? nullptr : hxb->data(), + this->cx_desc_, cxb == nullptr ? nullptr : cxb->data(), + this->weight_desc_, wb->data(), + this->y_descs_, outb->mutable_data(), + this->hy_desc_, hyb->mutable_data(), + this->cy_desc_, cyb == nullptr ? nullptr : cyb->mutable_data(), + wspace->mutable_data(), + this->workspace_.Size(), rspace->mutable_data(), + this->reserve_space_.Size()); + // clang-format on + }, + {inb, wb, hxb, cxb}, {outb, hyb, cyb, wspace, rspace}); + buf_.push(input); + buf_.push(output); + buf_.push(hx); + buf_.push(cx); + } else { + dev->Exec([inb, outb, wb, hxb, cxb, hyb, cyb, wspace, this](Context *ctx) { + // clang-format off + cudnnRNNForwardInference( + ctx->cudnn_handle, + this->rnn_desc_, + this->seq_length_, + this->x_descs_, inb->data(), + this->hx_desc_, hxb == nullptr ? nullptr : hxb->data(), + this->cx_desc_, cxb == nullptr ? nullptr : cxb->data(), + this->weight_desc_, wb->data(), + this->y_descs_, outb->mutable_data(), + this->hy_desc_, hyb->mutable_data(), + this->cy_desc_, cyb == nullptr ? nullptr : cyb->mutable_data(), + wspace->mutable_data(), this->workspace_.Size()); + // clang-format on + }, {inb, wb, hxb, cxb}, {outb, hyb, cyb, wspace}); + } + auto outputs = + SplitOutput(num_x, hidden_dim_ * num_directions_, inputs, output); + outputs.push_back(hy); + if (has_cell_) outputs.push_back(cy); + return outputs; } +// TODO(wangwei) check Tensor device to be on cuda? const std::pair, vector> CudnnRNN::Backward( - int flag, const vector& grads) { - CHECK(has_init_cudnn_); - singa::Tensor grad = grads[0]; - singa::Tensor dhy = grads[1]; - singa::Tensor dcy = grads[2]; - CHECK_EQ(grad.device()->lang(), kCuda); - CHECK_EQ(grad.nDim(), 3u); - CHECK(!buf_.empty()); - Tensor cy = buf_.top(); + int flag, const vector &grads) { + // dhy (and dcy) is at last + const Tensor cx = buf_.top(); // cannot use const Tensor& due to pop() buf_.pop(); - CHECK(!buf_.empty()); - Tensor cx = buf_.top(); + const Tensor hx = buf_.top(); buf_.pop(); - CHECK(!buf_.empty()); - Tensor hy = buf_.top(); + const Tensor y = buf_.top(); buf_.pop(); - CHECK(!buf_.empty()); - Tensor hx = buf_.top(); + const Tensor x = buf_.top(); buf_.pop(); - CHECK(!buf_.empty()); - Tensor src_output = buf_.top(); - buf_.pop(); - CHECK(!buf_.empty()); - Tensor src_data = buf_.top(); - buf_.pop(); - vector param_grad; - vector data_grad; - Tensor dx; - dx.ResetLike(src_data); - Tensor dw; - dw.ResetLike(weight_); - Tensor dhx; - dhx.ResetLike(hx); + + auto dev = y.device(); + auto dtype = y.data_type(); + + CHECK_GT(grads.size(), 1u + has_cell_); + size_t num_dy = grads.size() - has_cell_ - 1; + CHECK_EQ(num_dy, seq_length_); + const Tensor dy = MergeInputs(num_dy, grads); + CHECK_EQ(dy.Size(), y.Size()); + const Tensor dhy = grads.at(num_dy); + Tensor dcy; + if (has_cell_) + dcy = grads.at(num_dy + 1); + + Shape xshape{y.Size() * input_dim_ / hidden_dim_ / num_directions_}; + Tensor dx(xshape, dev, dtype); + Tensor dw(weight_.shape(), dev, dtype); + Shape state_shape{num_stacks_ * num_directions_, batch_size_, hidden_dim_}; + Tensor dhx(state_shape, dev, dtype); Tensor dcx; - dcx.ResetLike(cx); + if (has_cell_) + dcx.ResetLike(dhx); + dw.SetValue(0.0f); + Block *yb = y.block(), *dyb = dy.block(), *dhyb = dhy.block(), + *dcyb = dcy.block(), *xb = x.block(), *cxb = cx.block(), + *wb = weight_.block(), *dwb = dw.block(), *hxb = hx.block(), + *dxb = dx.block(), *dhxb = dhx.block(), *dcxb = dcx.block(), + *wspace = workspace_.block(), *rspace = reserve_space_.block(); + y.device()->Exec( + [yb, dyb, dhyb, dcyb, xb, cxb, wb, dwb, hxb, dxb, dhxb, dcxb, wspace, + rspace, this](Context *ctx) { + // clang-format off + cudnnRNNBackwardData( + ctx->cudnn_handle, + this->rnn_desc_, + this->seq_length_, + this->y_descs_, yb->data(), + this->dy_descs_, dyb->data(), + this->dhy_desc_, dhyb == nullptr ? nullptr : dhyb->data(), + this->dcy_desc_, dcyb == nullptr ? nullptr : dcyb->data(), + this->weight_desc_, wb->data(), + this->hx_desc_, hxb == nullptr ? nullptr : hxb->data(), + this->cx_desc_, cxb == nullptr ? nullptr : cxb->data(), + this->dx_descs_, dxb->mutable_data(), + this->dhx_desc_, dhxb->mutable_data(), + this->dcx_desc_, dcxb == nullptr ? nullptr : dcxb->mutable_data(), + wspace->mutable_data(), this->workspace_.Size(), + rspace->mutable_data(), this->reserve_space_.Size()); + cudnnRNNBackwardWeights( + ctx->cudnn_handle, + this->rnn_desc_, + this->seq_length_, + this->x_descs_, xb->data(), + this->hx_desc_, hxb == nullptr ? nullptr : hxb->data(), + this->y_descs_, yb->data(), + wspace->data(), this->workspace_.Size(), + this->dweight_desc_, dwb->mutable_data(), + rspace->data(), this->reserve_space_.Size()); + // clang-format on + }, + {yb, dyb, dhyb, dcyb, xb, wb, wspace, rspace}, + {dxb, dwb, dhxb, dcxb, wspace, rspace}); - dx.device()->Exec([grad, dw, src_data, src_output, hx, this](Context *ctx) { - Block *inblock = src_data.block(), *srcoutblock = src_output.block(), - *dwblock = dw.block(), *hxblock = hx.block(); - cudnnRNNBackwardWeights( - ctx->cudnn_handle, this->rnn_desc_, seqLength_, this->x_descs_, - inblock->data(), this->hx_desc_, hxblock->data(), this->y_descs_, - srcoutblock->data(), this->workspace_.block()->mutable_data(), - this->workspace_count_ * sizeof(float), this->weight_desc_, - dwblock->mutable_data(), this->reserve_.block()->mutable_data(), - this->ReserveSize_ * sizeof(float)); - }, {src_data.block(), hx.block(), src_output.block()}, {dw.block(), workspace_.block()}); - - // LOG(ERROR) << "backward src"; - dx.device()->Exec([grad, dw, src_output, dx, cy, cx, hy, hx, dhy, dcy, dhx, dcx, this](Context *ctx) { - Block *srcoutblock = src_output.block(), *wblock = this->weight_.block(), *dxblock = dx.block(), - *dyblock = grad.block(), *cxblock = cx.block(), *hxblock = hx.block(), *dhyblock = dhy.block(), - *dcyblock = dcy.block(), *dhxblock = dhx.block(), *dcxblock = dcx.block(); - cudnnRNNBackwardData( - ctx->cudnn_handle, this->rnn_desc_, seqLength_, this->y_descs_, srcoutblock->data(), - this->y_descs_, dyblock->data(), this->hy_desc_, dhyblock->data(), - this->cy_desc_, dcyblock->data(), this->weight_desc_, wblock->data(), - this->hx_desc_, hxblock->data(), this->cx_desc_, cxblock->data(), - this->x_descs_, dxblock->mutable_data(), this->hx_desc_, dhxblock->mutable_data(), - this->cx_desc_, dcxblock->mutable_data(), this->workspace_.block()->mutable_data(), - this->workspace_count_ * sizeof(float), this->reserve_.block()->mutable_data(), - this->ReserveSize_ * sizeof(float)); - }, {hx.block(), src_output.block(), grad.block(), grad.block(), dhy.block(), dcy.block(), - this->weight_.block(), hx.block(), cx.block()}, - {dx.block(), dhx.block(), dcx.block(), reserve_.block(), workspace_.block()}); - param_grad.push_back(dw); - data_grad.push_back(dx); - data_grad.push_back(dhx); - data_grad.push_back(dcx); - return std::make_pair(data_grad, param_grad); + vector param_grad{dw}; + auto data_grads = SplitOutput(num_dy, input_dim_, grads, dx); + data_grads.push_back(dhx); + if (has_cell_) + data_grads.push_back(dcx); + return std::make_pair(data_grads, param_grad); } } // namespace singa diff --git a/src/model/layer/cudnn_rnn.h b/src/model/layer/cudnn_rnn.h index b1e9f43e59..d2f8db5754 100644 --- a/src/model/layer/cudnn_rnn.h +++ b/src/model/layer/cudnn_rnn.h @@ -20,6 +20,7 @@ #define SRC_MODEL_LAYER_CUDNN_RNN_H_ #include "singa/singa_config.h" #ifdef USE_CUDNN +#if CUDNN_VERSION_MAJOR >= 5 #include #include #include @@ -41,45 +42,46 @@ class CudnnRNN : public RNN { const std::string layer_type() const override { return "CudnnRNN"; } const vector Forward(int flag, const vector& inputs) override; - const std::pair, vector> Backward(int flag, const vector& grads) override; - - /// \copydoc Layer::Setup(const LayerConf&); - void Setup(const Shape& in_sample, const LayerConf &conf) override; + const std::pair, vector> Backward( + int flag, const vector& grads) override; void ToDevice(std::shared_ptr device) override; - size_t workspace_byte_limit() { return workspace_byte_limit_; } - // string prefer() { return prefer_; } - string inputMode() const { return inputMode_; } - string direction() const { return direction_; } - string mode() const { return mode_; } - - protected: - /// Init cudnn related data structures. - void InitCudnn(const Tensor& input); + void SetRNNDescriptor(shared_ptr dev); + void ResetHiddenAndCellDescriptors(size_t batch_size); + void DestroyIODescriptors(); + void UpdateIODescriptors(size_t num, const vector& inputs); + void UpdateSpaces(size_t num, shared_ptr dev); + void UpdateStates(size_t num, const vector& inputs); + Tensor MergeInputs(size_t num, const vector& in); + vector SplitOutput(size_t num, size_t dim, const vector& in, + const Tensor output); protected: - bool has_init_cudnn_ = false; cudnnTensorDescriptor_t* x_descs_ = nullptr; + cudnnTensorDescriptor_t* dx_descs_ = nullptr; cudnnTensorDescriptor_t* y_descs_ = nullptr; + cudnnTensorDescriptor_t* dy_descs_ = nullptr; cudnnTensorDescriptor_t hx_desc_ = nullptr; + cudnnTensorDescriptor_t dhx_desc_ = nullptr; cudnnTensorDescriptor_t cx_desc_ = nullptr; + cudnnTensorDescriptor_t dcx_desc_ = nullptr; cudnnTensorDescriptor_t hy_desc_ = nullptr; + cudnnTensorDescriptor_t dhy_desc_ = nullptr; cudnnTensorDescriptor_t cy_desc_ = nullptr; + cudnnTensorDescriptor_t dcy_desc_ = nullptr; cudnnFilterDescriptor_t weight_desc_ = nullptr; + cudnnFilterDescriptor_t dweight_desc_ = nullptr; cudnnRNNDescriptor_t rnn_desc_ = nullptr; cudnnDropoutDescriptor_t dropout_desc_ = nullptr; - size_t workspace_byte_limit_, workspace_count_; - size_t ReserveSize_; + cudnnDataType_t dtype_ = CUDNN_DATA_FLOAT; Tensor workspace_; - string inputMode_; - string direction_; - string mode_; - Tensor reserve_; - Tensor dropoutStates_; + Tensor reserve_space_; + Tensor dropout_state_; }; } // namespace singa +#endif // CUDNN_VERSION_MAJOR >= 5 #endif // USE_CUDNN #endif // SRC_MODEL_LAYER_CUDNN_RNN_H_ diff --git a/src/model/layer/rnn.cc b/src/model/layer/rnn.cc index 493a5e42b5..6b831a7d86 100644 --- a/src/model/layer/rnn.cc +++ b/src/model/layer/rnn.cc @@ -19,20 +19,64 @@ #include "./rnn.h" #include #include "singa/model/layer.h" +#include "singa/utils/string.h" namespace singa { void RNN::Setup(const Shape& in_sample, const LayerConf &conf) { Layer::Setup(in_sample, conf); + RNNConf rnn_conf = conf.rnn_conf(); - hiddenSize_ = rnn_conf.hiddensize(); - CHECK_GT(hiddenSize_, 0u); + hidden_dim_ = rnn_conf.hidden_dim(); + CHECK_GT(hidden_dim_, 0u); + num_stacks_ = rnn_conf.num_stacks(); + CHECK_GT(num_stacks_, 0u); + input_dim_ = Product(in_sample); + CHECK_GT(input_dim_, 0u); + dropout_ = rnn_conf.dropout(); + CHECK_GE(dropout_, 0); - numLayers_ = rnn_conf.numlayers(); - CHECK_GT(numLayers_, 0u); + input_mode_ = ToLowerCase(rnn_conf.input_mode()); + CHECK(input_mode_ == "linear" || input_mode_ == "skip") + << "Input mode of " << input_mode_ << " is not supported; Please use " + << "'linear' and 'skip'"; - dropout_ = rnn_conf.dropout(); - CHECK_GE(dropout_, 0u); + direction_ = ToLowerCase(rnn_conf.direction()); + if (direction_ == "unidirectional") + num_directions_ = 1; + else if (direction_ == "bidirectional") + num_directions_ = 2; + else + LOG(FATAL) << "Direction of " << direction_ + << " is not supported; Please use unidirectional or bidirectional"; + + rnn_mode_ = ToLowerCase(rnn_conf.rnn_mode()); + if (rnn_mode_ == "lstm") { + has_cell_ = true; + } else if (rnn_mode_ !="relu" && rnn_mode_ != "tanh" && rnn_mode_ != "gru") { + LOG(FATAL) << "RNN memory unit (mode) of " << rnn_mode_ + << " is not supported Please use 'relu', 'tanh', 'lstm' and 'gru'"; + } + // the first constant (4) is the size of float + // the second constant (2, 8, 6) is the number of sets of params + int mult = 1; + if (rnn_mode_ == "relu" || rnn_mode_ == "tanh") + mult *= 1; + else if (rnn_mode_ == "lstm") + mult *= 4; + else if (rnn_mode_ == "gru") + mult *= 3; + if (direction_ == "bidirectional") + mult *= 2; + + size_t weight_size = 0; + for (size_t i = 0; i < num_stacks_; i++) { + size_t dim = hidden_dim_ * (in_sample[0] + hidden_dim_ + 2); + if (i > 0) + dim = hidden_dim_ * (hidden_dim_ + hidden_dim_ + 2); + weight_size += mult * dim; + } + weight_.Reshape(Shape{weight_size}); } const vector RNN::Forward(int flag, const vector& inputs) { @@ -40,7 +84,8 @@ const vector RNN::Forward(int flag, const vector& inputs) { return data_output; } -const std::pair, vector> RNN::Backward(int flag, const vector& grads) { +const std::pair, vector> RNN::Backward(int flag, + const vector& grads) { vector param_grad; vector data_grad; return std::make_pair(data_grad, param_grad); diff --git a/src/model/layer/rnn.h b/src/model/layer/rnn.h index ec5a35d71b..375002110d 100644 --- a/src/model/layer/rnn.h +++ b/src/model/layer/rnn.h @@ -47,38 +47,37 @@ class RNN : public Layer { const std::pair, vector> Backward( int flag, const vector& grads) override; - - size_t hiddenSize() const { return hiddenSize_; } - size_t numLayers() const { return numLayers_; } - size_t weightSize() const { return weightSize_; } - float dropout() const { return dropout_; } - void set_weight(Tensor w) { weight_.ResetLike(w); weight_.CopyData(w); } - + const vector param_values() override { + return vector{weight_}; + } void ToDevice(std::shared_ptr device) override; /// Return the internal state stack, which should be empty at the beginning - /// of - /// one iteration. + /// of one iteration. // std::stack states() const { return states_; } + string input_mode() const { return input_mode_; } + string direction() const { return direction_; } + string rnn_mode() const { return rnn_mode_; } + protected: /// Storing input or output from Forward(), which are used in Backward(). /// Rules: /// 1. push the 'input' or 'output' into states_ if the flag of Forward() is /// for kTrain and 'input' or 'output' is necessary for Backward(). /// 2. pop data out in Backward(). - // std::stack states_; std::stack buf_; - size_t hiddenSize_; - size_t numLayers_; - size_t numLinearLayer_; - size_t seqLength_; - size_t weightSize_; /*all the weights and biases*/ - float dropout_; + bool has_cell_ = false; + size_t num_directions_ = 1; + size_t input_dim_ = 0, hidden_dim_ = 0, num_stacks_ = 0, seq_length_ = 0; + size_t batch_size_ = 0; + size_t seed_ = 0x1234567; + float dropout_ = 0.0f; + string input_mode_, direction_, rnn_mode_; Tensor weight_; }; } // namespace singa diff --git a/src/model/optimizer/adagrad.cc b/src/model/optimizer/adagrad.cc index 3ed1855b29..cdb3fac785 100644 --- a/src/model/optimizer/adagrad.cc +++ b/src/model/optimizer/adagrad.cc @@ -27,8 +27,10 @@ void AdaGrad::Setup(const OptimizerConf& conf) { delta_ = conf.delta(); } // value = value - lr*grad/sqrt(history+delta) void AdaGrad::Apply(int step, float lr, const string& name, const Tensor& grad, Tensor& value) { - if (history_gradient_.find(name) == history_gradient_.end()) + if (history_gradient_.find(name) == history_gradient_.end()) { history_gradient_[name].ResetLike(value); + history_gradient_[name].SetValue(0.0f); + } Tensor& history = history_gradient_[name]; Tensor tmp = Square(grad); history += tmp; diff --git a/src/model/optimizer/nesterov.cc b/src/model/optimizer/nesterov.cc index e5354b1b6d..051499bb8f 100644 --- a/src/model/optimizer/nesterov.cc +++ b/src/model/optimizer/nesterov.cc @@ -34,8 +34,10 @@ void Nesterov::Apply(int step, float lr, const string& name, const Tensor& grad, Tensor& value) { if (momentum_generator_) { float mom = momentum_generator_(step); - if (history_gradient_.find(name) == history_gradient_.end()) + if (history_gradient_.find(name) == history_gradient_.end()) { history_gradient_[name].ResetLike(value); + history_gradient_[name].SetValue(0.0f); + } Tensor& history = history_gradient_[name]; Tensor tmp = history.Clone(); history *= mom; diff --git a/src/model/optimizer/rmsprop.cc b/src/model/optimizer/rmsprop.cc index 6d77ccde64..13e2a755bc 100644 --- a/src/model/optimizer/rmsprop.cc +++ b/src/model/optimizer/rmsprop.cc @@ -32,6 +32,7 @@ void RMSProp::Apply(int step, float lr, const string& name, const Tensor& grad, Tensor& value) { if (history_gradient_.find(name) == history_gradient_.end()) { history_gradient_[name].ResetLike(value); + history_gradient_[name].SetValue(0.0f); } Tensor& history = history_gradient_[name]; history *= rho_; diff --git a/src/model/optimizer/sgd.cc b/src/model/optimizer/sgd.cc index 2797fc6821..d78d5b8d06 100644 --- a/src/model/optimizer/sgd.cc +++ b/src/model/optimizer/sgd.cc @@ -36,8 +36,10 @@ void SGD::Apply(int step, float lr, const string& name, const Tensor& grad, if (momentum_generator_) { float mom = momentum_generator_(step); if (mom != 0) { - if (history_gradient_.find(name) == history_gradient_.end()) + if (history_gradient_.find(name) == history_gradient_.end()) { history_gradient_[name].ResetLike(value); + history_gradient_[name].SetValue(0.0f); + } Tensor& history = history_gradient_[name]; history *= mom; Axpy(lr, grad, &history); diff --git a/src/proto/model.proto b/src/proto/model.proto index d8193f17fa..31ebfc356f 100644 --- a/src/proto/model.proto +++ b/src/proto/model.proto @@ -393,19 +393,19 @@ message ConvolutionConf { } message RNNConf { - optional uint32 hiddensize = 1; // The number of hiddensize - optional uint32 numlayers = 2; // The number of stacked RNN layers + optional uint32 hidden_dim = 1; // The number of hiddensize + optional uint32 num_stacks = 2; // The number of stacked RNN layers optional float dropout = 3 [default = 0]; - optional int32 workspace_byte_limit = 50 [default = 512]; + optional bool remember_state = 4 [default = false]; // cudnn inputmode - // options: "cudnn_linear_input", "cudnn_skip_input" - optional string inputmode = 51 [default = "cudnn_linear_input"]; + // options: "linear", "skip" + optional string input_mode = 7 [default = "linear"]; // cudnn direction - // options: "cudnn_undirectional", "cudnn_bidirectional" - optional string direction = 52 [default = "cudnn_undirectional"]; + // options: "unidirectional", "bidirectional" + optional string direction = 8 [default = "unidirectional"]; // cudnn RNN mode - // options: "cudnn_rnn_relu", "cudnn_rnn_tanh", "cudnn_lstm", "cudnn_gru" - optional string mode = 53 [default = "cudnn_rnn_relu"]; + // options: "relu", "tanh", "lstm", "gru" + optional string rnn_mode = 9 [default = "relu"]; } /* diff --git a/test/singa/test_cudnn_rnn.cc b/test/singa/test_cudnn_rnn.cc index 1a79d7b0fa..ebbf0aab3e 100644 --- a/test/singa/test_cudnn_rnn.cc +++ b/test/singa/test_cudnn_rnn.cc @@ -26,187 +26,154 @@ using singa::CudnnRNN; using singa::Shape; -TEST(CudnnRNN, Setup) { +using singa::Tensor; +class TestCudnnRNN : public ::testing::Test { + protected: + virtual void SetUp() { + singa::RNNConf *rnnconf = conf.mutable_rnn_conf(); + rnnconf->set_hidden_dim(hidden_dim); + rnnconf->set_num_stacks(1); + rnnconf->set_dropout(1); + rnnconf->set_input_mode("linear"); + rnnconf->set_direction("unidirectional"); + rnnconf->set_rnn_mode("tanh"); + } + singa::LayerConf conf; + size_t hidden_dim = 4; +}; + +TEST_F(TestCudnnRNN, Setup) { CudnnRNN rnn; EXPECT_EQ("CudnnRNN", rnn.layer_type()); - - singa::LayerConf conf; - singa::RNNConf *rnnconf = conf.mutable_rnn_conf(); - rnnconf->set_hiddensize(2); - rnnconf->set_numlayers(1); - rnnconf->set_dropout(0); - rnnconf->set_inputmode("cudnn_linear_input"); - rnnconf->set_direction("cudnn_undirectional"); - rnnconf->set_mode("cudnn_rnn_tanh"); - // MB - rnnconf->set_workspace_byte_limit(256); - rnn.Setup(Shape{4, 1, 2}, conf); - - EXPECT_EQ(2u, rnn.hiddenSize()); - EXPECT_EQ(1u, rnn.numLayers()); - EXPECT_EQ(0u, rnn.dropout()); - EXPECT_EQ("cudnn_linear_input", rnn.inputMode()); - EXPECT_EQ("cudnn_undirectional", rnn.direction()); - EXPECT_EQ("cudnn_rnn_tanh", rnn.mode()); - EXPECT_EQ(256u << 20, rnn.workspace_byte_limit()); + rnn.Setup(Shape{2}, conf); + auto weight = rnn.param_values().at(0); + EXPECT_EQ(weight.Size(), hidden_dim * (2 + hidden_dim + 2)); } -TEST(CudnnRNN, Forward) { +TEST_F(TestCudnnRNN, Forward) { auto cuda = std::make_shared(); const size_t seqLength = 4, batchsize = 1, dim = 2; - const size_t numLayers = 1, hiddensize = 2, numDirections = 1; const float x[seqLength * batchsize * dim] = {1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f}; - singa::Tensor in(singa::Shape{seqLength, batchsize, dim}, cuda); - in.CopyDataFromHostPtr(x, seqLength * batchsize * dim); + vector inputs; + for (size_t i = 0; i < seqLength; i++) { + Tensor t(Shape{batchsize, dim}, cuda); + t.CopyDataFromHostPtr(x + i * t.Size(), t.Size()); + inputs.push_back(t); + } - - const float hx_data[numLayers * batchsize * hiddensize * numDirections] = {1.0f, 1.0f}; - singa::Tensor hx(singa::Shape{numLayers, batchsize, hiddensize * numDirections}, cuda); - hx.CopyDataFromHostPtr(hx_data, numLayers * batchsize * hiddensize * numDirections); + singa::Tensor hx; + inputs.push_back(hx); - const float cx_data[numLayers * batchsize * hiddensize * numDirections] = {1.0f, 1.0f}; - singa::Tensor cx(singa::Shape{numLayers, batchsize, hiddensize * numDirections}, cuda); - cx.CopyDataFromHostPtr(cx_data, numLayers * batchsize * hiddensize * numDirections); - CudnnRNN rnn; - - singa::LayerConf conf; - singa::RNNConf *rnnconf = conf.mutable_rnn_conf(); - rnnconf->set_hiddensize(2); - rnnconf->set_numlayers(1); - rnnconf->set_dropout(0); - rnnconf->set_inputmode("cudnn_linear_input"); - rnnconf->set_direction("cudnn_undirectional"); - rnnconf->set_mode("cudnn_rnn_tanh"); - // MB - rnnconf->set_workspace_byte_limit(256); - rnn.Setup(Shape{4, 1, 2}, conf); - - - size_t weightSize = rnn.weightSize(); + rnn.Setup(Shape{dim}, conf); + rnn.ToDevice(cuda); + + auto weight = rnn.param_values().at(0); + size_t weightSize = weight.Size(); float we[weightSize]; + float wvalue = 0.1f; for (size_t i = 0; i < weightSize; i++) - we[i] = 1.0f; - singa::Tensor weight(singa::Shape{weightSize, 1, 1}, cuda); + we[i] = wvalue; weight.CopyDataFromHostPtr(we, weightSize); - rnn.set_weight(weight); - - vector input_array; - input_array.push_back(in); - input_array.push_back(hx); - input_array.push_back(cx); - const auto ret = rnn.Forward(singa::kTrain, input_array); - // singa::CppCPU host(0, 1); - singa::Tensor out1 = ret[0]; - out1.ToHost(); - const float *outptr1 = out1.data(); - EXPECT_EQ(8u, out1.Size()); - EXPECT_NEAR(1.0f, outptr1[0], 0.0001); // tanh 6 - EXPECT_NEAR(1.0f, outptr1[1], 0.0001); - EXPECT_NEAR(1.0f, outptr1[2], 0.0001); - EXPECT_NEAR(1.0f, outptr1[3], 0.0001); - EXPECT_NEAR(1.0f, outptr1[4], 0.0001); - EXPECT_NEAR(1.0f, outptr1[5], 0.0001); - EXPECT_NEAR(1.0f, outptr1[6], 0.0001); - EXPECT_NEAR(1.0f, outptr1[7], 0.0001); - - singa::Tensor hy1 = ret[1]; - hy1.ToHost(); - const float *hyptr1 = hy1.data(); - EXPECT_EQ(2u, hy1.Size()); - EXPECT_NEAR(1.0f, hyptr1[0], 0.0001); - EXPECT_NEAR(1.0f, hyptr1[1], 0.0001); + + const auto ret = rnn.Forward(singa::kEval, inputs); + EXPECT_EQ(ret.size(), seqLength + 1); + vector hxptr(hidden_dim, 0.0f); + for (size_t i = 0; i < seqLength; i++) { + auto y = ret[i]; + y.ToHost(); + auto yptr = y.data(); + vector tmp; + for (size_t j = 0; j < hidden_dim; j++) { + float ty = 0; + for (size_t k = 0; k < dim; k++) { + ty += x[i * dim + k] * wvalue; + } + ty += wvalue; + for (size_t k = 0; k < hidden_dim; k++) { + ty += hxptr[k] * wvalue; + } + ty += wvalue; + ty = tanh(ty); + EXPECT_NEAR(ty, yptr[j], 1e-4); + tmp.push_back(ty); + } + std::copy(tmp.begin(), tmp.end(), hxptr.begin()); + } } -TEST(CudnnRNN, Backward) { - // src_data +TEST_F(TestCudnnRNN, Backward) { auto cuda = std::make_shared(); const size_t seqLength = 4, batchsize = 1, dim = 2; - const size_t numLayers = 1, hiddensize = 2, numDirections = 1; const float x[seqLength * batchsize * dim] = {1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f}; - singa::Tensor in(singa::Shape{seqLength, batchsize, dim}, cuda); - in.CopyDataFromHostPtr(x, seqLength * batchsize * dim); - const float hx_data[numLayers * batchsize * hiddensize * numDirections] = {1.0f, 1.0f}; - singa::Tensor hx(singa::Shape{numLayers, batchsize, hiddensize * numDirections}, cuda); - hx.CopyDataFromHostPtr(hx_data, numLayers * batchsize * hiddensize * numDirections); + vector inputs; + for (size_t i = 0; i < seqLength; i++) { + Tensor t(Shape{batchsize, dim}, cuda); + t.CopyDataFromHostPtr(x + i * t.Size(), t.Size()); + inputs.push_back(t); + } - const float cx_data[numLayers * batchsize * hiddensize * numDirections] = {1.0f, 1.0f}; - singa::Tensor cx(singa::Shape{numLayers, batchsize, hiddensize * numDirections}, cuda); - cx.CopyDataFromHostPtr(cx_data, numLayers * batchsize * hiddensize * numDirections); + singa::Tensor hx; + inputs.push_back(hx); CudnnRNN rnn; + rnn.Setup(Shape{dim}, conf); + rnn.ToDevice(cuda); - singa::LayerConf conf; - singa::RNNConf *rnnconf = conf.mutable_rnn_conf(); - rnnconf->set_hiddensize(2); - rnnconf->set_numlayers(1); - rnnconf->set_dropout(0); - rnnconf->set_inputmode("cudnn_linear_input"); - rnnconf->set_direction("cudnn_undirectional"); - rnnconf->set_mode("cudnn_rnn_tanh"); - // MB - rnnconf->set_workspace_byte_limit(256); - rnn.Setup(Shape{4, 1, 2}, conf); - - size_t weightSize = rnn.weightSize(); + auto weight = rnn.param_values().at(0); + size_t weightSize = weight.Size(); float we[weightSize]; + float wvalue = 0.1f; for (size_t i = 0; i < weightSize; i++) - we[i] = 1.0f; - singa::Tensor weight(singa::Shape{weightSize, 1, 1}, cuda); + we[i] = wvalue; weight.CopyDataFromHostPtr(we, weightSize); - rnn.set_weight(weight); - - - vector input_array; - input_array.push_back(in); - input_array.push_back(hx); - input_array.push_back(cx); - const auto ret = rnn.Forward(singa::kTrain, input_array); - - // grad - const float dy[seqLength * batchsize * hiddensize * numDirections] = {1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f}; - singa::Tensor grad(singa::Shape{seqLength, batchsize, hiddensize * numDirections}, - cuda); - grad.CopyDataFromHostPtr(dy, seqLength * batchsize * hiddensize * numDirections); - - const float dhy_data[numLayers * batchsize * hiddensize * numDirections] = {1.0f, 1.0f}; - singa::Tensor dhy(singa::Shape{numLayers, batchsize, hiddensize * numDirections}, - cuda); - dhy.CopyDataFromHostPtr(dhy_data, numLayers * batchsize * hiddensize * numDirections); - - const float dcy_data[numLayers * batchsize * hiddensize * numDirections] = {1.0f, 1.0f}; - singa::Tensor dcy(singa::Shape{numLayers, batchsize, hiddensize * numDirections}, - cuda); - dcy.CopyDataFromHostPtr(dcy_data, numLayers * batchsize * hiddensize * numDirections); - - vector grad_array; - grad_array.push_back(grad); - grad_array.push_back(dhy); - grad_array.push_back(dcy); - const auto ret_back = rnn.Backward(singa::kTrain, grad_array); - // singa::CppCPU host(0, 1); - singa::Tensor in_grad = ret_back.first[0]; - in_grad.ToHost(); - const float *dx = in_grad.data(); - EXPECT_EQ(8u, in_grad.Size()); - EXPECT_NEAR(0.14, dx[0], 0.0001); - EXPECT_NEAR(0.14, dx[1], 0.0001); - EXPECT_NEAR(0.1596, dx[2], 0.0001); - EXPECT_NEAR(0.1596, dx[3], 0.0001); - EXPECT_NEAR(0.1623, dx[4], 0.0001); - EXPECT_NEAR(0.1623, dx[5], 0.0001); - EXPECT_NEAR(0.1627, dx[6], 0.0001); - EXPECT_NEAR(0.1627, dx[7], 0.0001); - - singa::Tensor dhx_grad = ret_back.first[1]; - dhx_grad.ToHost(); - const float *dhx = dhx_grad.data(); - EXPECT_EQ(2u, dhx_grad.Size()); - EXPECT_NEAR(0.1627, dhx[0], 0.0001); - EXPECT_NEAR(0.1627, dhx[1], 0.0001); + + const auto outs = rnn.Forward(singa::kTrain, inputs); + + float dyptr[seqLength * batchsize * hidden_dim]; + for (size_t i = 0; i < seqLength * batchsize * hidden_dim; i++) + dyptr[i] = i * 0.1f; + vector grads; + for (size_t i = 0; i < seqLength; i++) { + Tensor dy(Shape{batchsize, hidden_dim}, cuda); + dy.CopyDataFromHostPtr(dyptr + i * dy.Size(), dy.Size()); + grads.push_back(dy); + } + Tensor dhy; + grads.push_back(dhy); + vector dhyptr(hidden_dim, 0.0f); + const auto ret = rnn.Backward(singa::kTrain, grads); + for (size_t i = seqLength - 1; i > 0 ; i --) { + auto dx = ret.first[i]; + auto y = outs[i].Clone(); + y.ToHost(); + dx.ToHost(); + auto dxptr = dx.data(); + auto yptr = y.data(); + for (size_t j = 0; j < hidden_dim; j++) { + dhyptr[j] += dyptr[i * hidden_dim + j]; + dhyptr[j] *= 1 - yptr[j] * yptr[j]; + } + for (size_t k = 0; k < dim; k++) { + float tdx = 0; + for (size_t j = 0; j < hidden_dim; j++) { + tdx += dhyptr[j] * wvalue; + } + EXPECT_NEAR(tdx, dxptr[k], 1e-4); + } + vector tmp; + for (size_t k = 0; k < hidden_dim; k++) { + float tdhy = 0; + for (size_t j = 0; j < hidden_dim; j++) { + tdhy += dhyptr[j] * wvalue; + } + tmp.push_back(tdhy); + } + std::copy(tmp.begin(), tmp.end(), dhyptr.begin()); + } } #endif // USE_CUDNN From d8a977e5497a8e946903f0d26e18f0c2e83f5c18 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Mon, 8 Aug 2016 17:41:26 +0800 Subject: [PATCH 08/16] SINGA-218 Implementation for RNN CUDNN version Add an example using the char-rnn model. The trained model (with 2 stacks of lstm) over linux kernel source code could generate source code with some meaning full patterns, e.g., indention, comments, variable definition, assignments. --- examples/char-rnn/README.md | 31 +++++ examples/char-rnn/sample.py | 102 +++++++++++++++++ examples/char-rnn/train.py | 207 ++++++++++++++++++++++++++++++++++ src/core/tensor/tensor.cc | 3 +- src/io/csv_encoder.cc | 2 +- src/model/layer/cudnn_rnn.cc | 34 +++--- src/model/layer/rnn.cc | 16 +-- src/model/layer/rnn.h | 30 +++-- src/proto/model.proto | 2 +- src/python/singa/layer.py | 71 +++++++++++- src/python/swig/model_layer.i | 60 ++++++---- test/singa/test_cudnn_rnn.cc | 34 +++--- 12 files changed, 519 insertions(+), 73 deletions(-) create mode 100644 examples/char-rnn/README.md create mode 100644 examples/char-rnn/sample.py create mode 100644 examples/char-rnn/train.py diff --git a/examples/char-rnn/README.md b/examples/char-rnn/README.md new file mode 100644 index 0000000000..c5cdbb8fc8 --- /dev/null +++ b/examples/char-rnn/README.md @@ -0,0 +1,31 @@ +# Train Char-RNN using SINGA + +Recurrent neural networks (RNN) are widely used for modelling sequential data, +e.g., natural language sentences. This example describe how to implement a RNN +application (or model) using SINGA's RNN layers. +We will use the [char-rnn](https://github.com/karpathy/char-rnn) modle as an +example, which trains over setences or +source code, with each character as an input unit. Particularly, we will train +a RNN using GRU over Linux kernel source code. After training, we expect to +generate meaningful code from the model. + + +## Instructions + +* Compile and install SINGA. Currently the RNN implmentation depends on Cudnn V5. + +* Prepare the dataset. Download the [kernel source code](http://cs.stanford.edu/people/karpathy/char-rnn/). +Other plain text files can also be used. + +* Start the training, + + python train.py input_linux.txt + + Some hyper-parameters could be set through command line, + + python train.py -h + + +* Sample characters from the model by providing num of characters and the seed string. + + python sample.py 100 --seed '#include 0: + for c in seed_text: + x = np.zeros((1, vocab_size), dtype=np.float32) + x[0, char_to_idx[c]] = 1 + tx=tensor.from_numpy(x) + tx.to_device(cuda) + inputs=[tx, hx, cx] + outputs=rnn.forward(model_pb2.kEval, inputs) + y = dense.forward(model_pb2.kEval, outputs[0]) + y = tensor.softmax(y) + hx = outputs[1] + cx = outputs[2] + sys.stdout.write(seed_text) + else: + y = tensor.Tensor((1, vocab_size), cuda) + y.set_value(1.0 / vocab_size) + + for i in range(nsamples): + y.to_host() + prob = tensor.to_numpy(y)[0] + if do_sample: + cur=np.random.choice(vocab_size, 1, p=prob)[0] + else: + cur = np.argmax(prob) + sys.stdout.write(idx_to_char[cur]) + x = np.zeros((1, vocab_size), dtype=np.float32) + x[0, cur] = 1 + tx=tensor.from_numpy(x) + tx.to_device(cuda) + inputs=[tx, hx, cx] + outputs=rnn.forward(model_pb2.kEval, inputs) + y = dense.forward(model_pb2.kEval, outputs[0]) + y = tensor.softmax(y) + hx = outputs[1] + cx = outputs[2] + print '' + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='sample chars from char-rnn') + parser.add_argument('--seed', help='seed text string which warms up the rnn'\ + ' states for sampling', default='') + parser.add_argument('n', type=int, help='num of characters to sample') + args = parser.parse_args() + assert args.n > 0, 'n must > 0' + sample('model.bin', args.n, seed_text=args.seed) diff --git a/examples/char-rnn/train.py b/examples/char-rnn/train.py new file mode 100644 index 0000000000..22fdc827db --- /dev/null +++ b/examples/char-rnn/train.py @@ -0,0 +1,207 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +'''Train a Char-RNN model using plain text files. +The model is created following https://github.com/karpathy/char-rnn +The train file could be any text file, +e.g., http://cs.stanford.edu/people/karpathy/char-rnn/ +''' +import sys +import os +import cPickle as pickle +import numpy as np +import argparse + +sys.path.append(os.path.join(os.path.dirname(__file__), '../../build/python')) +from singa import layer +from singa import loss +from singa import device +from singa import tensor +from singa import optimizer +from singa import initializer +from singa.proto import core_pb2 +from singa.proto import model_pb2 +from singa import utils + + +class Data(object): + def __init__(self, fpath, batch_size=32, seq_length=100, train_ratio=0.8): + '''Data object for loading a plain text file. + + Args: + fpath, path to the text file. + train_ratio, split the text file into train and test sets, where + train_ratio of the characters are in the train set. + ''' + self.raw_data = open(fpath, 'r').read() # read text file + chars = list(set(self.raw_data)) + self.vocab_size = len(chars) + self.char_to_idx = {ch:i for i, ch in enumerate(chars)} + self.idx_to_char = {i:ch for i, ch in enumerate(chars)} + data = [self.char_to_idx[c] for c in self.raw_data] + # seq_length + 1 for the data + label + nsamples = len(data) / (1 + seq_length) + data = data[0:nsamples * (1 + seq_length)] + data = np.asarray(data, dtype=np.int32) + data = np.reshape(data, (-1, seq_length + 1)) + # shuffle all sequences + np.random.shuffle(data) + self.train_dat = data[0:int(data.shape[0]*train_ratio)] + self.num_train_batch = self.train_dat.shape[0] / batch_size + self.val_dat = data[self.train_dat.shape[0]:] + self.num_test_batch = self.val_dat.shape[0] / batch_size + print 'train dat', self.train_dat.shape + print 'val dat', self.val_dat.shape + + +def numpy2tensors(npx, npy, dev): + '''batch, seq, dim -- > seq, batch, dim''' + tmpx=np.swapaxes(npx, 0, 1) + tmpy=np.swapaxes(npy, 0, 1) + inputs=[] + labels=[] + for t in range(tmpx.shape[0]): + x = tensor.from_numpy(tmpx[t]) + y = tensor.from_numpy(tmpy[t]) + x.to_device(dev) + y.to_device(dev) + inputs.append(x) + labels.append(y) + return inputs, labels + + +def convert(batch, batch_size, seq_length, vocab_size, dev): + '''convert a batch of data into a sequence of input tensors''' + y = batch[:, 1:] + x1 = batch[:, :seq_length] + x = np.zeros((batch_size, seq_length, vocab_size), dtype=np.float32) + for b in range(batch_size): + for t in range(seq_length): + c = x1[b, t] + x[b, t, c] = 1 + return numpy2tensors(x, y, dev) + + +def get_lr(epoch): + return 0.001 / float(1 << (epoch / 50)) + + +def train(data, max_epoch, hidden_size =100, seq_length=100, batch_size=16, + num_stacks=1, lr=0.001, dropout = 0.5, model_path='model.bin'): + # SGD with L2 gradient normalization + opt = optimizer.SGD(constraint=optimizer.L2Constraint(5)) + cuda = device.create_cuda_gpu() + rnn = layer.LSTM(name='lstm', hidden_size=hidden_size, num_stacks=num_stacks, + dropout=dropout, input_sample_shape=(data.vocab_size,)) + rnn.to_device(cuda) + print 'created rnn' + rnn_w = rnn.param_values()[0] + initializer.uniform(rnn_w, -0.08, 0.08) # init all rnn parameters + print 'rnn weight l1 = %f' % (rnn_w.l1()) + dense = layer.Dense('dense', data.vocab_size, input_sample_shape=(hidden_size,)) + dense.to_device(cuda) + dense_w = dense.param_values()[0] + dense_b = dense.param_values()[1] + print 'dense w ', dense_w.shape + print 'dense b ', dense_b.shape + initializer.xavier(dense_w) # init weight matrix using Xavier + print 'dense weight l1 = %f' % (dense_w.l1()) + dense_b.set_value(0.0) + print 'dense b l1 = %f' % (dense_b.l1()) + + g_dense_w = tensor.Tensor(dense_w.shape, cuda) + g_dense_b = tensor.Tensor(dense_b.shape, cuda) + + lossfun = loss.SoftmaxCrossEntropy(); + for epoch in range(max_epoch): + train_loss = 0 + for b in range(data.num_train_batch): + batch = data.train_dat[b * batch_size: (b + 1) * batch_size] + inputs, labels = convert(batch, batch_size, seq_length, + data.vocab_size, cuda) + inputs.append(tensor.Tensor()) + inputs.append(tensor.Tensor()) + + outputs = rnn.forward(model_pb2.kTrain, inputs)[0:-2] + grads=[] + batch_loss = 0 + g_dense_w.set_value(0.0) + g_dense_b.set_value(0.0) + for output, label in zip(outputs, labels): + act = dense.forward(model_pb2.kTrain, output) + lvalue = lossfun.forward(model_pb2.kTrain, act, label) + batch_loss += lvalue.l1() + grad = lossfun.backward() + grad, gwb = dense.backward(model_pb2.kTrain, grad) + grads.append(grad) + g_dense_w += gwb[0] + g_dense_b += gwb[1] + #print output.l1(), act.l1() + utils.update_progress(b * 1.0 / data.num_train_batch, + 'training loss = %f' % (batch_loss / seq_length)) + train_loss += batch_loss + + grads.append(tensor.Tensor()) + grads.append(tensor.Tensor()) + g_rnn_w=rnn.backward(model_pb2.kTrain, grads)[1][0] + dense_w, dense_b = dense.param_values() + opt.apply_with_lr(epoch, get_lr(epoch), g_rnn_w, rnn_w, 'rnnw') + opt.apply_with_lr(epoch, get_lr(epoch), g_dense_w, dense_w, 'dense_w') + opt.apply_with_lr(epoch, get_lr(epoch), g_dense_b, dense_b, 'dense_b') + print '\nEpoch %d, train loss is %f' % (epoch, + train_loss / data.num_train_batch / seq_length) + eval_loss = 0 + for b in range(data.num_test_batch): + batch = data.val_dat[b * batch_size: (b + 1) * batch_size] + inputs, labels = convert(batch, batch_size, seq_length, + data.vocab_size, cuda) + inputs.append(tensor.Tensor()) + inputs.append(tensor.Tensor()) + outputs = rnn.forward(model_pb2.kEval, inputs)[0:-2] + for output, label in zip(outputs, labels): + output = dense.forward(model_pb2.kEval, output) + eval_loss += lossfun.forward(model_pb2.kEval, output, label).l1() + print 'Epoch %d, evaluation loss is %f' % (epoch, + eval_loss / data.num_test_batch / seq_length) + + # checkpoint the file model + with open(model_path, 'wb') as fd: + print 'saving model to %s' % model_path + d={} + for name, w in zip(['rnn_w', 'dense_w', 'dense_b'], [rnn_w, dense_w, dense_b]): + w.to_host() + d[name]=tensor.to_numpy(w) + d['idx_to_char']=data.idx_to_char + d['char_to_idx']=data.char_to_idx + d['hidden_size']=hidden_size + d['num_stacks']=num_stacks + d['dropout']=dropout + + pickle.dump(d, fd) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Train multi-stack LSTM for '\ + 'modeling character sequence from plain text files') + parser.add_argument('data', type=string, help='training file') + parser.add_argument('-b', type=int, default=32, help='batch_size') + parser.add_argument('-l', type=int, default=64, help='sequence length') + parser.add_argument('-d', type=int, default=128, help='hidden size') + parser.add_argument('-s', type=int, default=2, help='num of stacks') + parser.add_argument('-m', type=int, default=50, help='max num of epoch') + args = parser.parse_args() + data = Data(args.data, batch_size=args.b, seq_length=args.l) + train(data, args.m, hidden_size=args.d, num_stacks=args.s, + seq_length=args.l, batch_size=args.b) diff --git a/src/core/tensor/tensor.cc b/src/core/tensor/tensor.cc index 846bda715c..e5bd67374b 100644 --- a/src/core/tensor/tensor.cc +++ b/src/core/tensor/tensor.cc @@ -301,7 +301,8 @@ Tensor &Tensor::operator=(const Tensor &in) { shape_ = in.shape_; device_ = in.device_; block_ = in.block(); - block_->IncRefCount(); + if (block_ != nullptr) + block_->IncRefCount(); return *this; } diff --git a/src/io/csv_encoder.cc b/src/io/csv_encoder.cc index 1b797a9a2b..6089ab5f6c 100644 --- a/src/io/csv_encoder.cc +++ b/src/io/csv_encoder.cc @@ -22,7 +22,7 @@ namespace singa { std::string CSVEncoder::Encode(vector& data) { - CHECK_GE(data.size(), 1); + CHECK_GE(data.size(), 1u); size_t size = data[0].Size(); const float* value = data[0].data(); std::string des = ""; diff --git a/src/model/layer/cudnn_rnn.cc b/src/model/layer/cudnn_rnn.cc index 242a342e92..896c1e9184 100644 --- a/src/model/layer/cudnn_rnn.cc +++ b/src/model/layer/cudnn_rnn.cc @@ -17,6 +17,7 @@ */ #include "./cudnn_rnn.h" #ifdef USE_CUDNN +#if CUDNN_VERSION_MAJOR >= 5 #include #include #include "./cudnn_utils.h" @@ -92,7 +93,7 @@ void CudnnRNN::UpdateIODescriptors(size_t len, const vector &inputs) { } for (size_t i = 0; i < len; i++) { - CHECK_EQ(inputs[i].shape(1), input_dim_); + CHECK_EQ(inputs[i].shape(1), input_size_); if (inputs[i].shape(0) != batch_size_ || reset) { int d[3] = {1, 1, 1}, s[3] = {1, 1, 1}; d[0] = static_cast(inputs[i].shape(0)); @@ -104,7 +105,7 @@ void CudnnRNN::UpdateIODescriptors(size_t len, const vector &inputs) { CUDNN_CHECK(cudnnSetTensorNdDescriptor(dx_descs_[i], dtype_, 3, d, s)); d[0] = static_cast(inputs[i].shape(0)); - d[1] = static_cast(hidden_dim_ * num_directions_); + d[1] = static_cast(hidden_size_ * num_directions_); s[0] = d[1] * d[2]; s[1] = d[2]; CUDNN_CHECK(cudnnSetTensorNdDescriptor(y_descs_[i], dtype_, 3, d, s)); @@ -121,7 +122,7 @@ void CudnnRNN::SetRNNDescriptor(shared_ptr dev) { CUDNN_CHECK(cudnnDropoutGetStatesSize(ctx->cudnn_handle, &state_size)); dropout_state_ = Tensor(Shape{state_size}, dev, kChar); CUDNN_CHECK(cudnnSetDropoutDescriptor( - dropout_desc_, ctx->cudnn_handle, dropout_, + dropout_desc_, ctx->cudnn_handle, 1 - dropout_, // keep probability dropout_state_.block()->mutable_data(), state_size, seed_)); CUDNN_CHECK(cudnnCreateRNNDescriptor(&rnn_desc_)); @@ -146,7 +147,7 @@ void CudnnRNN::SetRNNDescriptor(shared_ptr dev) { rnn_mode = CUDNN_LSTM; else if (rnn_mode_ == "gru") rnn_mode = CUDNN_GRU; - CUDNN_CHECK(cudnnSetRNNDescriptor(rnn_desc_, hidden_dim_, num_stacks_, + CUDNN_CHECK(cudnnSetRNNDescriptor(rnn_desc_, hidden_size_, num_stacks_, dropout_desc_, input_mode, direction, rnn_mode, dtype_)); @@ -176,7 +177,7 @@ void CudnnRNN::ResetHiddenAndCellDescriptors(size_t batch_size) { int dim[3] = {1, 1, 1}; dim[0] = static_cast(num_stacks_ * num_directions_); dim[1] = static_cast(batch_size); - dim[2] = static_cast(hidden_dim_); + dim[2] = static_cast(hidden_size_); int stride[3] = {1, 1, 1}; stride[0] = dim[1] * dim[2]; stride[1] = dim[2]; @@ -238,7 +239,7 @@ vector CudnnRNN::SplitOutput(size_t num, size_t dim, const Tensor output) { vector outputs; if (num == 1) { - outputs.push_back(output); + outputs.push_back(Reshape(output, Shape{in.at(0).shape(0), dim})); } else { for (size_t i = 0, offset = 0; offset < output.Size(); i++) { Shape s{in.at(i).shape(0), dim}; @@ -261,7 +262,7 @@ const vector CudnnRNN::Forward(int flag, const vector &inputs) { CHECK_GT(inputs.size(), 1u + has_cell_); size_t num_x = inputs.size() - has_cell_ - 1; Tensor input = MergeInputs(num_x, inputs); - LOG(INFO) << "input size " << input.Size() << " value " << input.L1(); + // LOG(INFO) << "input size " << input.Size() << " value " << input.L1(); if (rnn_desc_ != nullptr) CHECK_EQ(dtype_, GetCudnnDataType(dtype)) @@ -273,11 +274,11 @@ const vector CudnnRNN::Forward(int flag, const vector &inputs) { UpdateStates(num_x, inputs); // CheckFowardShapes(); - Shape outshape{input.Size() * hidden_dim_ / input_dim_ * num_directions_}; + Shape outshape{input.Size() * hidden_size_ / input_size_ * num_directions_}; Tensor output(outshape, dev, dtype); - LOG(INFO) << "output size " << output.Size(); + // LOG(INFO) << "output size " << output.Size(); Tensor hx = inputs.at(num_x); - Shape state_shape{num_stacks_ * num_directions_, batch_size_, hidden_dim_}; + Shape state_shape{num_stacks_ * num_directions_, batch_size_, hidden_size_}; Tensor hy(state_shape, dev, dtype); Tensor cy, cx; if (has_cell_) { @@ -285,8 +286,8 @@ const vector CudnnRNN::Forward(int flag, const vector &inputs) { cy.ResetLike(hy); } - LOG(INFO) << "hidden size " << hy.Size(); - LOG(INFO) << "weight size " << weight_.Size() << " value " << weight_.L1(); + // LOG(INFO) << "hidden size " << hy.Size(); + // LOG(INFO) << "weight size " << weight_.Size() << " value " << weight_.L1(); Block *inb = input.block(), *outb = output.block(), *wb = this->weight_.block(), *hxb = hx.block(), *cxb = cx.block(), *hyb = hy.block(), *cyb = cy.block(), @@ -336,7 +337,7 @@ const vector CudnnRNN::Forward(int flag, const vector &inputs) { }, {inb, wb, hxb, cxb}, {outb, hyb, cyb, wspace}); } auto outputs = - SplitOutput(num_x, hidden_dim_ * num_directions_, inputs, output); + SplitOutput(num_x, hidden_size_ * num_directions_, inputs, output); outputs.push_back(hy); if (has_cell_) outputs.push_back(cy); return outputs; @@ -368,10 +369,10 @@ const std::pair, vector> CudnnRNN::Backward( if (has_cell_) dcy = grads.at(num_dy + 1); - Shape xshape{y.Size() * input_dim_ / hidden_dim_ / num_directions_}; + Shape xshape{y.Size() * input_size_ / hidden_size_ / num_directions_}; Tensor dx(xshape, dev, dtype); Tensor dw(weight_.shape(), dev, dtype); - Shape state_shape{num_stacks_ * num_directions_, batch_size_, hidden_dim_}; + Shape state_shape{num_stacks_ * num_directions_, batch_size_, hidden_size_}; Tensor dhx(state_shape, dev, dtype); Tensor dcx; if (has_cell_) @@ -419,7 +420,7 @@ const std::pair, vector> CudnnRNN::Backward( {dxb, dwb, dhxb, dcxb, wspace, rspace}); vector param_grad{dw}; - auto data_grads = SplitOutput(num_dy, input_dim_, grads, dx); + auto data_grads = SplitOutput(num_dy, input_size_, grads, dx); data_grads.push_back(dhx); if (has_cell_) data_grads.push_back(dcx); @@ -427,4 +428,5 @@ const std::pair, vector> CudnnRNN::Backward( } } // namespace singa +#endif // CUDNN_VERSION_MAJOR >= 5 #endif // USE_CUDNN diff --git a/src/model/layer/rnn.cc b/src/model/layer/rnn.cc index 6b831a7d86..424c20bf7c 100644 --- a/src/model/layer/rnn.cc +++ b/src/model/layer/rnn.cc @@ -27,13 +27,13 @@ void RNN::Setup(const Shape& in_sample, const LayerConf &conf) { Layer::Setup(in_sample, conf); RNNConf rnn_conf = conf.rnn_conf(); - hidden_dim_ = rnn_conf.hidden_dim(); - CHECK_GT(hidden_dim_, 0u); + hidden_size_ = rnn_conf.hidden_size(); + CHECK_GT(hidden_size_, 0u); num_stacks_ = rnn_conf.num_stacks(); CHECK_GT(num_stacks_, 0u); - input_dim_ = Product(in_sample); - CHECK_GT(input_dim_, 0u); - dropout_ = rnn_conf.dropout(); + input_size_ = Product(in_sample); + CHECK_GT(input_size_, 0u); + dropout_ = rnn_conf.dropout(); // drop probability CHECK_GE(dropout_, 0); input_mode_ = ToLowerCase(rnn_conf.input_mode()); @@ -71,9 +71,9 @@ void RNN::Setup(const Shape& in_sample, const LayerConf &conf) { size_t weight_size = 0; for (size_t i = 0; i < num_stacks_; i++) { - size_t dim = hidden_dim_ * (in_sample[0] + hidden_dim_ + 2); + size_t dim = hidden_size_ * (in_sample[0] + hidden_size_ + 2); if (i > 0) - dim = hidden_dim_ * (hidden_dim_ + hidden_dim_ + 2); + dim = hidden_size_ * (hidden_size_ + hidden_size_ + 2); weight_size += mult * dim; } weight_.Reshape(Shape{weight_size}); @@ -81,6 +81,7 @@ void RNN::Setup(const Shape& in_sample, const LayerConf &conf) { const vector RNN::Forward(int flag, const vector& inputs) { vector data_output; + LOG(FATAL) << "CPU RNN is not implemented!"; return data_output; } @@ -88,6 +89,7 @@ const std::pair, vector> RNN::Backward(int flag, const vector& grads) { vector param_grad; vector data_grad; + LOG(FATAL) << "CPU RNN is not implemented!"; return std::make_pair(data_grad, param_grad); } diff --git a/src/model/layer/rnn.h b/src/model/layer/rnn.h index 375002110d..1b5dad7988 100644 --- a/src/model/layer/rnn.h +++ b/src/model/layer/rnn.h @@ -37,20 +37,32 @@ class RNN : public Layer { /// \copydoc Layer::layer_type() const std::string layer_type() const override { return "RNN"; } - /// \copydoc Layer::Setup(const LayerConf&); - void Setup(const vector& in_shape, const LayerConf& conf) override; + /// Setup the RNN layer. + /// in_shape is the shape of a single training instance from one timestep, + void Setup(const Shape& in_shape, const LayerConf& conf) override; - /// \copydoc Layer::Forward(int flag, const vector&) + /// The inputs vector includes where xi is the input + /// tensor at the i-th time step. hx is used to initialize the hidden tensor, + /// which could be a dummy tensor (like Tensor hx;). cx is used to initialize + /// the cell tensor, which could be a dummy tensor( like Tensor cx;). For + /// dummy tensors, 0's would be used during computation. + /// cx is missing for gru/relu/tanh RNNs, and is valid for lstm. + /// The dim order of xi is , and the batchsize of xi must be + /// >= that of x(i+1). + /// The output vector includes where yi is the output + /// tensor at the i-th time step. hy is the final hidden tensor, cy is the + /// final cell tensor. cy is missing for gru/relu/tanh RNNs and is valid for + /// lstm. const vector Forward(int flag, const vector& inputs) override; - /// \copydoc Layer::Backward(int, const vector&); + /// The grads vector includes , the symbols are + /// similar to those for Forward. dcy is missing for gru/relu/tanh RNNs and is + /// valid for lstm. + /// The first vector of the output includes . + /// The second vector of the output includes the gradients of all parameters. const std::pair, vector> Backward( int flag, const vector& grads) override; - void set_weight(Tensor w) { - weight_.ResetLike(w); - weight_.CopyData(w); - } const vector param_values() override { return vector{weight_}; } @@ -73,7 +85,7 @@ class RNN : public Layer { std::stack buf_; bool has_cell_ = false; size_t num_directions_ = 1; - size_t input_dim_ = 0, hidden_dim_ = 0, num_stacks_ = 0, seq_length_ = 0; + size_t input_size_ = 0, hidden_size_ = 0, num_stacks_ = 0, seq_length_ = 0; size_t batch_size_ = 0; size_t seed_ = 0x1234567; float dropout_ = 0.0f; diff --git a/src/proto/model.proto b/src/proto/model.proto index 31ebfc356f..692382010d 100644 --- a/src/proto/model.proto +++ b/src/proto/model.proto @@ -393,7 +393,7 @@ message ConvolutionConf { } message RNNConf { - optional uint32 hidden_dim = 1; // The number of hiddensize + optional uint32 hidden_size = 1; // The hidden feature size optional uint32 num_stacks = 2; // The number of stacked RNN layers optional float dropout = 3 [default = 0]; optional bool remember_state = 4 [default = false]; diff --git a/src/python/singa/layer.py b/src/python/singa/layer.py index a443e1ab97..a87eb1036f 100644 --- a/src/python/singa/layer.py +++ b/src/python/singa/layer.py @@ -403,7 +403,7 @@ def __init__(self, name, num_output, use_bias=True, if W_specs is None: W_specs = {'init': 'xavier'} if b_specs is None: - b_specs = {'init': 'constant'} + b_specs = {'init': 'constant', 'value': 0} if 'name' not in W_specs: W_specs['name'] = name + '_weight' if 'name' not in b_specs: @@ -502,6 +502,71 @@ def __init__(self, name, axis=1, engine='cudnn', input_sample_shape=None): self.setup(input_sample_shape) +class RNN(Layer): + def __init__(self, name, hidden_size, rnn_mode='lstm', engine='cudnn', + dropout=0.0, num_stacks=1, input_mode='linear', bidirectional=False, + param_specs=None, input_sample_shape=None): + super(RNN, self).__init__(name) + conf = self.conf.rnn_conf + assert hidden_size > 0, 'Hidden feature size must > 0' + conf.hidden_size = hidden_size + assert rnn_mode in Set(['lstm', 'gru', 'tanh', 'relu']), \ + 'rnn mode %s is not available' %s (rnn_mode) + conf.rnn_mode = rnn_mode + conf.num_stacks = num_stacks + conf.dropout = dropout + conf.input_mode = input_mode + conf.direction = 'unidirectional' + if bidirectional: + conf.direction = 'bidirectional' + _check_engine(engine, ['cudnn']) + if param_specs is None: + param_specs = {'name': name + '-weight', + 'init': 'uniform', 'low':0, 'high':1}; + self.conf.param.extend([_construct_param_specs_from_dict(param_specs)]) + self.param_specs.append(_construct_param_specs_from_dict(param_specs)) + + self.layer = singa_wrap.CudnnRNN() + if input_sample_shape is not None: + self.setup(input_sample_shape) + + def forward(self, flag, inputs): + assert self.has_setup, 'Must call setup() before forward()' + assert len(inputs) > 1, 'The input to RNN must include at '\ + 'least one input tensor '\ + 'and one hidden state tensor (could be a dummy tensor)' + tensors = [] + for t in inputs: + assert isinstance(t, tensor.Tensor), 'input must be py Tensor %s' % (type(t)) + tensors.append(t.singa_tensor) + y = self.layer.Forward(flag, tensors) + return tensor.from_raw_tensors(y) + + def backward(self, flag, grad): + tensors = [] + for t in grad: + assert isinstance(t, tensor.Tensor), 'grad must be py Tensor' + tensors.append(t.singa_tensor) + ret = self.layer.Backward(flag, tensors) + return tensor.from_raw_tensors(ret[0]), tensor.from_raw_tensors(ret[1]) + +class LSTM(RNN): + def __init__(self, name, hidden_size, engine='cudnn', + dropout=0.0, num_stacks=1, input_mode='linear', bidirectional=False, + param_specs=None, input_sample_shape=None): + super(LSTM, self).__init__(name, hidden_size, 'lstm', engine, dropout, + num_stacks, input_mode, bidirectional, param_specs, + input_sample_shape) + +class GRU(RNN): + def __init__(self, name, hidden_size, engine='cudnn', + dropout=0.0, num_stacks=1, input_mode='linear', bidirectional=False, + param_specs=None, input_sample_shape=None): + super(GRU, self).__init__(name, hidden_size, 'gru', engine, dropout, + num_stacks, input_mode, bidirectional, param_specs, + input_sample_shape) + + def _check_engine(engine, allowed_engines): assert engine.lower() in Set(allowed_engines), \ '%s is not a supported engine. Pls use one of %s' % \ @@ -585,8 +650,8 @@ def _construct_param_specs_from_dict(specs): if specs['init'].lower() == 'uniform': assert 'low' in specs and 'high' in specs, \ 'low and high are required for "uniform" init method' - filler.low = specs['low'] - filler.high = specs['high'] + filler.min = specs['low'] + filler.max = specs['high'] elif specs['init'].lower() == 'gaussian': assert 'mean' in specs and 'std' in specs, \ 'std and mean are required for "gaussian" init method' diff --git a/src/python/swig/model_layer.i b/src/python/swig/model_layer.i index 873ebc9b24..9d3930179e 100644 --- a/src/python/swig/model_layer.i +++ b/src/python/swig/model_layer.i @@ -30,6 +30,8 @@ %{ #include "singa/model/layer.h" +#include "../src/model/layer/rnn.h" +#include "../src/model/layer/cudnn_rnn.h" #include "singa/core/tensor.h" #include "singa/proto/model.pb.h" using singa::Tensor; @@ -40,6 +42,8 @@ using singa::LayerConf; %} %shared_ptr(singa::Layer) +%shared_ptr(singa::RNN) +%shared_ptr(singa::CudnnRNN) namespace std { %template(strVector) vector; @@ -52,26 +56,44 @@ namespace std { namespace singa { - class Layer { - public: - Layer(); +class Layer { + public: + Layer(); // virtual void Setup(const std::vector>&, const string&); - virtual void Setup(const std::vector& in_sample_shape, - const std::string& proto_str); - const std::vector param_values(); - virtual const std::vector GetOutputSampleShape() const; - virtual void ToDevice(std::shared_ptr device); - virtual void AsType(DataType dtype); - virtual const Tensor Forward(int flag, const Tensor& input); - virtual const std::vector Forward( - int flag, const std::vector& inputs); - virtual const std::pair> Backward( - int flag, const Tensor& grad); - virtual const std::pair, std::vector> - Backward(int flag, const vector& grads); + void Setup(const std::vector& in_sample_shape, + const std::string& proto_str); + virtual const std::vector param_values(); + virtual const std::vector GetOutputSampleShape() const; + virtual void ToDevice(std::shared_ptr device); + virtual void AsType(DataType dtype); + virtual const Tensor Forward(int flag, const Tensor& input); + virtual const std::vector Forward( + int flag, const std::vector& inputs); + virtual const std::pair> Backward( + int flag, const Tensor& grad); + virtual const std::pair, std::vector> + Backward(int flag, const vector& grads); +}; + +std::shared_ptr CreateLayer(const std::string& type); +const std::vector GetRegisteredLayers(); +class RNN : public Layer { + /* + public: + void Setup(const std::vector& in_sample_shape, + const std::string& proto_str) override; + */ +}; +class CudnnRNN : public RNN { + public: + // note: Must use std::vector instead of vector. + const std::vector Forward(int flag, const std::vector& inputs) override; + const std::pair, std::vector> Backward( + int flag, const std::vector& grads) override; + void ToDevice(std::shared_ptr device) override; + const std::vector param_values() override; + const std::vector GetOutputSampleShape() const override; +}; - }; - std::shared_ptr CreateLayer(const std::string& type); - const std::vector GetRegisteredLayers(); } diff --git a/test/singa/test_cudnn_rnn.cc b/test/singa/test_cudnn_rnn.cc index ebbf0aab3e..e0de02e7fd 100644 --- a/test/singa/test_cudnn_rnn.cc +++ b/test/singa/test_cudnn_rnn.cc @@ -21,6 +21,7 @@ #include "../src/model/layer/cudnn_rnn.h" #ifdef USE_CUDNN +#if CUDNN_VERSION_MAJOR >= 5 #include "gtest/gtest.h" @@ -31,15 +32,15 @@ class TestCudnnRNN : public ::testing::Test { protected: virtual void SetUp() { singa::RNNConf *rnnconf = conf.mutable_rnn_conf(); - rnnconf->set_hidden_dim(hidden_dim); + rnnconf->set_hidden_size(hidden_size); rnnconf->set_num_stacks(1); - rnnconf->set_dropout(1); + rnnconf->set_dropout(0); rnnconf->set_input_mode("linear"); rnnconf->set_direction("unidirectional"); rnnconf->set_rnn_mode("tanh"); } singa::LayerConf conf; - size_t hidden_dim = 4; + size_t hidden_size = 4; }; TEST_F(TestCudnnRNN, Setup) { @@ -47,7 +48,7 @@ TEST_F(TestCudnnRNN, Setup) { EXPECT_EQ("CudnnRNN", rnn.layer_type()); rnn.Setup(Shape{2}, conf); auto weight = rnn.param_values().at(0); - EXPECT_EQ(weight.Size(), hidden_dim * (2 + hidden_dim + 2)); + EXPECT_EQ(weight.Size(), hidden_size * (2 + hidden_size + 2)); } TEST_F(TestCudnnRNN, Forward) { @@ -80,19 +81,19 @@ TEST_F(TestCudnnRNN, Forward) { const auto ret = rnn.Forward(singa::kEval, inputs); EXPECT_EQ(ret.size(), seqLength + 1); - vector hxptr(hidden_dim, 0.0f); + vector hxptr(hidden_size, 0.0f); for (size_t i = 0; i < seqLength; i++) { auto y = ret[i]; y.ToHost(); auto yptr = y.data(); vector tmp; - for (size_t j = 0; j < hidden_dim; j++) { + for (size_t j = 0; j < hidden_size; j++) { float ty = 0; for (size_t k = 0; k < dim; k++) { ty += x[i * dim + k] * wvalue; } ty += wvalue; - for (size_t k = 0; k < hidden_dim; k++) { + for (size_t k = 0; k < hidden_size; k++) { ty += hxptr[k] * wvalue; } ty += wvalue; @@ -134,18 +135,18 @@ TEST_F(TestCudnnRNN, Backward) { const auto outs = rnn.Forward(singa::kTrain, inputs); - float dyptr[seqLength * batchsize * hidden_dim]; - for (size_t i = 0; i < seqLength * batchsize * hidden_dim; i++) + float dyptr[seqLength * batchsize * hidden_size]; + for (size_t i = 0; i < seqLength * batchsize * hidden_size; i++) dyptr[i] = i * 0.1f; vector grads; for (size_t i = 0; i < seqLength; i++) { - Tensor dy(Shape{batchsize, hidden_dim}, cuda); + Tensor dy(Shape{batchsize, hidden_size}, cuda); dy.CopyDataFromHostPtr(dyptr + i * dy.Size(), dy.Size()); grads.push_back(dy); } Tensor dhy; grads.push_back(dhy); - vector dhyptr(hidden_dim, 0.0f); + vector dhyptr(hidden_size, 0.0f); const auto ret = rnn.Backward(singa::kTrain, grads); for (size_t i = seqLength - 1; i > 0 ; i --) { auto dx = ret.first[i]; @@ -154,21 +155,21 @@ TEST_F(TestCudnnRNN, Backward) { dx.ToHost(); auto dxptr = dx.data(); auto yptr = y.data(); - for (size_t j = 0; j < hidden_dim; j++) { - dhyptr[j] += dyptr[i * hidden_dim + j]; + for (size_t j = 0; j < hidden_size; j++) { + dhyptr[j] += dyptr[i * hidden_size + j]; dhyptr[j] *= 1 - yptr[j] * yptr[j]; } for (size_t k = 0; k < dim; k++) { float tdx = 0; - for (size_t j = 0; j < hidden_dim; j++) { + for (size_t j = 0; j < hidden_size; j++) { tdx += dhyptr[j] * wvalue; } EXPECT_NEAR(tdx, dxptr[k], 1e-4); } vector tmp; - for (size_t k = 0; k < hidden_dim; k++) { + for (size_t k = 0; k < hidden_size; k++) { float tdhy = 0; - for (size_t j = 0; j < hidden_dim; j++) { + for (size_t j = 0; j < hidden_size; j++) { tdhy += dhyptr[j] * wvalue; } tmp.push_back(tdhy); @@ -176,4 +177,5 @@ TEST_F(TestCudnnRNN, Backward) { std::copy(tmp.begin(), tmp.end(), dhyptr.begin()); } } +#endif // CUDNN_VERSION_MAJOR >= 5 #endif // USE_CUDNN From 58f65c7c293263e0cb3e76dad26b09095784238a Mon Sep 17 00:00:00 2001 From: caiqc Date: Thu, 4 Aug 2016 15:51:47 +0800 Subject: [PATCH 09/16] SINGA-233 A new communication framework for SINGA In this ticket, we implement a new communication framework for SINGA. We abstract each physical computing node as an endpoint, and add two interfaces, i.e., send and recv, to the endpoint so that users can directly call them to accomplish data transfer. --- CMakeLists.txt | 11 +- include/singa/io/network/endpoint.h | 130 ++++++ include/singa/io/network/integer.h | 73 ++++ include/singa/io/network/message.h | 75 ++++ src/CMakeLists.txt | 1 + src/io/network/endpoint.cc | 650 ++++++++++++++++++++++++++++ src/io/network/message.cc | 115 +++++ test/singa/test_ep.cc | 69 +++ 8 files changed, 1120 insertions(+), 4 deletions(-) create mode 100644 include/singa/io/network/endpoint.h create mode 100644 include/singa/io/network/integer.h create mode 100644 include/singa/io/network/message.h create mode 100644 src/io/network/endpoint.cc create mode 100644 src/io/network/message.cc create mode 100644 test/singa/test_ep.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 38014ce097..e688b50074 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,12 +18,12 @@ SET(SINGA_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/include;${CMAKE_SOURCE_DIR}/lib/cnmem/include;${PROJECT_BINARY_DIR}") INCLUDE_DIRECTORIES(${SINGA_INCLUDE_DIR}) -OPTION(USE_CBLAS "Use CBlas libs" ON) -OPTION(USE_CUDA "Use Cuda libs" ON) -OPTION(USE_CUDNN "Use Cudnn libs" ON) +OPTION(USE_CBLAS "Use CBlas libs" OFF) +OPTION(USE_CUDA "Use Cuda libs" OFF) +OPTION(USE_CUDNN "Use Cudnn libs" OFF) OPTION(USE_OPENCV "Use opencv" OFF) OPTION(USE_LMDB "Use LMDB libs" OFF) -OPTION(USE_PYTHON "Generate py wrappers" ON) +OPTION(USE_PYTHON "Generate py wrappers" OFF) OPTION(USE_OPENCL "Use OpenCL" OFF) #OPTION(BUILD_OPENCL_TESTS "Build OpenCL tests" OFF) @@ -46,6 +46,9 @@ IF (USE_CUDA) ADD_SUBDIRECTORY(lib/cnmem) LIST(APPEND SINGA_LINKER_LIBS cnmem) ENDIF() + +LIST(APPEND SINGA_LINKER_LIBS ev) + ADD_SUBDIRECTORY(src) ADD_SUBDIRECTORY(test) ADD_SUBDIRECTORY(examples) diff --git a/include/singa/io/network/endpoint.h b/include/singa/io/network/endpoint.h new file mode 100644 index 0000000000..1079fcc41e --- /dev/null +++ b/include/singa/io/network/endpoint.h @@ -0,0 +1,130 @@ +/************************************************************ +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*************************************************************/ + +#ifndef SINGA_COMM_END_POINT_H_ +#define SINGA_COMM_END_POINT_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "singa/io/network/message.h" + +namespace singa { + +#define LOCKED 1 +#define UNLOCKED 0 + +#define SIG_EP 1 +#define SIG_MSG 2 + +#define CONN_INIT 0 +#define CONN_PENDING 1 +#define CONN_EST 2 +#define CONN_ERROR 3 + +#define MAX_RETRY_CNT 3 + +class NetworkThread; +class EndPointFactory; + +class EndPoint { + private: + std::queue send_; + std::queue recv_; + std::queue to_ack_; + std::condition_variable cv_; + std::mutex mtx_; + struct sockaddr_in addr_; + int fd_[2] = {-1, -1}; // two endpoints simultaneously connect to each other + int conn_status_ = CONN_INIT; + int pending_cnt_ = 0; + int retry_cnt_ = 0; + NetworkThread* thread_ = nullptr; + EndPoint(NetworkThread* t):thread_(t){} + ~EndPoint(); + friend class NetworkThread; + friend class EndPointFactory; + public: + int send(Message*); + Message* recv(); +}; + +class EndPointFactory { + private: + std::unordered_map ip_ep_map_; + std::condition_variable map_cv_; + std::mutex map_mtx_; + NetworkThread* thread_; + EndPoint* getEp(uint32_t ip); + EndPoint* getOrCreateEp(uint32_t ip); + friend class NetworkThread; + public: + EndPointFactory(NetworkThread* thread) : thread_(thread) {} + ~EndPointFactory(); + EndPoint* getEp(const char* host); + void getNewEps(std::vector& neps); +}; + +class NetworkThread{ + private: + struct ev_loop *loop_; + ev_async ep_sig_; + ev_async msg_sig_; + + std::thread* thread_; + + std::unordered_map fd_wwatcher_map_; + std::unordered_map fd_rwatcher_map_; + std::unordered_map fd_ip_map_; + + std::map pending_msgs_; + + ev_io socket_watcher_; + int port_; + int socket_fd_; + + void handleConnLost(int, EndPoint*, bool = true); + void doWork(); + int asyncSend(int); + void asyncSendPendingMsg(EndPoint*); + public: + EndPointFactory* epf_; + + NetworkThread(int); + //void join(); + void notify(int signal); + + void onRecv(int fd); + void onSend(int fd = -1); + void onConnEst(int fd); + void onNewEp(); + void onNewConn(); +}; +} +#endif diff --git a/include/singa/io/network/integer.h b/include/singa/io/network/integer.h new file mode 100644 index 0000000000..9c2799d510 --- /dev/null +++ b/include/singa/io/network/integer.h @@ -0,0 +1,73 @@ +/************************************************************ + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + *************************************************************/ + +#ifndef INTEGER_H_ +#define INTEGER_H_ + +#include + +namespace singa{ +static bool isNetworkOrder() { + int test = 1; + return (1 != *(uint8_t*)&test); +} + +template +static inline T byteSwap(const T& v) { + int size = sizeof(v); + T ret; + uint8_t *dest = reinterpret_cast(&ret); + uint8_t *src = const_cast(reinterpret_cast(&v)); + for (int i = 0; i < size; ++i) { + dest[i] = src[size - i - 1]; + } + return ret; +} + +template +static inline T hton(const T& v) +{ + return isNetworkOrder() ? v : byteSwap(v); +} + +template +static inline T ntoh(const T& v) +{ + return hton(v); +} + +static inline int appendInteger(char* buf) {return 0;} +static inline int readInteger(char* buf) {return 0;} + +template +static int appendInteger(char* buf, Type value, Types... values) { + *(Type*)buf = hton(value); + return sizeof(Type) + appendInteger(buf + sizeof(Type), values...); +} + +template +static int readInteger(char* buf, Type& value, Types&... values) { + value = ntoh(*(Type*)buf); + return sizeof(Type) + readInteger(buf + sizeof(Type), values...); +} + +} +#endif diff --git a/include/singa/io/network/message.h b/include/singa/io/network/message.h new file mode 100644 index 0000000000..0f691c0b49 --- /dev/null +++ b/include/singa/io/network/message.h @@ -0,0 +1,75 @@ +/************************************************************ +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*************************************************************/ + +#ifndef SINGA_COMM_MESSAGE_H_ +#define SINGA_COMM_MESSAGE_H_ + +#include +#include + +namespace singa { + +#define MSG_DATA 0 +#define MSG_ACK 1 + +class NetworkThread; +class EndPoint; +class Message{ + private: + uint8_t type_; + uint32_t id_; + std::size_t msize_ = 0; + std::size_t psize_ = 0; + std::size_t processed_ = 0; + char* msg_ = nullptr; + static const int hsize_ = sizeof(id_) + 2 * sizeof(std::size_t) + sizeof(type_); + char mdata_[hsize_]; + friend class NetworkThread; + friend class EndPoint; + public: + Message(int = MSG_DATA, uint32_t = 0); + Message(const Message&) = delete; + Message(Message&&); + ~Message(); + + void setMetadata(const void*, int); + void setPayload(const void*, int); + + std::size_t getMetadata(void**); + std::size_t getPayload(void**); + + std::size_t getSize(); + void setId(uint32_t); +}; + +class MessageQueue +{ + public: + void push(Message&); + Message& front(); + void pop(); + std::size_t size(); + private: + std::mutex lock_; + std::queue mqueue_; +}; +} +#endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 38e6aa36d1..06f177d358 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -69,6 +69,7 @@ TARGET_LINK_LIBRARIES(singa_model ${SINGA_LINKER_LIBS}) LIST(APPEND SINGA_LINKER_LIBS singa_model) AUX_SOURCE_DIRECTORY(io io_source) +AUX_SOURCE_DIRECTORY(io/network io_source) ADD_LIBRARY(singa_io SHARED ${io_source}) TARGET_LINK_LIBRARIES(singa_io ${SINGA_LINKER_LIBS}) LIST(APPEND SINGA_LINKER_LIBS singa_io) diff --git a/src/io/network/endpoint.cc b/src/io/network/endpoint.cc new file mode 100644 index 0000000000..2926a056c1 --- /dev/null +++ b/src/io/network/endpoint.cc @@ -0,0 +1,650 @@ +/************************************************************ +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*************************************************************/ + +#include "singa/io/network/endpoint.h" +#include "singa/io/network/integer.h" +#include "singa/utils/logging.h" + +#include +#include +#include +#include +#include +#include + +#include + +namespace singa { + +static void async_ep_cb(struct ev_loop* loop, ev_async* ev, int revent) { + reinterpret_cast(ev_userdata(loop))->onNewEp(); +} + +static void async_msg_cb(struct ev_loop* loop, ev_async* ev, int revent) { + reinterpret_cast(ev_userdata(loop))->onSend(); +} + +static void writable_cb(struct ev_loop* loop, ev_io* ev, int revent) { + reinterpret_cast(ev_userdata(loop))->onSend(ev->fd); +} + +static void readable_cb(struct ev_loop* loop, ev_io* ev, int revent) { + reinterpret_cast(ev_userdata(loop))->onRecv(ev->fd); +} + +static void conn_cb(struct ev_loop* loop, ev_io* ev, int revent) { + reinterpret_cast(ev_userdata(loop))->onConnEst(ev->fd); +} + +static void accept_cb(struct ev_loop* loop, ev_io* ev, int revent) { + reinterpret_cast(ev_userdata(loop))->onNewConn(); +} + +EndPoint::~EndPoint() { + while(!recv_.empty()) { + delete send_.front(); + send_.pop(); + } + while(!to_ack_.empty()) { + delete send_.front(); + send_.pop(); + } + while(!send_.empty()) { + delete send_.front(); + send_.pop(); + } +} + +int EndPoint::send(Message* msg) { + CHECK(msg->type_ == MSG_DATA); + static std::atomic id(0); + std::unique_lock lock(this->mtx_); + + if (this->conn_status_ == CONN_ERROR) { + LOG(INFO) << "EndPoint " << inet_ntoa(addr_.sin_addr) << " is disconnected"; + return -1; + } + + if (msg->psize_ == 0 && msg->msize_ == 0) + // no data to send + return 0; + + msg->setId(id++); + + send_.push(new Message(static_cast(*msg))); + + thread_->notify(SIG_MSG); + return msg->getSize(); +} + +Message* EndPoint::recv() { + std::unique_lock lock(this->mtx_); + while(this->recv_.empty() && conn_status_ != CONN_ERROR) + this->cv_.wait(lock); + + Message* ret = nullptr; + if (!recv_.empty()) { + ret = recv_.front(); + recv_.pop(); + } + return ret; +} + +EndPointFactory::~EndPointFactory() { + for (auto& p : ip_ep_map_) + { + delete p.second; + } +} + +EndPoint* EndPointFactory::getOrCreateEp(uint32_t ip) { + std::unique_lock lock(map_mtx_); + if (0 == ip_ep_map_.count(ip)) { + ip_ep_map_[ip] = new EndPoint(this->thread_); + } + return ip_ep_map_[ip]; +} + +EndPoint* EndPointFactory::getEp(uint32_t ip) { + std::unique_lock lock(map_mtx_); + if (0 == ip_ep_map_.count(ip)) { + return nullptr; + } + return ip_ep_map_[ip]; +} + +EndPoint* EndPointFactory::getEp(const char* host) { + // get the ip address of host + struct hostent *he; + struct in_addr **list; + + if ((he = gethostbyname(host)) == nullptr) { + LOG(INFO) << "Unable to resolve host " << host; + return nullptr; + } + + list = (struct in_addr**) he->h_addr_list; + uint32_t ip = ntohl(list[0]->s_addr); + + EndPoint* ep = nullptr; + map_mtx_.lock(); + if (0 == ip_ep_map_.count(ip)) { + ep = new EndPoint(this->thread_); + ep->thread_ = this->thread_; + ip_ep_map_[ip] = ep; + + // copy the address info + bcopy(list[0], &ep->addr_.sin_addr, sizeof(struct in_addr)); + + thread_->notify(SIG_EP); + } + ep = ip_ep_map_[ip]; + map_mtx_.unlock(); + + std::unique_lock eplock(ep->mtx_); + while (ep->conn_status_ == CONN_PENDING || ep->conn_status_ == CONN_INIT) { + ep->pending_cnt_++; + ep->cv_.wait(eplock); + ep->pending_cnt_--; + } + + if (ep->conn_status_ == CONN_ERROR) { + ep = nullptr; + } + + return ep; +} + +void EndPointFactory::getNewEps(std::vector& neps) { + std::unique_lock lock(this->map_mtx_); + for (auto& p : this->ip_ep_map_) { + EndPoint* ep = p.second; + std::unique_lock eplock(ep->mtx_); + if (ep->conn_status_ == CONN_INIT) { + neps.push_back(ep); + } + } +} + +NetworkThread::NetworkThread(int port) { + this->port_ = port; + thread_ = new std::thread([this] {doWork();}); + this->epf_ = new EndPointFactory(this); +} + +void NetworkThread::doWork() { + + // prepare event loop + if (!(loop_ = ev_default_loop(0))) { + // log here + } + + ev_async_init(&ep_sig_, async_ep_cb); + ev_async_start(loop_, &ep_sig_); + + ev_async_init(&msg_sig_, async_msg_cb); + ev_async_start(loop_, &msg_sig_); + + // bind and listen + struct sockaddr_in addr; + if ((socket_fd_ = socket(AF_INET, SOCK_STREAM, 0)) < 0) { + LOG(FATAL) << "Socket Error: " << strerror(errno); + } + + bzero(&addr, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(this->port_); + addr.sin_addr.s_addr = INADDR_ANY; + + if (bind(socket_fd_, (struct sockaddr*)&addr, sizeof(addr))) { + LOG(FATAL) << "Bind Error: " << strerror(errno); + } + + if (listen(socket_fd_, 10)) { + LOG(FATAL) << "Listen Error: " << strerror(errno); + } + + ev_io_init(&socket_watcher_, accept_cb, socket_fd_, EV_READ); + ev_io_start(loop_, &socket_watcher_); + + ev_set_userdata(loop_, this); + + while(1) + ev_run(loop_, 0); +} + +void NetworkThread::notify(int signal) { + switch(signal) { + case SIG_EP: + ev_async_send(this->loop_, &this->ep_sig_); + break; + case SIG_MSG: + ev_async_send(this->loop_, &this->msg_sig_); + break; + default: + break; + } +} + +void NetworkThread::onNewEp() { + std::vector neps; + this->epf_->getNewEps(neps); + + for (auto& ep : neps) { + std::unique_lock ep_lock(ep->mtx_); + int& fd = ep->fd_[0]; + if (ep->conn_status_ == CONN_INIT) { + + fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) { + // resources not available + LOG(FATAL) << "Unable to create socket"; + } + + // set this fd non-blocking + fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK); + + this->fd_ip_map_[fd] = ntohl(ep->addr_.sin_addr.s_addr); + + // initialize the addess + ep->addr_.sin_family = AF_INET; + ep->addr_.sin_port = htons(port_); + bzero(&(ep->addr_.sin_zero), 8); + + if (connect(fd, (struct sockaddr*)&ep->addr_, + sizeof(struct sockaddr)) ) { + LOG(INFO) << "Connect Error: " << strerror(errno); + if (errno != EINPROGRESS) { + ep->conn_status_ = CONN_ERROR; + ep->cv_.notify_all(); + continue; + } else { + ep->conn_status_ = CONN_PENDING; + ev_io_init(&this->fd_wwatcher_map_[fd], conn_cb, fd, EV_WRITE); + ev_io_start(this->loop_, &this->fd_wwatcher_map_[fd]); + } + } else { + // connection established immediately + LOG(INFO) << "Connected to " << inet_ntoa(ep->addr_.sin_addr) << " fd = "<< fd; + ep->conn_status_ = CONN_EST; + ev_io_stop(this->loop_, &this->fd_wwatcher_map_[fd]); + + // poll for new msgs + ev_io_init(&this->fd_rwatcher_map_[fd], readable_cb, fd, EV_READ); + ev_io_start(this->loop_, &this->fd_rwatcher_map_[fd]); + + asyncSendPendingMsg(ep); + ep->cv_.notify_all(); + } + } + } +} + +void NetworkThread::onConnEst(int fd) { + + EndPoint* ep = epf_->getEp(this->fd_ip_map_[fd]); + + std::unique_lock lock(ep->mtx_); + + if (connect(fd, (struct sockaddr*)&ep->addr_, sizeof(struct sockaddr)) < 0 && errno != EISCONN) { + LOG(INFO) << "Unable to connect to " << inet_ntoa(ep->addr_.sin_addr) << ": "<< strerror(errno); + if (errno == EINPROGRESS) { + // continue to watch this socket + return; + } + + handleConnLost(ep->fd_[0], ep); + + switch(ep->conn_status_) { + case CONN_INIT: + case CONN_PENDING: + return; + default: + break; + } + + } else { + LOG(INFO) << "Connected to " << inet_ntoa(ep->addr_.sin_addr) << ", fd = "<< fd; + ep->conn_status_ = CONN_EST; + // connect established; poll for new msgs + ev_io_stop(this->loop_, &this->fd_wwatcher_map_[fd]); + + ev_io_init(&this->fd_rwatcher_map_[fd], readable_cb, fd, EV_READ); + ev_io_start(this->loop_, &this->fd_rwatcher_map_[fd]); + } + + if (ep->conn_status_ == CONN_EST && ep->to_ack_.size() > 0) + // if there are pending message, it means these msgs were sent over + // previous sockets that have been lost now + // we need to resend these msgs to the remote side + asyncSendPendingMsg(ep); + + // Finally notify all waiting threads + ep->cv_.notify_all(); +} + +void NetworkThread::onNewConn() { + // accept new tcp connection + struct sockaddr_in addr; + socklen_t len = sizeof(addr); + int fd = accept(socket_fd_, (struct sockaddr*)&addr, &len); + if (fd < 0) { + LOG(INFO) << "Accept Error: " << strerror(errno); + return; + } + + LOG(INFO) << "Accept a client from " << inet_ntoa(addr.sin_addr) << ", fd = " << fd; + + // set this fd as non-blocking + fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK); + + EndPoint* ep; + uint32_t a = ntohl(addr.sin_addr.s_addr); + + ep = epf_->getOrCreateEp(a); + std::unique_lock lock(ep->mtx_); + + if (ep->fd_[1] >= 0) { + // the previous connection is lost + handleConnLost(ep->fd_[1], ep, false); + } + + if (ep->fd_[0] == fd) { + // this fd is reused + handleConnLost(fd, ep, false); + } + + fd_ip_map_[fd] = a; + ev_io_init(&fd_rwatcher_map_[fd], readable_cb, fd, EV_READ); + ev_io_start(loop_, &fd_rwatcher_map_[fd]); + + // record the remote address + bcopy(&addr, &ep->addr_, len); + + ep->conn_status_ = CONN_EST; + ep->fd_[1] = fd; + + if (ep->to_ack_.size() > 0) + // see if there are any messages waiting for ack + // if yes, resend them + asyncSendPendingMsg(ep); + + // this connection is initiaed by remote side, + // so we dont need to notify the waiting thread + // later threads wanting to send to this ep, however, + // are able to reuse this ep +} + +void NetworkThread::onSend(int fd) { + std::vector invalid_fd; + + if (fd == -1) { + // this is a signal of new message to send + for(auto& p : fd_ip_map_) { + // send message + if (asyncSend(p.first) < 0) + invalid_fd.push_back(p.first); + } + } else { + if (asyncSend(fd) < 0) + invalid_fd.push_back(fd); + } + + for (auto& p : invalid_fd) { + EndPoint* ep = epf_->getEp(fd_ip_map_.at(p)); + std::unique_lock lock(ep->mtx_); + handleConnLost(p, ep); + } +} + +void NetworkThread::asyncSendPendingMsg(EndPoint* ep) { + // simply put the pending msgs to the send queue + + LOG(INFO) << "There are " << ep->send_.size() << " to-send msgs, and " << ep->to_ack_.size() << " to-ack msgs"; + + while (!ep->send_.empty()) { + ep->to_ack_.push(ep->send_.front()); + ep->send_.pop(); + } + + std::swap(ep->send_, ep->to_ack_); + + if (ep->send_.size() > 0) { + notify(SIG_MSG); + } +} + +/** + * @brief non-locking send; + * + * @param ep + * + */ +int NetworkThread::asyncSend(int fd) { + + EndPoint* ep = epf_->getEp(fd_ip_map_[fd]); + + std::unique_lock ep_lock(ep->mtx_); + + if (ep->conn_status_ != CONN_EST) + goto out; + + while(!ep->send_.empty()) { + + Message& msg = *ep->send_.front(); + int nbytes; + + while(msg.processed_ < msg.getSize()) { + if (msg.type_ == MSG_ACK) { + nbytes = write(fd, msg.mdata_ + msg.processed_, msg.getSize() - msg.processed_); + } + else + nbytes = write(fd, msg.msg_ + msg.processed_, msg.getSize() - msg.processed_); + + LOG(INFO) << "Send " << nbytes << " bytes to " << inet_ntoa(ep->addr_.sin_addr) << " over fd " << fd; + + if (nbytes == -1) { + if (errno == EWOULDBLOCK) { + ev_io_init(&fd_wwatcher_map_[fd], writable_cb, fd, EV_WRITE); + ev_io_start(loop_, &fd_wwatcher_map_[fd]); + goto out; + } else { + // this connection is lost; reset the send status + // so that next time the whole msg would be sent entirely + msg.processed_ = 0; + goto err; + } + } else + msg.processed_ += nbytes; + } + + CHECK(msg.processed_ == msg.getSize()); + + if (msg.type_ != MSG_ACK) { + msg.processed_ = 0; + ep->to_ack_.push(&msg); + } else { + delete &msg; + } + + ep->send_.pop(); + + // for test + if (ep->retry_cnt_ == 0) + close(fd); + } +out: + if (ep->send_.empty()) + ev_io_stop(loop_, &this->fd_wwatcher_map_[fd]); + return 0; +err: + return -1; +} + +void NetworkThread::onRecv(int fd) { + + Message* m = &pending_msgs_[fd]; + Message& msg = (*m); + int nread; + EndPoint* ep = epf_->getEp(fd_ip_map_[fd]); + + LOG(INFO) << "Start to read from EndPoint " << inet_ntoa(ep->addr_.sin_addr) << " over fd " << fd; + + std::unique_lock lock(ep->mtx_); + while(1) { + + if (msg.processed_ < Message::hsize_) { + nread = read(fd, msg.mdata_ + msg.processed_, + Message::hsize_ - msg.processed_); + + if (nread <= 0) { + if (errno != EWOULDBLOCK || nread == 0) { + // socket error or shuts down + if (nread < 0) + LOG(INFO) << "Faile to receive from EndPoint " << inet_ntoa(ep->addr_.sin_addr) << ": " << strerror(errno); + else + LOG(INFO) << "Faile to receive from EndPoint " << inet_ntoa(ep->addr_.sin_addr) << ": Connection reset by remote side"; + handleConnLost(fd, ep); + } + break; + } + + msg.processed_ += nread; + while (msg.processed_ >= sizeof(msg.type_) + sizeof(msg.id_)) { + readInteger(msg.mdata_, msg.type_, msg.id_); + if(msg.type_ == MSG_ACK) { + LOG(INFO) << "Receive an ACK message from " << inet_ntoa(ep->addr_.sin_addr) << " for MSG " << msg.id_; + while (!ep->to_ack_.empty()) { + Message* m = ep->to_ack_.front(); + if (m->id_ <= msg.id_) { + delete m; + ep->to_ack_.pop(); + } else { + break; + } + } + + // reset + msg.processed_ -= sizeof(msg.type_) + sizeof(msg.id_); + memmove(msg.mdata_, + msg.mdata_ + sizeof(msg.type_) + sizeof(msg.id_), + msg.processed_); + + } else break; + } + + if (msg.processed_ < Message::hsize_) { + continue; + } + + // got the whole metadata; + readInteger(msg.mdata_, msg.type_, msg.id_, msg.msize_, msg.psize_); + LOG(INFO) << "Receive a message: id = " << msg.id_ << ", msize_ = " << msg.msize_ << ", psize_ = " << msg.psize_; + } + + // start reading the real data + if (msg.msg_ == nullptr) { + msg.msg_ = new char[msg.getSize()]; + memcpy(msg.msg_, msg.mdata_, Message::hsize_); + } + + nread = read(fd, msg.msg_ + msg.processed_, msg.getSize() - msg.processed_); + if (nread <= 0) { + if (errno != EWOULDBLOCK || nread == 0) { + // socket error or shuts down + if (nread < 0) + LOG(INFO) << "Faile to receive from EndPoint " << inet_ntoa(ep->addr_.sin_addr) << ": " << strerror(errno); + else + LOG(INFO) << "Faile to receive from EndPoint " << inet_ntoa(ep->addr_.sin_addr) << ": Connection reset by remote side"; + handleConnLost(fd, ep); + } + break; + } + + msg.processed_ += nread; + if (msg.processed_ == msg.getSize()) { + LOG(INFO) << "Receive a " << msg.processed_ << " bytes DATA message from " << inet_ntoa(ep->addr_.sin_addr) << " with id " << msg.id_; + ep->recv_.push(new Message(static_cast(msg))); + // notify of waiting thread + ep->cv_.notify_one(); + ep->send_.push(new Message(MSG_ACK, msg.id_)); + msg.processed_ = 0; + } + } +} + +/** + * @brief clean up for the lost connection; the caller should acquire the lock + * for the respective endpoint + * + * @param fd + * @param ep + * @param reconn + */ +void NetworkThread::handleConnLost(int fd, EndPoint* ep, bool reconn) { + CHECK(fd >= 0); + LOG(INFO) << "Lost connection to EndPoint " << inet_ntoa(ep->addr_.sin_addr) << ", fd = " << fd; + + this->pending_msgs_.erase(fd); + this->fd_ip_map_.erase(fd); + ev_io_stop(loop_, &this->fd_wwatcher_map_[fd]); + ev_io_stop(loop_, &this->fd_rwatcher_map_[fd]); + fd_wwatcher_map_.erase(fd); + fd_rwatcher_map_.erase(fd); + close(fd); + + if (fd == ep->fd_[0] || ep->fd_[0] < 0) { + if (!ep->send_.empty()) + ep->send_.front()->processed_ = 0; + } + + if (reconn) { + + int sfd = (fd == ep->fd_[0]) ? ep->fd_[1] : ep->fd_[0]; + + if (fd == ep->fd_[0]) + ep->fd_[0] = -1; + else + ep->fd_[1] = -1; + + // see if the other fd is ok or not + if (sfd < 0) { + if (ep->retry_cnt_ < MAX_RETRY_CNT) { + // notify myself for retry + ep->retry_cnt_++; + ep->conn_status_ = CONN_INIT; + LOG(INFO) << "Reconnect to EndPoint " << inet_ntoa(ep->addr_.sin_addr); + this->notify(SIG_EP); + } else { + LOG(INFO) << "Maximum retry count achieved for EndPoint " << inet_ntoa(ep->addr_.sin_addr); + ep->conn_status_ = CONN_ERROR; + // notify all threads that this ep is no longer connected + ep->cv_.notify_all(); + } + } else { + // if there is another working fd, try to send data over this fd + if (!ep->send_.empty()) + this->notify(SIG_MSG); + } + } +} + +} diff --git a/src/io/network/message.cc b/src/io/network/message.cc new file mode 100644 index 0000000000..ac3fc14739 --- /dev/null +++ b/src/io/network/message.cc @@ -0,0 +1,115 @@ +/************************************************************ +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*************************************************************/ + +#include +#include + +#include + +#include "singa/io/network/message.h" +#include "singa/io/network/integer.h" + +namespace singa { + +Message::Message(Message&& msg) { + std::swap(msize_, msg.msize_); + std::swap(psize_, msg.psize_); + std::swap(msg_, msg.msg_); + std::swap(type_, msg.type_); +} + +Message::Message(int type, uint32_t ack_msg_id): type_(type), id_(ack_msg_id) { + if (type_ == MSG_ACK) + appendInteger(mdata_, type_, id_); +} + +Message::~Message() { + if (msg_) + free(msg_); +} + +std::size_t Message::getSize() { + if (type_ == MSG_ACK) + return sizeof(type_) + sizeof(id_); + else + return this->hsize_ + this->psize_ + this->msize_; +} + +void Message::setId(uint32_t id) { + this->id_ = id; + appendInteger(msg_, type_, id_); +} + +void Message::setMetadata(const void* buf, int size) { + this->msize_ = size; + msg_ = (char*) malloc (this->getSize()); + appendInteger(msg_, type_, id_, msize_, psize_); + memcpy(msg_ + hsize_, buf, size); +} + +void Message::setPayload(const void* buf, int size) { + this->psize_ = size; + msg_ = (char*) realloc(msg_, this->getSize()); + appendInteger(msg_ + hsize_ - sizeof(psize_), psize_); + memcpy(msg_ + hsize_ + msize_, buf, size); +} + +std::size_t Message::getMetadata(void** p) { + if (this->msize_ == 0) + *p = nullptr; + else + *p = msg_ + hsize_; + return this->msize_; +} + +std::size_t Message::getPayload(void** p) { + if (this->psize_ == 0) + *p = nullptr; + else + *p = msg_ + hsize_ + msize_; + return this->psize_; +} + +void MessageQueue::push(Message& msg) { + this->lock_.lock(); + this->mqueue_.push(static_cast(msg)); + this->lock_.unlock(); +} + +void MessageQueue::pop() { + this->lock_.lock(); + this->mqueue_.pop(); + this->lock_.unlock(); +} + +Message& MessageQueue::front() { + this->lock_.lock(); + Message& ret = this->mqueue_.front(); + this->lock_.unlock(); + return ret; +} + +std::size_t MessageQueue::size() { + std::unique_lock lock(lock_); + return mqueue_.size(); +} + +} diff --git a/test/singa/test_ep.cc b/test/singa/test_ep.cc new file mode 100644 index 0000000000..2435f28f5a --- /dev/null +++ b/test/singa/test_ep.cc @@ -0,0 +1,69 @@ +#include "singa/io/network/endpoint.h" +#include "singa/io/network/integer.h" +#include "singa/io/network/message.h" +#include +#include + +#include "singa/utils/logging.h" + +#define SIZE 100 +#define PORT 10000 + +using namespace singa; +int main(int argc, char** argv) { + char md[SIZE]; + char payload[SIZE]; + + char* host = "localhost"; + int port = PORT; + + for (int i = 1; i < argc; ++i) + { + if (strcmp(argv[i], "-p") == 0) + port = atoi(argv[++i]); + else if (strcmp(argv[i], "-h") == 0) + host = argv[++i]; + else + fprintf(stderr, "Invalid option %s\n", argv[i]); + } + + memset(md, 'a', SIZE); + memset(payload, 'b', SIZE); + + Message* m = new Message(); + m->setMetadata(md, SIZE); + m->setPayload(payload, SIZE); + + NetworkThread* t = new NetworkThread(port); + + EndPointFactory* epf = t->epf_; + + // sleep + sleep(3); + + EndPoint* ep = epf->getEp(host); + + int cnt = 0; + + while(ep && cnt++ <= 100 && ep->send(m) > 0 ) { + + LOG(INFO) << "Send a " << m->getSize() << " bytes message"; + + Message* m1 = ep->recv(); + + if (!m1) + break; + + char *p; + + LOG(INFO) << "Receive a " << m1->getSize() << " bytes message"; + + CHECK(m1->getMetadata((void**)&p) == SIZE); + CHECK(0 == strncmp(p, md, SIZE)); + CHECK(m1->getPayload((void**)&p) == SIZE); + CHECK(0 == strncmp(p, payload, SIZE)); + + delete m; + m = m1; + } +} From 9135a8e39b3d851e3ba738282e09a572dca485e7 Mon Sep 17 00:00:00 2001 From: caiqc Date: Thu, 4 Aug 2016 17:21:47 +0800 Subject: [PATCH 10/16] SINGA-233 Fix a minor bug in processing lost connections --- src/io/network/endpoint.cc | 43 ++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/src/io/network/endpoint.cc b/src/io/network/endpoint.cc index 2926a056c1..7fe72b3bc7 100644 --- a/src/io/network/endpoint.cc +++ b/src/io/network/endpoint.cc @@ -269,6 +269,7 @@ void NetworkThread::onNewEp() { ep->addr_.sin_port = htons(port_); bzero(&(ep->addr_.sin_zero), 8); + LOG(INFO) << "Connecting to " << inet_ntoa(ep->addr_.sin_addr) << " fd = "<< fd; if (connect(fd, (struct sockaddr*)&ep->addr_, sizeof(struct sockaddr)) ) { LOG(INFO) << "Connect Error: " << strerror(errno); @@ -397,9 +398,11 @@ void NetworkThread::onSend(int fd) { std::vector invalid_fd; if (fd == -1) { + //LOG(INFO) << "There are " << fd_ip_map_.size() << " connections"; // this is a signal of new message to send for(auto& p : fd_ip_map_) { // send message + //LOG(INFO) << "Try to send over fd " << p.first; if (asyncSend(p.first) < 0) invalid_fd.push_back(p.first); } @@ -440,6 +443,9 @@ void NetworkThread::asyncSendPendingMsg(EndPoint* ep) { */ int NetworkThread::asyncSend(int fd) { + if (fd_ip_map_.count(fd) == 0) + return 0; + EndPoint* ep = epf_->getEp(fd_ip_map_[fd]); std::unique_lock ep_lock(ep->mtx_); @@ -459,7 +465,7 @@ int NetworkThread::asyncSend(int fd) { else nbytes = write(fd, msg.msg_ + msg.processed_, msg.getSize() - msg.processed_); - LOG(INFO) << "Send " << nbytes << " bytes to " << inet_ntoa(ep->addr_.sin_addr) << " over fd " << fd; + // LOG(INFO) << "Send " << nbytes << " bytes to " << inet_ntoa(ep->addr_.sin_addr) << " over fd " << fd; if (nbytes == -1) { if (errno == EWOULDBLOCK) { @@ -488,8 +494,11 @@ int NetworkThread::asyncSend(int fd) { ep->send_.pop(); // for test - if (ep->retry_cnt_ == 0) - close(fd); + // if (ep->retry_cnt_ == 0) { + // LOG(INFO) << "Disconnect with Endpoint " << inet_ntoa(ep->addr_.sin_addr) << " over fd " << fd; + // close(fd); + // handleConnLost(fd, ep); + // } } out: if (ep->send_.empty()) @@ -506,7 +515,7 @@ void NetworkThread::onRecv(int fd) { int nread; EndPoint* ep = epf_->getEp(fd_ip_map_[fd]); - LOG(INFO) << "Start to read from EndPoint " << inet_ntoa(ep->addr_.sin_addr) << " over fd " << fd; + //LOG(INFO) << "Start to read from EndPoint " << inet_ntoa(ep->addr_.sin_addr) << " over fd " << fd; std::unique_lock lock(ep->mtx_); while(1) { @@ -519,9 +528,9 @@ void NetworkThread::onRecv(int fd) { if (errno != EWOULDBLOCK || nread == 0) { // socket error or shuts down if (nread < 0) - LOG(INFO) << "Faile to receive from EndPoint " << inet_ntoa(ep->addr_.sin_addr) << ": " << strerror(errno); + LOG(INFO) << "Fail to receive from EndPoint " << inet_ntoa(ep->addr_.sin_addr) << ": " << strerror(errno); else - LOG(INFO) << "Faile to receive from EndPoint " << inet_ntoa(ep->addr_.sin_addr) << ": Connection reset by remote side"; + LOG(INFO) << "Fail to receive from EndPoint " << inet_ntoa(ep->addr_.sin_addr) << ": Connection reset by remote side"; handleConnLost(fd, ep); } break; @@ -557,7 +566,7 @@ void NetworkThread::onRecv(int fd) { // got the whole metadata; readInteger(msg.mdata_, msg.type_, msg.id_, msg.msize_, msg.psize_); - LOG(INFO) << "Receive a message: id = " << msg.id_ << ", msize_ = " << msg.msize_ << ", psize_ = " << msg.psize_; + // LOG(INFO) << "Receive a message: id = " << msg.id_ << ", msize_ = " << msg.msize_ << ", psize_ = " << msg.psize_; } // start reading the real data @@ -571,9 +580,9 @@ void NetworkThread::onRecv(int fd) { if (errno != EWOULDBLOCK || nread == 0) { // socket error or shuts down if (nread < 0) - LOG(INFO) << "Faile to receive from EndPoint " << inet_ntoa(ep->addr_.sin_addr) << ": " << strerror(errno); + LOG(INFO) << "Fail to receive from EndPoint " << inet_ntoa(ep->addr_.sin_addr) << ": " << strerror(errno); else - LOG(INFO) << "Faile to receive from EndPoint " << inet_ntoa(ep->addr_.sin_addr) << ": Connection reset by remote side"; + LOG(INFO) << "Fail to receive from EndPoint " << inet_ntoa(ep->addr_.sin_addr) << ": Connection reset by remote side"; handleConnLost(fd, ep); } break; @@ -609,22 +618,20 @@ void NetworkThread::handleConnLost(int fd, EndPoint* ep, bool reconn) { ev_io_stop(loop_, &this->fd_rwatcher_map_[fd]); fd_wwatcher_map_.erase(fd); fd_rwatcher_map_.erase(fd); - close(fd); + //close(fd); if (fd == ep->fd_[0] || ep->fd_[0] < 0) { if (!ep->send_.empty()) ep->send_.front()->processed_ = 0; } - if (reconn) { - - int sfd = (fd == ep->fd_[0]) ? ep->fd_[1] : ep->fd_[0]; - - if (fd == ep->fd_[0]) - ep->fd_[0] = -1; - else - ep->fd_[1] = -1; + int sfd = (fd == ep->fd_[0]) ? ep->fd_[1] : ep->fd_[0]; + if (fd == ep->fd_[0]) + ep->fd_[0] = -1; + else + ep->fd_[1] = -1; + if (reconn) { // see if the other fd is ok or not if (sfd < 0) { if (ep->retry_cnt_ < MAX_RETRY_CNT) { From 3da9e3471051d636f58443f589e429494b0afa0a Mon Sep 17 00:00:00 2001 From: caiqc Date: Fri, 5 Aug 2016 00:03:24 +0800 Subject: [PATCH 11/16] SINGA-233 Fix bugs in sending large messages --- include/singa/io/network/endpoint.h | 5 + src/io/network/endpoint.cc | 178 +++++++++++++++++++--------- test/CMakeLists.txt | 15 ++- test/singa/test_ep.cc | 8 +- 4 files changed, 142 insertions(+), 64 deletions(-) diff --git a/include/singa/io/network/endpoint.h b/include/singa/io/network/endpoint.h index 1079fcc41e..063ca11fe0 100644 --- a/include/singa/io/network/endpoint.h +++ b/include/singa/io/network/endpoint.h @@ -50,6 +50,9 @@ namespace singa { #define MAX_RETRY_CNT 3 +#define EV_WATCHER_STOP 0 +#define EV_WATCHER_START 1 + class NetworkThread; class EndPointFactory; @@ -62,6 +65,7 @@ class EndPoint { std::mutex mtx_; struct sockaddr_in addr_; int fd_[2] = {-1, -1}; // two endpoints simultaneously connect to each other + int pfd_ = -1; int conn_status_ = CONN_INIT; int pending_cnt_ = 0; int retry_cnt_ = 0; @@ -113,6 +117,7 @@ class NetworkThread{ void doWork(); int asyncSend(int); void asyncSendPendingMsg(EndPoint*); + void afterConnEst(EndPoint* ep, int fd, bool active); public: EndPointFactory* epf_; diff --git a/src/io/network/endpoint.cc b/src/io/network/endpoint.cc index 7fe72b3bc7..a74f5c2c9d 100644 --- a/src/io/network/endpoint.cc +++ b/src/io/network/endpoint.cc @@ -283,17 +283,21 @@ void NetworkThread::onNewEp() { ev_io_start(this->loop_, &this->fd_wwatcher_map_[fd]); } } else { + afterConnEst(ep, fd, true); + // connection established immediately - LOG(INFO) << "Connected to " << inet_ntoa(ep->addr_.sin_addr) << " fd = "<< fd; - ep->conn_status_ = CONN_EST; - ev_io_stop(this->loop_, &this->fd_wwatcher_map_[fd]); + // LOG(INFO) << "Connected to " << inet_ntoa(ep->addr_.sin_addr) << " fd = "<< fd; + // ep->conn_status_ = CONN_EST; - // poll for new msgs - ev_io_init(&this->fd_rwatcher_map_[fd], readable_cb, fd, EV_READ); - ev_io_start(this->loop_, &this->fd_rwatcher_map_[fd]); + // //ev_io_stop(this->loop_, &this->fd_wwatcher_map_[fd]); + // ev_io_init(&fd_wwatcher_map_[fd], writable_cb, fd, EV_WRITE); - asyncSendPendingMsg(ep); - ep->cv_.notify_all(); + // // poll for new msgs + // ev_io_init(&this->fd_rwatcher_map_[fd], readable_cb, fd, EV_READ); + // ev_io_start(this->loop_, &this->fd_rwatcher_map_[fd]); + + // asyncSendPendingMsg(ep); + // ep->cv_.notify_all(); } } } @@ -314,32 +318,21 @@ void NetworkThread::onConnEst(int fd) { handleConnLost(ep->fd_[0], ep); - switch(ep->conn_status_) { - case CONN_INIT: - case CONN_PENDING: - return; - default: - break; - } + if (ep->conn_status_ == CONN_EST && ep->conn_status_ == CONN_ERROR) + ep->cv_.notify_all(); } else { - LOG(INFO) << "Connected to " << inet_ntoa(ep->addr_.sin_addr) << ", fd = "<< fd; - ep->conn_status_ = CONN_EST; - // connect established; poll for new msgs - ev_io_stop(this->loop_, &this->fd_wwatcher_map_[fd]); - ev_io_init(&this->fd_rwatcher_map_[fd], readable_cb, fd, EV_READ); - ev_io_start(this->loop_, &this->fd_rwatcher_map_[fd]); - } + afterConnEst(ep, fd, true); - if (ep->conn_status_ == CONN_EST && ep->to_ack_.size() > 0) - // if there are pending message, it means these msgs were sent over - // previous sockets that have been lost now - // we need to resend these msgs to the remote side - asyncSendPendingMsg(ep); + //ep->conn_status_ = CONN_EST; + //// connect established; poll for new msgs + //ev_io_stop(this->loop_, &this->fd_wwatcher_map_[fd]); + //ev_io_init(&fd_wwatcher_map_[fd], writable_cb, fd, EV_WRITE); - // Finally notify all waiting threads - ep->cv_.notify_all(); + //ev_io_init(&this->fd_rwatcher_map_[fd], readable_cb, fd, EV_READ); + //ev_io_start(this->loop_, &this->fd_rwatcher_map_[fd]); + } } void NetworkThread::onNewConn() { @@ -363,35 +356,93 @@ void NetworkThread::onNewConn() { ep = epf_->getOrCreateEp(a); std::unique_lock lock(ep->mtx_); - if (ep->fd_[1] >= 0) { - // the previous connection is lost - handleConnLost(ep->fd_[1], ep, false); + // Passive connection + afterConnEst(ep, fd, false); + + // This should not be put before afterConnEst otherwise it may have no + // effect + fd_ip_map_[fd] = a; + + // record the remote address + bcopy(&addr, &ep->addr_, len); +} + +/** + * @brief The processing for a connected socket + * + * @param ep + * @param fd + * @param active indicate whethen this socket is locally initiated or not + */ +void NetworkThread::afterConnEst(EndPoint* ep, int fd, bool active) { + + if (active) + LOG(INFO) << "Connected to " << inet_ntoa(ep->addr_.sin_addr) << ", fd = "<< fd; + + int sfd; + + if (active) { + ep->fd_[0] = fd; + sfd = ep->fd_[1]; + } else { + if (ep->fd_[1] >= 0) { + // the previous connection is lost + handleConnLost(ep->fd_[1], ep, false); + } + ep->fd_[1] = fd; + sfd = ep->fd_[0]; } - if (ep->fd_[0] == fd) { - // this fd is reused + if (sfd == fd) + // this fd is a reuse of a previous socket fd + // so we first need to clean the resouce for that fd handleConnLost(fd, ep, false); - } - fd_ip_map_[fd] = a; + // initialize io watchers and add the read watcher to the ev loop ev_io_init(&fd_rwatcher_map_[fd], readable_cb, fd, EV_READ); ev_io_start(loop_, &fd_rwatcher_map_[fd]); - // record the remote address - bcopy(&addr, &ep->addr_, len); + // stop watching the writable watcher if necessary + if (active) + ev_io_stop(loop_, &fd_wwatcher_map_[fd]); + ev_io_init(&fd_wwatcher_map_[fd], writable_cb, fd, EV_WRITE); + + // see whether there is already a established connection for this fd + if (ep->conn_status_ == CONN_EST && sfd >= 0) { + // check if fd and sfd are associate with the same socket + struct sockaddr_in addr; + socklen_t len; + if (getsockname(fd, (struct sockaddr*)&addr, &len)) { + LOG(INFO) << "Unable to get local socket address: " << strerror(errno); + return; + } - ep->conn_status_ = CONN_EST; - ep->fd_[1] = fd; + // see whether the local address of fd is the same as the remote side + // of sfd, which has already been stored in ep->addr_ + if (addr.sin_addr.s_addr == ep->addr_.sin_addr.s_addr && addr.sin_port == ep->addr_.sin_port) { + LOG(INFO) << fd << " and " << sfd << " are associated with the same socket"; + } else { + // this socket is redundant, we close it maunally + close(fd); + handleConnLost(fd, ep); + } + } else { + ep->pfd_ = fd; // set the primary fd + ep->conn_status_ = CONN_EST; + } - if (ep->to_ack_.size() > 0) - // see if there are any messages waiting for ack - // if yes, resend them - asyncSendPendingMsg(ep); + if (fd == ep->pfd_) { + this->asyncSendPendingMsg(ep); + } - // this connection is initiaed by remote side, - // so we dont need to notify the waiting thread + // Finally notify all waiting threads + // if this connection is initiaed by remote side, + // we dont need to notify the waiting thread // later threads wanting to send to this ep, however, // are able to reuse this ep + if (active) { + ep->cv_.notify_all(); + } } void NetworkThread::onSend(int fd) { @@ -423,6 +474,9 @@ void NetworkThread::asyncSendPendingMsg(EndPoint* ep) { LOG(INFO) << "There are " << ep->send_.size() << " to-send msgs, and " << ep->to_ack_.size() << " to-ack msgs"; + if (ep->to_ack_.empty()) + return; + while (!ep->send_.empty()) { ep->to_ack_.push(ep->send_.front()); ep->send_.pop(); @@ -443,14 +497,16 @@ void NetworkThread::asyncSendPendingMsg(EndPoint* ep) { */ int NetworkThread::asyncSend(int fd) { - if (fd_ip_map_.count(fd) == 0) - return 0; - EndPoint* ep = epf_->getEp(fd_ip_map_[fd]); std::unique_lock ep_lock(ep->mtx_); + if (fd != ep->pfd_) + // we only send over the primary fd + return 0; + if (ep->conn_status_ != CONN_EST) + // This happens during reconnection goto out; while(!ep->send_.empty()) { @@ -465,12 +521,10 @@ int NetworkThread::asyncSend(int fd) { else nbytes = write(fd, msg.msg_ + msg.processed_, msg.getSize() - msg.processed_); - // LOG(INFO) << "Send " << nbytes << " bytes to " << inet_ntoa(ep->addr_.sin_addr) << " over fd " << fd; - if (nbytes == -1) { if (errno == EWOULDBLOCK) { - ev_io_init(&fd_wwatcher_map_[fd], writable_cb, fd, EV_WRITE); - ev_io_start(loop_, &fd_wwatcher_map_[fd]); + if (!ev_is_active(&fd_wwatcher_map_[fd]) && !ev_is_pending(&fd_wwatcher_map_[fd])) + ev_io_start(loop_, &fd_wwatcher_map_[fd]); goto out; } else { // this connection is lost; reset the send status @@ -480,14 +534,24 @@ int NetworkThread::asyncSend(int fd) { } } else msg.processed_ += nbytes; + + //std::size_t m, p; + //uint8_t type; + //uint32_t id; + //if (msg.msg_) { + // readInteger(msg.msg_, type, id, m, p); + // LOG(INFO) << "Send " << msg.processed_ << " bytes to " << inet_ntoa(ep->addr_.sin_addr) << " over fd " << fd << " for the current DATA MSG " << msg.id_ << ", " << id << ", " << m << ", " << p; + //} } CHECK(msg.processed_ == msg.getSize()); if (msg.type_ != MSG_ACK) { + LOG(INFO) << "Send a DATA message to " << inet_ntoa(ep->addr_.sin_addr) << " for MSG " << msg.id_ << ", len = " << msg.getSize();; msg.processed_ = 0; ep->to_ack_.push(&msg); } else { + //LOG(INFO) << "Send an ACK message to " << inet_ntoa(ep->addr_.sin_addr) << " for MSG " << msg.id_; delete &msg; } @@ -497,7 +561,7 @@ int NetworkThread::asyncSend(int fd) { // if (ep->retry_cnt_ == 0) { // LOG(INFO) << "Disconnect with Endpoint " << inet_ntoa(ep->addr_.sin_addr) << " over fd " << fd; // close(fd); - // handleConnLost(fd, ep); + // goto err; // } } out: @@ -566,7 +630,8 @@ void NetworkThread::onRecv(int fd) { // got the whole metadata; readInteger(msg.mdata_, msg.type_, msg.id_, msg.msize_, msg.psize_); - // LOG(INFO) << "Receive a message: id = " << msg.id_ << ", msize_ = " << msg.msize_ << ", psize_ = " << msg.psize_; + + //LOG(INFO) << "Receive a message: id = " << msg.id_ << ", msize_ = " << msg.msize_ << ", psize_ = " << msg.psize_ << " from " << inet_ntoa(ep->addr_.sin_addr) << " over fd " << fd; } // start reading the real data @@ -589,6 +654,9 @@ void NetworkThread::onRecv(int fd) { } msg.processed_ += nread; + + //LOG(INFO) << "Receive a message: id = " << msg.id_ << ", msize_ = " << msg.msize_ << ", psize_ = " << msg.psize_ << ", processed_ = " << msg.processed_ << " from " << inet_ntoa(ep->addr_.sin_addr) << " over fd " << fd; + if (msg.processed_ == msg.getSize()) { LOG(INFO) << "Receive a " << msg.processed_ << " bytes DATA message from " << inet_ntoa(ep->addr_.sin_addr) << " with id " << msg.id_; ep->recv_.push(new Message(static_cast(msg))); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 044d65aafe..3bfd36c203 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -9,9 +9,14 @@ IF(NOT USE_OPENCL) LIST(REMOVE_ITEM singa_test_source "singa/test_opencl.cc") ENDIF() -ADD_EXECUTABLE(test_singa "gtest/gtest_main.cc" ${singa_test_source}) -ADD_DEPENDENCIES(test_singa singa_core singa_utils) -MESSAGE(STATUS "link libs" ${singa_linker_libs}) -TARGET_LINK_LIBRARIES(test_singa gtest singa_core singa_utils singa_model +#ADD_EXECUTABLE(test_singa "gtest/gtest_main.cc" ${singa_test_source}) +#ADD_DEPENDENCIES(test_singa singa_core singa_utils) +#MESSAGE(STATUS "link libs" ${singa_linker_libs}) +#TARGET_LINK_LIBRARIES(test_singa gtest singa_core singa_utils singa_model +# singa_io proto protobuf ${SINGA_LINKER_LIBS}) +#SET_TARGET_PROPERTIES(test_singa PROPERTIES LINK_FLAGS "${LINK_FLAGS} -pthread") + +ADD_EXECUTABLE(test_ep "singa/test_ep.cc") +ADD_DEPENDENCIES(test_ep singa_io) +TARGET_LINK_LIBRARIES(test_ep singa_core singa_utils singa_model singa_io proto protobuf ${SINGA_LINKER_LIBS}) -SET_TARGET_PROPERTIES(test_singa PROPERTIES LINK_FLAGS "${LINK_FLAGS} -pthread") diff --git a/test/singa/test_ep.cc b/test/singa/test_ep.cc index 2435f28f5a..e6d3e071ea 100644 --- a/test/singa/test_ep.cc +++ b/test/singa/test_ep.cc @@ -6,13 +6,13 @@ #include "singa/utils/logging.h" -#define SIZE 100 +#define SIZE 10000000 #define PORT 10000 using namespace singa; int main(int argc, char** argv) { - char md[SIZE]; - char payload[SIZE]; + char* md = new char[SIZE]; + char* payload = new char[SIZE]; char* host = "localhost"; int port = PORT; @@ -45,7 +45,7 @@ int main(int argc, char** argv) { int cnt = 0; - while(ep && cnt++ <= 100 && ep->send(m) > 0 ) { + while(ep && cnt++ <= 5 && ep->send(m) > 0 ) { LOG(INFO) << "Send a " << m->getSize() << " bytes message"; From fe5b2112ced472c15136b81eaa386c0ec19b2e4f Mon Sep 17 00:00:00 2001 From: caiqc Date: Fri, 5 Aug 2016 16:41:52 +0800 Subject: [PATCH 12/16] SINGA-233 Add the support for the detection of endpoint timeout --- include/singa/io/network/endpoint.h | 12 ++- src/io/network/endpoint.cc | 158 ++++++++++++++++++++-------- src/io/network/message.cc | 1 + test/singa/test_ep.cc | 61 +++++++---- 4 files changed, 165 insertions(+), 67 deletions(-) diff --git a/include/singa/io/network/endpoint.h b/include/singa/io/network/endpoint.h index 063ca11fe0..ac243ffd77 100644 --- a/include/singa/io/network/endpoint.h +++ b/include/singa/io/network/endpoint.h @@ -50,8 +50,7 @@ namespace singa { #define MAX_RETRY_CNT 3 -#define EV_WATCHER_STOP 0 -#define EV_WATCHER_START 1 +#define EP_TIMEOUT 5. class NetworkThread; class EndPointFactory; @@ -64,13 +63,16 @@ class EndPoint { std::condition_variable cv_; std::mutex mtx_; struct sockaddr_in addr_; + ev_timer timer_; + ev_tstamp last_msg_time_; int fd_[2] = {-1, -1}; // two endpoints simultaneously connect to each other int pfd_ = -1; + bool is_socket_loop_ = false; int conn_status_ = CONN_INIT; int pending_cnt_ = 0; int retry_cnt_ = 0; NetworkThread* thread_ = nullptr; - EndPoint(NetworkThread* t):thread_(t){} + EndPoint(NetworkThread* t); ~EndPoint(); friend class NetworkThread; friend class EndPointFactory; @@ -105,7 +107,8 @@ class NetworkThread{ std::unordered_map fd_wwatcher_map_; std::unordered_map fd_rwatcher_map_; - std::unordered_map fd_ip_map_; + + std::unordered_map fd_ep_map_; std::map pending_msgs_; @@ -130,6 +133,7 @@ class NetworkThread{ void onConnEst(int fd); void onNewEp(); void onNewConn(); + void onTimeout(struct ev_timer* timer); }; } #endif diff --git a/src/io/network/endpoint.cc b/src/io/network/endpoint.cc index a74f5c2c9d..c7e06f9ca0 100644 --- a/src/io/network/endpoint.cc +++ b/src/io/network/endpoint.cc @@ -58,6 +58,14 @@ static void accept_cb(struct ev_loop* loop, ev_io* ev, int revent) { reinterpret_cast(ev_userdata(loop))->onNewConn(); } +static void timeout_cb(struct ev_loop* loop, ev_timer* ev, int revent) { + reinterpret_cast(ev_userdata(loop))->onTimeout(ev); +} + +EndPoint::EndPoint(NetworkThread* t) : thread_(t) { + this->timer_.data = reinterpret_cast(this); +} + EndPoint::~EndPoint() { while(!recv_.empty()) { delete send_.front(); @@ -262,7 +270,7 @@ void NetworkThread::onNewEp() { // set this fd non-blocking fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK); - this->fd_ip_map_[fd] = ntohl(ep->addr_.sin_addr.s_addr); + this->fd_ep_map_[fd] = ep; // initialize the addess ep->addr_.sin_family = AF_INET; @@ -305,7 +313,9 @@ void NetworkThread::onNewEp() { void NetworkThread::onConnEst(int fd) { - EndPoint* ep = epf_->getEp(this->fd_ip_map_[fd]); + //EndPoint* ep = epf_->getEp(this->fd_ip_map_[fd]); + CHECK(fd_ep_map_.count(fd) > 0); + EndPoint* ep = fd_ep_map_.at(fd); std::unique_lock lock(ep->mtx_); @@ -359,14 +369,41 @@ void NetworkThread::onNewConn() { // Passive connection afterConnEst(ep, fd, false); - // This should not be put before afterConnEst otherwise it may have no - // effect - fd_ip_map_[fd] = a; - // record the remote address bcopy(&addr, &ep->addr_, len); } +void NetworkThread::onTimeout(struct ev_timer* timer) { + + EndPoint* ep = reinterpret_cast(timer->data); + + ev_tstamp timeout = EP_TIMEOUT + ep->last_msg_time_; + ev_tstamp now = ev_now(loop_); + + std::unique_lock lock(ep->mtx_); + if (now > timeout) { + if (!ep->to_ack_.empty() || !ep->send_.empty()) { + + LOG(INFO) << "EndPoint " << inet_ntoa(ep->addr_.sin_addr) << " timeouts"; + // we consider this ep has been disconnected + for (int i = 0; i < 2; ++i) + { + int fd = ep->fd_[i]; + if (fd >= 0) + handleConnLost(fd, ep); + } + return; + } + + timer->repeat = EP_TIMEOUT; + + } else { + timer->repeat = timeout - now; + } + + ev_timer_again(loop_, &ep->timer_); +} + /** * @brief The processing for a connected socket * @@ -393,10 +430,14 @@ void NetworkThread::afterConnEst(EndPoint* ep, int fd, bool active) { sfd = ep->fd_[0]; } - if (sfd == fd) + if (sfd == fd) { // this fd is a reuse of a previous socket fd // so we first need to clean the resouce for that fd - handleConnLost(fd, ep, false); + // we duplicate this fd to let the resouce of the oldf fd can be freed + // also indicate there is no need to reconnect + fd = dup(fd); + handleConnLost(sfd, ep, false); + } // initialize io watchers and add the read watcher to the ev loop ev_io_init(&fd_rwatcher_map_[fd], readable_cb, fd, EV_READ); @@ -407,6 +448,8 @@ void NetworkThread::afterConnEst(EndPoint* ep, int fd, bool active) { ev_io_stop(loop_, &fd_wwatcher_map_[fd]); ev_io_init(&fd_wwatcher_map_[fd], writable_cb, fd, EV_WRITE); + ep->last_msg_time_ = ev_now(loop_); + // see whether there is already a established connection for this fd if (ep->conn_status_ == CONN_EST && sfd >= 0) { // check if fd and sfd are associate with the same socket @@ -414,27 +457,37 @@ void NetworkThread::afterConnEst(EndPoint* ep, int fd, bool active) { socklen_t len; if (getsockname(fd, (struct sockaddr*)&addr, &len)) { LOG(INFO) << "Unable to get local socket address: " << strerror(errno); - return; - } - - // see whether the local address of fd is the same as the remote side - // of sfd, which has already been stored in ep->addr_ - if (addr.sin_addr.s_addr == ep->addr_.sin_addr.s_addr && addr.sin_port == ep->addr_.sin_port) { - LOG(INFO) << fd << " and " << sfd << " are associated with the same socket"; } else { - // this socket is redundant, we close it maunally - close(fd); - handleConnLost(fd, ep); + // see whether the local address of fd is the same as the remote side + // of sfd, which has already been stored in ep->addr_ + if (addr.sin_addr.s_addr == ep->addr_.sin_addr.s_addr && addr.sin_port == ep->addr_.sin_port) { + LOG(INFO) << fd << " and " << sfd << " are associated with the same socket"; + ep->is_socket_loop_ = true; + } else { + // this socket is redundant, we close it maunally if the local ip + // is smaller than the peer ip + if ((addr.sin_addr.s_addr < ep->addr_.sin_addr.s_addr) + || (addr.sin_addr.s_addr == ep->addr_.sin_addr.s_addr && addr.sin_port < ep->addr_.sin_port)) + handleConnLost(fd, ep, false); + } } } else { ep->pfd_ = fd; // set the primary fd ep->conn_status_ = CONN_EST; + + // start timeout watcher to detect the liveness of EndPoint + ev_init(&ep->timer_, timeout_cb); + ep->timer_.repeat = EP_TIMEOUT; + ev_timer_start(loop_, &ep->timer_); + //timeout_cb(loop_, &ep->timer_, EV_TIMER); } if (fd == ep->pfd_) { this->asyncSendPendingMsg(ep); } + fd_ep_map_[fd] = ep; + // Finally notify all waiting threads // if this connection is initiaed by remote side, // we dont need to notify the waiting thread @@ -451,7 +504,7 @@ void NetworkThread::onSend(int fd) { if (fd == -1) { //LOG(INFO) << "There are " << fd_ip_map_.size() << " connections"; // this is a signal of new message to send - for(auto& p : fd_ip_map_) { + for(auto& p : fd_ep_map_) { // send message //LOG(INFO) << "Try to send over fd " << p.first; if (asyncSend(p.first) < 0) @@ -463,7 +516,8 @@ void NetworkThread::onSend(int fd) { } for (auto& p : invalid_fd) { - EndPoint* ep = epf_->getEp(fd_ip_map_.at(p)); + //EndPoint* ep = epf_->getEp(fd_ip_map_.at(p)); + EndPoint* ep = fd_ep_map_.at(p); std::unique_lock lock(ep->mtx_); handleConnLost(p, ep); } @@ -474,16 +528,14 @@ void NetworkThread::asyncSendPendingMsg(EndPoint* ep) { LOG(INFO) << "There are " << ep->send_.size() << " to-send msgs, and " << ep->to_ack_.size() << " to-ack msgs"; - if (ep->to_ack_.empty()) - return; - - while (!ep->send_.empty()) { - ep->to_ack_.push(ep->send_.front()); - ep->send_.pop(); + if (!ep->to_ack_.empty()) { + while (!ep->send_.empty()) { + ep->to_ack_.push(ep->send_.front()); + ep->send_.pop(); + } + std::swap(ep->send_, ep->to_ack_); } - std::swap(ep->send_, ep->to_ack_); - if (ep->send_.size() > 0) { notify(SIG_MSG); } @@ -497,13 +549,16 @@ void NetworkThread::asyncSendPendingMsg(EndPoint* ep) { */ int NetworkThread::asyncSend(int fd) { - EndPoint* ep = epf_->getEp(fd_ip_map_[fd]); + //EndPoint* ep = epf_->getEp(fd_ip_map_[fd]); + CHECK(fd_ep_map_.count(fd) > 0); + EndPoint* ep = fd_ep_map_.at(fd); std::unique_lock ep_lock(ep->mtx_); - if (fd != ep->pfd_) + if (fd != ep->pfd_ ) // we only send over the primary fd - return 0; + // return -1 to indicate this fd is redundant + return ep->is_socket_loop_ ? 0 : -1; if (ep->conn_status_ != CONN_EST) // This happens during reconnection @@ -532,8 +587,10 @@ int NetworkThread::asyncSend(int fd) { msg.processed_ = 0; goto err; } - } else + } else { + ep->last_msg_time_ = ev_now(loop_); msg.processed_ += nbytes; + } //std::size_t m, p; //uint8_t type; @@ -547,7 +604,7 @@ int NetworkThread::asyncSend(int fd) { CHECK(msg.processed_ == msg.getSize()); if (msg.type_ != MSG_ACK) { - LOG(INFO) << "Send a DATA message to " << inet_ntoa(ep->addr_.sin_addr) << " for MSG " << msg.id_ << ", len = " << msg.getSize();; + LOG(INFO) << "Send a DATA message to " << inet_ntoa(ep->addr_.sin_addr) << " for MSG " << msg.id_ << ", len = " << msg.getSize() << " over fd " << fd; msg.processed_ = 0; ep->to_ack_.push(&msg); } else { @@ -557,7 +614,7 @@ int NetworkThread::asyncSend(int fd) { ep->send_.pop(); - // for test + //for test // if (ep->retry_cnt_ == 0) { // LOG(INFO) << "Disconnect with Endpoint " << inet_ntoa(ep->addr_.sin_addr) << " over fd " << fd; // close(fd); @@ -577,13 +634,17 @@ void NetworkThread::onRecv(int fd) { Message* m = &pending_msgs_[fd]; Message& msg = (*m); int nread; - EndPoint* ep = epf_->getEp(fd_ip_map_[fd]); + //EndPoint* ep = epf_->getEp(fd_ip_map_[fd]); + + CHECK(fd_ep_map_.count(fd) > 0); + EndPoint* ep = fd_ep_map_.at(fd); //LOG(INFO) << "Start to read from EndPoint " << inet_ntoa(ep->addr_.sin_addr) << " over fd " << fd; std::unique_lock lock(ep->mtx_); - while(1) { + ep->last_msg_time_ = ev_now(loop_); + while(1) { if (msg.processed_ < Message::hsize_) { nread = read(fd, msg.mdata_ + msg.processed_, Message::hsize_ - msg.processed_); @@ -631,7 +692,7 @@ void NetworkThread::onRecv(int fd) { // got the whole metadata; readInteger(msg.mdata_, msg.type_, msg.id_, msg.msize_, msg.psize_); - //LOG(INFO) << "Receive a message: id = " << msg.id_ << ", msize_ = " << msg.msize_ << ", psize_ = " << msg.psize_ << " from " << inet_ntoa(ep->addr_.sin_addr) << " over fd " << fd; + LOG(INFO) << "Receive a message: id = " << msg.id_ << ", msize_ = " << msg.msize_ << ", psize_ = " << msg.psize_ << " from " << inet_ntoa(ep->addr_.sin_addr) << " over fd " << fd; } // start reading the real data @@ -681,14 +742,14 @@ void NetworkThread::handleConnLost(int fd, EndPoint* ep, bool reconn) { LOG(INFO) << "Lost connection to EndPoint " << inet_ntoa(ep->addr_.sin_addr) << ", fd = " << fd; this->pending_msgs_.erase(fd); - this->fd_ip_map_.erase(fd); + this->fd_ep_map_.erase(fd); ev_io_stop(loop_, &this->fd_wwatcher_map_[fd]); ev_io_stop(loop_, &this->fd_rwatcher_map_[fd]); fd_wwatcher_map_.erase(fd); fd_rwatcher_map_.erase(fd); - //close(fd); + close(fd); - if (fd == ep->fd_[0] || ep->fd_[0] < 0) { + if (fd == ep->pfd_) { if (!ep->send_.empty()) ep->send_.front()->processed_ = 0; } @@ -700,8 +761,10 @@ void NetworkThread::handleConnLost(int fd, EndPoint* ep, bool reconn) { ep->fd_[1] = -1; if (reconn) { - // see if the other fd is ok or not + // see if the other fd is alive or not if (sfd < 0) { + if (ep->conn_status_ == CONN_EST) + ev_timer_stop(loop_, &ep->timer_); if (ep->retry_cnt_ < MAX_RETRY_CNT) { // notify myself for retry ep->retry_cnt_++; @@ -711,13 +774,20 @@ void NetworkThread::handleConnLost(int fd, EndPoint* ep, bool reconn) { } else { LOG(INFO) << "Maximum retry count achieved for EndPoint " << inet_ntoa(ep->addr_.sin_addr); ep->conn_status_ = CONN_ERROR; + // notify all threads that this ep is no longer connected ep->cv_.notify_all(); } } else { - // if there is another working fd, try to send data over this fd - if (!ep->send_.empty()) - this->notify(SIG_MSG); + if (!ep->is_socket_loop_) { + // if there is another working fd, set this fd as primary and + // send data over this fd + ep->pfd_ = sfd; + ep->last_msg_time_ = ev_now(loop_); + asyncSendPendingMsg(ep); + } else { + handleConnLost(sfd, ep); + } } } } diff --git a/src/io/network/message.cc b/src/io/network/message.cc index ac3fc14739..e63d176fce 100644 --- a/src/io/network/message.cc +++ b/src/io/network/message.cc @@ -34,6 +34,7 @@ Message::Message(Message&& msg) { std::swap(psize_, msg.psize_); std::swap(msg_, msg.msg_); std::swap(type_, msg.type_); + std::swap(id_, msg.id_); } Message::Message(int type, uint32_t ack_msg_id): type_(type), id_(ack_msg_id) { diff --git a/test/singa/test_ep.cc b/test/singa/test_ep.cc index e6d3e071ea..63dbb4303f 100644 --- a/test/singa/test_ep.cc +++ b/test/singa/test_ep.cc @@ -8,6 +8,7 @@ #define SIZE 10000000 #define PORT 10000 +#define ITER 10 using namespace singa; int main(int argc, char** argv) { @@ -30,10 +31,6 @@ int main(int argc, char** argv) { memset(md, 'a', SIZE); memset(payload, 'b', SIZE); - Message* m = new Message(); - m->setMetadata(md, SIZE); - m->setPayload(payload, SIZE); - NetworkThread* t = new NetworkThread(port); EndPointFactory* epf = t->epf_; @@ -43,27 +40,53 @@ int main(int argc, char** argv) { EndPoint* ep = epf->getEp(host); - int cnt = 0; + Message* m[ITER]; + for (int i = 0; i < ITER; ++i) + { + m[i] = new Message(); + m[i]->setMetadata(md, SIZE); + m[i]->setPayload(payload, SIZE); + } + + while (1) { + for (int i = 0; i < ITER; ++i) + { + if (ep->send(m[i]) < 0) return 1; + delete m[i]; + } + + for (int i = 0; i < ITER; ++i) + { + m[i] = ep->recv(); + if (!m[i]) + return 1; + char *p; + CHECK(m[i]->getMetadata((void**)&p) == SIZE); + CHECK(0 == strncmp(p, md, SIZE)); + CHECK(m[i]->getPayload((void**)&p) == SIZE); + CHECK(0 == strncmp(p, payload, SIZE)); + } + } - while(ep && cnt++ <= 5 && ep->send(m) > 0 ) { + //while(ep && cnt++ <= 5 && ep->send(m) > 0 ) { - LOG(INFO) << "Send a " << m->getSize() << " bytes message"; + // LOG(INFO) << "Send a " << m->getSize() << " bytes message"; - Message* m1 = ep->recv(); + // Message* m1 = ep->recv(); - if (!m1) - break; + // if (!m1) + // break; - char *p; + // char *p; - LOG(INFO) << "Receive a " << m1->getSize() << " bytes message"; + // LOG(INFO) << "Receive a " << m1->getSize() << " bytes message"; - CHECK(m1->getMetadata((void**)&p) == SIZE); - CHECK(0 == strncmp(p, md, SIZE)); - CHECK(m1->getPayload((void**)&p) == SIZE); - CHECK(0 == strncmp(p, payload, SIZE)); + // CHECK(m1->getMetadata((void**)&p) == SIZE); + // CHECK(0 == strncmp(p, md, SIZE)); + // CHECK(m1->getPayload((void**)&p) == SIZE); + // CHECK(0 == strncmp(p, payload, SIZE)); - delete m; - m = m1; - } + // delete m; + // m = m1; + //} } From 24574af0ca3adb830237a0cb46638080dba11a26 Mon Sep 17 00:00:00 2001 From: caiqc Date: Tue, 9 Aug 2016 22:45:47 +0800 Subject: [PATCH 13/16] SINGA-233 Some code cleaning --- include/singa/io/{network => }/integer.h | 0 .../io/{network/endpoint.h => network.h} | 51 +++++++++---- include/singa/io/network/message.h | 75 ------------------- src/io/network/endpoint.cc | 4 +- src/io/network/message.cc | 28 +------ test/CMakeLists.txt | 18 ++--- test/singa/test_ep.cc | 26 ++++++- 7 files changed, 74 insertions(+), 128 deletions(-) rename include/singa/io/{network => }/integer.h (100%) rename include/singa/io/{network/endpoint.h => network.h} (80%) delete mode 100644 include/singa/io/network/message.h diff --git a/include/singa/io/network/integer.h b/include/singa/io/integer.h similarity index 100% rename from include/singa/io/network/integer.h rename to include/singa/io/integer.h diff --git a/include/singa/io/network/endpoint.h b/include/singa/io/network.h similarity index 80% rename from include/singa/io/network/endpoint.h rename to include/singa/io/network.h index ac243ffd77..846c94bdce 100644 --- a/include/singa/io/network/endpoint.h +++ b/include/singa/io/network.h @@ -19,8 +19,8 @@ * *************************************************************/ -#ifndef SINGA_COMM_END_POINT_H_ -#define SINGA_COMM_END_POINT_H_ +#ifndef SINGA_COMM_NETWORK_H_ +#define SINGA_COMM_NETWORK_H_ #include #include @@ -32,8 +32,7 @@ #include #include #include - -#include "singa/io/network/message.h" +#include namespace singa { @@ -52,9 +51,41 @@ namespace singa { #define EP_TIMEOUT 5. +#define MSG_DATA 0 +#define MSG_ACK 1 + class NetworkThread; +class EndPoint; class EndPointFactory; +class Message{ + private: + uint8_t type_; + uint32_t id_; + std::size_t msize_ = 0; + std::size_t psize_ = 0; + std::size_t processed_ = 0; + char* msg_ = nullptr; + static const int hsize_ = sizeof(id_) + 2 * sizeof(std::size_t) + sizeof(type_); + char mdata_[hsize_]; + friend class NetworkThread; + friend class EndPoint; + public: + Message(int = MSG_DATA, uint32_t = 0); + Message(const Message&) = delete; + Message(Message&&); + ~Message(); + + void setMetadata(const void*, int); + void setPayload(const void*, int); + + std::size_t getMetadata(void**); + std::size_t getPayload(void**); + + std::size_t getSize(); + void setId(uint32_t); +}; + class EndPoint { private: std::queue send_; @@ -102,20 +133,15 @@ class NetworkThread{ struct ev_loop *loop_; ev_async ep_sig_; ev_async msg_sig_; - + ev_io socket_watcher_; + int port_; + int socket_fd_; std::thread* thread_; - std::unordered_map fd_wwatcher_map_; std::unordered_map fd_rwatcher_map_; - std::unordered_map fd_ep_map_; - std::map pending_msgs_; - ev_io socket_watcher_; - int port_; - int socket_fd_; - void handleConnLost(int, EndPoint*, bool = true); void doWork(); int asyncSend(int); @@ -125,7 +151,6 @@ class NetworkThread{ EndPointFactory* epf_; NetworkThread(int); - //void join(); void notify(int signal); void onRecv(int fd); diff --git a/include/singa/io/network/message.h b/include/singa/io/network/message.h deleted file mode 100644 index 0f691c0b49..0000000000 --- a/include/singa/io/network/message.h +++ /dev/null @@ -1,75 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#ifndef SINGA_COMM_MESSAGE_H_ -#define SINGA_COMM_MESSAGE_H_ - -#include -#include - -namespace singa { - -#define MSG_DATA 0 -#define MSG_ACK 1 - -class NetworkThread; -class EndPoint; -class Message{ - private: - uint8_t type_; - uint32_t id_; - std::size_t msize_ = 0; - std::size_t psize_ = 0; - std::size_t processed_ = 0; - char* msg_ = nullptr; - static const int hsize_ = sizeof(id_) + 2 * sizeof(std::size_t) + sizeof(type_); - char mdata_[hsize_]; - friend class NetworkThread; - friend class EndPoint; - public: - Message(int = MSG_DATA, uint32_t = 0); - Message(const Message&) = delete; - Message(Message&&); - ~Message(); - - void setMetadata(const void*, int); - void setPayload(const void*, int); - - std::size_t getMetadata(void**); - std::size_t getPayload(void**); - - std::size_t getSize(); - void setId(uint32_t); -}; - -class MessageQueue -{ - public: - void push(Message&); - Message& front(); - void pop(); - std::size_t size(); - private: - std::mutex lock_; - std::queue mqueue_; -}; -} -#endif diff --git a/src/io/network/endpoint.cc b/src/io/network/endpoint.cc index c7e06f9ca0..96a2e4ab55 100644 --- a/src/io/network/endpoint.cc +++ b/src/io/network/endpoint.cc @@ -19,8 +19,8 @@ * *************************************************************/ -#include "singa/io/network/endpoint.h" -#include "singa/io/network/integer.h" +#include "singa/io/network.h" +#include "singa/io/integer.h" #include "singa/utils/logging.h" #include diff --git a/src/io/network/message.cc b/src/io/network/message.cc index e63d176fce..5bf9b8e0bb 100644 --- a/src/io/network/message.cc +++ b/src/io/network/message.cc @@ -24,8 +24,8 @@ #include -#include "singa/io/network/message.h" -#include "singa/io/network/integer.h" +#include "singa/io/network.h" +#include "singa/io/integer.h" namespace singa { @@ -89,28 +89,4 @@ std::size_t Message::getPayload(void** p) { return this->psize_; } -void MessageQueue::push(Message& msg) { - this->lock_.lock(); - this->mqueue_.push(static_cast(msg)); - this->lock_.unlock(); -} - -void MessageQueue::pop() { - this->lock_.lock(); - this->mqueue_.pop(); - this->lock_.unlock(); -} - -Message& MessageQueue::front() { - this->lock_.lock(); - Message& ret = this->mqueue_.front(); - this->lock_.unlock(); - return ret; -} - -std::size_t MessageQueue::size() { - std::unique_lock lock(lock_); - return mqueue_.size(); -} - } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3bfd36c203..fda871d07a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -9,14 +9,14 @@ IF(NOT USE_OPENCL) LIST(REMOVE_ITEM singa_test_source "singa/test_opencl.cc") ENDIF() -#ADD_EXECUTABLE(test_singa "gtest/gtest_main.cc" ${singa_test_source}) -#ADD_DEPENDENCIES(test_singa singa_core singa_utils) -#MESSAGE(STATUS "link libs" ${singa_linker_libs}) -#TARGET_LINK_LIBRARIES(test_singa gtest singa_core singa_utils singa_model -# singa_io proto protobuf ${SINGA_LINKER_LIBS}) +ADD_EXECUTABLE(test_singa "gtest/gtest_main.cc" ${singa_test_source}) +ADD_DEPENDENCIES(test_singa singa_core singa_utils) +MESSAGE(STATUS "link libs" ${singa_linker_libs}) +TARGET_LINK_LIBRARIES(test_singa gtest singa_core singa_utils singa_model + singa_io proto protobuf ${SINGA_LINKER_LIBS}) #SET_TARGET_PROPERTIES(test_singa PROPERTIES LINK_FLAGS "${LINK_FLAGS} -pthread") -ADD_EXECUTABLE(test_ep "singa/test_ep.cc") -ADD_DEPENDENCIES(test_ep singa_io) -TARGET_LINK_LIBRARIES(test_ep singa_core singa_utils singa_model - singa_io proto protobuf ${SINGA_LINKER_LIBS}) +#ADD_EXECUTABLE(test_ep "singa/test_ep.cc") +#ADD_DEPENDENCIES(test_ep singa_io) +#TARGET_LINK_LIBRARIES(test_ep singa_core singa_utils singa_model +# singa_io proto protobuf ${SINGA_LINKER_LIBS}) diff --git a/test/singa/test_ep.cc b/test/singa/test_ep.cc index 63dbb4303f..cc04064906 100644 --- a/test/singa/test_ep.cc +++ b/test/singa/test_ep.cc @@ -1,6 +1,26 @@ -#include "singa/io/network/endpoint.h" -#include "singa/io/network/integer.h" -#include "singa/io/network/message.h" +/************************************************************ +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*************************************************************/ + +#include "singa/io/integer.h" +#include "singa/io/network.h" #include #include From afa886ec5cb712888eb6f431a8175d2ae47d241b Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Wed, 10 Aug 2016 12:00:55 +0800 Subject: [PATCH 14/16] SINGA-218 Implementation for RNN CUDNN version Add checks for cudnn version, which should be >= 5.05 to compile cudnnrnn code --- CMakeLists.txt | 2 +- cmake/Templates/singa_config.h.in | 2 ++ src/model/layer/cudnn_rnn.cc | 4 ++-- src/model/layer/cudnn_rnn.h | 4 ++-- src/python/swig/model_layer.i | 10 +++++----- test/singa/test_cudnn_rnn.cc | 4 ++-- 6 files changed, 14 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e688b50074..980d389341 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.6) PROJECT(singa) -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -g") +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -g -O2 ") LIST(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Thirdparty) #message(STATUS "module path: ${CMAKE_MODULE_PATH}") diff --git a/cmake/Templates/singa_config.h.in b/cmake/Templates/singa_config.h.in index 0220d18925..47ae694670 100644 --- a/cmake/Templates/singa_config.h.in +++ b/cmake/Templates/singa_config.h.in @@ -14,6 +14,8 @@ #cmakedefine USE_CUDNN #cmakedefine CUDNN_VERSION_MAJOR @CUDNN_VERSION_MAJOR@ +#cmakedefine CUDNN_VERSION_MINOR @CUDNN_VERSION_MINOR@ +#cmakedefine CUDNN_VERSION_PATCH @CUDNN_VERSION_PATCH@ #cmakedefine USE_OPENCL diff --git a/src/model/layer/cudnn_rnn.cc b/src/model/layer/cudnn_rnn.cc index 896c1e9184..bfbfa48617 100644 --- a/src/model/layer/cudnn_rnn.cc +++ b/src/model/layer/cudnn_rnn.cc @@ -17,7 +17,7 @@ */ #include "./cudnn_rnn.h" #ifdef USE_CUDNN -#if CUDNN_VERSION_MAJOR >= 5 +#if CUDNN_VERSION_MAJOR >= 5 && CUDNN_VERSION_PATCH >= 5 #include #include #include "./cudnn_utils.h" @@ -428,5 +428,5 @@ const std::pair, vector> CudnnRNN::Backward( } } // namespace singa -#endif // CUDNN_VERSION_MAJOR >= 5 +#endif // CUDNN_VERSION_MAJOR >= 5 && CUDNN_VERSION_PATCH >= 5 #endif // USE_CUDNN diff --git a/src/model/layer/cudnn_rnn.h b/src/model/layer/cudnn_rnn.h index d2f8db5754..cfb8aac236 100644 --- a/src/model/layer/cudnn_rnn.h +++ b/src/model/layer/cudnn_rnn.h @@ -20,7 +20,7 @@ #define SRC_MODEL_LAYER_CUDNN_RNN_H_ #include "singa/singa_config.h" #ifdef USE_CUDNN -#if CUDNN_VERSION_MAJOR >= 5 +#if CUDNN_VERSION_MAJOR >= 5 && CUDNN_VERSION_PATCH >= 5 #include #include #include @@ -82,6 +82,6 @@ class CudnnRNN : public RNN { } // namespace singa -#endif // CUDNN_VERSION_MAJOR >= 5 +#endif // CUDNN_VERSION_MAJOR >= 5 && CUDNN_VERSION_PATCH >= 5 #endif // USE_CUDNN #endif // SRC_MODEL_LAYER_CUDNN_RNN_H_ diff --git a/src/python/swig/model_layer.i b/src/python/swig/model_layer.i index 9d3930179e..6cbfe8ff54 100644 --- a/src/python/swig/model_layer.i +++ b/src/python/swig/model_layer.i @@ -34,6 +34,7 @@ #include "../src/model/layer/cudnn_rnn.h" #include "singa/core/tensor.h" #include "singa/proto/model.pb.h" +#include "singa/singa_config.h" using singa::Tensor; using singa::ParamSpec; using singa::DataType; @@ -78,12 +79,9 @@ class Layer { std::shared_ptr CreateLayer(const std::string& type); const std::vector GetRegisteredLayers(); class RNN : public Layer { - /* - public: - void Setup(const std::vector& in_sample_shape, - const std::string& proto_str) override; - */ }; + +#if CUDNN_VERSION_MINOR >= 5 && CUDNN_VERSION_PATCH >= 5 class CudnnRNN : public RNN { public: // note: Must use std::vector instead of vector. @@ -95,5 +93,7 @@ class CudnnRNN : public RNN { const std::vector GetOutputSampleShape() const override; }; +#endif // CUDNN_VERSION_MINOR >= 5 && CUDNN_VERSION_PATCH >= 5 + } diff --git a/test/singa/test_cudnn_rnn.cc b/test/singa/test_cudnn_rnn.cc index e0de02e7fd..effb3b185e 100644 --- a/test/singa/test_cudnn_rnn.cc +++ b/test/singa/test_cudnn_rnn.cc @@ -21,7 +21,7 @@ #include "../src/model/layer/cudnn_rnn.h" #ifdef USE_CUDNN -#if CUDNN_VERSION_MAJOR >= 5 +#if CUDNN_VERSION_MAJOR >= 5 && CUDNN_VERSION_PATCH >= 5 #include "gtest/gtest.h" @@ -177,5 +177,5 @@ TEST_F(TestCudnnRNN, Backward) { std::copy(tmp.begin(), tmp.end(), dhyptr.begin()); } } -#endif // CUDNN_VERSION_MAJOR >= 5 +#endif // CUDNN_VERSION_MAJOR >= 5 && CUDNN_VERSION_PATCH >= 5 #endif // USE_CUDNN From ca3540adff16abc84250486255d4c1637f699ddf Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Wed, 10 Aug 2016 13:22:47 +0800 Subject: [PATCH 15/16] SINGA-233 New communication interface for SINGA Reformat code following google style. Update cmake files to ignore communication code if ENABLE_DIST is off. --- CMakeLists.txt | 6 +- cmake/Templates/singa_config.h.in | 2 + include/singa/io/network.h | 195 ++--- src/io/network/endpoint.cc | 1248 +++++++++++++++-------------- src/io/network/message.cc | 85 +- test/CMakeLists.txt | 17 +- test/singa/test_convolution.cc | 4 + test/singa/test_ep.cc | 141 ++-- 8 files changed, 880 insertions(+), 818 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 980d389341..c1d05215fb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,6 +25,7 @@ OPTION(USE_OPENCV "Use opencv" OFF) OPTION(USE_LMDB "Use LMDB libs" OFF) OPTION(USE_PYTHON "Generate py wrappers" OFF) OPTION(USE_OPENCL "Use OpenCL" OFF) +OPTION(ENABLE_DIST "enable distributed training" OFF) #OPTION(BUILD_OPENCL_TESTS "Build OpenCL tests" OFF) INCLUDE("cmake/Dependencies.cmake") @@ -47,7 +48,10 @@ IF (USE_CUDA) LIST(APPEND SINGA_LINKER_LIBS cnmem) ENDIF() -LIST(APPEND SINGA_LINKER_LIBS ev) +# TODO(wangwei) detect the ev lib +IF (ENABLE_DIST) + LIST(APPEND SINGA_LINKER_LIBS ev) +ENDIF() ADD_SUBDIRECTORY(src) ADD_SUBDIRECTORY(test) diff --git a/cmake/Templates/singa_config.h.in b/cmake/Templates/singa_config.h.in index 47ae694670..d03d58b068 100644 --- a/cmake/Templates/singa_config.h.in +++ b/cmake/Templates/singa_config.h.in @@ -19,6 +19,8 @@ #cmakedefine USE_OPENCL +#cmakedefine ENABLE_DIST + // lmdb #cmakedefine USE_LMDB diff --git a/include/singa/io/network.h b/include/singa/io/network.h index 846c94bdce..63983addf4 100644 --- a/include/singa/io/network.h +++ b/include/singa/io/network.h @@ -21,7 +21,8 @@ #ifndef SINGA_COMM_NETWORK_H_ #define SINGA_COMM_NETWORK_H_ - +#include "singa/singa_config.h" +#ifdef ENABLE_DIST #include #include #include @@ -58,107 +59,113 @@ class NetworkThread; class EndPoint; class EndPointFactory; -class Message{ - private: - uint8_t type_; - uint32_t id_; - std::size_t msize_ = 0; - std::size_t psize_ = 0; - std::size_t processed_ = 0; - char* msg_ = nullptr; - static const int hsize_ = sizeof(id_) + 2 * sizeof(std::size_t) + sizeof(type_); - char mdata_[hsize_]; - friend class NetworkThread; - friend class EndPoint; - public: - Message(int = MSG_DATA, uint32_t = 0); - Message(const Message&) = delete; - Message(Message&&); - ~Message(); - - void setMetadata(const void*, int); - void setPayload(const void*, int); - - std::size_t getMetadata(void**); - std::size_t getPayload(void**); - - std::size_t getSize(); - void setId(uint32_t); +class Message { +private: + uint8_t type_; + uint32_t id_; + std::size_t msize_ = 0; + std::size_t psize_ = 0; + std::size_t processed_ = 0; + char *msg_ = nullptr; + static const int hsize_ = + sizeof(id_) + 2 * sizeof(std::size_t) + sizeof(type_); + char mdata_[hsize_]; + friend class NetworkThread; + friend class EndPoint; + +public: + Message(int = MSG_DATA, uint32_t = 0); + Message(const Message &) = delete; + Message(Message &&); + ~Message(); + + void setMetadata(const void *, int); + void setPayload(const void *, int); + + std::size_t getMetadata(void **); + std::size_t getPayload(void **); + + std::size_t getSize(); + void setId(uint32_t); }; class EndPoint { - private: - std::queue send_; - std::queue recv_; - std::queue to_ack_; - std::condition_variable cv_; - std::mutex mtx_; - struct sockaddr_in addr_; - ev_timer timer_; - ev_tstamp last_msg_time_; - int fd_[2] = {-1, -1}; // two endpoints simultaneously connect to each other - int pfd_ = -1; - bool is_socket_loop_ = false; - int conn_status_ = CONN_INIT; - int pending_cnt_ = 0; - int retry_cnt_ = 0; - NetworkThread* thread_ = nullptr; - EndPoint(NetworkThread* t); - ~EndPoint(); - friend class NetworkThread; - friend class EndPointFactory; - public: - int send(Message*); - Message* recv(); +private: + std::queue send_; + std::queue recv_; + std::queue to_ack_; + std::condition_variable cv_; + std::mutex mtx_; + struct sockaddr_in addr_; + ev_timer timer_; + ev_tstamp last_msg_time_; + int fd_[2] = { -1, -1 }; // two endpoints simultaneously connect to each other + int pfd_ = -1; + bool is_socket_loop_ = false; + int conn_status_ = CONN_INIT; + int pending_cnt_ = 0; + int retry_cnt_ = 0; + NetworkThread *thread_ = nullptr; + EndPoint(NetworkThread *t); + ~EndPoint(); + friend class NetworkThread; + friend class EndPointFactory; + +public: + int send(Message *); + Message *recv(); }; class EndPointFactory { - private: - std::unordered_map ip_ep_map_; - std::condition_variable map_cv_; - std::mutex map_mtx_; - NetworkThread* thread_; - EndPoint* getEp(uint32_t ip); - EndPoint* getOrCreateEp(uint32_t ip); - friend class NetworkThread; - public: - EndPointFactory(NetworkThread* thread) : thread_(thread) {} - ~EndPointFactory(); - EndPoint* getEp(const char* host); - void getNewEps(std::vector& neps); +private: + std::unordered_map ip_ep_map_; + std::condition_variable map_cv_; + std::mutex map_mtx_; + NetworkThread *thread_; + EndPoint *getEp(uint32_t ip); + EndPoint *getOrCreateEp(uint32_t ip); + friend class NetworkThread; + +public: + EndPointFactory(NetworkThread *thread) : thread_(thread) {} + ~EndPointFactory(); + EndPoint *getEp(const char *host); + void getNewEps(std::vector &neps); }; -class NetworkThread{ - private: - struct ev_loop *loop_; - ev_async ep_sig_; - ev_async msg_sig_; - ev_io socket_watcher_; - int port_; - int socket_fd_; - std::thread* thread_; - std::unordered_map fd_wwatcher_map_; - std::unordered_map fd_rwatcher_map_; - std::unordered_map fd_ep_map_; - std::map pending_msgs_; - - void handleConnLost(int, EndPoint*, bool = true); - void doWork(); - int asyncSend(int); - void asyncSendPendingMsg(EndPoint*); - void afterConnEst(EndPoint* ep, int fd, bool active); - public: - EndPointFactory* epf_; - - NetworkThread(int); - void notify(int signal); - - void onRecv(int fd); - void onSend(int fd = -1); - void onConnEst(int fd); - void onNewEp(); - void onNewConn(); - void onTimeout(struct ev_timer* timer); +class NetworkThread { +private: + struct ev_loop *loop_; + ev_async ep_sig_; + ev_async msg_sig_; + ev_io socket_watcher_; + int port_; + int socket_fd_; + std::thread *thread_; + std::unordered_map fd_wwatcher_map_; + std::unordered_map fd_rwatcher_map_; + std::unordered_map fd_ep_map_; + std::map pending_msgs_; + + void handleConnLost(int, EndPoint *, bool = true); + void doWork(); + int asyncSend(int); + void asyncSendPendingMsg(EndPoint *); + void afterConnEst(EndPoint *ep, int fd, bool active); + +public: + EndPointFactory *epf_; + + NetworkThread(int); + void notify(int signal); + + void onRecv(int fd); + void onSend(int fd = -1); + void onConnEst(int fd); + void onNewEp(); + void onNewConn(); + void onTimeout(struct ev_timer *timer); }; } +#endif // ENABLE_DIST #endif diff --git a/src/io/network/endpoint.cc b/src/io/network/endpoint.cc index 96a2e4ab55..e61acdbe93 100644 --- a/src/io/network/endpoint.cc +++ b/src/io/network/endpoint.cc @@ -18,9 +18,11 @@ * under the License. * *************************************************************/ +#include "singa/singa_config.h" +#ifdef ENABLE_DIST #include "singa/io/network.h" -#include "singa/io/integer.h" +#include "singa/utils/integer.h" #include "singa/utils/logging.h" #include @@ -34,374 +36,376 @@ namespace singa { -static void async_ep_cb(struct ev_loop* loop, ev_async* ev, int revent) { - reinterpret_cast(ev_userdata(loop))->onNewEp(); +static void async_ep_cb(struct ev_loop *loop, ev_async *ev, int revent) { + reinterpret_cast(ev_userdata(loop))->onNewEp(); } -static void async_msg_cb(struct ev_loop* loop, ev_async* ev, int revent) { - reinterpret_cast(ev_userdata(loop))->onSend(); +static void async_msg_cb(struct ev_loop *loop, ev_async *ev, int revent) { + reinterpret_cast(ev_userdata(loop))->onSend(); } -static void writable_cb(struct ev_loop* loop, ev_io* ev, int revent) { - reinterpret_cast(ev_userdata(loop))->onSend(ev->fd); +static void writable_cb(struct ev_loop *loop, ev_io *ev, int revent) { + reinterpret_cast(ev_userdata(loop))->onSend(ev->fd); } -static void readable_cb(struct ev_loop* loop, ev_io* ev, int revent) { - reinterpret_cast(ev_userdata(loop))->onRecv(ev->fd); +static void readable_cb(struct ev_loop *loop, ev_io *ev, int revent) { + reinterpret_cast(ev_userdata(loop))->onRecv(ev->fd); } -static void conn_cb(struct ev_loop* loop, ev_io* ev, int revent) { - reinterpret_cast(ev_userdata(loop))->onConnEst(ev->fd); +static void conn_cb(struct ev_loop *loop, ev_io *ev, int revent) { + reinterpret_cast(ev_userdata(loop))->onConnEst(ev->fd); } -static void accept_cb(struct ev_loop* loop, ev_io* ev, int revent) { - reinterpret_cast(ev_userdata(loop))->onNewConn(); +static void accept_cb(struct ev_loop *loop, ev_io *ev, int revent) { + reinterpret_cast(ev_userdata(loop))->onNewConn(); } -static void timeout_cb(struct ev_loop* loop, ev_timer* ev, int revent) { - reinterpret_cast(ev_userdata(loop))->onTimeout(ev); +static void timeout_cb(struct ev_loop *loop, ev_timer *ev, int revent) { + reinterpret_cast(ev_userdata(loop))->onTimeout(ev); } -EndPoint::EndPoint(NetworkThread* t) : thread_(t) { - this->timer_.data = reinterpret_cast(this); +EndPoint::EndPoint(NetworkThread *t) : thread_(t) { + this->timer_.data = reinterpret_cast(this); } EndPoint::~EndPoint() { - while(!recv_.empty()) { - delete send_.front(); - send_.pop(); - } - while(!to_ack_.empty()) { - delete send_.front(); - send_.pop(); - } - while(!send_.empty()) { - delete send_.front(); - send_.pop(); - } + while (!recv_.empty()) { + delete send_.front(); + send_.pop(); + } + while (!to_ack_.empty()) { + delete send_.front(); + send_.pop(); + } + while (!send_.empty()) { + delete send_.front(); + send_.pop(); + } } -int EndPoint::send(Message* msg) { - CHECK(msg->type_ == MSG_DATA); - static std::atomic id(0); - std::unique_lock lock(this->mtx_); +int EndPoint::send(Message *msg) { + CHECK(msg->type_ == MSG_DATA); + static std::atomic id(0); + std::unique_lock lock(this->mtx_); - if (this->conn_status_ == CONN_ERROR) { - LOG(INFO) << "EndPoint " << inet_ntoa(addr_.sin_addr) << " is disconnected"; - return -1; - } + if (this->conn_status_ == CONN_ERROR) { + LOG(INFO) << "EndPoint " << inet_ntoa(addr_.sin_addr) << " is disconnected"; + return -1; + } - if (msg->psize_ == 0 && msg->msize_ == 0) - // no data to send - return 0; + if (msg->psize_ == 0 && msg->msize_ == 0) + // no data to send + return 0; - msg->setId(id++); + msg->setId(id++); - send_.push(new Message(static_cast(*msg))); + send_.push(new Message(static_cast(*msg))); - thread_->notify(SIG_MSG); - return msg->getSize(); + thread_->notify(SIG_MSG); + return msg->getSize(); } -Message* EndPoint::recv() { - std::unique_lock lock(this->mtx_); - while(this->recv_.empty() && conn_status_ != CONN_ERROR) - this->cv_.wait(lock); - - Message* ret = nullptr; - if (!recv_.empty()) { - ret = recv_.front(); - recv_.pop(); - } - return ret; +Message *EndPoint::recv() { + std::unique_lock lock(this->mtx_); + while (this->recv_.empty() && conn_status_ != CONN_ERROR) + this->cv_.wait(lock); + + Message *ret = nullptr; + if (!recv_.empty()) { + ret = recv_.front(); + recv_.pop(); + } + return ret; } EndPointFactory::~EndPointFactory() { - for (auto& p : ip_ep_map_) - { - delete p.second; - } + for (auto &p : ip_ep_map_) { + delete p.second; + } } -EndPoint* EndPointFactory::getOrCreateEp(uint32_t ip) { - std::unique_lock lock(map_mtx_); - if (0 == ip_ep_map_.count(ip)) { - ip_ep_map_[ip] = new EndPoint(this->thread_); - } - return ip_ep_map_[ip]; +EndPoint *EndPointFactory::getOrCreateEp(uint32_t ip) { + std::unique_lock lock(map_mtx_); + if (0 == ip_ep_map_.count(ip)) { + ip_ep_map_[ip] = new EndPoint(this->thread_); + } + return ip_ep_map_[ip]; } -EndPoint* EndPointFactory::getEp(uint32_t ip) { - std::unique_lock lock(map_mtx_); - if (0 == ip_ep_map_.count(ip)) { - return nullptr; - } - return ip_ep_map_[ip]; +EndPoint *EndPointFactory::getEp(uint32_t ip) { + std::unique_lock lock(map_mtx_); + if (0 == ip_ep_map_.count(ip)) { + return nullptr; + } + return ip_ep_map_[ip]; } -EndPoint* EndPointFactory::getEp(const char* host) { - // get the ip address of host - struct hostent *he; - struct in_addr **list; - - if ((he = gethostbyname(host)) == nullptr) { - LOG(INFO) << "Unable to resolve host " << host; - return nullptr; - } - - list = (struct in_addr**) he->h_addr_list; - uint32_t ip = ntohl(list[0]->s_addr); - - EndPoint* ep = nullptr; - map_mtx_.lock(); - if (0 == ip_ep_map_.count(ip)) { - ep = new EndPoint(this->thread_); - ep->thread_ = this->thread_; - ip_ep_map_[ip] = ep; - - // copy the address info - bcopy(list[0], &ep->addr_.sin_addr, sizeof(struct in_addr)); - - thread_->notify(SIG_EP); - } - ep = ip_ep_map_[ip]; - map_mtx_.unlock(); +EndPoint *EndPointFactory::getEp(const char *host) { + // get the ip address of host + struct hostent *he; + struct in_addr **list; + + if ((he = gethostbyname(host)) == nullptr) { + LOG(INFO) << "Unable to resolve host " << host; + return nullptr; + } + + list = (struct in_addr **)he->h_addr_list; + uint32_t ip = ntohl(list[0]->s_addr); + + EndPoint *ep = nullptr; + map_mtx_.lock(); + if (0 == ip_ep_map_.count(ip)) { + ep = new EndPoint(this->thread_); + ep->thread_ = this->thread_; + ip_ep_map_[ip] = ep; + + // copy the address info + bcopy(list[0], &ep->addr_.sin_addr, sizeof(struct in_addr)); + + thread_->notify(SIG_EP); + } + ep = ip_ep_map_[ip]; + map_mtx_.unlock(); + + std::unique_lock eplock(ep->mtx_); + while (ep->conn_status_ == CONN_PENDING || ep->conn_status_ == CONN_INIT) { + ep->pending_cnt_++; + ep->cv_.wait(eplock); + ep->pending_cnt_--; + } + + if (ep->conn_status_ == CONN_ERROR) { + ep = nullptr; + } + + return ep; +} +void EndPointFactory::getNewEps(std::vector &neps) { + std::unique_lock lock(this->map_mtx_); + for (auto &p : this->ip_ep_map_) { + EndPoint *ep = p.second; std::unique_lock eplock(ep->mtx_); - while (ep->conn_status_ == CONN_PENDING || ep->conn_status_ == CONN_INIT) { - ep->pending_cnt_++; - ep->cv_.wait(eplock); - ep->pending_cnt_--; + if (ep->conn_status_ == CONN_INIT) { + neps.push_back(ep); } - - if (ep->conn_status_ == CONN_ERROR) { - ep = nullptr; - } - - return ep; -} - -void EndPointFactory::getNewEps(std::vector& neps) { - std::unique_lock lock(this->map_mtx_); - for (auto& p : this->ip_ep_map_) { - EndPoint* ep = p.second; - std::unique_lock eplock(ep->mtx_); - if (ep->conn_status_ == CONN_INIT) { - neps.push_back(ep); - } - } + } } NetworkThread::NetworkThread(int port) { - this->port_ = port; - thread_ = new std::thread([this] {doWork();}); - this->epf_ = new EndPointFactory(this); + this->port_ = port; + thread_ = new std::thread([this] { doWork(); }); + this->epf_ = new EndPointFactory(this); } void NetworkThread::doWork() { - // prepare event loop - if (!(loop_ = ev_default_loop(0))) { - // log here - } + // prepare event loop + if (!(loop_ = ev_default_loop(0))) { + // log here + } - ev_async_init(&ep_sig_, async_ep_cb); - ev_async_start(loop_, &ep_sig_); + ev_async_init(&ep_sig_, async_ep_cb); + ev_async_start(loop_, &ep_sig_); - ev_async_init(&msg_sig_, async_msg_cb); - ev_async_start(loop_, &msg_sig_); + ev_async_init(&msg_sig_, async_msg_cb); + ev_async_start(loop_, &msg_sig_); - // bind and listen - struct sockaddr_in addr; - if ((socket_fd_ = socket(AF_INET, SOCK_STREAM, 0)) < 0) { - LOG(FATAL) << "Socket Error: " << strerror(errno); - } + // bind and listen + struct sockaddr_in addr; + if ((socket_fd_ = socket(AF_INET, SOCK_STREAM, 0)) < 0) { + LOG(FATAL) << "Socket Error: " << strerror(errno); + } - bzero(&addr, sizeof(addr)); - addr.sin_family = AF_INET; - addr.sin_port = htons(this->port_); - addr.sin_addr.s_addr = INADDR_ANY; + bzero(&addr, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(this->port_); + addr.sin_addr.s_addr = INADDR_ANY; - if (bind(socket_fd_, (struct sockaddr*)&addr, sizeof(addr))) { - LOG(FATAL) << "Bind Error: " << strerror(errno); - } + if (bind(socket_fd_, (struct sockaddr *)&addr, sizeof(addr))) { + LOG(FATAL) << "Bind Error: " << strerror(errno); + } - if (listen(socket_fd_, 10)) { - LOG(FATAL) << "Listen Error: " << strerror(errno); - } + if (listen(socket_fd_, 10)) { + LOG(FATAL) << "Listen Error: " << strerror(errno); + } - ev_io_init(&socket_watcher_, accept_cb, socket_fd_, EV_READ); - ev_io_start(loop_, &socket_watcher_); + ev_io_init(&socket_watcher_, accept_cb, socket_fd_, EV_READ); + ev_io_start(loop_, &socket_watcher_); - ev_set_userdata(loop_, this); + ev_set_userdata(loop_, this); - while(1) - ev_run(loop_, 0); + while (1) + ev_run(loop_, 0); } void NetworkThread::notify(int signal) { - switch(signal) { - case SIG_EP: - ev_async_send(this->loop_, &this->ep_sig_); - break; - case SIG_MSG: - ev_async_send(this->loop_, &this->msg_sig_); - break; - default: - break; - } + switch (signal) { + case SIG_EP: + ev_async_send(this->loop_, &this->ep_sig_); + break; + case SIG_MSG: + ev_async_send(this->loop_, &this->msg_sig_); + break; + default: + break; + } } void NetworkThread::onNewEp() { - std::vector neps; - this->epf_->getNewEps(neps); - - for (auto& ep : neps) { - std::unique_lock ep_lock(ep->mtx_); - int& fd = ep->fd_[0]; - if (ep->conn_status_ == CONN_INIT) { - - fd = socket(AF_INET, SOCK_STREAM, 0); - if (fd < 0) { - // resources not available - LOG(FATAL) << "Unable to create socket"; - } + std::vector neps; + this->epf_->getNewEps(neps); - // set this fd non-blocking - fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK); - - this->fd_ep_map_[fd] = ep; - - // initialize the addess - ep->addr_.sin_family = AF_INET; - ep->addr_.sin_port = htons(port_); - bzero(&(ep->addr_.sin_zero), 8); - - LOG(INFO) << "Connecting to " << inet_ntoa(ep->addr_.sin_addr) << " fd = "<< fd; - if (connect(fd, (struct sockaddr*)&ep->addr_, - sizeof(struct sockaddr)) ) { - LOG(INFO) << "Connect Error: " << strerror(errno); - if (errno != EINPROGRESS) { - ep->conn_status_ = CONN_ERROR; - ep->cv_.notify_all(); - continue; - } else { - ep->conn_status_ = CONN_PENDING; - ev_io_init(&this->fd_wwatcher_map_[fd], conn_cb, fd, EV_WRITE); - ev_io_start(this->loop_, &this->fd_wwatcher_map_[fd]); - } - } else { - afterConnEst(ep, fd, true); + for (auto &ep : neps) { + std::unique_lock ep_lock(ep->mtx_); + int &fd = ep->fd_[0]; + if (ep->conn_status_ == CONN_INIT) { + + fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) { + // resources not available + LOG(FATAL) << "Unable to create socket"; + } + + // set this fd non-blocking + fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK); + + this->fd_ep_map_[fd] = ep; + + // initialize the addess + ep->addr_.sin_family = AF_INET; + ep->addr_.sin_port = htons(port_); + bzero(&(ep->addr_.sin_zero), 8); + + LOG(INFO) << "Connecting to " << inet_ntoa(ep->addr_.sin_addr) + << " fd = " << fd; + if (connect(fd, (struct sockaddr *)&ep->addr_, sizeof(struct sockaddr))) { + LOG(INFO) << "Connect Error: " << strerror(errno); + if (errno != EINPROGRESS) { + ep->conn_status_ = CONN_ERROR; + ep->cv_.notify_all(); + continue; + } else { + ep->conn_status_ = CONN_PENDING; + ev_io_init(&this->fd_wwatcher_map_[fd], conn_cb, fd, EV_WRITE); + ev_io_start(this->loop_, &this->fd_wwatcher_map_[fd]); + } + } else { + afterConnEst(ep, fd, true); - // connection established immediately - // LOG(INFO) << "Connected to " << inet_ntoa(ep->addr_.sin_addr) << " fd = "<< fd; - // ep->conn_status_ = CONN_EST; + // connection established immediately + // LOG(INFO) << "Connected to " << inet_ntoa(ep->addr_.sin_addr) << " fd + // = "<< fd; + // ep->conn_status_ = CONN_EST; - // //ev_io_stop(this->loop_, &this->fd_wwatcher_map_[fd]); - // ev_io_init(&fd_wwatcher_map_[fd], writable_cb, fd, EV_WRITE); + // //ev_io_stop(this->loop_, &this->fd_wwatcher_map_[fd]); + // ev_io_init(&fd_wwatcher_map_[fd], writable_cb, fd, EV_WRITE); - // // poll for new msgs - // ev_io_init(&this->fd_rwatcher_map_[fd], readable_cb, fd, EV_READ); - // ev_io_start(this->loop_, &this->fd_rwatcher_map_[fd]); + // // poll for new msgs + // ev_io_init(&this->fd_rwatcher_map_[fd], readable_cb, fd, EV_READ); + // ev_io_start(this->loop_, &this->fd_rwatcher_map_[fd]); - // asyncSendPendingMsg(ep); - // ep->cv_.notify_all(); - } - } + // asyncSendPendingMsg(ep); + // ep->cv_.notify_all(); + } } + } } void NetworkThread::onConnEst(int fd) { - //EndPoint* ep = epf_->getEp(this->fd_ip_map_[fd]); - CHECK(fd_ep_map_.count(fd) > 0); - EndPoint* ep = fd_ep_map_.at(fd); + // EndPoint* ep = epf_->getEp(this->fd_ip_map_[fd]); + CHECK(fd_ep_map_.count(fd) > 0); + EndPoint *ep = fd_ep_map_.at(fd); - std::unique_lock lock(ep->mtx_); + std::unique_lock lock(ep->mtx_); - if (connect(fd, (struct sockaddr*)&ep->addr_, sizeof(struct sockaddr)) < 0 && errno != EISCONN) { - LOG(INFO) << "Unable to connect to " << inet_ntoa(ep->addr_.sin_addr) << ": "<< strerror(errno); - if (errno == EINPROGRESS) { - // continue to watch this socket - return; - } + if (connect(fd, (struct sockaddr *)&ep->addr_, sizeof(struct sockaddr)) < 0 && + errno != EISCONN) { + LOG(INFO) << "Unable to connect to " << inet_ntoa(ep->addr_.sin_addr) + << ": " << strerror(errno); + if (errno == EINPROGRESS) { + // continue to watch this socket + return; + } - handleConnLost(ep->fd_[0], ep); + handleConnLost(ep->fd_[0], ep); - if (ep->conn_status_ == CONN_EST && ep->conn_status_ == CONN_ERROR) - ep->cv_.notify_all(); + if (ep->conn_status_ == CONN_EST && ep->conn_status_ == CONN_ERROR) + ep->cv_.notify_all(); - } else { + } else { - afterConnEst(ep, fd, true); + afterConnEst(ep, fd, true); - //ep->conn_status_ = CONN_EST; - //// connect established; poll for new msgs - //ev_io_stop(this->loop_, &this->fd_wwatcher_map_[fd]); - //ev_io_init(&fd_wwatcher_map_[fd], writable_cb, fd, EV_WRITE); + // ep->conn_status_ = CONN_EST; + //// connect established; poll for new msgs + // ev_io_stop(this->loop_, &this->fd_wwatcher_map_[fd]); + // ev_io_init(&fd_wwatcher_map_[fd], writable_cb, fd, EV_WRITE); - //ev_io_init(&this->fd_rwatcher_map_[fd], readable_cb, fd, EV_READ); - //ev_io_start(this->loop_, &this->fd_rwatcher_map_[fd]); - } + // ev_io_init(&this->fd_rwatcher_map_[fd], readable_cb, fd, EV_READ); + // ev_io_start(this->loop_, &this->fd_rwatcher_map_[fd]); + } } void NetworkThread::onNewConn() { - // accept new tcp connection - struct sockaddr_in addr; - socklen_t len = sizeof(addr); - int fd = accept(socket_fd_, (struct sockaddr*)&addr, &len); - if (fd < 0) { - LOG(INFO) << "Accept Error: " << strerror(errno); - return; - } + // accept new tcp connection + struct sockaddr_in addr; + socklen_t len = sizeof(addr); + int fd = accept(socket_fd_, (struct sockaddr *)&addr, &len); + if (fd < 0) { + LOG(INFO) << "Accept Error: " << strerror(errno); + return; + } - LOG(INFO) << "Accept a client from " << inet_ntoa(addr.sin_addr) << ", fd = " << fd; + LOG(INFO) << "Accept a client from " << inet_ntoa(addr.sin_addr) + << ", fd = " << fd; - // set this fd as non-blocking - fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK); + // set this fd as non-blocking + fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK); - EndPoint* ep; - uint32_t a = ntohl(addr.sin_addr.s_addr); + EndPoint *ep; + uint32_t a = ntohl(addr.sin_addr.s_addr); - ep = epf_->getOrCreateEp(a); - std::unique_lock lock(ep->mtx_); + ep = epf_->getOrCreateEp(a); + std::unique_lock lock(ep->mtx_); - // Passive connection - afterConnEst(ep, fd, false); + // Passive connection + afterConnEst(ep, fd, false); - // record the remote address - bcopy(&addr, &ep->addr_, len); + // record the remote address + bcopy(&addr, &ep->addr_, len); } -void NetworkThread::onTimeout(struct ev_timer* timer) { - - EndPoint* ep = reinterpret_cast(timer->data); +void NetworkThread::onTimeout(struct ev_timer *timer) { - ev_tstamp timeout = EP_TIMEOUT + ep->last_msg_time_; - ev_tstamp now = ev_now(loop_); + EndPoint *ep = reinterpret_cast(timer->data); - std::unique_lock lock(ep->mtx_); - if (now > timeout) { - if (!ep->to_ack_.empty() || !ep->send_.empty()) { - - LOG(INFO) << "EndPoint " << inet_ntoa(ep->addr_.sin_addr) << " timeouts"; - // we consider this ep has been disconnected - for (int i = 0; i < 2; ++i) - { - int fd = ep->fd_[i]; - if (fd >= 0) - handleConnLost(fd, ep); - } - return; - } + ev_tstamp timeout = EP_TIMEOUT + ep->last_msg_time_; + ev_tstamp now = ev_now(loop_); - timer->repeat = EP_TIMEOUT; + std::unique_lock lock(ep->mtx_); + if (now > timeout) { + if (!ep->to_ack_.empty() || !ep->send_.empty()) { - } else { - timer->repeat = timeout - now; + LOG(INFO) << "EndPoint " << inet_ntoa(ep->addr_.sin_addr) << " timeouts"; + // we consider this ep has been disconnected + for (int i = 0; i < 2; ++i) { + int fd = ep->fd_[i]; + if (fd >= 0) + handleConnLost(fd, ep); + } + return; } - ev_timer_again(loop_, &ep->timer_); + timer->repeat = EP_TIMEOUT; + + } else { + timer->repeat = timeout - now; + } + + ev_timer_again(loop_, &ep->timer_); } /** @@ -411,322 +415,351 @@ void NetworkThread::onTimeout(struct ev_timer* timer) { * @param fd * @param active indicate whethen this socket is locally initiated or not */ -void NetworkThread::afterConnEst(EndPoint* ep, int fd, bool active) { - - if (active) - LOG(INFO) << "Connected to " << inet_ntoa(ep->addr_.sin_addr) << ", fd = "<< fd; - - int sfd; - - if (active) { - ep->fd_[0] = fd; - sfd = ep->fd_[1]; - } else { - if (ep->fd_[1] >= 0) { - // the previous connection is lost - handleConnLost(ep->fd_[1], ep, false); - } - ep->fd_[1] = fd; - sfd = ep->fd_[0]; - } - - if (sfd == fd) { - // this fd is a reuse of a previous socket fd - // so we first need to clean the resouce for that fd - // we duplicate this fd to let the resouce of the oldf fd can be freed - // also indicate there is no need to reconnect - fd = dup(fd); - handleConnLost(sfd, ep, false); - } - - // initialize io watchers and add the read watcher to the ev loop - ev_io_init(&fd_rwatcher_map_[fd], readable_cb, fd, EV_READ); - ev_io_start(loop_, &fd_rwatcher_map_[fd]); - - // stop watching the writable watcher if necessary - if (active) - ev_io_stop(loop_, &fd_wwatcher_map_[fd]); - ev_io_init(&fd_wwatcher_map_[fd], writable_cb, fd, EV_WRITE); - - ep->last_msg_time_ = ev_now(loop_); - - // see whether there is already a established connection for this fd - if (ep->conn_status_ == CONN_EST && sfd >= 0) { - // check if fd and sfd are associate with the same socket - struct sockaddr_in addr; - socklen_t len; - if (getsockname(fd, (struct sockaddr*)&addr, &len)) { - LOG(INFO) << "Unable to get local socket address: " << strerror(errno); - } else { - // see whether the local address of fd is the same as the remote side - // of sfd, which has already been stored in ep->addr_ - if (addr.sin_addr.s_addr == ep->addr_.sin_addr.s_addr && addr.sin_port == ep->addr_.sin_port) { - LOG(INFO) << fd << " and " << sfd << " are associated with the same socket"; - ep->is_socket_loop_ = true; - } else { - // this socket is redundant, we close it maunally if the local ip - // is smaller than the peer ip - if ((addr.sin_addr.s_addr < ep->addr_.sin_addr.s_addr) - || (addr.sin_addr.s_addr == ep->addr_.sin_addr.s_addr && addr.sin_port < ep->addr_.sin_port)) - handleConnLost(fd, ep, false); - } - } +void NetworkThread::afterConnEst(EndPoint *ep, int fd, bool active) { + + if (active) + LOG(INFO) << "Connected to " << inet_ntoa(ep->addr_.sin_addr) + << ", fd = " << fd; + + int sfd; + + if (active) { + ep->fd_[0] = fd; + sfd = ep->fd_[1]; + } else { + if (ep->fd_[1] >= 0) { + // the previous connection is lost + handleConnLost(ep->fd_[1], ep, false); + } + ep->fd_[1] = fd; + sfd = ep->fd_[0]; + } + + if (sfd == fd) { + // this fd is a reuse of a previous socket fd + // so we first need to clean the resouce for that fd + // we duplicate this fd to let the resouce of the oldf fd can be freed + // also indicate there is no need to reconnect + fd = dup(fd); + handleConnLost(sfd, ep, false); + } + + // initialize io watchers and add the read watcher to the ev loop + ev_io_init(&fd_rwatcher_map_[fd], readable_cb, fd, EV_READ); + ev_io_start(loop_, &fd_rwatcher_map_[fd]); + + // stop watching the writable watcher if necessary + if (active) + ev_io_stop(loop_, &fd_wwatcher_map_[fd]); + ev_io_init(&fd_wwatcher_map_[fd], writable_cb, fd, EV_WRITE); + + ep->last_msg_time_ = ev_now(loop_); + + // see whether there is already a established connection for this fd + if (ep->conn_status_ == CONN_EST && sfd >= 0) { + // check if fd and sfd are associate with the same socket + struct sockaddr_in addr; + socklen_t len; + if (getsockname(fd, (struct sockaddr *)&addr, &len)) { + LOG(INFO) << "Unable to get local socket address: " << strerror(errno); } else { - ep->pfd_ = fd; // set the primary fd - ep->conn_status_ = CONN_EST; - - // start timeout watcher to detect the liveness of EndPoint - ev_init(&ep->timer_, timeout_cb); - ep->timer_.repeat = EP_TIMEOUT; - ev_timer_start(loop_, &ep->timer_); - //timeout_cb(loop_, &ep->timer_, EV_TIMER); - } - - if (fd == ep->pfd_) { - this->asyncSendPendingMsg(ep); - } - - fd_ep_map_[fd] = ep; - - // Finally notify all waiting threads - // if this connection is initiaed by remote side, - // we dont need to notify the waiting thread - // later threads wanting to send to this ep, however, - // are able to reuse this ep - if (active) { - ep->cv_.notify_all(); - } + // see whether the local address of fd is the same as the remote side + // of sfd, which has already been stored in ep->addr_ + if (addr.sin_addr.s_addr == ep->addr_.sin_addr.s_addr && + addr.sin_port == ep->addr_.sin_port) { + LOG(INFO) << fd << " and " << sfd + << " are associated with the same socket"; + ep->is_socket_loop_ = true; + } else { + // this socket is redundant, we close it maunally if the local ip + // is smaller than the peer ip + if ((addr.sin_addr.s_addr < ep->addr_.sin_addr.s_addr) || + (addr.sin_addr.s_addr == ep->addr_.sin_addr.s_addr && + addr.sin_port < ep->addr_.sin_port)) + handleConnLost(fd, ep, false); + } + } + } else { + ep->pfd_ = fd; // set the primary fd + ep->conn_status_ = CONN_EST; + + // start timeout watcher to detect the liveness of EndPoint + ev_init(&ep->timer_, timeout_cb); + ep->timer_.repeat = EP_TIMEOUT; + ev_timer_start(loop_, &ep->timer_); + // timeout_cb(loop_, &ep->timer_, EV_TIMER); + } + + if (fd == ep->pfd_) { + this->asyncSendPendingMsg(ep); + } + + fd_ep_map_[fd] = ep; + + // Finally notify all waiting threads + // if this connection is initiaed by remote side, + // we dont need to notify the waiting thread + // later threads wanting to send to this ep, however, + // are able to reuse this ep + if (active) { + ep->cv_.notify_all(); + } } void NetworkThread::onSend(int fd) { - std::vector invalid_fd; - - if (fd == -1) { - //LOG(INFO) << "There are " << fd_ip_map_.size() << " connections"; - // this is a signal of new message to send - for(auto& p : fd_ep_map_) { - // send message - //LOG(INFO) << "Try to send over fd " << p.first; - if (asyncSend(p.first) < 0) - invalid_fd.push_back(p.first); - } - } else { - if (asyncSend(fd) < 0) - invalid_fd.push_back(fd); - } - - for (auto& p : invalid_fd) { - //EndPoint* ep = epf_->getEp(fd_ip_map_.at(p)); - EndPoint* ep = fd_ep_map_.at(p); - std::unique_lock lock(ep->mtx_); - handleConnLost(p, ep); - } + std::vector invalid_fd; + + if (fd == -1) { + // LOG(INFO) << "There are " << fd_ip_map_.size() << " connections"; + // this is a signal of new message to send + for (auto &p : fd_ep_map_) { + // send message + // LOG(INFO) << "Try to send over fd " << p.first; + if (asyncSend(p.first) < 0) + invalid_fd.push_back(p.first); + } + } else { + if (asyncSend(fd) < 0) + invalid_fd.push_back(fd); + } + + for (auto &p : invalid_fd) { + // EndPoint* ep = epf_->getEp(fd_ip_map_.at(p)); + EndPoint *ep = fd_ep_map_.at(p); + std::unique_lock lock(ep->mtx_); + handleConnLost(p, ep); + } } -void NetworkThread::asyncSendPendingMsg(EndPoint* ep) { - // simply put the pending msgs to the send queue +void NetworkThread::asyncSendPendingMsg(EndPoint *ep) { + // simply put the pending msgs to the send queue - LOG(INFO) << "There are " << ep->send_.size() << " to-send msgs, and " << ep->to_ack_.size() << " to-ack msgs"; + LOG(INFO) << "There are " << ep->send_.size() << " to-send msgs, and " + << ep->to_ack_.size() << " to-ack msgs"; - if (!ep->to_ack_.empty()) { - while (!ep->send_.empty()) { - ep->to_ack_.push(ep->send_.front()); - ep->send_.pop(); - } - std::swap(ep->send_, ep->to_ack_); + if (!ep->to_ack_.empty()) { + while (!ep->send_.empty()) { + ep->to_ack_.push(ep->send_.front()); + ep->send_.pop(); } + std::swap(ep->send_, ep->to_ack_); + } - if (ep->send_.size() > 0) { - notify(SIG_MSG); - } + if (ep->send_.size() > 0) { + notify(SIG_MSG); + } } /** - * @brief non-locking send; + * @brief non-locking send; * * @param ep * */ int NetworkThread::asyncSend(int fd) { - //EndPoint* ep = epf_->getEp(fd_ip_map_[fd]); - CHECK(fd_ep_map_.count(fd) > 0); - EndPoint* ep = fd_ep_map_.at(fd); - - std::unique_lock ep_lock(ep->mtx_); - - if (fd != ep->pfd_ ) - // we only send over the primary fd - // return -1 to indicate this fd is redundant - return ep->is_socket_loop_ ? 0 : -1; + // EndPoint* ep = epf_->getEp(fd_ip_map_[fd]); + CHECK(fd_ep_map_.count(fd) > 0); + EndPoint *ep = fd_ep_map_.at(fd); - if (ep->conn_status_ != CONN_EST) - // This happens during reconnection - goto out; + std::unique_lock ep_lock(ep->mtx_); - while(!ep->send_.empty()) { + if (fd != ep->pfd_) + // we only send over the primary fd + // return -1 to indicate this fd is redundant + return ep->is_socket_loop_ ? 0 : -1; - Message& msg = *ep->send_.front(); - int nbytes; + if (ep->conn_status_ != CONN_EST) + // This happens during reconnection + goto out; - while(msg.processed_ < msg.getSize()) { - if (msg.type_ == MSG_ACK) { - nbytes = write(fd, msg.mdata_ + msg.processed_, msg.getSize() - msg.processed_); - } - else - nbytes = write(fd, msg.msg_ + msg.processed_, msg.getSize() - msg.processed_); - - if (nbytes == -1) { - if (errno == EWOULDBLOCK) { - if (!ev_is_active(&fd_wwatcher_map_[fd]) && !ev_is_pending(&fd_wwatcher_map_[fd])) - ev_io_start(loop_, &fd_wwatcher_map_[fd]); - goto out; - } else { - // this connection is lost; reset the send status - // so that next time the whole msg would be sent entirely - msg.processed_ = 0; - goto err; - } - } else { - ep->last_msg_time_ = ev_now(loop_); - msg.processed_ += nbytes; - } + while (!ep->send_.empty()) { - //std::size_t m, p; - //uint8_t type; - //uint32_t id; - //if (msg.msg_) { - // readInteger(msg.msg_, type, id, m, p); - // LOG(INFO) << "Send " << msg.processed_ << " bytes to " << inet_ntoa(ep->addr_.sin_addr) << " over fd " << fd << " for the current DATA MSG " << msg.id_ << ", " << id << ", " << m << ", " << p; - //} - } + Message &msg = *ep->send_.front(); + int nbytes; - CHECK(msg.processed_ == msg.getSize()); + while (msg.processed_ < msg.getSize()) { + if (msg.type_ == MSG_ACK) { + nbytes = write(fd, msg.mdata_ + msg.processed_, + msg.getSize() - msg.processed_); + } else + nbytes = write(fd, msg.msg_ + msg.processed_, + msg.getSize() - msg.processed_); - if (msg.type_ != MSG_ACK) { - LOG(INFO) << "Send a DATA message to " << inet_ntoa(ep->addr_.sin_addr) << " for MSG " << msg.id_ << ", len = " << msg.getSize() << " over fd " << fd; - msg.processed_ = 0; - ep->to_ack_.push(&msg); + if (nbytes == -1) { + if (errno == EWOULDBLOCK) { + if (!ev_is_active(&fd_wwatcher_map_[fd]) && + !ev_is_pending(&fd_wwatcher_map_[fd])) + ev_io_start(loop_, &fd_wwatcher_map_[fd]); + goto out; } else { - //LOG(INFO) << "Send an ACK message to " << inet_ntoa(ep->addr_.sin_addr) << " for MSG " << msg.id_; - delete &msg; + // this connection is lost; reset the send status + // so that next time the whole msg would be sent entirely + msg.processed_ = 0; + goto err; } + } else { + ep->last_msg_time_ = ev_now(loop_); + msg.processed_ += nbytes; + } + + // std::size_t m, p; + // uint8_t type; + // uint32_t id; + // if (msg.msg_) { + // readInteger(msg.msg_, type, id, m, p); + // LOG(INFO) << "Send " << msg.processed_ << " bytes to " << + // inet_ntoa(ep->addr_.sin_addr) << " over fd " << fd << " for the current + // DATA MSG " << msg.id_ << ", " << id << ", " << m << ", " << p; + //} + } + + CHECK(msg.processed_ == msg.getSize()); + + if (msg.type_ != MSG_ACK) { + LOG(INFO) << "Send a DATA message to " << inet_ntoa(ep->addr_.sin_addr) + << " for MSG " << msg.id_ << ", len = " << msg.getSize() + << " over fd " << fd; + msg.processed_ = 0; + ep->to_ack_.push(&msg); + } else { + // LOG(INFO) << "Send an ACK message to " << inet_ntoa(ep->addr_.sin_addr) + // << " for MSG " << msg.id_; + delete &msg; + } - ep->send_.pop(); + ep->send_.pop(); - //for test - // if (ep->retry_cnt_ == 0) { - // LOG(INFO) << "Disconnect with Endpoint " << inet_ntoa(ep->addr_.sin_addr) << " over fd " << fd; - // close(fd); - // goto err; - // } - } + // for test + // if (ep->retry_cnt_ == 0) { + // LOG(INFO) << "Disconnect with Endpoint " << + // inet_ntoa(ep->addr_.sin_addr) << " over fd " << fd; + // close(fd); + // goto err; + // } + } out: - if (ep->send_.empty()) - ev_io_stop(loop_, &this->fd_wwatcher_map_[fd]); - return 0; + if (ep->send_.empty()) + ev_io_stop(loop_, &this->fd_wwatcher_map_[fd]); + return 0; err: - return -1; + return -1; } void NetworkThread::onRecv(int fd) { - Message* m = &pending_msgs_[fd]; - Message& msg = (*m); - int nread; - //EndPoint* ep = epf_->getEp(fd_ip_map_[fd]); - - CHECK(fd_ep_map_.count(fd) > 0); - EndPoint* ep = fd_ep_map_.at(fd); - - //LOG(INFO) << "Start to read from EndPoint " << inet_ntoa(ep->addr_.sin_addr) << " over fd " << fd; - - std::unique_lock lock(ep->mtx_); - - ep->last_msg_time_ = ev_now(loop_); - while(1) { - if (msg.processed_ < Message::hsize_) { - nread = read(fd, msg.mdata_ + msg.processed_, - Message::hsize_ - msg.processed_); - - if (nread <= 0) { - if (errno != EWOULDBLOCK || nread == 0) { - // socket error or shuts down - if (nread < 0) - LOG(INFO) << "Fail to receive from EndPoint " << inet_ntoa(ep->addr_.sin_addr) << ": " << strerror(errno); - else - LOG(INFO) << "Fail to receive from EndPoint " << inet_ntoa(ep->addr_.sin_addr) << ": Connection reset by remote side"; - handleConnLost(fd, ep); - } - break; - } - - msg.processed_ += nread; - while (msg.processed_ >= sizeof(msg.type_) + sizeof(msg.id_)) { - readInteger(msg.mdata_, msg.type_, msg.id_); - if(msg.type_ == MSG_ACK) { - LOG(INFO) << "Receive an ACK message from " << inet_ntoa(ep->addr_.sin_addr) << " for MSG " << msg.id_; - while (!ep->to_ack_.empty()) { - Message* m = ep->to_ack_.front(); - if (m->id_ <= msg.id_) { - delete m; - ep->to_ack_.pop(); - } else { - break; - } - } - - // reset - msg.processed_ -= sizeof(msg.type_) + sizeof(msg.id_); - memmove(msg.mdata_, - msg.mdata_ + sizeof(msg.type_) + sizeof(msg.id_), - msg.processed_); - - } else break; - } - - if (msg.processed_ < Message::hsize_) { - continue; - } - - // got the whole metadata; - readInteger(msg.mdata_, msg.type_, msg.id_, msg.msize_, msg.psize_); - - LOG(INFO) << "Receive a message: id = " << msg.id_ << ", msize_ = " << msg.msize_ << ", psize_ = " << msg.psize_ << " from " << inet_ntoa(ep->addr_.sin_addr) << " over fd " << fd; + Message *m = &pending_msgs_[fd]; + Message &msg = (*m); + int nread; + // EndPoint* ep = epf_->getEp(fd_ip_map_[fd]); + + CHECK(fd_ep_map_.count(fd) > 0); + EndPoint *ep = fd_ep_map_.at(fd); + + // LOG(INFO) << "Start to read from EndPoint " << + // inet_ntoa(ep->addr_.sin_addr) << " over fd " << fd; + + std::unique_lock lock(ep->mtx_); + + ep->last_msg_time_ = ev_now(loop_); + while (1) { + if (msg.processed_ < Message::hsize_) { + nread = read(fd, msg.mdata_ + msg.processed_, + Message::hsize_ - msg.processed_); + + if (nread <= 0) { + if (errno != EWOULDBLOCK || nread == 0) { + // socket error or shuts down + if (nread < 0) + LOG(INFO) << "Fail to receive from EndPoint " + << inet_ntoa(ep->addr_.sin_addr) << ": " + << strerror(errno); + else + LOG(INFO) << "Fail to receive from EndPoint " + << inet_ntoa(ep->addr_.sin_addr) + << ": Connection reset by remote side"; + handleConnLost(fd, ep); } - - // start reading the real data - if (msg.msg_ == nullptr) { - msg.msg_ = new char[msg.getSize()]; - memcpy(msg.msg_, msg.mdata_, Message::hsize_); - } - - nread = read(fd, msg.msg_ + msg.processed_, msg.getSize() - msg.processed_); - if (nread <= 0) { - if (errno != EWOULDBLOCK || nread == 0) { - // socket error or shuts down - if (nread < 0) - LOG(INFO) << "Fail to receive from EndPoint " << inet_ntoa(ep->addr_.sin_addr) << ": " << strerror(errno); - else - LOG(INFO) << "Fail to receive from EndPoint " << inet_ntoa(ep->addr_.sin_addr) << ": Connection reset by remote side"; - handleConnLost(fd, ep); + break; + } + + msg.processed_ += nread; + while (msg.processed_ >= sizeof(msg.type_) + sizeof(msg.id_)) { + readInteger(msg.mdata_, msg.type_, msg.id_); + if (msg.type_ == MSG_ACK) { + LOG(INFO) << "Receive an ACK message from " + << inet_ntoa(ep->addr_.sin_addr) << " for MSG " << msg.id_; + while (!ep->to_ack_.empty()) { + Message *m = ep->to_ack_.front(); + if (m->id_ <= msg.id_) { + delete m; + ep->to_ack_.pop(); + } else { + break; } - break; - } - - msg.processed_ += nread; - - //LOG(INFO) << "Receive a message: id = " << msg.id_ << ", msize_ = " << msg.msize_ << ", psize_ = " << msg.psize_ << ", processed_ = " << msg.processed_ << " from " << inet_ntoa(ep->addr_.sin_addr) << " over fd " << fd; - - if (msg.processed_ == msg.getSize()) { - LOG(INFO) << "Receive a " << msg.processed_ << " bytes DATA message from " << inet_ntoa(ep->addr_.sin_addr) << " with id " << msg.id_; - ep->recv_.push(new Message(static_cast(msg))); - // notify of waiting thread - ep->cv_.notify_one(); - ep->send_.push(new Message(MSG_ACK, msg.id_)); - msg.processed_ = 0; - } - } + } + + // reset + msg.processed_ -= sizeof(msg.type_) + sizeof(msg.id_); + memmove(msg.mdata_, msg.mdata_ + sizeof(msg.type_) + sizeof(msg.id_), + msg.processed_); + + } else + break; + } + + if (msg.processed_ < Message::hsize_) { + continue; + } + + // got the whole metadata; + readInteger(msg.mdata_, msg.type_, msg.id_, msg.msize_, msg.psize_); + + LOG(INFO) << "Receive a message: id = " << msg.id_ + << ", msize_ = " << msg.msize_ << ", psize_ = " << msg.psize_ + << " from " << inet_ntoa(ep->addr_.sin_addr) << " over fd " + << fd; + } + + // start reading the real data + if (msg.msg_ == nullptr) { + msg.msg_ = new char[msg.getSize()]; + memcpy(msg.msg_, msg.mdata_, Message::hsize_); + } + + nread = read(fd, msg.msg_ + msg.processed_, msg.getSize() - msg.processed_); + if (nread <= 0) { + if (errno != EWOULDBLOCK || nread == 0) { + // socket error or shuts down + if (nread < 0) + LOG(INFO) << "Fail to receive from EndPoint " + << inet_ntoa(ep->addr_.sin_addr) << ": " << strerror(errno); + else + LOG(INFO) << "Fail to receive from EndPoint " + << inet_ntoa(ep->addr_.sin_addr) + << ": Connection reset by remote side"; + handleConnLost(fd, ep); + } + break; + } + + msg.processed_ += nread; + + // LOG(INFO) << "Receive a message: id = " << msg.id_ << ", msize_ = " << + // msg.msize_ << ", psize_ = " << msg.psize_ << ", processed_ = " << + // msg.processed_ << " from " << inet_ntoa(ep->addr_.sin_addr) << " over fd + // " << fd; + + if (msg.processed_ == msg.getSize()) { + LOG(INFO) << "Receive a " << msg.processed_ << " bytes DATA message from " + << inet_ntoa(ep->addr_.sin_addr) << " with id " << msg.id_; + ep->recv_.push(new Message(static_cast(msg))); + // notify of waiting thread + ep->cv_.notify_one(); + ep->send_.push(new Message(MSG_ACK, msg.id_)); + msg.processed_ = 0; + } + } } /** @@ -737,59 +770,62 @@ void NetworkThread::onRecv(int fd) { * @param ep * @param reconn */ -void NetworkThread::handleConnLost(int fd, EndPoint* ep, bool reconn) { - CHECK(fd >= 0); - LOG(INFO) << "Lost connection to EndPoint " << inet_ntoa(ep->addr_.sin_addr) << ", fd = " << fd; - - this->pending_msgs_.erase(fd); - this->fd_ep_map_.erase(fd); - ev_io_stop(loop_, &this->fd_wwatcher_map_[fd]); - ev_io_stop(loop_, &this->fd_rwatcher_map_[fd]); - fd_wwatcher_map_.erase(fd); - fd_rwatcher_map_.erase(fd); - close(fd); - - if (fd == ep->pfd_) { - if (!ep->send_.empty()) - ep->send_.front()->processed_ = 0; - } - - int sfd = (fd == ep->fd_[0]) ? ep->fd_[1] : ep->fd_[0]; - if (fd == ep->fd_[0]) - ep->fd_[0] = -1; - else - ep->fd_[1] = -1; - - if (reconn) { - // see if the other fd is alive or not - if (sfd < 0) { - if (ep->conn_status_ == CONN_EST) - ev_timer_stop(loop_, &ep->timer_); - if (ep->retry_cnt_ < MAX_RETRY_CNT) { - // notify myself for retry - ep->retry_cnt_++; - ep->conn_status_ = CONN_INIT; - LOG(INFO) << "Reconnect to EndPoint " << inet_ntoa(ep->addr_.sin_addr); - this->notify(SIG_EP); - } else { - LOG(INFO) << "Maximum retry count achieved for EndPoint " << inet_ntoa(ep->addr_.sin_addr); - ep->conn_status_ = CONN_ERROR; - - // notify all threads that this ep is no longer connected - ep->cv_.notify_all(); - } - } else { - if (!ep->is_socket_loop_) { - // if there is another working fd, set this fd as primary and - // send data over this fd - ep->pfd_ = sfd; - ep->last_msg_time_ = ev_now(loop_); - asyncSendPendingMsg(ep); - } else { - handleConnLost(sfd, ep); - } - } - } +void NetworkThread::handleConnLost(int fd, EndPoint *ep, bool reconn) { + CHECK(fd >= 0); + LOG(INFO) << "Lost connection to EndPoint " << inet_ntoa(ep->addr_.sin_addr) + << ", fd = " << fd; + + this->pending_msgs_.erase(fd); + this->fd_ep_map_.erase(fd); + ev_io_stop(loop_, &this->fd_wwatcher_map_[fd]); + ev_io_stop(loop_, &this->fd_rwatcher_map_[fd]); + fd_wwatcher_map_.erase(fd); + fd_rwatcher_map_.erase(fd); + close(fd); + + if (fd == ep->pfd_) { + if (!ep->send_.empty()) + ep->send_.front()->processed_ = 0; + } + + int sfd = (fd == ep->fd_[0]) ? ep->fd_[1] : ep->fd_[0]; + if (fd == ep->fd_[0]) + ep->fd_[0] = -1; + else + ep->fd_[1] = -1; + + if (reconn) { + // see if the other fd is alive or not + if (sfd < 0) { + if (ep->conn_status_ == CONN_EST) + ev_timer_stop(loop_, &ep->timer_); + if (ep->retry_cnt_ < MAX_RETRY_CNT) { + // notify myself for retry + ep->retry_cnt_++; + ep->conn_status_ = CONN_INIT; + LOG(INFO) << "Reconnect to EndPoint " << inet_ntoa(ep->addr_.sin_addr); + this->notify(SIG_EP); + } else { + LOG(INFO) << "Maximum retry count achieved for EndPoint " + << inet_ntoa(ep->addr_.sin_addr); + ep->conn_status_ = CONN_ERROR; + + // notify all threads that this ep is no longer connected + ep->cv_.notify_all(); + } + } else { + if (!ep->is_socket_loop_) { + // if there is another working fd, set this fd as primary and + // send data over this fd + ep->pfd_ = sfd; + ep->last_msg_time_ = ev_now(loop_); + asyncSendPendingMsg(ep); + } else { + handleConnLost(sfd, ep); + } + } + } } - } + +#endif // ENABLE_DIST diff --git a/src/io/network/message.cc b/src/io/network/message.cc index 5bf9b8e0bb..32f29b7671 100644 --- a/src/io/network/message.cc +++ b/src/io/network/message.cc @@ -18,6 +18,8 @@ * under the License. * *************************************************************/ +#include "singa/singa_config.h" +#ifdef ENABLE_DIST #include #include @@ -25,68 +27,69 @@ #include #include "singa/io/network.h" -#include "singa/io/integer.h" +#include "singa/utils/integer.h" namespace singa { -Message::Message(Message&& msg) { - std::swap(msize_, msg.msize_); - std::swap(psize_, msg.psize_); - std::swap(msg_, msg.msg_); - std::swap(type_, msg.type_); - std::swap(id_, msg.id_); +Message::Message(Message &&msg) { + std::swap(msize_, msg.msize_); + std::swap(psize_, msg.psize_); + std::swap(msg_, msg.msg_); + std::swap(type_, msg.type_); + std::swap(id_, msg.id_); } -Message::Message(int type, uint32_t ack_msg_id): type_(type), id_(ack_msg_id) { - if (type_ == MSG_ACK) - appendInteger(mdata_, type_, id_); +Message::Message(int type, uint32_t ack_msg_id) : type_(type), id_(ack_msg_id) { + if (type_ == MSG_ACK) + appendInteger(mdata_, type_, id_); } Message::~Message() { - if (msg_) - free(msg_); + if (msg_) + free(msg_); } std::size_t Message::getSize() { - if (type_ == MSG_ACK) - return sizeof(type_) + sizeof(id_); - else - return this->hsize_ + this->psize_ + this->msize_; + if (type_ == MSG_ACK) + return sizeof(type_) + sizeof(id_); + else + return this->hsize_ + this->psize_ + this->msize_; } void Message::setId(uint32_t id) { - this->id_ = id; - appendInteger(msg_, type_, id_); + this->id_ = id; + appendInteger(msg_, type_, id_); } -void Message::setMetadata(const void* buf, int size) { - this->msize_ = size; - msg_ = (char*) malloc (this->getSize()); - appendInteger(msg_, type_, id_, msize_, psize_); - memcpy(msg_ + hsize_, buf, size); +void Message::setMetadata(const void *buf, int size) { + this->msize_ = size; + msg_ = (char *)malloc(this->getSize()); + appendInteger(msg_, type_, id_, msize_, psize_); + memcpy(msg_ + hsize_, buf, size); } -void Message::setPayload(const void* buf, int size) { - this->psize_ = size; - msg_ = (char*) realloc(msg_, this->getSize()); - appendInteger(msg_ + hsize_ - sizeof(psize_), psize_); - memcpy(msg_ + hsize_ + msize_, buf, size); +void Message::setPayload(const void *buf, int size) { + this->psize_ = size; + msg_ = (char *)realloc(msg_, this->getSize()); + appendInteger(msg_ + hsize_ - sizeof(psize_), psize_); + memcpy(msg_ + hsize_ + msize_, buf, size); } -std::size_t Message::getMetadata(void** p) { - if (this->msize_ == 0) - *p = nullptr; - else - *p = msg_ + hsize_; - return this->msize_; +std::size_t Message::getMetadata(void **p) { + if (this->msize_ == 0) + *p = nullptr; + else + *p = msg_ + hsize_; + return this->msize_; } -std::size_t Message::getPayload(void** p) { - if (this->psize_ == 0) - *p = nullptr; - else - *p = msg_ + hsize_ + msize_; - return this->psize_; +std::size_t Message::getPayload(void **p) { + if (this->psize_ == 0) + *p = nullptr; + else + *p = msg_ + hsize_ + msize_; + return this->psize_; } - } + +#endif // ENABLE_DIST diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index fda871d07a..f196928904 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,22 +1,27 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/include) + +IF(ENABLE_DIST) + ADD_EXECUTABLE(test_ep "singa/test_ep.cc") + ADD_DEPENDENCIES(test_ep singa_io) + TARGET_LINK_LIBRARIES(test_ep singa_utils singa_io protobuf ${SINGA_LINKER_LIBS}) +ENDIF() + ADD_LIBRARY(gtest STATIC EXCLUDE_FROM_ALL "gtest/gtest.h" "gtest/gtest-all.cc") AUX_SOURCE_DIRECTORY(singa singa_test_source) +LIST(REMOVE_ITEM singa_test_source "singa/test_ep.cc") IF(NOT USE_OPENCL) MESSAGE(STATUS "Skipping OpenCL tests") LIST(REMOVE_ITEM singa_test_source "singa/test_opencl.cc") ENDIF() + ADD_EXECUTABLE(test_singa "gtest/gtest_main.cc" ${singa_test_source}) ADD_DEPENDENCIES(test_singa singa_core singa_utils) -MESSAGE(STATUS "link libs" ${singa_linker_libs}) +#MESSAGE(STATUS "link libs" ${singa_linker_libs}) TARGET_LINK_LIBRARIES(test_singa gtest singa_core singa_utils singa_model singa_io proto protobuf ${SINGA_LINKER_LIBS}) -#SET_TARGET_PROPERTIES(test_singa PROPERTIES LINK_FLAGS "${LINK_FLAGS} -pthread") +SET_TARGET_PROPERTIES(test_singa PROPERTIES LINK_FLAGS "${LINK_FLAGS} -pthread ") -#ADD_EXECUTABLE(test_ep "singa/test_ep.cc") -#ADD_DEPENDENCIES(test_ep singa_io) -#TARGET_LINK_LIBRARIES(test_ep singa_core singa_utils singa_model -# singa_io proto protobuf ${SINGA_LINKER_LIBS}) diff --git a/test/singa/test_convolution.cc b/test/singa/test_convolution.cc index b5f36055f7..c3ddceea69 100644 --- a/test/singa/test_convolution.cc +++ b/test/singa/test_convolution.cc @@ -18,6 +18,9 @@ * under the License. * *************************************************************/ +#include "singa/singa_config.h" + +#ifdef USE_CBLAS #include "../src/model/layer/convolution.h" #include "gtest/gtest.h" @@ -202,3 +205,4 @@ TEST(Convolution, Backward) { dwptr[7]); EXPECT_FLOAT_EQ(dy[0] * x[4] + dy[4] * x[13], dwptr[8]); } +#endif // USE_CBLAS diff --git a/test/singa/test_ep.cc b/test/singa/test_ep.cc index cc04064906..0d862e534c 100644 --- a/test/singa/test_ep.cc +++ b/test/singa/test_ep.cc @@ -18,95 +18,96 @@ * under the License. * *************************************************************/ - -#include "singa/io/integer.h" +#include "singa/singa_config.h" +#ifdef ENABLE_DIST #include "singa/io/network.h" +#include "singa/utils/integer.h" +#include "singa/utils/logging.h" #include #include +#include +#include -#include "singa/utils/logging.h" #define SIZE 10000000 #define PORT 10000 #define ITER 10 using namespace singa; -int main(int argc, char** argv) { - char* md = new char[SIZE]; - char* payload = new char[SIZE]; - - char* host = "localhost"; - int port = PORT; - - for (int i = 1; i < argc; ++i) - { - if (strcmp(argv[i], "-p") == 0) - port = atoi(argv[++i]); - else if (strcmp(argv[i], "-h") == 0) - host = argv[++i]; - else - fprintf(stderr, "Invalid option %s\n", argv[i]); - } - - memset(md, 'a', SIZE); - memset(payload, 'b', SIZE); - - NetworkThread* t = new NetworkThread(port); - - EndPointFactory* epf = t->epf_; - - // sleep - sleep(3); - - EndPoint* ep = epf->getEp(host); - - Message* m[ITER]; - for (int i = 0; i < ITER; ++i) - { - m[i] = new Message(); - m[i]->setMetadata(md, SIZE); - m[i]->setPayload(payload, SIZE); +int main(int argc, char **argv) { + char *md = new char[SIZE]; + char *payload = new char[SIZE]; + + const char *host = "localhost"; + int port = PORT; + + for (int i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-p") == 0) + port = atoi(argv[++i]); + else if (strcmp(argv[i], "-h") == 0) + host = argv[++i]; + else + fprintf(stderr, "Invalid option %s\n", argv[i]); + } + + memset(md, 'a', SIZE); + memset(payload, 'b', SIZE); + + NetworkThread *t = new NetworkThread(port); + + EndPointFactory *epf = t->epf_; + + // sleep + sleep(3); + + EndPoint *ep = epf->getEp(host); + + Message *m[ITER]; + for (int i = 0; i < ITER; ++i) { + m[i] = new Message(); + m[i]->setMetadata(md, SIZE); + m[i]->setPayload(payload, SIZE); + } + + while (1) { + for (int i = 0; i < ITER; ++i) { + if (ep->send(m[i]) < 0) + return 1; + delete m[i]; } - while (1) { - for (int i = 0; i < ITER; ++i) - { - if (ep->send(m[i]) < 0) return 1; - delete m[i]; - } - - for (int i = 0; i < ITER; ++i) - { - m[i] = ep->recv(); - if (!m[i]) - return 1; - char *p; - CHECK(m[i]->getMetadata((void**)&p) == SIZE); - CHECK(0 == strncmp(p, md, SIZE)); - CHECK(m[i]->getPayload((void**)&p) == SIZE); - CHECK(0 == strncmp(p, payload, SIZE)); - } + for (int i = 0; i < ITER; ++i) { + m[i] = ep->recv(); + if (!m[i]) + return 1; + char *p; + CHECK(m[i]->getMetadata((void **)&p) == SIZE); + CHECK(0 == strncmp(p, md, SIZE)); + CHECK(m[i]->getPayload((void **)&p) == SIZE); + CHECK(0 == strncmp(p, payload, SIZE)); } + } - //while(ep && cnt++ <= 5 && ep->send(m) > 0 ) { + // while(ep && cnt++ <= 5 && ep->send(m) > 0 ) { - // LOG(INFO) << "Send a " << m->getSize() << " bytes message"; + // LOG(INFO) << "Send a " << m->getSize() << " bytes message"; - // Message* m1 = ep->recv(); + // Message* m1 = ep->recv(); - // if (!m1) - // break; + // if (!m1) + // break; - // char *p; + // char *p; - // LOG(INFO) << "Receive a " << m1->getSize() << " bytes message"; + // LOG(INFO) << "Receive a " << m1->getSize() << " bytes message"; - // CHECK(m1->getMetadata((void**)&p) == SIZE); - // CHECK(0 == strncmp(p, md, SIZE)); - // CHECK(m1->getPayload((void**)&p) == SIZE); - // CHECK(0 == strncmp(p, payload, SIZE)); + // CHECK(m1->getMetadata((void**)&p) == SIZE); + // CHECK(0 == strncmp(p, md, SIZE)); + // CHECK(m1->getPayload((void**)&p) == SIZE); + // CHECK(0 == strncmp(p, payload, SIZE)); - // delete m; - // m = m1; - //} + // delete m; + // m = m1; + //} } +#endif // ENABLE_DIST From e906b84dbba902a1edd4cd10e7e8534b0846ab39 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Wed, 10 Aug 2016 21:05:22 +0800 Subject: [PATCH 16/16] SINGA-235 - Unify the engines for cudnn and singa layers For most layers, we would have multiple implementations, e.g., using cudnn for nvidia gpu, using cpp for cpu and using opencl for other gpus. These layers have different classes. They are registered with different identifiers. This ticket would unify the layer identifiers for each engine: 1. cudnn layers are registered with identifier = cudnn_xxx, e.g., cudnn_convolution for the CudnnConvolution layer. 2. singa layers are registered with identifier = singa_xxx, e.g., singa_convolution for the Convolution layer. cudnn engine must run on cuda devices. and singa engine could run on cuda-gpu device or cpp-cpu device depending on the layer type. For instance, the Convolution layer must run on cpp-cpu device, and Dense layer can run on both devices and would select the correct device automatically. Users need to make sure the engine and the device of the tensors. Both CPP and Python code is updated. Users have to compose the layer identifier manually for CPP version. For Python version, users can set layer.engine='cudnn' or 'singa'. All identifiers are case insensitive. --- examples/char-rnn/README.md | 17 +- examples/char-rnn/train.py | 2 +- examples/cifar10/alexnet-parallel.cc | 84 ++++----- examples/cifar10/alexnet.cc | 58 +++---- examples/cifar10/alexnet.py | 5 +- examples/cifar10/train.py | 33 ++-- examples/cifar10/vgg-parallel.cc | 91 +++++----- examples/cifar10/vgg.py | 24 +-- examples/imagenet/alexnet.cc | 22 +-- include/singa/core/device.h | 30 ++-- include/singa/model/feed_forward_net.h | 12 +- include/singa/model/layer.h | 4 +- include/singa/utils/integer.h | 73 ++++++++ src/core/device/platform.cc | 4 +- src/core/tensor/tensor.cc | 4 +- src/core/tensor/tensor_math_cpp.h | 2 +- src/model/feed_forward_net.cc | 17 +- src/model/layer/activation.cc | 27 ++- src/model/layer/activation.h | 2 +- src/model/layer/batchnorm.cc | 6 +- src/model/layer/batchnorm.h | 2 +- src/model/layer/convolution.cc | 2 +- src/model/layer/convolution.h | 2 +- src/model/layer/cudnn_activation.cc | 10 +- src/model/layer/cudnn_activation.h | 2 +- src/model/layer/cudnn_batchnorm.cc | 2 +- src/model/layer/cudnn_batchnorm.h | 2 +- src/model/layer/cudnn_convolution.cc | 2 +- src/model/layer/cudnn_convolution.h | 2 +- src/model/layer/cudnn_dropout.cc | 2 +- src/model/layer/cudnn_dropout.h | 2 +- src/model/layer/cudnn_lrn.cc | 2 +- src/model/layer/cudnn_lrn.h | 2 +- src/model/layer/cudnn_pooling.cc | 2 +- src/model/layer/cudnn_pooling.h | 2 +- src/model/layer/cudnn_rnn.cc | 17 +- src/model/layer/cudnn_rnn.h | 2 +- src/model/layer/cudnn_softmax.cc | 2 +- src/model/layer/cudnn_softmax.h | 2 +- src/model/layer/cudnn_utils.h | 2 +- src/model/layer/dense.cc | 2 +- src/model/layer/dense.h | 2 +- src/model/layer/dropout.cc | 2 +- src/model/layer/dropout.h | 2 +- src/model/layer/flatten.cc | 2 +- src/model/layer/flatten.h | 2 +- src/model/layer/lrn.cc | 2 +- src/model/layer/lrn.h | 4 +- src/model/layer/pooling.cc | 2 +- src/model/layer/pooling.h | 2 +- src/model/layer/prelu.cc | 4 +- src/model/layer/prelu.h | 2 +- src/model/layer/rnn.cc | 2 +- src/model/layer/rnn.h | 2 +- src/model/layer/softmax.cc | 2 +- src/model/layer/softmax.h | 2 +- src/python/singa/device.py | 13 ++ src/python/singa/layer.py | 226 ++++++++++++++++--------- src/python/singa/net.py | 6 +- src/python/singa/tensor.py | 7 +- src/python/swig/core_device.i | 4 + src/python/swig/model_layer.i | 3 - test/singa/test_activation.cc | 26 +-- test/singa/test_batchnorm.cc | 26 +-- test/singa/test_convolution.cc | 2 +- test/singa/test_cudnn_activation.cc | 28 +-- test/singa/test_cudnn_batchnorm.cc | 2 +- test/singa/test_cudnn_convolution.cc | 4 +- test/singa/test_cudnn_dropout.cc | 2 +- test/singa/test_cudnn_lrn.cc | 2 +- test/singa/test_cudnn_pooling.cc | 2 +- test/singa/test_cudnn_rnn.cc | 2 +- test/singa/test_cudnn_softmax.cc | 2 +- test/singa/test_dense.cc | 2 +- test/singa/test_dropout.cc | 2 +- test/singa/test_flatten.cc | 2 +- test/singa/test_layer.cc | 19 +-- test/singa/test_lrn.cc | 2 +- test/singa/test_pooling.cc | 2 +- test/singa/test_prelu.cc | 2 +- test/singa/test_softmax.cc | 2 +- 81 files changed, 573 insertions(+), 433 deletions(-) create mode 100644 include/singa/utils/integer.h diff --git a/examples/char-rnn/README.md b/examples/char-rnn/README.md index c5cdbb8fc8..d4cfa30728 100644 --- a/examples/char-rnn/README.md +++ b/examples/char-rnn/README.md @@ -1,10 +1,10 @@ # Train Char-RNN using SINGA Recurrent neural networks (RNN) are widely used for modelling sequential data, -e.g., natural language sentences. This example describe how to implement a RNN +e.g., natural language sentences. This example describes how to implement a RNN application (or model) using SINGA's RNN layers. -We will use the [char-rnn](https://github.com/karpathy/char-rnn) modle as an -example, which trains over setences or +We will use the [char-rnn](https://github.com/karpathy/char-rnn) model as an +example, which trains over sentences or source code, with each character as an input unit. Particularly, we will train a RNN using GRU over Linux kernel source code. After training, we expect to generate meaningful code from the model. @@ -12,20 +12,19 @@ generate meaningful code from the model. ## Instructions -* Compile and install SINGA. Currently the RNN implmentation depends on Cudnn V5. +* Compile and install SINGA. Currently the RNN implementation depends on Cudnn with version >= 5.05. * Prepare the dataset. Download the [kernel source code](http://cs.stanford.edu/people/karpathy/char-rnn/). Other plain text files can also be used. * Start the training, - python train.py input_linux.txt + python train.py input_linux.txt Some hyper-parameters could be set through command line, - python train.py -h + python train.py -h +* Sample characters from the model by providing the number of characters to sample and the seed string. -* Sample characters from the model by providing num of characters and the seed string. - - python sample.py 100 --seed '#include #include + namespace singa { +const std::string engine = "cudnn"; LayerConf GenConvConf(string name, int nb_filter, int kernel, int stride, int pad, float std) { LayerConf conf; conf.set_name(name); - conf.set_type("CudnnConvolution"); + conf.set_type(engine + "_convolution"); ConvolutionConf *conv = conf.mutable_convolution_conf(); conv->set_num_output(nb_filter); conv->add_kernel_size(kernel); @@ -67,7 +63,7 @@ LayerConf GenPoolingConf(string name, bool max_pool, int kernel, int stride, int pad) { LayerConf conf; conf.set_name(name); - conf.set_type("CudnnPooling"); + conf.set_type(engine + "_pooling"); PoolingConf *pool = conf.mutable_pooling_conf(); pool->set_kernel_size(kernel); pool->set_stride(stride); @@ -79,14 +75,14 @@ LayerConf GenPoolingConf(string name, bool max_pool, int kernel, int stride, LayerConf GenReLUConf(string name) { LayerConf conf; conf.set_name(name); - conf.set_type("RELU"); + conf.set_type(engine + "_relu"); return conf; } LayerConf GenDenseConf(string name, int num_output, float std, float wd) { LayerConf conf; conf.set_name(name); - conf.set_type("Dense"); + conf.set_type("singa_dense"); DenseConf *dense = conf.mutable_dense_conf(); dense->set_num_output(num_output); @@ -108,7 +104,7 @@ LayerConf GenDenseConf(string name, int num_output, float std, float wd) { LayerConf GenLRNConf(string name) { LayerConf conf; conf.set_name(name); - conf.set_type("CudnnLRN"); + conf.set_type(engine + "_lrn"); LRNConf *lrn = conf.mutable_lrn_conf(); lrn->set_local_size(3); lrn->set_alpha(5e-05); @@ -119,7 +115,7 @@ LayerConf GenLRNConf(string name) { LayerConf GenFlattenConf(string name) { LayerConf conf; conf.set_name(name); - conf.set_type("Flatten"); + conf.set_type("singa_flatten"); return conf; } @@ -127,20 +123,19 @@ FeedForwardNet CreateNet() { FeedForwardNet net; Shape s{3, 32, 32}; - net.Add(new CudnnConvolution(), GenConvConf("conv1", 32, 5, 1, 2, 0.0001), - &s); - net.Add(new CudnnPooling(), GenPoolingConf("pool1", true, 3, 2, 1)); - net.Add(new CudnnActivation(), GenReLUConf("relu1")); - net.Add(new CudnnLRN(), GenLRNConf("lrn1")); - net.Add(new CudnnConvolution(), GenConvConf("conv2", 32, 5, 1, 2, 0.01)); - net.Add(new CudnnActivation(), GenReLUConf("relu2")); - net.Add(new CudnnPooling(), GenPoolingConf("pool2", false, 3, 2, 1)); - net.Add(new CudnnLRN(), GenLRNConf("lrn2")); - net.Add(new CudnnConvolution, GenConvConf("conv3", 64, 5, 1, 2, 0.01)); - net.Add(new CudnnActivation(), GenReLUConf("relu3")); - net.Add(new CudnnPooling(), GenPoolingConf("pool3", false, 3, 2, 1)); - net.Add(new Flatten(), GenFlattenConf("flat")); - net.Add(new Dense(), GenDenseConf("ip", 10, 0.01, 250)); + net.Add(GenConvConf("conv1", 32, 5, 1, 2, 0.0001), &s); + net.Add(GenPoolingConf("pool1", true, 3, 2, 1)); + net.Add(GenReLUConf("relu1")); + net.Add(GenLRNConf("lrn1")); + net.Add(GenConvConf("conv2", 32, 5, 1, 2, 0.01)); + net.Add(GenReLUConf("relu2")); + net.Add(GenPoolingConf("pool2", false, 3, 2, 1)); + net.Add(GenLRNConf("lrn2")); + net.Add(GenConvConf("conv3", 64, 5, 1, 2, 0.01)); + net.Add(GenReLUConf("relu3")); + net.Add(GenPoolingConf("pool3", false, 3, 2, 1)); + net.Add(GenFlattenConf("flat")); + net.Add(GenDenseConf("ip", 10, 0.01, 250)); return net; } @@ -228,35 +223,18 @@ void Train(float lr, int num_epoch, string data_dir) { mem_conf.add_device(0); mem_conf.add_device(1); std::shared_ptr mem_pool(new CnMemPool(mem_conf)); - std::shared_ptr cuda_1(new CudaGPU(0, mem_pool)); - std::shared_ptr cuda_2(new CudaGPU(1, mem_pool)); - net_1.ToDevice(cuda_1); - net_2.ToDevice(cuda_2); - - /* - // this does not work for net_2 - train_x_2.ResetLike(train_x); - train_y_2.ResetLike(train_y); - test_x_2.ResetLike(test_x); - test_y_2.ResetLike(test_y); - - train_x.ToDevice(cuda_1); - train_y.ToDevice(cuda_1); - test_x.ToDevice(cuda_1); - test_y.ToDevice(cuda_1); + std::shared_ptr dev_1(new CudaGPU(0, mem_pool)); + std::shared_ptr dev_2(new CudaGPU(1, mem_pool)); - train_x_2.ToDevice(cuda_2); - train_y_2.ToDevice(cuda_2); - test_x_2.ToDevice(cuda_2); - test_y_2.ToDevice(cuda_2); - */ + net_1.ToDevice(dev_1); + net_2.ToDevice(dev_2); - train_x_1.ToDevice(cuda_1); - train_y_1.ToDevice(cuda_1); - test_x.ToDevice(cuda_1); - test_y.ToDevice(cuda_1); - train_x_2.ToDevice(cuda_2); - train_y_2.ToDevice(cuda_2); + train_x_1.ToDevice(dev_1); + train_y_1.ToDevice(dev_1); + test_x.ToDevice(dev_1); + test_y.ToDevice(dev_1); + train_x_2.ToDevice(dev_2); + train_y_2.ToDevice(dev_2); // net.Train(100, num_epoch, train_x, train_y, test_x, test_y); diff --git a/examples/cifar10/alexnet.cc b/examples/cifar10/alexnet.cc index 6480557d41..e1363e4345 100644 --- a/examples/cifar10/alexnet.cc +++ b/examples/cifar10/alexnet.cc @@ -26,19 +26,14 @@ #include "singa/model/metric.h" #include "singa/utils/channel.h" #include "singa/utils/string.h" -#include "../../src/model/layer/cudnn_convolution.h" -#include "../../src/model/layer/cudnn_activation.h" -#include "../../src/model/layer/cudnn_pooling.h" -#include "../../src/model/layer/cudnn_lrn.h" -#include "../../src/model/layer/dense.h" -#include "../../src/model/layer/flatten.h" namespace singa { +const std::string engine = "cudnn"; LayerConf GenConvConf(string name, int nb_filter, int kernel, int stride, int pad, float std) { LayerConf conf; conf.set_name(name); - conf.set_type("CudnnConvolution"); + conf.set_type(engine + "_convolution"); ConvolutionConf *conv = conf.mutable_convolution_conf(); conv->set_num_output(nb_filter); conv->add_kernel_size(kernel); @@ -63,7 +58,7 @@ LayerConf GenPoolingConf(string name, bool max_pool, int kernel, int stride, int pad) { LayerConf conf; conf.set_name(name); - conf.set_type("CudnnPooling"); + conf.set_type(engine + "_pooling"); PoolingConf *pool = conf.mutable_pooling_conf(); pool->set_kernel_size(kernel); pool->set_stride(stride); @@ -75,14 +70,14 @@ LayerConf GenPoolingConf(string name, bool max_pool, int kernel, int stride, LayerConf GenReLUConf(string name) { LayerConf conf; conf.set_name(name); - conf.set_type("RELU"); + conf.set_type(engine + "_relu"); return conf; } LayerConf GenDenseConf(string name, int num_output, float std, float wd) { LayerConf conf; conf.set_name(name); - conf.set_type("Dense"); + conf.set_type("singa_dense"); DenseConf *dense = conf.mutable_dense_conf(); dense->set_num_output(num_output); @@ -104,7 +99,7 @@ LayerConf GenDenseConf(string name, int num_output, float std, float wd) { LayerConf GenLRNConf(string name) { LayerConf conf; conf.set_name(name); - conf.set_type("CudnnLRN"); + conf.set_type(engine + "_lrn"); LRNConf *lrn = conf.mutable_lrn_conf(); lrn->set_local_size(3); lrn->set_alpha(5e-05); @@ -115,7 +110,7 @@ LayerConf GenLRNConf(string name) { LayerConf GenFlattenConf(string name) { LayerConf conf; conf.set_name(name); - conf.set_type("Flatten"); + conf.set_type("singa_flatten"); return conf; } @@ -123,20 +118,19 @@ FeedForwardNet CreateNet() { FeedForwardNet net; Shape s{3, 32, 32}; - net.Add(new CudnnConvolution(), GenConvConf("conv1", 32, 5, 1, 2, 0.0001), - &s); - net.Add(new CudnnPooling(), GenPoolingConf("pool1", true, 3, 2, 1)); - net.Add(new CudnnActivation(), GenReLUConf("relu1")); - net.Add(new CudnnLRN(), GenLRNConf("lrn1")); - net.Add(new CudnnConvolution(), GenConvConf("conv2", 32, 5, 1, 2, 0.01)); - net.Add(new CudnnActivation(), GenReLUConf("relu2")); - net.Add(new CudnnPooling(), GenPoolingConf("pool2", false, 3, 2, 1)); - net.Add(new CudnnLRN(), GenLRNConf("lrn2")); - net.Add(new CudnnConvolution, GenConvConf("conv3", 64, 5, 1, 2, 0.01)); - net.Add(new CudnnActivation(), GenReLUConf("relu3")); - net.Add(new CudnnPooling(), GenPoolingConf("pool3", false, 3, 2, 1)); - net.Add(new Flatten(), GenFlattenConf("flat")); - net.Add(new Dense(), GenDenseConf("ip", 10, 0.01, 250)); + net.Add(GenConvConf("conv1", 32, 5, 1, 2, 0.0001), &s); + net.Add(GenPoolingConf("pool1", true, 3, 2, 1)); + net.Add(GenReLUConf("relu1")); + net.Add(GenLRNConf("lrn1")); + net.Add(GenConvConf("conv2", 32, 5, 1, 2, 0.01)); + net.Add(GenReLUConf("relu2")); + net.Add(GenPoolingConf("pool2", false, 3, 2, 1)); + net.Add(GenLRNConf("lrn2")); + net.Add(GenConvConf("conv3", 64, 5, 1, 2, 0.01)); + net.Add(GenReLUConf("relu3")); + net.Add(GenPoolingConf("pool3", false, 3, 2, 1)); + net.Add(GenFlattenConf("flat")); + net.Add(GenDenseConf("ip", 10, 0.01, 250)); return net; } @@ -184,12 +178,12 @@ void Train(float lr, int num_epoch, string data_dir) { Accuracy acc; net.Compile(true, &sgd, &loss, &acc); - auto cuda = std::make_shared(); - net.ToDevice(cuda); - train_x.ToDevice(cuda); - train_y.ToDevice(cuda); - test_x.ToDevice(cuda); - test_y.ToDevice(cuda); + auto dev = std::make_shared(); + net.ToDevice(dev); + train_x.ToDevice(dev); + train_y.ToDevice(dev); + test_x.ToDevice(dev); + test_y.ToDevice(dev); net.Train(100, num_epoch, train_x, train_y, test_x, test_y); } } diff --git a/examples/cifar10/alexnet.py b/examples/cifar10/alexnet.py index 96c339a261..9ed55992b2 100644 --- a/examples/cifar10/alexnet.py +++ b/examples/cifar10/alexnet.py @@ -31,7 +31,10 @@ from singa import net as ffnet -def create_net(): +def create_net(use_cpu=False): + if use_cpu: + layer.engine = 'singa' + net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy()) W0_specs = {'init': 'gaussian', 'mean': 0, 'std': 0.0001} W1_specs = {'init': 'gaussian', 'mean': 0, 'std': 0.01} diff --git a/examples/cifar10/train.py b/examples/cifar10/train.py index cb4110db34..3285651a0d 100644 --- a/examples/cifar10/train.py +++ b/examples/cifar10/train.py @@ -96,16 +96,23 @@ def alexnet_lr(epoch): return 0.00001 -def train(data, net, max_epoch, get_lr, weight_decay, batch_size=100): +def train(data, net, max_epoch, get_lr, weight_decay, batch_size=100, + use_cpu=False): print 'Start intialization............' - cuda = device.create_cuda_gpu() - net.to_device(cuda) + if use_cpu: + print 'Using CPU' + dev = device.get_default_device() + else: + print 'Using GPU' + dev = device.create_cuda_gpu() + + net.to_device(dev) opt = optimizer.SGD(momentum=0.9, weight_decay=0.004) for (p, specs) in zip(net.param_values(), net.param_specs()): opt.register(p, specs) - tx = tensor.Tensor((batch_size, 3, 32, 32), cuda) - ty = tensor.Tensor((batch_size,), cuda, core_pb2.kInt) + tx = tensor.Tensor((batch_size, 3, 32, 32), dev) + ty = tensor.Tensor((batch_size,), dev, core_pb2.kInt) train_x, train_y, test_x, test_y = data num_train_batch = train_x.shape[0] / batch_size num_test_batch = test_x.shape[0] / batch_size @@ -127,7 +134,7 @@ def train(data, net, max_epoch, get_lr, weight_decay, batch_size=100): # update progress bar utils.update_progress(b * 1.0 / num_train_batch, 'training loss = %f, accuracy = %f' % (l, a)) - info = 'training loss = %f, training accuracy = %f' \ + info = '\ntraining loss = %f, training accuracy = %f' \ % (loss / num_train_batch, acc / num_train_batch) print info @@ -146,9 +153,11 @@ def train(data, net, max_epoch, get_lr, weight_decay, batch_size=100): net.save('model.bin') # save model params into checkpoint file if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Train vgg/alexnet for cifar10') + parser = argparse.ArgumentParser(description='Train vgg/alexnet for ' + 'cifar10 dataset') parser.add_argument('model', choices=['vgg', 'alexnet'], default='alexnet') parser.add_argument('data', default='cifar-10-batches-py') + parser.add_argument('--use_cpu', action='store_true') args = parser.parse_args() assert os.path.exists(args.data), \ 'Pls download the cifar10 dataset via "download_data.py py"' @@ -157,9 +166,11 @@ def train(data, net, max_epoch, get_lr, weight_decay, batch_size=100): test_x, test_y = load_test_data(args.data) if args.model == 'alexnet': train_x, test_x = normalize_for_alexnet(train_x, test_x) - net = alexnet.create_net() - train((train_x, train_y, test_x, test_y), net, 140, alexnet_lr, 0.004) + net = alexnet.create_net(args.use_cpu) + train((train_x, train_y, test_x, test_y), net, 140, alexnet_lr, 0.004, + use_cpu=args.use_cpu) else: train_x, test_x = normalize_for_vgg(train_x, test_x) - net = vgg.create_net() - train((train_x, train_y, test_x, test_y), net, 250, vgg_lr, 0.0005) + net = vgg.create_net(args.use_cpu) + train((train_x, train_y, test_x, test_y), net, 250, vgg_lr, 0.0005, + use_cpu=args.use_cpu) diff --git a/examples/cifar10/vgg-parallel.cc b/examples/cifar10/vgg-parallel.cc index c6b7fa1a8d..149cb21196 100644 --- a/examples/cifar10/vgg-parallel.cc +++ b/examples/cifar10/vgg-parallel.cc @@ -28,27 +28,20 @@ #include "singa/utils/channel.h" #include "singa/utils/string.h" #include "singa/core/memory.h" -#include "../../src/model/layer/cudnn_convolution.h" -#include "../../src/model/layer/cudnn_activation.h" -#include "../../src/model/layer/cudnn_pooling.h" -#include "../../src/model/layer/cudnn_lrn.h" -#include "../../src/model/layer/dropout.h" -#include "../../src/model/layer/cudnn_batchnorm.h" -#include "../../src/model/layer/dense.h" -#include "../../src/model/layer/flatten.h" #include #include #include namespace singa { +const std::string engine = "cudnn"; const float default_wd = 0.0005f; LayerConf GenConvConf(string name, int nb_filter, int kernel, int stride, int pad, float std = .02f, float bias = .0f) { LayerConf conf; conf.set_name(name); - conf.set_type("CudnnConvolution"); + conf.set_type(engine + "_convolution"); ConvolutionConf *conv = conf.mutable_convolution_conf(); conv->set_num_output(nb_filter); conv->add_kernel_size(kernel); @@ -75,7 +68,7 @@ LayerConf GenPoolingConf(string name, bool max_pool, int kernel, int stride, int pad) { LayerConf conf; conf.set_name(name); - conf.set_type("CudnnPooling"); + conf.set_type(engine + "_pooling"); PoolingConf *pool = conf.mutable_pooling_conf(); pool->set_kernel_size(kernel); pool->set_stride(stride); @@ -87,14 +80,14 @@ LayerConf GenPoolingConf(string name, bool max_pool, int kernel, int stride, LayerConf GenReLUConf(string name) { LayerConf conf; conf.set_name(name); - conf.set_type("RELU"); + conf.set_type(engine + "_relu"); return conf; } LayerConf GenDenseConf(string name, int num_output, float std, float wd = default_wd) { LayerConf conf; conf.set_name(name); - conf.set_type("Dense"); + conf.set_type("singa_dense"); DenseConf *dense = conf.mutable_dense_conf(); dense->set_num_output(num_output); @@ -116,14 +109,14 @@ LayerConf GenDenseConf(string name, int num_output, float std, float wd = defaul LayerConf GenFlattenConf(string name) { LayerConf conf; conf.set_name(name); - conf.set_type("Flatten"); + conf.set_type("singa_flatten"); return conf; } LayerConf GenBatchNormConf(string name) { LayerConf conf; conf.set_name(name); - conf.set_type("CudnnBatchNorm"); + conf.set_type(engine + "_batchnorm"); ParamSpec *gammaspec = conf.add_param(); gammaspec->set_name(name + "_gamma"); auto gammafill = gammaspec->mutable_filler(); @@ -155,7 +148,7 @@ LayerConf GenBatchNormConf(string name) { LayerConf GenDropoutConf(string name, float dropout_ratio) { LayerConf conf; conf.set_name(name); - conf.set_type("Dropout"); + conf.set_type(engine + "_dropout"); DropoutConf *dropout = conf.mutable_dropout_conf(); dropout->set_dropout_ratio(dropout_ratio); @@ -163,47 +156,47 @@ LayerConf GenDropoutConf(string name, float dropout_ratio) { } void ConvBNReLU(FeedForwardNet& net, string name, int nb_filter, Shape* shape = nullptr) { - net.Add(new CudnnConvolution(), GenConvConf(name+"_conv", nb_filter, 3, 1, 1), shape); - net.Add(new CudnnBatchNorm(), GenBatchNormConf(name+"_bn")); - net.Add(new CudnnActivation(), GenReLUConf(name+"_relu")); + net.Add(GenConvConf(name+"_conv", nb_filter, 3, 1, 1), shape); + net.Add(GenBatchNormConf(name+"_bn")); + net.Add(GenReLUConf(name+"_relu")); } FeedForwardNet CreateNet() { FeedForwardNet net; Shape s{3, 32, 32}; ConvBNReLU(net, "conv1_1", 64, &s); - net.Add(new Dropout(), GenDropoutConf("drop1", 0.3)); + net.Add(GenDropoutConf("drop1", 0.3)); ConvBNReLU(net, "conv1_2", 64); - net.Add(new CudnnPooling(), GenPoolingConf("pool1", true, 2, 2, 0)); + net.Add(GenPoolingConf("pool1", true, 2, 2, 0)); ConvBNReLU(net, "conv2_1", 128); - net.Add(new Dropout(), GenDropoutConf("drop2", 0.4)); + net.Add(GenDropoutConf("drop2", 0.4)); ConvBNReLU(net, "conv2_2", 128); - net.Add(new CudnnPooling(), GenPoolingConf("pool2", true, 2, 2, 0)); + net.Add(GenPoolingConf("pool2", true, 2, 2, 0)); ConvBNReLU(net, "conv3_1", 256); - net.Add(new Dropout(), GenDropoutConf("drop3_1", 0.4)); + net.Add(GenDropoutConf("drop3_1", 0.4)); ConvBNReLU(net, "conv3_2", 256); - net.Add(new Dropout(), GenDropoutConf("drop3_2", 0.4)); + net.Add(GenDropoutConf("drop3_2", 0.4)); ConvBNReLU(net, "conv3_3", 256); - net.Add(new CudnnPooling(), GenPoolingConf("pool3", true, 2, 2, 0)); + net.Add(GenPoolingConf("pool3", true, 2, 2, 0)); ConvBNReLU(net, "conv4_1", 512); - net.Add(new Dropout(), GenDropoutConf("drop4_1", 0.4)); + net.Add(GenDropoutConf("drop4_1", 0.4)); ConvBNReLU(net, "conv4_2", 512); - net.Add(new Dropout(), GenDropoutConf("drop4_2", 0.4)); + net.Add(GenDropoutConf("drop4_2", 0.4)); ConvBNReLU(net, "conv4_3", 512); - net.Add(new CudnnPooling(), GenPoolingConf("pool4", true, 2, 2, 0)); + net.Add(GenPoolingConf("pool4", true, 2, 2, 0)); ConvBNReLU(net, "conv5_1", 512); - net.Add(new Dropout(), GenDropoutConf("drop5_1", 0.4)); + net.Add(GenDropoutConf("drop5_1", 0.4)); ConvBNReLU(net, "conv5_2", 512); - net.Add(new Dropout(), GenDropoutConf("drop5_2", 0.4)); + net.Add(GenDropoutConf("drop5_2", 0.4)); ConvBNReLU(net, "conv5_3", 512); - net.Add(new CudnnPooling(), GenPoolingConf("pool5", true, 2, 2, 0)); - net.Add(new Flatten(), GenFlattenConf("flat")); - net.Add(new Dropout(), GenDropoutConf("flat_drop", 0.5)); - net.Add(new Dense(), GenDenseConf("ip1", 512, 0.02)); - net.Add(new CudnnBatchNorm(), GenBatchNormConf("ip1_bn")); - net.Add(new CudnnActivation(), GenReLUConf("ip1_relu")); - net.Add(new Dropout(), GenDropoutConf("ip1_drop", 0.5)); - net.Add(new Dense(), GenDenseConf("ip2", 10, 0.02)); + net.Add(GenPoolingConf("pool5", true, 2, 2, 0)); + net.Add(GenFlattenConf("flat")); + net.Add(GenDropoutConf("flat_drop", 0.5)); + net.Add(GenDenseConf("ip1", 512, 0.02)); + net.Add(GenBatchNormConf("ip1_bn")); + net.Add(GenReLUConf("ip1_relu")); + net.Add(GenDropoutConf("ip1_drop", 0.5)); + net.Add(GenDenseConf("ip2", 10, 0.02)); return net; } @@ -294,17 +287,17 @@ void Train(float lr, int num_epoch, string data_dir) { mem_conf.add_device(0); mem_conf.add_device(1); std::shared_ptr mem_pool(new CnMemPool(mem_conf)); - std::shared_ptr cuda_1(new CudaGPU(0, mem_pool)); - std::shared_ptr cuda_2(new CudaGPU(1, mem_pool)); - net_1.ToDevice(cuda_1); - net_2.ToDevice(cuda_2); - - train_x_1.ToDevice(cuda_1); - train_y_1.ToDevice(cuda_1); - test_x.ToDevice(cuda_1); - test_y.ToDevice(cuda_1); - train_x_2.ToDevice(cuda_2); - train_y_2.ToDevice(cuda_2); + std::shared_ptr dev_1(new CudaGPU(0, mem_pool)); + std::shared_ptr dev_2(new CudaGPU(1, mem_pool)); + net_1.ToDevice(dev_1); + net_2.ToDevice(dev_2); + + train_x_1.ToDevice(dev_1); + train_y_1.ToDevice(dev_1); + test_x.ToDevice(dev_1); + test_y.ToDevice(dev_1); + train_x_2.ToDevice(dev_2); + train_y_2.ToDevice(dev_2); LOG(INFO) << "Launching thread..."; std::thread t1 = diff --git a/examples/cifar10/vgg.py b/examples/cifar10/vgg.py index 0b9bb56717..97e690cad7 100644 --- a/examples/cifar10/vgg.py +++ b/examples/cifar10/vgg.py @@ -40,40 +40,42 @@ def ConvBnReLU(net, name, nb_filers, sample_shape=None): net.add(layer.Activation(name + '_3')) -def create_net(): +def create_net(use_cpu=False): + if use_cpu: + layer.engine = 'singa' net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy()) ConvBnReLU(net, 'conv1_1', 64, (3, 32, 32)) - net.add(layer.Dropout('drop1', 0.3, engine='cuda')) + net.add(layer.Dropout('drop1', 0.3)) ConvBnReLU(net, 'conv1_2', 64) net.add(layer.MaxPooling2D('pool1', 2, 2, border_mode='valid')) ConvBnReLU(net, 'conv2_1', 128) - net.add(layer.Dropout('drop2_1', 0.4, engine='cuda')) + net.add(layer.Dropout('drop2_1', 0.4)) ConvBnReLU(net, 'conv2_2', 128) net.add(layer.MaxPooling2D('pool2', 2, 2, border_mode='valid')) ConvBnReLU(net, 'conv3_1', 256) - net.add(layer.Dropout('drop3_1', 0.4, engine='cuda')) + net.add(layer.Dropout('drop3_1', 0.4)) ConvBnReLU(net, 'conv3_2', 256) - net.add(layer.Dropout('drop3_2', 0.4, engine='cuda')) + net.add(layer.Dropout('drop3_2', 0.4)) ConvBnReLU(net, 'conv3_3', 256) net.add(layer.MaxPooling2D('pool3', 2, 2, border_mode='valid')) ConvBnReLU(net, 'conv4_1', 512) - net.add(layer.Dropout('drop4_1', 0.4, engine='cuda')) + net.add(layer.Dropout('drop4_1', 0.4)) ConvBnReLU(net, 'conv4_2', 512) - net.add(layer.Dropout('drop4_2', 0.4, engine='cuda')) + net.add(layer.Dropout('drop4_2', 0.4)) ConvBnReLU(net, 'conv4_3', 512) net.add(layer.MaxPooling2D('pool4', 2, 2, border_mode='valid')) ConvBnReLU(net, 'conv5_1', 512) - net.add(layer.Dropout('drop5_1', 0.4, engine='cuda')) + net.add(layer.Dropout('drop5_1', 0.4)) ConvBnReLU(net, 'conv5_2', 512) - net.add(layer.Dropout('drop5_2', 0.4, engine='cuda')) + net.add(layer.Dropout('drop5_2', 0.4)) ConvBnReLU(net, 'conv5_3', 512) net.add(layer.MaxPooling2D('pool5', 2, 2, border_mode='valid')) net.add(layer.Flatten('flat')) - net.add(layer.Dropout('drop_flat', 0.5, engine='cuda')) + net.add(layer.Dropout('drop_flat', 0.5)) net.add(layer.Dense('ip1', 512)) net.add(layer.BatchNormalization('batchnorm_ip1')) net.add(layer.Activation('relu_ip1')) - net.add(layer.Dropout('drop_ip2', 0.5, engine='cuda')) + net.add(layer.Dropout('drop_ip2', 0.5)) net.add(layer.Dense('ip2', 10)) print 'Start intialization............' for (p, name) in zip(net.param_values(), net.param_names()): diff --git a/examples/imagenet/alexnet.cc b/examples/imagenet/alexnet.cc index 270312c5d9..3fb5d04697 100644 --- a/examples/imagenet/alexnet.cc +++ b/examples/imagenet/alexnet.cc @@ -22,13 +22,6 @@ #include "singa/singa_config.h" #ifdef USE_OPENCV #include -#include "../../src/model/layer/cudnn_activation.h" -#include "../../src/model/layer/cudnn_convolution.h" -#include "../../src/model/layer/dropout.h" -#include "../../src/model/layer/cudnn_lrn.h" -#include "../../src/model/layer/cudnn_pooling.h" -#include "../../src/model/layer/dense.h" -#include "../../src/model/layer/flatten.h" #include "./ilsvrc12.h" #include "singa/io/snapshot.h" #include "singa/model/feed_forward_net.h" @@ -40,11 +33,12 @@ #include "singa/utils/timer.h" namespace singa { +const std::string engine = "cudnn"; LayerConf GenConvConf(string name, int nb_filter, int kernel, int stride, int pad, float std, float bias = .0f) { LayerConf conf; conf.set_name(name); - conf.set_type("CudnnConvolution"); + conf.set_type(engine + "_convolution"); ConvolutionConf *conv = conf.mutable_convolution_conf(); conv->set_num_output(nb_filter); conv->add_kernel_size(kernel); @@ -71,7 +65,7 @@ LayerConf GenPoolingConf(string name, bool max_pool, int kernel, int stride, int pad) { LayerConf conf; conf.set_name(name); - conf.set_type("CudnnPooling"); + conf.set_type(engine + "_pooling"); PoolingConf *pool = conf.mutable_pooling_conf(); pool->set_kernel_size(kernel); pool->set_stride(stride); @@ -83,7 +77,7 @@ LayerConf GenPoolingConf(string name, bool max_pool, int kernel, int stride, LayerConf GenReLUConf(string name) { LayerConf conf; conf.set_name(name); - conf.set_type("RELU"); + conf.set_type(engine + "_relu"); return conf; } @@ -91,7 +85,7 @@ LayerConf GenDenseConf(string name, int num_output, float std, float wd, float bias = .0f) { LayerConf conf; conf.set_name(name); - conf.set_type("Dense"); + conf.set_type("singa_dense"); DenseConf *dense = conf.mutable_dense_conf(); dense->set_num_output(num_output); @@ -115,7 +109,7 @@ LayerConf GenDenseConf(string name, int num_output, float std, float wd, LayerConf GenLRNConf(string name) { LayerConf conf; conf.set_name(name); - conf.set_type("CudnnLRN"); + conf.set_type(engine + "_lrn"); LRNConf *lrn = conf.mutable_lrn_conf(); lrn->set_local_size(5); lrn->set_alpha(1e-04); @@ -126,14 +120,14 @@ LayerConf GenLRNConf(string name) { LayerConf GenFlattenConf(string name) { LayerConf conf; conf.set_name(name); - conf.set_type("Flatten"); + conf.set_type("singa_flatten"); return conf; } LayerConf GenDropoutConf(string name, float dropout_ratio) { LayerConf conf; conf.set_name(name); - conf.set_type("Dropout"); + conf.set_type(engine + "_dropout"); DropoutConf *dropout = conf.mutable_dropout_conf(); dropout->set_dropout_ratio(dropout_ratio); return conf; diff --git a/include/singa/core/device.h b/include/singa/core/device.h index 778a13034e..4c461140bc 100644 --- a/include/singa/core/device.h +++ b/include/singa/core/device.h @@ -321,23 +321,33 @@ class Platform { /// Return a string containing all hardware info, e.g., version, memory size. static const std::string DeviceQuery(int id, bool verbose = false); + /// Return the defualt host device + static std::shared_ptr GetDefaultDevice() { + return defaultDevice; + } + /// Create a set of CudaGPU Device using 'num_devices' free GPUs. static const std::vector> CreateCudaGPUs(const size_t num_devices, size_t init_size = 0); /// Create a set of CudaGPU Device using given GPU IDs. static const std::vector> - CreateCudaGPUs(const std::vector &devices, size_t init_size = 0); - - /// Create a \p num_devices set of valid OpenCL devices, regardless of platforms. - /// If there are fewer valid devices than requested, then this method will return as many as possible. - /// If OpenCL is not in use, this method will return an empty array. - const std::vector> CreateOpenclDevices(const size_t num_devices); - - /// Create a set of valid OpenCL devices, regardless of platforms, assigning \p id to each device in sequence. - /// If there are fewer valid devices than requested, then this method will return as many as possible. + CreateCudaGPUsOn(const std::vector &devices, size_t init_size = 0); + + /// Create a \p num_devices set of valid OpenCL devices, regardless of + /// platforms. If there are fewer valid devices than requested, then this + /// method will return as many as possible.If OpenCL is not in use, this + /// method will return an empty array. + const std::vector > CreateOpenclDevices( + const size_t num_devices); + + /// Create a set of valid OpenCL devices, regardless of platforms, assigning + /// \p id to each device in sequence. + /// If there are fewer valid devices than requested, then this method will + /// return as many as possible. /// If OpenCL is not in use, this method will return an empty array. - const std::vector> CreateOpenclDevices(const vector& id); + const std::vector > + CreateOpenclDevices(const vector &id); /// This function is implementd by Caffe (http://caffe.berkeleyvision.org/). /// This function checks the availability of GPU #device_id. diff --git a/include/singa/model/feed_forward_net.h b/include/singa/model/feed_forward_net.h index 8adc259bd5..1bf112cc09 100644 --- a/include/singa/model/feed_forward_net.h +++ b/include/singa/model/feed_forward_net.h @@ -39,7 +39,7 @@ class FeedForwardNet { /// following the topological order. /// 2. this layer has already been setup (Setup function is called outside). /// The layer will be freed in the destructor of FeedForwardNet. - Layer* Add(Layer* layer); + std::shared_ptr Add(std::shared_ptr layer); // TODO(wangwei) add ConcatenateLayer and SliceLayer // AddConcatenateLayer(vector src, Layer *dst); @@ -49,11 +49,9 @@ class FeedForwardNet { /// Assume the layer is added in corret order. /// For the first layer, 'sample_shape' (the input sample shape) is necessary /// for calling Setup(). - Layer* Add(const LayerConf& conf, const Shape* sample_shape = nullptr); + std::shared_ptr Add(const LayerConf& conf, + const Shape* sample_shape = nullptr); - /// Add a layer, and call its Setup function. - Layer* Add(Layer* layer, const LayerConf& conf, - const Shape* sample_shape = nullptr); /// Set some fields used for training and evaluating the neural net. /// This method will instantiate an Updater ,then wrap the Optimier into /// Updater and always register the parameters of the net instance. @@ -147,13 +145,13 @@ class FeedForwardNet { return std::thread([=]() { Train(batchsize, nb_epoch, x, y); }); } - const vector layers() const { return layers_; } + const vector> layers() const { return layers_; } const vector GetParamNames() const; const vector GetParamSpecs() const; const vector GetParamValues() const; protected: - vector layers_; + vector> layers_; std::shared_ptr updater_; Loss* loss_; Metric* metric_; diff --git a/include/singa/model/layer.h b/include/singa/model/layer.h index d31bd955f4..58f0f4b675 100644 --- a/include/singa/model/layer.h +++ b/include/singa/model/layer.h @@ -222,8 +222,8 @@ class Layer { vector param_specs_; }; -#define RegisterLayerClass(SubLayer) \ - static Registra _##SubLayer##Layer(#SubLayer); +#define RegisterLayerClass(Name, SubLayer) \ + static Registra Name##SubLayer(#Name); inline std::shared_ptr CreateLayer(const std::string type) { std::shared_ptr layer(Factory::Create(type)); diff --git a/include/singa/utils/integer.h b/include/singa/utils/integer.h new file mode 100644 index 0000000000..9c2799d510 --- /dev/null +++ b/include/singa/utils/integer.h @@ -0,0 +1,73 @@ +/************************************************************ + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + *************************************************************/ + +#ifndef INTEGER_H_ +#define INTEGER_H_ + +#include + +namespace singa{ +static bool isNetworkOrder() { + int test = 1; + return (1 != *(uint8_t*)&test); +} + +template +static inline T byteSwap(const T& v) { + int size = sizeof(v); + T ret; + uint8_t *dest = reinterpret_cast(&ret); + uint8_t *src = const_cast(reinterpret_cast(&v)); + for (int i = 0; i < size; ++i) { + dest[i] = src[size - i - 1]; + } + return ret; +} + +template +static inline T hton(const T& v) +{ + return isNetworkOrder() ? v : byteSwap(v); +} + +template +static inline T ntoh(const T& v) +{ + return hton(v); +} + +static inline int appendInteger(char* buf) {return 0;} +static inline int readInteger(char* buf) {return 0;} + +template +static int appendInteger(char* buf, Type value, Types... values) { + *(Type*)buf = hton(value); + return sizeof(Type) + appendInteger(buf + sizeof(Type), values...); +} + +template +static int readInteger(char* buf, Type& value, Types&... values) { + value = ntoh(*(Type*)buf); + return sizeof(Type) + readInteger(buf + sizeof(Type), values...); +} + +} +#endif diff --git a/src/core/device/platform.cc b/src/core/device/platform.cc index a4561decd8..a3661f23b2 100644 --- a/src/core/device/platform.cc +++ b/src/core/device/platform.cc @@ -113,11 +113,11 @@ Platform::CreateCudaGPUs(const size_t num_devices, size_t init_size) { const vector gpus = GetGPUIDs(); CHECK_LE(num_devices, gpus.size()); vector use_gpus(gpus.begin(), gpus.begin() + num_devices); - return CreateCudaGPUs(use_gpus, init_size); + return CreateCudaGPUsOn(use_gpus, init_size); } const vector > -Platform::CreateCudaGPUs(const vector &devices, size_t init_size) { +Platform::CreateCudaGPUsOn(const vector &devices, size_t init_size) { MemPoolConf conf; if (init_size > 0) conf.set_init_size(init_size); diff --git a/src/core/tensor/tensor.cc b/src/core/tensor/tensor.cc index e5bd67374b..4141374710 100644 --- a/src/core/tensor/tensor.cc +++ b/src/core/tensor/tensor.cc @@ -454,7 +454,7 @@ float Tensor::L1() const { float nrm = 0.0f; TYPE_LANG_SWITCH(data_type_, DType, device_->lang(), Lang, { device_->Exec([&nrm, this](Context *ctx) { - DType ret; + DType ret = DType(0); Asum(this->Size(), this->block(), &ret, ctx); nrm = TypeCast(ret); }, {this->block()}, {}); @@ -467,7 +467,7 @@ float Tensor::L2() const { float nrm = 0.0f; TYPE_LANG_SWITCH(data_type_, DType, device_->lang(), Lang, { device_->Exec([&nrm, this](Context *ctx) { - DType ret; + DType ret = DType(0); Nrm2(this->Size(), this->block(), &ret, ctx); nrm = TypeCast(ret); }, {this->block()}, {}); diff --git a/src/core/tensor/tensor_math_cpp.h b/src/core/tensor/tensor_math_cpp.h index 941931d19f..a2802d51a6 100644 --- a/src/core/tensor/tensor_math_cpp.h +++ b/src/core/tensor/tensor_math_cpp.h @@ -239,7 +239,7 @@ void Sqrt(const size_t num, const Block *in, Block *out, float *outPtr = static_cast(out->mutable_data()); const float *inPtr = static_cast(in->data()); for (size_t i = 0; i < num; i++) { - CHECK_GT(inPtr[i], 0.f); + CHECK_GE(inPtr[i], 0.f); outPtr[i] = sqrt(inPtr[i]); } } diff --git a/src/model/feed_forward_net.cc b/src/model/feed_forward_net.cc index 9450c9eaf1..514d6e2b39 100644 --- a/src/model/feed_forward_net.cc +++ b/src/model/feed_forward_net.cc @@ -26,23 +26,16 @@ namespace singa { FeedForwardNet::~FeedForwardNet() { - for (auto layer : layers_) delete layer; -} -Layer* FeedForwardNet::Add(Layer* layer) { - layers_.push_back(layer); - return layer; } -Layer* FeedForwardNet::Add(const LayerConf& conf, const Shape* sample_shape) { - CHECK(sample_shape != nullptr || layers_.size()) - << "Must provide the input sample shape for the first layer"; - Layer* layer = nullptr; // TODO(wangwei) use CreateLayer(conf.type()); - Add(layer, conf, sample_shape); +std::shared_ptr FeedForwardNet::Add(std::shared_ptr layer) { + layers_.push_back(layer); return layer; } -Layer* FeedForwardNet::Add(Layer* layer, const LayerConf& conf, - const Shape* sample_shape) { +std::shared_ptr FeedForwardNet::Add(const LayerConf& conf, + const Shape* sample_shape) { + std::shared_ptr layer(CreateLayer(conf.type())); CHECK(conf.has_name()) << "Must set layer name"; if (sample_shape == nullptr) layer->Setup(layers_.back()->GetOutputSampleShape(), conf); diff --git a/src/model/layer/activation.cc b/src/model/layer/activation.cc index 2497c31fb3..aa40edb1a1 100644 --- a/src/model/layer/activation.cc +++ b/src/model/layer/activation.cc @@ -18,14 +18,23 @@ #include "singa/model/layer.h" #include "./activation.h" +#include "singa/utils/string.h" namespace singa { -RegisterLayerClass(Activation); +RegisterLayerClass(singa_relu, Activation); +RegisterLayerClass(singa_sigmoid, Activation); +RegisterLayerClass(singa_tanh, Activation); void Activation::Setup(const Shape& in_sample, const LayerConf& conf) { Layer::Setup(in_sample, conf); - mode_ = conf.type(); - if (mode_ == "RELU") { + auto pos = conf.type().find_first_of('_'); + CHECK_NE(pos, string::npos) << "There should be a '_' in the laye type " + << conf.type(); + mode_ = ToLowerCase(conf.type().substr(pos + 1)); + if (mode_ != "relu" && mode_ != "sigmoid" && mode_ != "tanh") + LOG(FATAL) << "Unkown activation type: " << conf.type() << " " << mode_ + << ". Please use singa_relu, singa_sigmoid, or singa_tanh"; + if (mode_ == "relu") { neg_slope_ = conf.relu_conf().negative_slope(); } out_sample_shape_ = in_sample; @@ -33,13 +42,13 @@ void Activation::Setup(const Shape& in_sample, const LayerConf& conf) { const Tensor Activation::Forward(int flag, const Tensor& input) { Tensor output; - if (mode_ == "SIGMOID") { + if (mode_ == "sigmoid") { output = Sigmoid(input); if (flag & kTrain) buf_.push(output); - } else if (mode_ == "TANH") { + } else if (mode_ == "tanh") { output = Tanh(input); if (flag & kTrain) buf_.push(output); - } else if (mode_ == "RELU") { + } else if (mode_ == "relu") { output = ReLU(input); if (flag & kTrain) buf_.push(input); } else @@ -55,11 +64,11 @@ const std::pair> Activation::Backward( // activation. Tensor input_grad, inout = buf_.top(); buf_.pop(); - if (mode_ == "SIGMOID") + if (mode_ == "sigmoid") input_grad = grad * inout * (inout * (-1.f) + 1.f); - else if (mode_ == "TANH") + else if (mode_ == "tanh") input_grad = grad * (inout * inout * (-1.f) + 1.f); - else if (mode_ == "RELU") + else if (mode_ == "relu") input_grad = grad * (inout > 0.f) + (inout <= 0.f) * neg_slope_; else LOG(FATAL) << "Unkown activation: " << mode_; return std::make_pair(input_grad, param_grad); diff --git a/src/model/layer/activation.h b/src/model/layer/activation.h index e3fb657773..7d15979d47 100644 --- a/src/model/layer/activation.h +++ b/src/model/layer/activation.h @@ -26,7 +26,7 @@ namespace singa { class Activation : public Layer { public: /// \copydoc Layer::layer_type() - const std::string layer_type() const override { return "Activation"; } + // const std::string layer_type() const override { return "Activation"; } /// \copydoc Layer::Setup(const LayerConf&); void Setup(const Shape& in_sample, const LayerConf& conf) override; diff --git a/src/model/layer/batchnorm.cc b/src/model/layer/batchnorm.cc index 6ea9f2af12..f34866187f 100644 --- a/src/model/layer/batchnorm.cc +++ b/src/model/layer/batchnorm.cc @@ -21,7 +21,7 @@ #include "batchnorm.h" namespace singa { -RegisterLayerClass(BatchNorm); +RegisterLayerClass(singa_batchnorm, BatchNorm); void BatchNorm::Setup(const Shape& in_sample, const LayerConf& conf) { Layer::Setup(in_sample, conf); out_sample_shape_ = in_sample; @@ -78,8 +78,8 @@ const Tensor BatchNorm::Forward(int flag, const Tensor& input) { runningVariance_ *= 1.0f - factor_; Axpy(factor_, var, &runningVariance_); Tensor tmp = var.Clone(); - tmp += 1e-6f; tmp = Sqrt(tmp); + tmp += 1e-6f; xnorm = x.Clone(); SubRow(mean, &xnorm); DivRow(tmp, &xnorm); @@ -94,8 +94,8 @@ const Tensor BatchNorm::Forward(int flag, const Tensor& input) { xnorm = x.Clone(); SubRow(runningMean_, &xnorm); Tensor tmp = runningVariance_.Clone(); - tmp += 1e-6f; tmp = Sqrt(tmp); + tmp += 1e-6f; DivRow(tmp, &xnorm); output = xnorm.Clone(); MultRow(bnScale_, &output); diff --git a/src/model/layer/batchnorm.h b/src/model/layer/batchnorm.h index f3d83abd12..c2cfde95f7 100644 --- a/src/model/layer/batchnorm.h +++ b/src/model/layer/batchnorm.h @@ -29,7 +29,7 @@ namespace singa { class BatchNorm : public Layer { public: /// \copydoc Layer::layer_type() - const std::string layer_type() const override { return "BatchNorm"; } + // const std::string layer_type() const override { return "BatchNorm"; } /// \copydoc Layer::Setup(const LayerConf&); void Setup(const Shape& in_sample, const LayerConf& conf) override; diff --git a/src/model/layer/convolution.cc b/src/model/layer/convolution.cc index 1bf6b39d20..4fc209fcef 100644 --- a/src/model/layer/convolution.cc +++ b/src/model/layer/convolution.cc @@ -23,7 +23,7 @@ namespace singa { using std::vector; -RegisterLayerClass(Convolution); +RegisterLayerClass(singa_convolution, Convolution); void Convolution::Setup(const Shape &in_sample, const LayerConf &conf) { Layer::Setup(in_sample, conf); ConvolutionConf conv_conf = conf.convolution_conf(); diff --git a/src/model/layer/convolution.h b/src/model/layer/convolution.h index 1383a66de7..d85a17ba0a 100644 --- a/src/model/layer/convolution.h +++ b/src/model/layer/convolution.h @@ -27,7 +27,7 @@ namespace singa { class Convolution : public Layer { public: /// \copydoc Layer::layer_type() - const std::string layer_type() const override { return "Convolution"; } + // const std::string layer_type() const override { return "Convolution"; } /// \copydoc Layer::Setup(const LayerConf&); void Setup(const vector& in_shape, const LayerConf& conf) override; diff --git a/src/model/layer/cudnn_activation.cc b/src/model/layer/cudnn_activation.cc index c86539dd3d..4ecb3756b3 100644 --- a/src/model/layer/cudnn_activation.cc +++ b/src/model/layer/cudnn_activation.cc @@ -25,7 +25,9 @@ #include "singa/utils/logging.h" namespace singa { -RegisterLayerClass(CudnnActivation); +RegisterLayerClass(cudnn_relu, CudnnActivation); +RegisterLayerClass(cudnn_sigmoid, CudnnActivation); +RegisterLayerClass(cudnn_tanh, CudnnActivation); CudnnActivation::~CudnnActivation() { if (acti_desc_ != nullptr) CUDNN_CHECK(cudnnDestroyActivationDescriptor(acti_desc_)); @@ -40,11 +42,11 @@ void CudnnActivation::InitCudnn(size_t size, DataType dtype) { CUDNN_CHECK(cudnnSetTensor4dDescriptor( desc_, CUDNN_TENSOR_NCHW, GetCudnnDataType(dtype), 1, 1, 1, size)); - if (mode_ == "SIGMOID") + if (mode_ == "sigmoid") cudnn_mode_ = CUDNN_ACTIVATION_SIGMOID; - else if (mode_ == "TANH") + else if (mode_ == "tanh") cudnn_mode_ = CUDNN_ACTIVATION_TANH; - else if (mode_ == "RELU") + else if (mode_ == "relu") cudnn_mode_ = CUDNN_ACTIVATION_RELU; else LOG(FATAL) << "Unkown activation: " << mode_; diff --git a/src/model/layer/cudnn_activation.h b/src/model/layer/cudnn_activation.h index 526e03fa50..c69d1575f6 100644 --- a/src/model/layer/cudnn_activation.h +++ b/src/model/layer/cudnn_activation.h @@ -35,7 +35,7 @@ class CudnnActivation : public Activation { public: ~CudnnActivation(); /// \copydoc Layer::layer_type() - const std::string layer_type() const override { return "CudnnActivation"; } + // const std::string layer_type() const override { return "CudnnActivation"; } const Tensor Forward(int flag, const Tensor& input) override; const std::pair> Backward(int flag, diff --git a/src/model/layer/cudnn_batchnorm.cc b/src/model/layer/cudnn_batchnorm.cc index 461f1b6da0..01682b7db8 100644 --- a/src/model/layer/cudnn_batchnorm.cc +++ b/src/model/layer/cudnn_batchnorm.cc @@ -23,7 +23,7 @@ namespace singa { -RegisterLayerClass(CudnnBatchNorm); +RegisterLayerClass(cudnn_batchnorm, CudnnBatchNorm); CudnnBatchNorm::~CudnnBatchNorm() { if (has_init_cudnn_) { CUDNN_CHECK(cudnnDestroyTensorDescriptor(shape_desc_)); diff --git a/src/model/layer/cudnn_batchnorm.h b/src/model/layer/cudnn_batchnorm.h index 4f46452ccd..c4390a1529 100644 --- a/src/model/layer/cudnn_batchnorm.h +++ b/src/model/layer/cudnn_batchnorm.h @@ -31,7 +31,7 @@ class CudnnBatchNorm : public BatchNorm { public: ~CudnnBatchNorm(); /// \copy doc Layer::layer_type() - const std::string layer_type() const override { return "CudnnBatchNorm"; } + // const std::string layer_type() const override { return "CudnnBatchNorm"; } void Setup(const Shape& in_sample, const LayerConf& conf) override; diff --git a/src/model/layer/cudnn_convolution.cc b/src/model/layer/cudnn_convolution.cc index e5efec01c9..ffd2ab7848 100644 --- a/src/model/layer/cudnn_convolution.cc +++ b/src/model/layer/cudnn_convolution.cc @@ -23,7 +23,7 @@ #include "singa/utils/logging.h" namespace singa { -RegisterLayerClass(CudnnConvolution); +RegisterLayerClass(cudnn_convolution, CudnnConvolution); CudnnConvolution::~CudnnConvolution() { if (bias_desc_ != nullptr) CUDNN_CHECK(cudnnDestroyTensorDescriptor(bias_desc_)); diff --git a/src/model/layer/cudnn_convolution.h b/src/model/layer/cudnn_convolution.h index cd0471f975..545fd5cc63 100644 --- a/src/model/layer/cudnn_convolution.h +++ b/src/model/layer/cudnn_convolution.h @@ -34,7 +34,7 @@ class CudnnConvolution : public Convolution { public: ~CudnnConvolution(); /// \copydoc Layer::layer_type() - const std::string layer_type() const override { return "CudnnConvolution"; } + // const std::string layer_type() const override { return "CudnnConvolution";} const Tensor Forward(int flag, const Tensor &input) override; const std::pair> Backward(int flag, diff --git a/src/model/layer/cudnn_dropout.cc b/src/model/layer/cudnn_dropout.cc index e6950caf33..c5b62cf68b 100644 --- a/src/model/layer/cudnn_dropout.cc +++ b/src/model/layer/cudnn_dropout.cc @@ -27,7 +27,7 @@ #include "singa/utils/logging.h" namespace singa { -RegisterLayerClass(CudnnDropout); +RegisterLayerClass(cudnn_dropout, CudnnDropout); CudnnDropout::~CudnnDropout() { if (drop_desc_ != nullptr) CUDNN_CHECK(cudnnDestroyDropoutDescriptor(drop_desc_)); diff --git a/src/model/layer/cudnn_dropout.h b/src/model/layer/cudnn_dropout.h index 9e0cb9e7cb..1241911366 100644 --- a/src/model/layer/cudnn_dropout.h +++ b/src/model/layer/cudnn_dropout.h @@ -36,7 +36,7 @@ class CudnnDropout : public Dropout { public: ~CudnnDropout(); /// \copydoc Layer::layer_type() - const std::string layer_type() const override { return "CudnnDropout"; } + // const std::string layer_type() const override { return "CudnnDropout"; } const Tensor Forward(int flag, const Tensor& input) override; const std::pair> Backward(int flag, diff --git a/src/model/layer/cudnn_lrn.cc b/src/model/layer/cudnn_lrn.cc index 540beb1a0f..ac7645e92b 100644 --- a/src/model/layer/cudnn_lrn.cc +++ b/src/model/layer/cudnn_lrn.cc @@ -23,7 +23,7 @@ #include "cudnn_utils.h" namespace singa { -RegisterLayerClass(CudnnLRN); +RegisterLayerClass(cudnn_lrn, CudnnLRN); CudnnLRN::~CudnnLRN() { if (has_init_cudnn_) { CUDNN_CHECK(cudnnDestroyLRNDescriptor(lrn_desc_)); diff --git a/src/model/layer/cudnn_lrn.h b/src/model/layer/cudnn_lrn.h index e2a5e54e3f..c48571d47b 100644 --- a/src/model/layer/cudnn_lrn.h +++ b/src/model/layer/cudnn_lrn.h @@ -31,7 +31,7 @@ class CudnnLRN : public LRN { public: ~CudnnLRN(); /// \copy doc Layer::layer_type() - const std::string layer_type() const override { return "CudnnLRN"; } + // const std::string layer_type() const override { return "CudnnLRN"; } const Tensor Forward(int flag, const Tensor& input) override; const std::pair> Backward(int flag, diff --git a/src/model/layer/cudnn_pooling.cc b/src/model/layer/cudnn_pooling.cc index 984427c2eb..895ce3c5b7 100644 --- a/src/model/layer/cudnn_pooling.cc +++ b/src/model/layer/cudnn_pooling.cc @@ -25,7 +25,7 @@ #include "singa/utils/logging.h" namespace singa { -RegisterLayerClass(CudnnPooling); +RegisterLayerClass(cudnn_pooling, CudnnPooling); CudnnPooling::~CudnnPooling() { if (pool_desc_ != nullptr) CUDNN_CHECK(cudnnDestroyPoolingDescriptor(pool_desc_)); diff --git a/src/model/layer/cudnn_pooling.h b/src/model/layer/cudnn_pooling.h index 90779f569a..2080db3097 100644 --- a/src/model/layer/cudnn_pooling.h +++ b/src/model/layer/cudnn_pooling.h @@ -35,7 +35,7 @@ class CudnnPooling : public Pooling { public: ~CudnnPooling(); /// \copydoc Layer::layer_type() - const std::string layer_type() const override { return "CudnnPooling"; } + // const std::string layer_type() const override { return "CudnnPooling"; } void Setup(const Shape& in_sample, const LayerConf &conf) override; const Tensor Forward(int flag, const Tensor &input) override; diff --git a/src/model/layer/cudnn_rnn.cc b/src/model/layer/cudnn_rnn.cc index bfbfa48617..9961df2404 100644 --- a/src/model/layer/cudnn_rnn.cc +++ b/src/model/layer/cudnn_rnn.cc @@ -24,6 +24,7 @@ #include "singa/utils/logging.h" namespace singa { +RegisterLayerClass(cudnn_rnn, CudnnRNN); CudnnRNN::~CudnnRNN() { if (weight_desc_ != nullptr) CUDNN_CHECK(cudnnDestroyFilterDescriptor(weight_desc_)); @@ -126,25 +127,19 @@ void CudnnRNN::SetRNNDescriptor(shared_ptr dev) { dropout_state_.block()->mutable_data(), state_size, seed_)); CUDNN_CHECK(cudnnCreateRNNDescriptor(&rnn_desc_)); - cudnnRNNInputMode_t input_mode; - if (input_mode_ == "linear") - input_mode = CUDNN_LINEAR_INPUT; - else if (input_mode_ == "skip") + cudnnRNNInputMode_t input_mode = CUDNN_LINEAR_INPUT; + if (input_mode_ == "skip") input_mode = CUDNN_SKIP_INPUT; - cudnnDirectionMode_t direction; - if (direction_ == "unidirectional") - direction = CUDNN_UNIDIRECTIONAL; - else if (direction_ == "bidirectional") + cudnnDirectionMode_t direction = CUDNN_UNIDIRECTIONAL; + if (direction_ == "bidirectional") direction = CUDNN_BIDIRECTIONAL; - cudnnRNNMode_t rnn_mode; + cudnnRNNMode_t rnn_mode = CUDNN_LSTM; if (rnn_mode_ == "relu") rnn_mode = CUDNN_RNN_RELU; else if (rnn_mode_ == "tanh") rnn_mode = CUDNN_RNN_TANH; - else if (rnn_mode_ == "lstm") - rnn_mode = CUDNN_LSTM; else if (rnn_mode_ == "gru") rnn_mode = CUDNN_GRU; CUDNN_CHECK(cudnnSetRNNDescriptor(rnn_desc_, hidden_size_, num_stacks_, diff --git a/src/model/layer/cudnn_rnn.h b/src/model/layer/cudnn_rnn.h index cfb8aac236..82c68b0a54 100644 --- a/src/model/layer/cudnn_rnn.h +++ b/src/model/layer/cudnn_rnn.h @@ -39,7 +39,7 @@ class CudnnRNN : public RNN { public: ~CudnnRNN(); /// \copydoc Layer::layer_type() - const std::string layer_type() const override { return "CudnnRNN"; } + // const std::string layer_type() const override { return "CudnnRNN"; } const vector Forward(int flag, const vector& inputs) override; const std::pair, vector> Backward( diff --git a/src/model/layer/cudnn_softmax.cc b/src/model/layer/cudnn_softmax.cc index 6dce68f233..f1a4a5bb43 100644 --- a/src/model/layer/cudnn_softmax.cc +++ b/src/model/layer/cudnn_softmax.cc @@ -23,7 +23,7 @@ #include "singa/utils/logging.h" namespace singa { -RegisterLayerClass(CudnnSoftmax); +RegisterLayerClass(cudnn_softmax, CudnnSoftmax); CudnnSoftmax::~CudnnSoftmax() { if (desc_ != nullptr) CUDNN_CHECK(cudnnDestroyTensorDescriptor(desc_)); } diff --git a/src/model/layer/cudnn_softmax.h b/src/model/layer/cudnn_softmax.h index aca3729245..532a643c0a 100644 --- a/src/model/layer/cudnn_softmax.h +++ b/src/model/layer/cudnn_softmax.h @@ -34,7 +34,7 @@ class CudnnSoftmax : public Softmax { public: ~CudnnSoftmax(); /// \copydoc Layer::layer_type() - const std::string layer_type() const override { return "CudnnSoftmax"; } + // const std::string layer_type() const override { return "CudnnSoftmax"; } /// \copydoc Layer::Setup(const LayerConf&); void Setup(const Shape& in_sample_shape, const LayerConf &conf) override; diff --git a/src/model/layer/cudnn_utils.h b/src/model/layer/cudnn_utils.h index 19c72ec100..64ee758cdb 100644 --- a/src/model/layer/cudnn_utils.h +++ b/src/model/layer/cudnn_utils.h @@ -26,7 +26,7 @@ #include "singa/utils/logging.h" namespace singa { inline cudnnDataType_t GetCudnnDataType(DataType dtype) { - cudnnDataType_t ret; + cudnnDataType_t ret = CUDNN_DATA_FLOAT; switch (dtype) { case kFloat32: ret = CUDNN_DATA_FLOAT; diff --git a/src/model/layer/dense.cc b/src/model/layer/dense.cc index 557d8bd004..1a2d16e47d 100644 --- a/src/model/layer/dense.cc +++ b/src/model/layer/dense.cc @@ -23,7 +23,7 @@ namespace singa { using std::vector; -RegisterLayerClass(Dense); +RegisterLayerClass(singa_dense, Dense); Dense::~Dense() { // delete weight_; // delete bias_; diff --git a/src/model/layer/dense.h b/src/model/layer/dense.h index bb5db66ea1..8a149a52d4 100644 --- a/src/model/layer/dense.h +++ b/src/model/layer/dense.h @@ -28,7 +28,7 @@ class Dense : public Layer { public: ~Dense(); /// \copydoc Layer::layer_type() - const std::string layer_type() const override { return "Dense"; } + // const std::string layer_type() const override { return "Dense"; } /// \copydoc Layer::Setup(const LayerConf&); void Setup(const Shape& in_sample, const LayerConf& conf) override; diff --git a/src/model/layer/dropout.cc b/src/model/layer/dropout.cc index 0a4b1dfc27..35801b443f 100644 --- a/src/model/layer/dropout.cc +++ b/src/model/layer/dropout.cc @@ -20,7 +20,7 @@ #include "./dropout.h" namespace singa { -RegisterLayerClass(Dropout); +RegisterLayerClass(singa_dropout, Dropout); void Dropout::Setup(const Shape& in_sample, const LayerConf& conf) { Layer::Setup(in_sample, conf); dropout_ratio_ = conf.dropout_conf().dropout_ratio(); diff --git a/src/model/layer/dropout.h b/src/model/layer/dropout.h index 1a4bdbf231..711c86b4de 100644 --- a/src/model/layer/dropout.h +++ b/src/model/layer/dropout.h @@ -26,7 +26,7 @@ namespace singa { class Dropout : public Layer { public: /// \copydoc Layer::layer_type() - const std::string layer_type() const override { return "Dropout"; } + // const std::string layer_type() const override { return "Dropout"; } /// \copydoc Layer::Setup(const LayerConf&); void Setup(const Shape& in_sample, const LayerConf& conf) override; diff --git a/src/model/layer/flatten.cc b/src/model/layer/flatten.cc index e7d8fa0a2c..d89361e0e4 100644 --- a/src/model/layer/flatten.cc +++ b/src/model/layer/flatten.cc @@ -20,7 +20,7 @@ #include "./flatten.h" namespace singa { -RegisterLayerClass(Flatten); +RegisterLayerClass(singa_flatten, Flatten); void Flatten::Setup(const Shape& in_sample, const LayerConf &conf) { Layer::Setup(in_sample, conf); axis_ = conf.flatten_conf().axis(); diff --git a/src/model/layer/flatten.h b/src/model/layer/flatten.h index 6ac90c2cb6..8bbf481e09 100644 --- a/src/model/layer/flatten.h +++ b/src/model/layer/flatten.h @@ -26,7 +26,7 @@ namespace singa { class Flatten : public Layer { public: /// \copydoc Layer::layer_type(); - const std::string layer_type() const override { return "Flatten"; } + // const std::string layer_type() const override { return "Flatten"; } /// \copydoc Layer::Setup(const LayerConf&); void Setup(const Shape& in_sample, const LayerConf& conf) override; diff --git a/src/model/layer/lrn.cc b/src/model/layer/lrn.cc index a6241471d4..6b5a618d3a 100644 --- a/src/model/layer/lrn.cc +++ b/src/model/layer/lrn.cc @@ -22,7 +22,7 @@ #include namespace singa { -RegisterLayerClass(LRN); +RegisterLayerClass(singa_lrn, LRN); void LRN::Setup(const Shape& in_sample, const LayerConf& conf) { Layer::Setup(in_sample, conf); out_sample_shape_ = in_sample; diff --git a/src/model/layer/lrn.h b/src/model/layer/lrn.h index 0632f8c5b0..57e26ba975 100644 --- a/src/model/layer/lrn.h +++ b/src/model/layer/lrn.h @@ -27,9 +27,7 @@ namespace singa { class LRN : public Layer { public: /// \copydoc Layer::layer_type() - const std::string layer_type() const override { - return "LRN"; - } + // const std::string layer_type() const override { return "LRN"; } /// \copydoc Layer::Setup(const LayerConf&); void Setup(const Shape& in_sample, const LayerConf& conf) override; diff --git a/src/model/layer/pooling.cc b/src/model/layer/pooling.cc index 943f9b23f0..5e7ba1d223 100644 --- a/src/model/layer/pooling.cc +++ b/src/model/layer/pooling.cc @@ -20,7 +20,7 @@ #include "singa/model/layer.h" namespace singa { -RegisterLayerClass(Pooling); +RegisterLayerClass(singa_pooling, Pooling); void Pooling::Setup(const Shape& in_sample, const LayerConf& conf) { Layer::Setup(in_sample, conf); PoolingConf pool_conf = conf.pooling_conf(); diff --git a/src/model/layer/pooling.h b/src/model/layer/pooling.h index 6df292a6ab..f84479993a 100644 --- a/src/model/layer/pooling.h +++ b/src/model/layer/pooling.h @@ -28,7 +28,7 @@ namespace singa { class Pooling : public Layer { public: /// \copydoc Layer::layer_type() - const std::string layer_type() const override { return "Pooling"; } + // const std::string layer_type() const override { return "Pooling"; } /// \copydoc Layer::Setup(const LayerConf&); void Setup(const Shape& in_sample, const LayerConf& conf) override; diff --git a/src/model/layer/prelu.cc b/src/model/layer/prelu.cc index 421bcaa0af..a20972c56e 100644 --- a/src/model/layer/prelu.cc +++ b/src/model/layer/prelu.cc @@ -20,7 +20,7 @@ #include "./prelu.h" namespace singa { -RegisterLayerClass(PReLU); +RegisterLayerClass(singa_prelu, PReLU); void PReLU::Setup(const Shape& in_sample, const LayerConf &conf) { Layer::Setup(in_sample, conf); out_sample_shape_ = in_sample; @@ -82,7 +82,7 @@ const std::pair > PReLU::Backward(int flag, Tensor da; da.ResetLike(a_); if (!channel_shared_) { - size_t n, c, h, w; + size_t n = 0, c = 0, h = 0, w = 0; Tensor temp1 = (input <= 0.f); if (temp1.nDim() == 4) { if (format_ == "NCHW") { diff --git a/src/model/layer/prelu.h b/src/model/layer/prelu.h index 70a9dcf95e..3041d1e221 100644 --- a/src/model/layer/prelu.h +++ b/src/model/layer/prelu.h @@ -27,7 +27,7 @@ namespace singa { class PReLU : public Layer { public: /// \copydoc Layer::layer_type() - const std::string layer_type() const override { return "PReLU"; } + // const std::string layer_type() const override { return "PReLU"; } /// \copydoc Layer::Setup(const LayerConf&); diff --git a/src/model/layer/rnn.cc b/src/model/layer/rnn.cc index 424c20bf7c..524b462b9c 100644 --- a/src/model/layer/rnn.cc +++ b/src/model/layer/rnn.cc @@ -22,7 +22,7 @@ #include "singa/utils/string.h" namespace singa { - +RegisterLayerClass(singa_rnn, RNN); void RNN::Setup(const Shape& in_sample, const LayerConf &conf) { Layer::Setup(in_sample, conf); diff --git a/src/model/layer/rnn.h b/src/model/layer/rnn.h index 1b5dad7988..3369a0052c 100644 --- a/src/model/layer/rnn.h +++ b/src/model/layer/rnn.h @@ -35,7 +35,7 @@ namespace singa { class RNN : public Layer { public: /// \copydoc Layer::layer_type() - const std::string layer_type() const override { return "RNN"; } + // const std::string layer_type() const override { return "RNN"; } /// Setup the RNN layer. /// in_shape is the shape of a single training instance from one timestep, diff --git a/src/model/layer/softmax.cc b/src/model/layer/softmax.cc index 6b1785cac1..6a49131404 100644 --- a/src/model/layer/softmax.cc +++ b/src/model/layer/softmax.cc @@ -19,7 +19,7 @@ #include "./softmax.h" namespace singa { -RegisterLayerClass(Softmax); +RegisterLayerClass(singa_softmax, Softmax); void Softmax::Setup(const Shape& in_sample, const LayerConf& conf) { Layer::Setup(in_sample, conf); CHECK_EQ(in_sample.size(), 1u); diff --git a/src/model/layer/softmax.h b/src/model/layer/softmax.h index 837b23aa26..cf71587c00 100644 --- a/src/model/layer/softmax.h +++ b/src/model/layer/softmax.h @@ -24,7 +24,7 @@ namespace singa { class Softmax : public Layer { public: /// \copydoc Layer::layer_type() - const std::string layer_type() const override { return "Softmax"; } + // const std::string layer_type() const override { return "Softmax"; } /// \copydoc Layer::Setup(const LayerConf&); void Setup(const Shape& in_sample, const LayerConf& conf) override; diff --git a/src/python/singa/device.py b/src/python/singa/device.py index 3db90bf8a9..aff3587818 100644 --- a/src/python/singa/device.py +++ b/src/python/singa/device.py @@ -73,3 +73,16 @@ def create_cuda_gpus(num): def create_cuda_gpu(): return singa.Platform.CreateCudaGPUs(1)[0] + + +def create_cuda_gpus_on(device_ids): + return singa.Platform.CreateCudaGPUsOn(device_ids) + + +def create_cuda_gpu_on(device_id): + devices = create_cuda_gpus_on([device_id]) + return devices[0] + + +def get_default_device(): + return singa.Platform.GetDefaultDevice() diff --git a/src/python/singa/layer.py b/src/python/singa/layer.py index a87eb1036f..c8c8c054c2 100644 --- a/src/python/singa/layer.py +++ b/src/python/singa/layer.py @@ -22,6 +22,12 @@ from .proto import model_pb2 import tensor +# engine could be 'cudnn', 'singa', which is used to create layers. +# e.g., CudnnConvolution layer is identified by 'cudnn_convolution' +# Convolution layer is identified by 'singa_convolution' +# engine is case insensitive +engine = 'cudnn' + class Layer(object): """Base Python layer class. @@ -78,12 +84,31 @@ def param_values(self): return tensor.from_raw_tensors(self.layer.param_values()) def forward(self, flag, input): + '''Forward propagate through this layer. + + Args: + flag, kTrain or kEval + input, an input tensor + + Return: + a tensor for the transformed feature + ''' assert self.has_setup, 'Must call setup() before forward()' assert isinstance(input, tensor.Tensor), 'input must be py Tensor' y = self.layer.Forward(flag, input.singa_tensor) return tensor.from_raw_tensor(y) def backward(self, flag, grad): + '''Backward propagate through this layer. + + Args: + flag, for future use. + grad, gradient of the returned values of the forward function. + + Return: + >, dx is the gradient of the input of the + forward function, dpi is the gradient of the i-th parameter + ''' assert isinstance(grad, tensor.Tensor), 'grad must be py Tensor' ret = self.layer.Backward(flag, grad.singa_tensor) return tensor.from_raw_tensor(ret[0]), tensor.from_raw_tensors(ret[1]) @@ -104,7 +129,7 @@ def __deepcopy__(self): class Conv2D(Layer): def __init__(self, name, nb_kernels, kernel=3, stride=1, border_mode='same', - engine='cudnn', cudnn_prefer='fatest', data_format='NCHW', + cudnn_prefer='fatest', data_format='NCHW', use_bias=True, W_specs=None, b_specs=None, pad=None, input_sample_shape=None): """Construct a layer for 2D convolution. @@ -117,8 +142,6 @@ def __init__(self, name, nb_kernels, kernel=3, stride=1, border_mode='same', 'valid' -> padding is 0 for height and width 'same' -> padding is half of the kernel (floor), the kernel must be odd number. - engine (string): implementation engin, could be 'cudnn' - (case insensitive) cudnn_prefer (string): the preferred algorithm for cudnn convolution which could be 'fatest', 'autotune', 'limited_workspace' and 'no_workspace' @@ -165,7 +188,7 @@ def __init__(self, name, nb_kernels, kernel=3, stride=1, border_mode='same', self.conf.param.extend([bspecs]) self.param_specs.append(bspecs) - _check_engine(engine, ['cudnn']) + _check_engine(engine, ['cudnn', 'singa']) self.layer = _create_layer(engine, 'Convolution') if input_sample_shape is not None: self.setup(input_sample_shape) @@ -174,7 +197,7 @@ def __init__(self, name, nb_kernels, kernel=3, stride=1, border_mode='same', class Conv1D(Conv2D): def __init__(self, name, nb_kernels, kernel=3, stride=1, - border_mode='same', engine='cudnn', cudnn_prefer='fatest', + border_mode='same', cudnn_prefer='fatest', use_bias=True, W_specs={'init': 'Xavier'}, b_specs={'init': 'Constant', 'value': 0}, pad=None, input_sample_shape=None): @@ -191,7 +214,7 @@ def __init__(self, name, nb_kernels, kernel=3, stride=1, if input_sample_shape is not None: input_sample_shape = (1, 1, input_sample_shape[0]) super(Conv1D, self).__init__(name, nb_kernels, (1, kernel), (0, stride), - border_mode, engine, cudnn_prefer, + border_mode, cudnn_prefer, use_bias=use_bias, pad=pad, W_specs=W_specs, b_specs=b_specs, input_sample_shape=input_sample_shape) @@ -206,15 +229,14 @@ def get_output_sample_shape(self): class Pooling2D(Layer): def __init__(self, name, mode, kernel=3, stride=2, border_mode='same', - pad=None, data_format='NCHW', engine='cudnn', - input_sample_shape=None): + pad=None, data_format='NCHW', input_sample_shape=None): super(Pooling2D, self).__init__(name) assert data_format == 'NCHW', 'Not supported data format: %s ' \ 'only "NCHW" is enabled currently' % (data_format) conf = self.conf.pooling_conf conf = _set_kernel_stride_pad(conf, kernel, stride, border_mode, pad) conf.pool = mode - _check_engine(engine, ['cudnn']) + _check_engine(engine, ['cudnn', 'singa']) self.layer = _create_layer(engine, 'Pooling') if input_sample_shape is not None: self.setup(input_sample_shape) @@ -223,27 +245,25 @@ def __init__(self, name, mode, kernel=3, stride=2, border_mode='same', class MaxPooling2D(Pooling2D): def __init__(self, name, kernel=3, stride=2, border_mode='same', pad=None, - data_format='NCHW', engine='cudnn', input_sample_shape=None): + data_format='NCHW', input_sample_shape=None): super(MaxPooling2D, self).__init__(name, model_pb2.PoolingConf.MAX, kernel, stride, border_mode, - pad, data_format, engine, - input_sample_shape) + pad, data_format, input_sample_shape) class AvgPooling2D(Pooling2D): def __init__(self, name, kernel=3, stride=2, border_mode='same', pad=None, - data_format='NCHW', engine='cudnn', input_sample_shape=None): + data_format='NCHW', input_sample_shape=None): super(AvgPooling2D, self).__init__(name, model_pb2.PoolingConf.AVE, kernel, stride, border_mode, - pad, data_format, engine, - input_sample_shape) + pad, data_format, input_sample_shape) class MaxPooling1D(MaxPooling2D): def __init__(self, name, kernel=3, stride=2, border_mode='same', pad=None, - data_format='NCHW', engine='cudnn', input_sample_shape=None): + data_format='NCHW', input_sample_shape=None): """Max pooling for 1D feature. Args: @@ -260,8 +280,7 @@ def __init__(self, name, kernel=3, stride=2, border_mode='same', pad=None, input_sample_shape = None super(MaxPooling1D, self).__init__(name, (1, kernel), (0, stride), border_mode, pad, - data_format, engine, - input_sample_shape) + data_format, input_sample_shape) def get_output_sample_shape(self): shape = self.layer.GetOutputSampleShape() @@ -271,7 +290,7 @@ def get_output_sample_shape(self): class AvgPooling1D(AvgPooling2D): def __init__(self, name, kernel=3, stride=2, border_mode='same', pad=None, - data_format='NCHW', engine='cudnn', input_sample_shape=None): + data_format='NCHW', input_sample_shape=None): """input_feature_length is a scalar value""" pad2 = None if pad is not None: @@ -285,8 +304,7 @@ def __init__(self, name, kernel=3, stride=2, border_mode='same', pad=None, super(AvgPooling1D, self).__init__(name, (kernel, 1), (0, stride), border_mode, pad2, - data_format, engine, - input_sample_shape) + data_format, input_sample_shape) def get_output_sample_shape(self): shape = self.layer.GetOutputSampleShape() @@ -296,7 +314,7 @@ def get_output_sample_shape(self): class BatchNormalization(Layer): # TODO(wangwei) add mode and epsilon arguments - def __init__(self, name, momentum=0.9, engine='cudnn', + def __init__(self, name, momentum=0.9, beta_specs=None, gamma_specs=None, input_sample_shape=None): """Batch-normalization. @@ -337,16 +355,15 @@ def __init__(self, name, momentum=0.9, engine='cudnn', self.param_specs.append(_construct_param_specs_from_dict(beta_specs)) self.param_specs.append(_construct_param_specs_from_dict(mean_specs)) self.param_specs.append(_construct_param_specs_from_dict(var_specs)) - _check_engine(engine, ['cudnn']) + _check_engine(engine, ['cudnn', 'singa']) self.layer = _create_layer(engine, 'BatchNorm') if input_sample_shape is not None: self.setup(input_sample_shape) class LRN(Layer): - def __init__(self, name, size=5, alpha=1, beta=0.75, mode='cross_channel', - k=1, engine='cudnn', input_sample_shape=None): + k=1, input_sample_shape=None): """Local response normalization. Args: @@ -364,7 +381,7 @@ def __init__(self, name, size=5, alpha=1, beta=0.75, mode='cross_channel', # TODO(wangwei) enable mode = 'within_channel' assert mode == 'cross_channel', 'only support mode="across_channel"' conf.norm_region = model_pb2.LRNConf.ACROSS_CHANNELS - _check_engine(engine, ['cudnn']) + _check_engine(engine, ['cudnn', 'singa']) self.layer = _create_layer(engine, 'LRN') if input_sample_shape is not None: self.setup(input_sample_shape) @@ -374,7 +391,7 @@ class Dense(Layer): def __init__(self, name, num_output, use_bias=True, W_specs=None, b_specs=None, - W_transpose=True, engine='cuda', input_sample_shape=None): + W_transpose=True, input_sample_shape=None): """Apply linear/affine transformation, also called inner-product or fully connected layer. @@ -392,7 +409,6 @@ def __init__(self, name, num_output, use_bias=True, 'regularizer' for regularization, currently support 'l2' b_specs (dict): specs for the bias vector, same fields as W_specs. W_transpose (bool): if true, output=x*W.T+b; - engine (string): could be 'cudnn', 'cuda' input_sample_shape (tuple): input feature length """ super(Dense, self).__init__(name) @@ -412,22 +428,19 @@ def __init__(self, name, num_output, use_bias=True, self.param_specs.append(_construct_param_specs_from_dict(W_specs)) self.conf.param.extend([_construct_param_specs_from_dict(b_specs)]) self.param_specs.append(_construct_param_specs_from_dict(b_specs)) - if engine == 'cudnn': - engine = 'cuda' - _check_engine(engine, ['cuda', 'cpp']) - self.layer = _create_layer(engine, 'Dense') + # dense layer is transparent to engine. + self.layer = _create_layer('singa', 'Dense') if input_sample_shape is not None: self.setup(input_sample_shape) class Dropout(Layer): - def __init__(self, name, p=0.5, engine='cuda', input_sample_shape=None): + def __init__(self, name, p=0.5, input_sample_shape=None): """Droput layer. Args: p (float): probability for dropping out the element, i.e., set to 0 - engine (string): 'cudnn' for cudnn version>=5; or 'cuda' name (string): layer name """ super(Dropout, self).__init__(name) @@ -436,7 +449,7 @@ def __init__(self, name, p=0.5, engine='cuda', input_sample_shape=None): # 'cudnn' works for v>=5.0 # if engine.lower() == 'cudnn': # engine = 'cuda' - _check_engine(engine, ['cudnn', 'cuda', 'cpp']) + _check_engine(engine, ['cudnn', 'singa']) self.layer = _create_layer(engine, 'Dropout') if input_sample_shape is not None: self.setup(input_sample_shape) @@ -444,28 +457,25 @@ def __init__(self, name, p=0.5, engine='cuda', input_sample_shape=None): class Activation(Layer): - def __init__(self, name, mode='relu', engine='cudnn', - input_sample_shape=None): + def __init__(self, name, mode='relu', input_sample_shape=None): """Activation layers. Args: - engine (string): 'cudnn' name (string): layer name mode (string): 'relu', 'sigmoid', or 'tanh' input_sample_shape (tuple): shape of a single sample """ super(Activation, self).__init__(name) - _check_engine(engine, ['cudnn', 'cuda', 'cpp']) - mode_dict = {'relu': 'RELU', 'sigmoid': 'SIGMOID', 'tanh': 'TANH'} - self.conf.type = mode_dict[mode.lower()] - self.layer = _create_layer(engine, 'Activation') + self.conf.type = (engine + '_' + mode).lower() + _check_engine(engine, ['cudnn', 'singa']) + self.layer = _create_layer(engine, mode) if input_sample_shape is not None: self.setup(input_sample_shape) class Softmax(Layer): - def __init__(self, name, axis=1, engine='cudnn', input_sample_shape=None): + def __init__(self, name, axis=1, input_sample_shape=None): """Apply softmax. Args: @@ -476,7 +486,7 @@ def __init__(self, name, axis=1, engine='cudnn', input_sample_shape=None): super(Softmax, self).__init__(name) # conf = self.conf.softmax_conf # conf.axis = axis - _check_engine(engine, ['cudnn', 'cuda', 'cpp']) + _check_engine(engine, ['cudnn', 'singa']) self.layer = _create_layer(engine, 'Softmax') if input_sample_shape is not None: self.setup(input_sample_shape) @@ -484,7 +494,7 @@ def __init__(self, name, axis=1, engine='cudnn', input_sample_shape=None): class Flatten(Layer): - def __init__(self, name, axis=1, engine='cudnn', input_sample_shape=None): + def __init__(self, name, axis=1, input_sample_shape=None): """Reshape the input tensor into a matrix. Args: axis (int): reshape the input as a matrix with the dimension @@ -494,24 +504,39 @@ def __init__(self, name, axis=1, engine='cudnn', input_sample_shape=None): super(Flatten, self).__init__(name) conf = self.conf.flatten_conf conf.axis = axis - _check_engine(engine, ['cudnn', 'cuda', 'cpp']) - if engine == 'cudnn': - engine = 'cuda' - self.layer = _create_layer(engine, 'Flatten') + # fltten layer is transparent to engine + self.layer = _create_layer('singa', 'Flatten') if input_sample_shape is not None: self.setup(input_sample_shape) class RNN(Layer): - def __init__(self, name, hidden_size, rnn_mode='lstm', engine='cudnn', - dropout=0.0, num_stacks=1, input_mode='linear', bidirectional=False, - param_specs=None, input_sample_shape=None): + def __init__(self, name, hidden_size, rnn_mode='lstm', dropout=0.0, + num_stacks=1, input_mode='linear', bidirectional=False, + param_specs=None, input_sample_shape=None): + '''Wrapper for singa::RNN class. + + Args: + hidden_size, hidden feature size, the same for all stacks of layers. + rnn_mode, decides the rnn unit, which could be one of 'lstm', 'gru', + 'tanh' and 'relu', refer to cudnn manual for each mode. + num_stacks, num of stacks of rnn layers. It is different to the + unrolling seqence length. + input_mode, 'linear' convert the input feature x by by a linear + transformation to get a feature vector of size hidden_size; + 'skip' does nothing but requires the input feature size equals + hidden_size + bidirection, True for bidirectional RNN + param_specs, config for initializing the RNN parameters. + input_sample_shape, includes a single integer for the input sample + feature size. + ''' super(RNN, self).__init__(name) conf = self.conf.rnn_conf assert hidden_size > 0, 'Hidden feature size must > 0' conf.hidden_size = hidden_size - assert rnn_mode in Set(['lstm', 'gru', 'tanh', 'relu']), \ - 'rnn mode %s is not available' %s (rnn_mode) + assert rnn_mode in Set(['lstm', 'gru', 'tanh', 'relu']), \ + 'rnn mode %s is not available' % (rnn_mode) conf.rnn_mode = rnn_mode conf.num_stacks = num_stacks conf.dropout = dropout @@ -519,10 +544,11 @@ def __init__(self, name, hidden_size, rnn_mode='lstm', engine='cudnn', conf.direction = 'unidirectional' if bidirectional: conf.direction = 'bidirectional' + # currently only has rnn layer implemented using cudnn _check_engine(engine, ['cudnn']) if param_specs is None: param_specs = {'name': name + '-weight', - 'init': 'uniform', 'low':0, 'high':1}; + 'init': 'uniform', 'low': 0, 'high': 1} self.conf.param.extend([_construct_param_specs_from_dict(param_specs)]) self.param_specs.append(_construct_param_specs_from_dict(param_specs)) @@ -531,18 +557,59 @@ def __init__(self, name, hidden_size, rnn_mode='lstm', engine='cudnn', self.setup(input_sample_shape) def forward(self, flag, inputs): + '''Forward inputs through the RNN. + + Args: + flag, kTrain or kEval. + inputs, , where xi is the input tensor for the + i-th position, its shape is (batch_size, input_feature_length); + the batch_size of xi must >= that of xi+1; hx is the initial + hidden state of shape (num_stacks * bidirection?2:1, batch_size, + hidden_size). cx is the initial cell state tensor of the same + shape as hy. cx is valid for only lstm. For other RNNs there is + no cx. Both hx and cx could be dummy tensors without shape and + data. + + Returns: + , where yi is the output tensor for the i-th + position, its shape is (batch_size, + hidden_size * bidirection?2:1). hy is the final hidden state + tensor. cx is the final cell state tensor. cx is only used for + lstm. + ''' assert self.has_setup, 'Must call setup() before forward()' assert len(inputs) > 1, 'The input to RNN must include at '\ - 'least one input tensor '\ - 'and one hidden state tensor (could be a dummy tensor)' + 'least one input tensor '\ + 'and one hidden state tensor (could be a dummy tensor)' tensors = [] for t in inputs: - assert isinstance(t, tensor.Tensor), 'input must be py Tensor %s' % (type(t)) + assert isinstance(t, tensor.Tensor), \ + 'input must be py Tensor %s' % (type(t)) tensors.append(t.singa_tensor) y = self.layer.Forward(flag, tensors) return tensor.from_raw_tensors(y) def backward(self, flag, grad): + '''Backward gradients through the RNN. + + Args: + flag, for future use. + grad, , where dyi is the gradient for the + i-th output, its shape is (batch_size, hidden_size*bidirection?2:1); + dhy is the gradient for the final hidden state, its shape is + (num_stacks * bidirection?2:1, batch_size, + hidden_size). dcy is the gradient for the final cell state. + cx is valid only for lstm. For other RNNs there is + no cx. Both dhy and dcy could be dummy tensors without shape and + data. + + Returns: + , where dxi is the gradient tensor for + the i-th input, its shape is (batch_size, + input_feature_length). dhx is the gradient for the initial + hidden state. dcx is the gradient for the initial cell state, + which is valid only for lstm. + ''' tensors = [] for t in grad: assert isinstance(t, tensor.Tensor), 'grad must be py Tensor' @@ -550,21 +617,23 @@ def backward(self, flag, grad): ret = self.layer.Backward(flag, tensors) return tensor.from_raw_tensors(ret[0]), tensor.from_raw_tensors(ret[1]) + class LSTM(RNN): - def __init__(self, name, hidden_size, engine='cudnn', - dropout=0.0, num_stacks=1, input_mode='linear', bidirectional=False, - param_specs=None, input_sample_shape=None): - super(LSTM, self).__init__(name, hidden_size, 'lstm', engine, dropout, - num_stacks, input_mode, bidirectional, param_specs, - input_sample_shape) + def __init__(self, name, hidden_size, dropout=0.0, num_stacks=1, + input_mode='linear', bidirectional=False, + param_specs=None, input_sample_shape=None): + super(LSTM, self).__init__(name, hidden_size, 'lstm', dropout, + num_stacks, input_mode, bidirectional, + param_specs, input_sample_shape) + class GRU(RNN): - def __init__(self, name, hidden_size, engine='cudnn', - dropout=0.0, num_stacks=1, input_mode='linear', bidirectional=False, - param_specs=None, input_sample_shape=None): - super(GRU, self).__init__(name, hidden_size, 'gru', engine, dropout, - num_stacks, input_mode, bidirectional, param_specs, - input_sample_shape) + def __init__(self, name, hidden_size, dropout=0.0, num_stacks=1, + input_mode='linear', bidirectional=False, param_specs=None, + input_sample_shape=None): + super(GRU, self).__init__(name, hidden_size, 'gru', dropout, + num_stacks, input_mode, bidirectional, + param_specs, input_sample_shape) def _check_engine(engine, allowed_engines): @@ -573,12 +642,17 @@ def _check_engine(engine, allowed_engines): (engine, ', '.join(allowed_engines)) -def _create_layer(engine, layer): - if engine == 'cuda' or engine == 'cpp': - layer_type = layer - else: - layer_type = engine.title() + layer - return singa_wrap.CreateLayer(layer_type) +def _create_layer(eng, layer): + ''' create singa wrap layer. + + Both arguments are case insensitive. + Args: + engine, implementation engine, either 'singa' or 'cudnn' + layer, layer type, e.g., 'convolution', 'pooling'; for activation + layers, use the specific activation mode, e.g. 'relu', 'tanh'. + ''' + layer_type = eng + '_' + layer + return singa_wrap.CreateLayer(layer_type.lower()) def _set_kernel_stride_pad(conf, kernel, stride, border_mode, pad): diff --git a/src/python/singa/net.py b/src/python/singa/net.py index c0ba61d2d9..1617717fa0 100644 --- a/src/python/singa/net.py +++ b/src/python/singa/net.py @@ -92,17 +92,17 @@ def predict(self, x): return tensor.softmax(xx) def forward(self, flag, x): - #print x.l1() + # print x.l1() for lyr in self.layers: x = lyr.forward(flag, x) # print lyr.name, x.l1() return x - def backward(self, flag=kTrain): + def backward(self): grad = self.loss.backward() pgrads = [] for lyr in reversed(self.layers): - grad, _pgrads = lyr.backward(flag, grad) + grad, _pgrads = lyr.backward(kTrain, grad) for g in reversed(_pgrads): pgrads.append(g) return reversed(pgrads) diff --git a/src/python/singa/tensor.py b/src/python/singa/tensor.py index 2d6fa5a99a..6e84a4f22c 100644 --- a/src/python/singa/tensor.py +++ b/src/python/singa/tensor.py @@ -39,7 +39,7 @@ def __init__(self, shape=None, device=None, dtype=core_pb2.kFloat32): return else: assert isinstance(shape, tuple), 'shape should be tuple' - vs = _tuple_to_vector(shape) + vs = list(shape) if device is None: self.singa_tensor = singa.Tensor(vs, dtype) else: @@ -111,8 +111,9 @@ def l1(self): return self.singa_tensor.L1() def set_value(self, x): - if isinstance(x, float): - self.singa_tensor.floatSetValue(x) + # assert type(x) == float, 'set value only accepts float input' + # if isinstance(x, float): + self.singa_tensor.floatSetValue(x) def copy_data(self, t): self.singa_tensor.CopyData(t.singa_tensor) diff --git a/src/python/swig/core_device.i b/src/python/swig/core_device.i index b79d37eb3b..a5d07315db 100644 --- a/src/python/swig/core_device.i +++ b/src/python/swig/core_device.i @@ -58,6 +58,10 @@ class Platform { static const std::string DeviceQuery(int id, bool verbose = false); static const std::vector > CreateCudaGPUs(const size_t num_devices, size_t init_size = 0); + static const std::vector> + CreateCudaGPUsOn(const std::vector &devices, size_t init_size = 0); + static std::shared_ptr GetDefaultDevice(); }; + } diff --git a/src/python/swig/model_layer.i b/src/python/swig/model_layer.i index 6cbfe8ff54..a6cdad1b38 100644 --- a/src/python/swig/model_layer.i +++ b/src/python/swig/model_layer.i @@ -81,7 +81,6 @@ const std::vector GetRegisteredLayers(); class RNN : public Layer { }; -#if CUDNN_VERSION_MINOR >= 5 && CUDNN_VERSION_PATCH >= 5 class CudnnRNN : public RNN { public: // note: Must use std::vector instead of vector. @@ -93,7 +92,5 @@ class CudnnRNN : public RNN { const std::vector GetOutputSampleShape() const override; }; -#endif // CUDNN_VERSION_MINOR >= 5 && CUDNN_VERSION_PATCH >= 5 - } diff --git a/test/singa/test_activation.cc b/test/singa/test_activation.cc index 001c49c7a5..bb8ad84759 100644 --- a/test/singa/test_activation.cc +++ b/test/singa/test_activation.cc @@ -27,15 +27,15 @@ using singa::Activation; using singa::Shape; TEST(Activation, Setup) { Activation acti; - EXPECT_EQ("Activation", acti.layer_type()); + // EXPECT_EQ("Activation", acti.layer_type()); singa::LayerConf conf; - conf.set_type("RELU"); + conf.set_type("singa_relu"); singa::ReLUConf* reluconf = conf.mutable_relu_conf(); reluconf->set_negative_slope(0.5); acti.Setup(Shape{3}, conf); - EXPECT_EQ("RELU", acti.Mode()); + EXPECT_EQ("relu", acti.Mode()); EXPECT_EQ(0.5f, acti.Negative_slope()); } @@ -46,13 +46,13 @@ TEST(Activation, Forward) { in.CopyDataFromHostPtr(x, n); float neg_slope = 0.5f; - std::string types[] = {"SIGMOID","TANH","RELU"}; + std::string types[] = {"singa_sigmoid", "singa_tanh", "singa_relu"}; for (int j = 0; j < 3; j++) { Activation acti; singa::LayerConf conf; std::string layertype = types[j]; conf.set_type(layertype); - if (layertype == "RELU") { + if (layertype == "relu") { singa::ReLUConf* reluconf = conf.mutable_relu_conf(); reluconf->set_negative_slope(neg_slope); } @@ -64,15 +64,15 @@ TEST(Activation, Forward) { EXPECT_EQ(n, out.Size()); float* y = new float[n]; - if (acti.Mode() == "SIGMOID") { + if (acti.Mode() == "sigmoid") { for (size_t i = 0; i < n; i++) y[i] = 1.f / (1.f + exp(-x[i])); } - else if (acti.Mode() == "TANH") { + else if (acti.Mode() == "tanh") { for (size_t i = 0; i < n; i++) y[i] = tanh(x[i]); } - else if (acti.Mode() == "RELU") { + else if (acti.Mode() == "relu") { for (size_t i = 0; i < n; i++) y[i] = (x[i] >= 0.f) ? x[i] : 0.f; } @@ -92,13 +92,13 @@ TEST(Activation, Backward) { in.CopyDataFromHostPtr(x, n); float neg_slope = 0.5f; - std::string types[] = {"SIGMOID","TANH","RELU"}; + std::string types[] = {"singa_sigmoid", "singa_tanh", "singa_relu"}; for (int j = 0; j < 3; j++) { Activation acti; singa::LayerConf conf; std::string layertype = types[j]; conf.set_type(layertype); - if (layertype == "RELU") { + if (layertype == "relu") { singa::ReLUConf* reluconf = conf.mutable_relu_conf(); reluconf->set_negative_slope(neg_slope); } @@ -114,15 +114,15 @@ TEST(Activation, Backward) { const float* xptr = in_diff.first.data(); float* dx = new float[n]; - if (acti.Mode() == "SIGMOID") { + if (acti.Mode() == "sigmoid") { for (size_t i = 0; i < n; i++) dx[i] = grad[i] * yptr[i] * (1. - yptr[i]); } - else if (acti.Mode() == "TANH") { + else if (acti.Mode() == "tanh") { for (size_t i = 0; i < n; i++) dx[i] = grad[i] * (1 - yptr[i] * yptr[i]); } - else if (acti.Mode() == "RELU") { + else if (acti.Mode() == "relu") { for (size_t i = 0; i < n; i++) dx[i] = grad[i] * (x[i] > 0.f) + acti.Negative_slope() * (x[i] <= 0.f); } diff --git a/test/singa/test_batchnorm.cc b/test/singa/test_batchnorm.cc index c72dc0f36e..a61f6f3252 100644 --- a/test/singa/test_batchnorm.cc +++ b/test/singa/test_batchnorm.cc @@ -27,7 +27,7 @@ using namespace singa; TEST(BatchNorm, Setup) { BatchNorm batchnorm; - EXPECT_EQ("BatchNorm", batchnorm.layer_type()); + // EXPECT_EQ("BatchNorm", batchnorm.layer_type()); singa::LayerConf conf; singa::BatchNormConf *batchnorm_conf = conf.mutable_batchnorm_conf(); @@ -68,10 +68,10 @@ TEST(BatchNorm, Forward) { EXPECT_EQ(1u, shape[1]); EXPECT_EQ(2u, shape[2]); EXPECT_EQ(1u, shape[3]); - EXPECT_NEAR(1.0f, outptr[0], 1e-6f); - EXPECT_NEAR(1.0f, outptr[1], 1e-6f); - EXPECT_NEAR(3.0f, outptr[2], 1e-6f); - EXPECT_NEAR(3.0f, outptr[3], 1e-6f); + EXPECT_NEAR(1.0f, outptr[0], 1e-4f); + EXPECT_NEAR(1.0f, outptr[1], 1e-4f); + EXPECT_NEAR(3.0f, outptr[2], 1e-4f); + EXPECT_NEAR(3.0f, outptr[3], 1e-4f); } TEST(BatchNorm, Backward) { @@ -107,10 +107,10 @@ TEST(BatchNorm, Backward) { EXPECT_EQ(2u, shape[2]); EXPECT_EQ(1u, shape[3]); const float *dxptr = ret.first.data(); - EXPECT_NEAR(.0f, dxptr[0], 1e-6f); - EXPECT_NEAR(.0f, dxptr[1], 1e-6f); - EXPECT_NEAR(.0f, dxptr[2], 1e-6f); - EXPECT_NEAR(.0f, dxptr[3], 1e-6f); + EXPECT_NEAR(.0f, dxptr[0], 1e-4f); + EXPECT_NEAR(.0f, dxptr[1], 1e-4f); + EXPECT_NEAR(.0f, dxptr[2], 1e-4f); + EXPECT_NEAR(.0f, dxptr[3], 1e-4f); Tensor dbnScale = ret.second.at(0); const float *dbnScaleptr = dbnScale.data(); @@ -118,8 +118,8 @@ TEST(BatchNorm, Backward) { EXPECT_EQ(1u, dbnScaleShape.size()); EXPECT_EQ(2u, dbnScaleShape[0]); - EXPECT_NEAR(-2.0f, dbnScaleptr[0], 1e-6f); - EXPECT_NEAR(-2.0f, dbnScaleptr[1], 1e-6f); + EXPECT_NEAR(-2.0f, dbnScaleptr[0], 1e-4f); + EXPECT_NEAR(-2.0f, dbnScaleptr[1], 1e-4f); Tensor dbnBias = ret.second.at(1); const float *dbnBiasptr = dbnBias.data(); @@ -127,6 +127,6 @@ TEST(BatchNorm, Backward) { EXPECT_EQ(1u, dbnBiasShape.size()); EXPECT_EQ(2u, dbnBiasShape[0]); - EXPECT_NEAR(6.0f, dbnBiasptr[0], 1e-6f); - EXPECT_NEAR(4.0f, dbnBiasptr[1], 1e-6f); + EXPECT_NEAR(6.0f, dbnBiasptr[0], 1e-4f); + EXPECT_NEAR(4.0f, dbnBiasptr[1], 1e-4f); } diff --git a/test/singa/test_convolution.cc b/test/singa/test_convolution.cc index c3ddceea69..4cfb38d96b 100644 --- a/test/singa/test_convolution.cc +++ b/test/singa/test_convolution.cc @@ -29,7 +29,7 @@ using singa::Convolution; using singa::Shape; TEST(Convolution, Setup) { Convolution conv; - EXPECT_EQ("Convolution", conv.layer_type()); + // EXPECT_EQ("Convolution", conv.layer_type()); singa::LayerConf conf; singa::ConvolutionConf *convconf = conf.mutable_convolution_conf(); diff --git a/test/singa/test_cudnn_activation.cc b/test/singa/test_cudnn_activation.cc index 9279d6c77f..6a989d1567 100644 --- a/test/singa/test_cudnn_activation.cc +++ b/test/singa/test_cudnn_activation.cc @@ -29,12 +29,12 @@ using singa::CudnnActivation; using singa::Shape; -TEST(TCudnnActivation, Setup) { +TEST(CudnnActivation, Setup) { CudnnActivation acti; - EXPECT_EQ("CudnnActivation", acti.layer_type()); + // EXPECT_EQ("CudnnActivation", acti.layer_type()); singa::LayerConf conf; - conf.set_type("RELU"); + conf.set_type("cudnn_relu"); singa::ReLUConf* reluconf = conf.mutable_relu_conf(); reluconf->set_negative_slope(0.5f); @@ -43,7 +43,7 @@ TEST(TCudnnActivation, Setup) { EXPECT_EQ(0.5f, acti.Negative_slope()); } -TEST(TCudnnActivation, Forward) { +TEST(CudnnActivation, Forward) { const float x[] = {1.0f, 2.0f, 3.0f, -2.0f, -3.0f, -4.0}; size_t n = sizeof(x) / sizeof(float); auto cuda = std::make_shared(); @@ -51,13 +51,13 @@ TEST(TCudnnActivation, Forward) { in.CopyDataFromHostPtr(x, n); float neg_slope = 0.5f; - std::string types[] = {"SIGMOID", "TANH", "RELU"}; + std::string types[] = {"cudnn_sigmoid", "cudnn_tanh", "cudnn_relu"}; for (int j = 0; j < 3; j++) { CudnnActivation acti; singa::LayerConf conf; std::string layertype = types[j]; conf.set_type(layertype); - if (layertype == "RELU") { + if (layertype == "relu") { singa::ReLUConf* reluconf = conf.mutable_relu_conf(); reluconf->set_negative_slope(neg_slope); } @@ -68,11 +68,11 @@ TEST(TCudnnActivation, Forward) { out.ToHost(); const float* yptr = out.data(); float* y = new float[n]; - if (acti.Mode() == "SIGMOID") { + if (acti.Mode() == "sigmoid") { for (size_t i = 0; i < n; i++) y[i] = 1.f / (1.f + exp(-x[i])); - } else if (acti.Mode() == "TANH") { + } else if (acti.Mode() == "tanh") { for (size_t i = 0; i < n; i++) y[i] = tanh(x[i]); - } else if (acti.Mode() == "RELU") { + } else if (acti.Mode() == "relu") { for (size_t i = 0; i < n; i++) y[i] = (x[i] >= 0.f) ? x[i] : 0.f; } else LOG(FATAL) << "Unkown activation: " << acti.Mode(); @@ -83,14 +83,14 @@ TEST(TCudnnActivation, Forward) { } } -TEST(TCudnnActivation, Backward) { +TEST(CudnnActivation, Backward) { const float x[] = {2.0f, 3.0f, 3.0f, 7.f, 0.0f, 5.0, 1.5, 2.5, -2.5, 1.5}; size_t n = sizeof(x) / sizeof(float); auto cuda = std::make_shared(); singa::Tensor in(singa::Shape{n}, cuda); in.CopyDataFromHostPtr(x, n); float neg_slope = 0.5f; - std::string types[] = {"SIGMOID", "TANH", "RELU"}; + std::string types[] = {"cudnn_sigmoid", "cudnn_tanh", "cudnn_relu"}; for (int j = 0; j < 3; j++) { CudnnActivation acti; singa::LayerConf conf; @@ -115,11 +115,11 @@ TEST(TCudnnActivation, Backward) { in_diff.ToHost(); const float* xptr = in_diff.data(); float* dx = new float[n]; - if (acti.Mode() == "SIGMOID") { + if (acti.Mode() == "sigmoid") { for (size_t i = 0; i < n; i++) dx[i] = grad[i] * yptr[i] * (1. - yptr[i]); - } else if (acti.Mode() == "TANH") { + } else if (acti.Mode() == "tanh") { for (size_t i = 0; i < n; i++) dx[i] = grad[i] * (1. - yptr[i] * yptr[i]); - } else if (acti.Mode() == "RELU") { + } else if (acti.Mode() == "relu") { for (size_t i = 0; i < n; i++) dx[i] = grad[i] * (x[i] > 0.f); //+ acti.Negative_slope() * (x[i] <= 0.f); diff --git a/test/singa/test_cudnn_batchnorm.cc b/test/singa/test_cudnn_batchnorm.cc index 4f6a38b1c6..b2746dcb82 100644 --- a/test/singa/test_cudnn_batchnorm.cc +++ b/test/singa/test_cudnn_batchnorm.cc @@ -28,7 +28,7 @@ using singa::CudnnBatchNorm; using singa::Shape; TEST(CudnnBatchNorm, Setup) { CudnnBatchNorm batchnorm; - EXPECT_EQ("CudnnBatchNorm", batchnorm.layer_type()); + // EXPECT_EQ("CudnnBatchNorm", batchnorm.layer_type()); singa::LayerConf conf; singa::BatchNormConf *batchnorm_conf = conf.mutable_batchnorm_conf(); diff --git a/test/singa/test_cudnn_convolution.cc b/test/singa/test_cudnn_convolution.cc index 66c62f68de..8dbee6358a 100644 --- a/test/singa/test_cudnn_convolution.cc +++ b/test/singa/test_cudnn_convolution.cc @@ -27,7 +27,7 @@ using singa::CudnnConvolution; using singa::Shape; TEST(CudnnConvolution, Setup) { CudnnConvolution conv; - EXPECT_EQ("CudnnConvolution", conv.layer_type()); + // EXPECT_EQ("CudnnConvolution", conv.layer_type()); singa::LayerConf conf; singa::ConvolutionConf *convconf = conf.mutable_convolution_conf(); @@ -199,7 +199,7 @@ TEST(CudnnConvolution, Backward) { // Tests for prefer=autotune TEST(CudnnConvolution_AT, Setup) { CudnnConvolution conv; - EXPECT_EQ("CudnnConvolution", conv.layer_type()); + // EXPECT_EQ("CudnnConvolution", conv.layer_type()); singa::LayerConf conf; singa::ConvolutionConf *convconf = conf.mutable_convolution_conf(); diff --git a/test/singa/test_cudnn_dropout.cc b/test/singa/test_cudnn_dropout.cc index 7f28aca866..4a89235ddc 100644 --- a/test/singa/test_cudnn_dropout.cc +++ b/test/singa/test_cudnn_dropout.cc @@ -36,7 +36,7 @@ using singa::CudnnDropout; using singa::Shape; TEST(CudnnDropout, Setup) { CudnnDropout drop; - EXPECT_EQ("CudnnDropout", drop.layer_type()); + // EXPECT_EQ("CudnnDropout", drop.layer_type()); singa::LayerConf conf; singa::DropoutConf* dropconf = conf.mutable_dropout_conf(); diff --git a/test/singa/test_cudnn_lrn.cc b/test/singa/test_cudnn_lrn.cc index 23fbe2e44c..04ca5f291a 100644 --- a/test/singa/test_cudnn_lrn.cc +++ b/test/singa/test_cudnn_lrn.cc @@ -30,7 +30,7 @@ using singa::CudnnLRN; using singa::Shape; TEST(CudnnLRN, Setup) { CudnnLRN lrn; - EXPECT_EQ("CudnnLRN", lrn.layer_type()); + // EXPECT_EQ("CudnnLRN", lrn.layer_type()); singa::LayerConf conf; singa::LRNConf *lrn_conf = conf.mutable_lrn_conf(); diff --git a/test/singa/test_cudnn_pooling.cc b/test/singa/test_cudnn_pooling.cc index 5c01889aa1..0e3314ee66 100644 --- a/test/singa/test_cudnn_pooling.cc +++ b/test/singa/test_cudnn_pooling.cc @@ -27,7 +27,7 @@ using singa::CudnnPooling; using singa::Shape; TEST(CudnnPooling, Setup) { CudnnPooling pool; - EXPECT_EQ("CudnnPooling", pool.layer_type()); + // EXPECT_EQ("CudnnPooling", pool.layer_type()); singa::LayerConf conf; singa::PoolingConf *poolconf = conf.mutable_pooling_conf(); diff --git a/test/singa/test_cudnn_rnn.cc b/test/singa/test_cudnn_rnn.cc index effb3b185e..e293cf7a78 100644 --- a/test/singa/test_cudnn_rnn.cc +++ b/test/singa/test_cudnn_rnn.cc @@ -45,7 +45,7 @@ class TestCudnnRNN : public ::testing::Test { TEST_F(TestCudnnRNN, Setup) { CudnnRNN rnn; - EXPECT_EQ("CudnnRNN", rnn.layer_type()); + // EXPECT_EQ("CudnnRNN", rnn.layer_type()); rnn.Setup(Shape{2}, conf); auto weight = rnn.param_values().at(0); EXPECT_EQ(weight.Size(), hidden_size * (2 + hidden_size + 2)); diff --git a/test/singa/test_cudnn_softmax.cc b/test/singa/test_cudnn_softmax.cc index 2b88843b22..6e0d5ab8cf 100644 --- a/test/singa/test_cudnn_softmax.cc +++ b/test/singa/test_cudnn_softmax.cc @@ -31,7 +31,7 @@ using singa::CudnnSoftmax; using singa::Shape; TEST(CudnnSoftmax, Setup) { CudnnSoftmax sft; - EXPECT_EQ("CudnnSoftmax", sft.layer_type()); + // EXPECT_EQ("CudnnSoftmax", sft.layer_type()); singa::LayerConf conf; singa::SoftmaxConf* softmaxconf = conf.mutable_softmax_conf(); diff --git a/test/singa/test_dense.cc b/test/singa/test_dense.cc index f4ecdfc500..17e161ad86 100644 --- a/test/singa/test_dense.cc +++ b/test/singa/test_dense.cc @@ -26,7 +26,7 @@ using singa::Dense; using singa::Shape; TEST(Dense, Setup) { Dense dense; - EXPECT_EQ("Dense", dense.layer_type()); + // EXPECT_EQ("Dense", dense.layer_type()); singa::LayerConf conf; singa::DenseConf *denseconf = conf.mutable_dense_conf(); diff --git a/test/singa/test_dropout.cc b/test/singa/test_dropout.cc index 3dd988aa6e..b0c34a33c6 100644 --- a/test/singa/test_dropout.cc +++ b/test/singa/test_dropout.cc @@ -26,7 +26,7 @@ using singa::Dropout; using singa::Shape; TEST(Dropout, Setup) { Dropout drop; - EXPECT_EQ("Dropout", drop.layer_type()); + // EXPECT_EQ("Dropout", drop.layer_type()); singa::LayerConf conf; singa::DropoutConf* dropconf = conf.mutable_dropout_conf(); diff --git a/test/singa/test_flatten.cc b/test/singa/test_flatten.cc index 25e00c4c26..65748f773a 100644 --- a/test/singa/test_flatten.cc +++ b/test/singa/test_flatten.cc @@ -26,7 +26,7 @@ using singa::Flatten; using singa::Shape; TEST(Flatten, Setup) { Flatten flt; - EXPECT_EQ("Flatten", flt.layer_type()); + // EXPECT_EQ("Flatten", flt.layer_type()); singa::LayerConf conf; singa::FlattenConf *flattenconf = conf.mutable_flatten_conf(); diff --git a/test/singa/test_layer.cc b/test/singa/test_layer.cc index 407176260e..aa0174656b 100644 --- a/test/singa/test_layer.cc +++ b/test/singa/test_layer.cc @@ -4,26 +4,25 @@ TEST(Layer, CreateLayer) { std::vector types{ - "Convolution", "Dense", "Dropout", "Activation", "BatchNorm", - "Flatten", "LRN", "Pooling", "PReLU", "Softmax"}; + "convolution", "dense", "dropout", "relu", "batchnorm", + "flatten", "lrn", "pooling", "prelu", "softmax"}; for (auto type : types) { - auto layer = singa::CreateLayer(type); - EXPECT_EQ(layer->layer_type(), type); + auto layer = singa::CreateLayer("singa_" + type); + // EXPECT_EQ(layer->layer_type(), type); } } #ifdef USE_CUDNN TEST(Layer, CreateCudnnLayer) { std::vector types{ - "CudnnConvolution", "CudnnActivation", - "CudnnBatchNorm", "Flatten", "CudnnLRN", - "CudnnPooling", "PReLU", "CudnnSoftmax"}; + "convolution", "dropout", "relu", "batchnorm", + "lrn", "pooling", "softmax"}; #if CUDNN_VERSION_MAJOR >= 5 - types.push_back("CudnnDropout"); + types.push_back("dropout"); #endif for (auto type : types) { - auto layer = singa::CreateLayer(type); - EXPECT_EQ(layer->layer_type(), type); + auto layer = singa::CreateLayer("cudnn_" + type); + // EXPECT_EQ(layer->layer_type(), type); } } #endif diff --git a/test/singa/test_lrn.cc b/test/singa/test_lrn.cc index 5de453530a..454e1a9a71 100644 --- a/test/singa/test_lrn.cc +++ b/test/singa/test_lrn.cc @@ -26,7 +26,7 @@ using namespace singa; TEST(LRN, Setup) { LRN lrn; - EXPECT_EQ("LRN", lrn.layer_type()); + // EXPECT_EQ("LRN", lrn.layer_type()); LayerConf conf; LRNConf *lrn_conf = conf.mutable_lrn_conf(); diff --git a/test/singa/test_pooling.cc b/test/singa/test_pooling.cc index 3089a90fdb..7ba56d1362 100644 --- a/test/singa/test_pooling.cc +++ b/test/singa/test_pooling.cc @@ -26,7 +26,7 @@ using singa::Pooling; using singa::Shape; TEST(Pooling, Setup) { Pooling pool; - EXPECT_EQ("Pooling", pool.layer_type()); + // EXPECT_EQ("Pooling", pool.layer_type()); singa::LayerConf conf; singa::PoolingConf *poolconf = conf.mutable_pooling_conf(); diff --git a/test/singa/test_prelu.cc b/test/singa/test_prelu.cc index dbb7cdef69..77b4b745e0 100644 --- a/test/singa/test_prelu.cc +++ b/test/singa/test_prelu.cc @@ -27,7 +27,7 @@ using singa::PReLU; using singa::Shape; TEST(PReLU, Setup) { PReLU prelu; - EXPECT_EQ("PReLU", prelu.layer_type()); + // EXPECT_EQ("PReLU", prelu.layer_type()); singa::LayerConf conf; singa::PReLUConf *preluconf = conf.mutable_prelu_conf(); diff --git a/test/singa/test_softmax.cc b/test/singa/test_softmax.cc index 00b837894d..8064b80984 100644 --- a/test/singa/test_softmax.cc +++ b/test/singa/test_softmax.cc @@ -27,7 +27,7 @@ using singa::Softmax; using singa::Shape; TEST(Softmax, Setup) { Softmax sft; - EXPECT_EQ("Softmax", sft.layer_type()); + // EXPECT_EQ("Softmax", sft.layer_type()); singa::LayerConf conf; sft.Setup(Shape{3}, conf);