From 3e551b358d7c289d4412e6abd6258b7b0f6e646f Mon Sep 17 00:00:00 2001 From: Oleksii Zaderykhin Date: Fri, 2 Sep 2022 17:55:26 +0300 Subject: [PATCH 1/8] The RegistersPool class was implemented. It is usefull for managing registers while developing jit kernels. --- .../src/nodes/kernels/registers_pool.hpp | 260 ++++++++++++++++++ 1 file changed, 260 insertions(+) create mode 100644 src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp diff --git a/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp b/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp new file mode 100644 index 00000000000000..1171c9a0454a17 --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp @@ -0,0 +1,260 @@ +// Copyright (C) 2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "cpu/x64/jit_generator.hpp" +#include +#include "ie_common.h" +#include + +namespace ov { +namespace intel_cpu { + +using namespace dnnl::impl::cpu; + +/** + * The RegistersPool is the base class for the IsaRegistersPool template: + * template + * class IsaRegistersPool : public RegistersPool; + * + * The registers pool must be created by instantiating the IsaRegistersPool template, like the next: + * RegistersPool::Ptr regPool = RegistersPool::create({ + * // the list of the registers to be excluded from pool + * Reg64(Operand::RAX), Reg64(Operand::RCX), Reg64(Operand::RDX), Reg64(Operand::RBX), + * Reg64(Operand::RSP), Reg64(Operand::RBP), Reg64(Operand::RSI), Reg64(Operand::RDI) + * }); + */ +class RegistersPool { +public: + using Ptr = std::shared_ptr; + + /** + * The scoped wrapper for the Xbyak registers. + * By creating it you are getting the register from the pool RegistersPool. + * It could be created by using constructor with RegistersPool as an argument, like the next: + * const RegistersPool::Reg reg {regPool}; + * The destructor will return the register to the pool. Or it could be returned manually: + * reg.returnToPool(); + * @tparam TReg Xbyak register class + */ + template + class Reg { + friend class RegistersPool; + public: + Reg(const RegistersPool::Ptr& regPool); + ~Reg() { returnToPool(); } + Reg& operator=(Reg&& other) noexcept { + reg = other.reg; + regPool = std::move(other.regPool); + return *this; + } + Reg(Reg&& other) noexcept : reg(other.reg), regPool(std::move(other.regPool)) {} + operator TReg&() { ensureValid(); return reg; } + operator const TReg&() const { ensureValid(); return reg; } + operator Xbyak::RegExp() const { ensureValid(); return reg; } + int getIdx() const { ensureValid(); return reg.getIdx(); } + friend Xbyak::RegExp operator+(const Reg& lhs, const Xbyak::RegExp& rhs) { + lhs.ensureValid(); + return lhs.operator Xbyak::RegExp() + rhs; + } + void returnToPool() { + if (regPool) { + regPool->returnToPool(reg); + } + regPool.reset(); + } + + private: + void ensureValid() const { + if (!regPool) { + IE_THROW() << "Invalid instance of RegistersPool::Reg was used"; + } + } + + private: + TReg reg; + RegistersPool::Ptr regPool; + }; + + virtual ~RegistersPool() { + checkUniqueAndUpdate(false); + } + + template + static Ptr create(std::initializer_list regsToExclude); + + template + size_t countFree() const { + if (std::is_same::value || std::is_same::value || std::is_same::value) { + return simdSet.countUnused(); + } else if (std::is_same::value || std::is_same::value) { + return generalSet.countUnused(); + } else if (std::is_same::value) { + return countUnusedOpmask(); + } + } + +protected: + class PhysicalSet { + public: + PhysicalSet(int size) + : size(size) { + for (int i = 0; i < size; ++i) { + unusedIndexes.emplace(i); + } + } + + void setAsUsed(int regIdx) { + assert(regIdx < size && regIdx >= 0); + auto it = unusedIndexes.find(regIdx); + if (it == unusedIndexes.end()) { + IE_THROW() << "Inconsistency in RegistersPool::PhysicalSet::setAsUsed()"; + } + unusedIndexes.erase(it); + } + + void setAsUnused(int regIdx) { + assert(regIdx < size && regIdx >= 0); + unusedIndexes.insert(regIdx); + } + + int getUnused() { + if (unusedIndexes.empty()) { + IE_THROW() << "Not enough registers in the RegistersPool"; + } + return *unusedIndexes.begin(); + } + + void exclude(Xbyak::Reg reg) { + unusedIndexes.erase(reg.getIdx()); + } + + size_t countUnused() const { + return unusedIndexes.size(); + } + + private: + std::unordered_set unusedIndexes; + int size; + }; + + virtual int getFreeOpmask() { IE_THROW() << "getFreeOpmask: The Opmask is not supported in current instruction set"; } + virtual void returnOpmaskToPool(int idx) { IE_THROW() << "returnOpmaskToPool: The Opmask is not supported in current instruction set"; } + virtual size_t countUnusedOpmask() const { IE_THROW() << "countUnusedOpmask: The Opmask is not supported in current instruction set"; } + + virtual void excludeOpmask(const Xbyak::Opmask& reg) { + } + + RegistersPool(std::initializer_list regsToExclude, int simdRegistersNumber) + : simdSet(simdRegistersNumber) { + checkUniqueAndUpdate(); + for (auto& reg : regsToExclude) { + if (reg.isXMM() || reg.isYMM() || reg.isZMM()) { + simdSet.exclude(reg); + } else if (reg.isOPMASK()) { + excludeOpmask(Xbyak::Opmask{reg.getIdx()}); + } else if (reg.isREG()) { + generalSet.exclude(reg); + } + } + generalSet.exclude(Xbyak::Reg64(Xbyak::Operand::RSP)); + } + +private: + template + int getFree() { + static_assert(std::is_same::value || std::is_same::value || std::is_same::value || + std::is_same::value || std::is_same::value || std::is_same::value, + "The AvxRegistersGuardPool::getFree() method unsupported type"); + if (std::is_same::value || std::is_same::value || std::is_same::value) { + auto idx = simdSet.getUnused(); + simdSet.setAsUsed(idx); + return idx; + } else if (std::is_same::value || std::is_same::value) { + auto idx = generalSet.getUnused(); + generalSet.setAsUsed(idx); + return idx; + } else if (std::is_same::value) { + return getFreeOpmask(); + } + } + + template + void returnToPool(TReg reg) { + if (std::is_same::value || std::is_same::value || std::is_same::value) { + simdSet.setAsUnused(reg.getIdx()); + } else if (std::is_same::value || std::is_same::value) { + generalSet.setAsUnused(reg.getIdx()); + } else if (std::is_same::value) { + returnOpmaskToPool(reg.getIdx()); + } + } + + void checkUniqueAndUpdate(bool isCtor = true) { + static thread_local bool isCreated = false; + if (isCtor) { + if (isCreated) { + IE_THROW() << "There should be only one instance of RegistersPool per thread"; + } + isCreated = true; + } else { + isCreated = false; + } + } + + PhysicalSet generalSet {16}; + PhysicalSet simdSet; +}; + +template +RegistersPool::Reg::Reg(const RegistersPool::Ptr& regPool) { + static_assert(std::is_same::value || std::is_same::value || std::is_same::value || + std::is_same::value || std::is_same::value || std::is_same::value, + "The type is not supported for the RegistersPool::Reg template"); + reg = TReg(regPool->template getFree()); + this->regPool = regPool; +} + +template +class IsaRegistersPool : public RegistersPool { +public: + IsaRegistersPool(std::initializer_list regsToExclude) : RegistersPool(regsToExclude, x64::cpu_isa_traits::n_vregs) {} +}; + +template <> +class IsaRegistersPool : public RegistersPool { +public: + IsaRegistersPool(std::initializer_list regsToExclude) + : RegistersPool(regsToExclude, x64::cpu_isa_traits::n_vregs) {} + + int getFreeOpmask() override { + auto idx = opmaskSet.getUnused(); + opmaskSet.setAsUsed(idx); + return idx; + } + + void returnOpmaskToPool(int idx) override { + opmaskSet.setAsUnused(idx); + } + + void excludeOpmask(const Xbyak::Opmask& reg) override { + opmaskSet.exclude(reg); + } + + size_t countUnusedOpmask() const override { + return opmaskSet.countUnused(); + } + +protected: + PhysicalSet opmaskSet {8}; +}; + +template +RegistersPool::Ptr RegistersPool::create(std::initializer_list regsToExclude) { + return std::make_shared>(regsToExclude); +} + +} // namespace intel_cpu +} // namespace ov From 209d60bd6dbb1816591011429e9f4a874a3b6ec3 Mon Sep 17 00:00:00 2001 From: Oleksii Zaderykhin Date: Tue, 6 Sep 2022 15:13:45 +0300 Subject: [PATCH 2/8] The unit tests for RegistersPool were added. --- .../src/nodes/kernels/registers_pool.hpp | 110 +++++++++------- .../intel_cpu/tests/unit/registers_pool.cpp | 118 ++++++++++++++++++ 2 files changed, 185 insertions(+), 43 deletions(-) create mode 100644 src/plugins/intel_cpu/tests/unit/registers_pool.cpp diff --git a/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp b/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp index 1171c9a0454a17..449f94a9da0793 100644 --- a/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp +++ b/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp @@ -29,6 +29,7 @@ using namespace dnnl::impl::cpu; class RegistersPool { public: using Ptr = std::shared_ptr; + static constexpr int anyIdx = -1; /** * The scoped wrapper for the Xbyak registers. @@ -36,15 +37,21 @@ class RegistersPool { * It could be created by using constructor with RegistersPool as an argument, like the next: * const RegistersPool::Reg reg {regPool}; * The destructor will return the register to the pool. Or it could be returned manually: - * reg.returnToPool(); + * reg.release(); * @tparam TReg Xbyak register class */ template class Reg { friend class RegistersPool; public: - Reg(const RegistersPool::Ptr& regPool); - ~Reg() { returnToPool(); } + Reg() { + static_assert(std::is_same::value || std::is_same::value || std::is_same::value || + std::is_same::value || std::is_same::value || std::is_same::value, + "The type is not supported for the RegistersPool::Reg template"); + } + Reg(const RegistersPool::Ptr& regPool) { initialize(regPool); } + Reg(const RegistersPool::Ptr& regPool, int requestedIdx) { initialize(regPool, requestedIdx); } + ~Reg() { release(); } Reg& operator=(Reg&& other) noexcept { reg = other.reg; regPool = std::move(other.regPool); @@ -59,20 +66,26 @@ class RegistersPool { lhs.ensureValid(); return lhs.operator Xbyak::RegExp() + rhs; } - void returnToPool() { + void release() { if (regPool) { regPool->returnToPool(reg); + regPool.reset(); } - regPool.reset(); } private: void ensureValid() const { if (!regPool) { - IE_THROW() << "Invalid instance of RegistersPool::Reg was used"; + IE_THROW() << "RegistersPool::Reg is either not initialized or released"; } } + void initialize(const RegistersPool::Ptr& pool, int requestedIdx = anyIdx) { + release(); + reg = TReg(pool->template getFree(requestedIdx)); + regPool = pool; + } + private: TReg reg; RegistersPool::Ptr regPool; @@ -99,48 +112,68 @@ class RegistersPool { protected: class PhysicalSet { public: - PhysicalSet(int size) - : size(size) { - for (int i = 0; i < size; ++i) { - unusedIndexes.emplace(i); - } - } + PhysicalSet(int size) : isFreeIndexVector(size, true) {} void setAsUsed(int regIdx) { - assert(regIdx < size && regIdx >= 0); - auto it = unusedIndexes.find(regIdx); - if (it == unusedIndexes.end()) { + if (regIdx >= isFreeIndexVector.size() || regIdx < 0) { + IE_THROW() << "regIdx is out of bounds in RegistersPool::PhysicalSet::setAsUsed()"; + } + if (!isFreeIndexVector[regIdx]) { IE_THROW() << "Inconsistency in RegistersPool::PhysicalSet::setAsUsed()"; } - unusedIndexes.erase(it); + isFreeIndexVector[regIdx] = false; } void setAsUnused(int regIdx) { - assert(regIdx < size && regIdx >= 0); - unusedIndexes.insert(regIdx); + if (regIdx >= isFreeIndexVector.size() || regIdx < 0) { + IE_THROW() << "regIdx is out of bounds in RegistersPool::PhysicalSet::setAsUsed()"; + } + if (isFreeIndexVector[regIdx]) { + IE_THROW() << "Inconsistency in RegistersPool::PhysicalSet::setAsUnused()"; + } + isFreeIndexVector[regIdx] = true; } - int getUnused() { - if (unusedIndexes.empty()) { - IE_THROW() << "Not enough registers in the RegistersPool"; + int getUnused(int requestedIdx) { + if (requestedIdx == anyIdx) { + return getFirstFreeIndex(); + } else { + if (!isFreeIndexVector[requestedIdx]) { + IE_THROW() << "The register with index #" << requestedIdx << " already used in the RegistersPool"; + } + return requestedIdx; } - return *unusedIndexes.begin(); } void exclude(Xbyak::Reg reg) { - unusedIndexes.erase(reg.getIdx()); + isFreeIndexVector.at(reg.getIdx()) = false; } size_t countUnused() const { - return unusedIndexes.size(); + size_t count = 0; + for (const auto& isFree : isFreeIndexVector) { + if (isFree) { + ++count; + } + } + return count; } private: - std::unordered_set unusedIndexes; - int size; + int getFirstFreeIndex() { + for (int c = 0; c < isFreeIndexVector.size(); ++c) { + if (isFreeIndexVector[c]) { + return c; + } + } + IE_THROW() << "Not enough registers in the RegistersPool"; + } + + private: + std::vector isFreeIndexVector; }; - virtual int getFreeOpmask() { IE_THROW() << "getFreeOpmask: The Opmask is not supported in current instruction set"; } + virtual int getFreeOpmask(int requestedIdx) { IE_THROW() << "getFreeOpmask: The Opmask is not supported in current instruction set"; } virtual void returnOpmaskToPool(int idx) { IE_THROW() << "returnOpmaskToPool: The Opmask is not supported in current instruction set"; } virtual size_t countUnusedOpmask() const { IE_THROW() << "countUnusedOpmask: The Opmask is not supported in current instruction set"; } @@ -164,20 +197,20 @@ class RegistersPool { private: template - int getFree() { + int getFree(int requestedIdx) { static_assert(std::is_same::value || std::is_same::value || std::is_same::value || std::is_same::value || std::is_same::value || std::is_same::value, - "The AvxRegistersGuardPool::getFree() method unsupported type"); + "Unsupported TReg by RegistersPool. Please, use the following registers either Reg32, Reg64, Xmm, Ymm, Zmm or Opmask"); if (std::is_same::value || std::is_same::value || std::is_same::value) { - auto idx = simdSet.getUnused(); + auto idx = simdSet.getUnused(requestedIdx); simdSet.setAsUsed(idx); return idx; } else if (std::is_same::value || std::is_same::value) { - auto idx = generalSet.getUnused(); + auto idx = generalSet.getUnused(requestedIdx); generalSet.setAsUsed(idx); return idx; } else if (std::is_same::value) { - return getFreeOpmask(); + return getFreeOpmask(requestedIdx); } } @@ -208,15 +241,6 @@ class RegistersPool { PhysicalSet simdSet; }; -template -RegistersPool::Reg::Reg(const RegistersPool::Ptr& regPool) { - static_assert(std::is_same::value || std::is_same::value || std::is_same::value || - std::is_same::value || std::is_same::value || std::is_same::value, - "The type is not supported for the RegistersPool::Reg template"); - reg = TReg(regPool->template getFree()); - this->regPool = regPool; -} - template class IsaRegistersPool : public RegistersPool { public: @@ -229,8 +253,8 @@ class IsaRegistersPool : public RegistersPool { IsaRegistersPool(std::initializer_list regsToExclude) : RegistersPool(regsToExclude, x64::cpu_isa_traits::n_vregs) {} - int getFreeOpmask() override { - auto idx = opmaskSet.getUnused(); + int getFreeOpmask(int requestedIdx) override { + auto idx = opmaskSet.getUnused(requestedIdx); opmaskSet.setAsUsed(idx); return idx; } diff --git a/src/plugins/intel_cpu/tests/unit/registers_pool.cpp b/src/plugins/intel_cpu/tests/unit/registers_pool.cpp new file mode 100644 index 00000000000000..3b5e1dfcba8977 --- /dev/null +++ b/src/plugins/intel_cpu/tests/unit/registers_pool.cpp @@ -0,0 +1,118 @@ +// Copyright (C) 2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include "nodes/kernels/registers_pool.hpp" + +using namespace ov::intel_cpu; +const int simdRegNumber = x64::cpu_isa_traits::n_vregs; +const int freeGeneralRegNumber = 15; + +TEST(RegistersPoolTests, get_return_by_scope) { + if (!x64::mayiuse(x64::avx2)) { return; } + RegistersPool::Ptr regPool = RegistersPool::create({}); + ASSERT_EQ(regPool->countFree(), simdRegNumber); + ASSERT_EQ(regPool->countFree(), freeGeneralRegNumber); + { + RegistersPool::Reg reg{regPool}; + ASSERT_NO_THROW(static_cast(reg)); + ASSERT_EQ(regPool->countFree(), simdRegNumber - 1); + ASSERT_EQ(regPool->countFree(), simdRegNumber - 1); + ASSERT_EQ(regPool->countFree(), freeGeneralRegNumber); + } + ASSERT_EQ(regPool->countFree(), simdRegNumber); +} + +TEST(RegistersPoolTests, get_return_by_method) { + if (!x64::mayiuse(x64::avx2)) { return; } + RegistersPool::Ptr regPool = RegistersPool::create({}); + ASSERT_EQ(regPool->countFree(), simdRegNumber); + RegistersPool::Reg reg{regPool}; + ASSERT_NO_THROW(static_cast(reg)); + ASSERT_EQ(regPool->countFree(), simdRegNumber - 1); + reg.release(); + ASSERT_ANY_THROW(static_cast(reg)); + ASSERT_EQ(regPool->countFree(), simdRegNumber); + reg = RegistersPool::Reg{regPool}; + ASSERT_NO_THROW(static_cast(reg)); + ASSERT_EQ(regPool->countFree(), simdRegNumber - 1); +} + +TEST(RegistersPoolTests, second_pool_exception) { + if (!x64::mayiuse(x64::avx2)) { return; } + RegistersPool::Ptr regPool = RegistersPool::create({}); + ASSERT_ANY_THROW(RegistersPool::create({})); +} + +TEST(RegistersPoolTests, default_ctor) { + if (!x64::mayiuse(x64::avx2)) { return; } + RegistersPool::Ptr regPool = RegistersPool::create({}); + RegistersPool::Reg reg; + ASSERT_EQ(regPool->countFree(), simdRegNumber); + ASSERT_ANY_THROW(static_cast(reg)); +} + +TEST(RegistersPoolTests, get_all) { + if (!x64::mayiuse(x64::avx2)) { return; } + RegistersPool::Ptr regPool = RegistersPool::create({}); + using Ptr = std::shared_ptr>; + std::vector regs(simdRegNumber); + std::vector idxs(simdRegNumber); + for (int c = 0; c < simdRegNumber; ++c) { + regs[c] = std::make_shared>(regPool); + idxs[c] = regs[c]->getIdx(); + } + ASSERT_EQ(regPool->countFree(), 0); + ASSERT_ANY_THROW(RegistersPool::Reg{regPool}); + std::sort(idxs.begin(), idxs.end()); + for (int c = 0; c < simdRegNumber; ++c) { + ASSERT_EQ(c, idxs[c]); + } + regs.clear(); + ASSERT_EQ(regPool->countFree(), simdRegNumber); +} + +TEST(RegistersPoolTests, move) { + if (!x64::mayiuse(x64::avx2)) { return; } + RegistersPool::Ptr regPool = RegistersPool::create({}); + RegistersPool::Reg reg{regPool}; + ASSERT_NO_THROW(static_cast(reg)); + ASSERT_EQ(regPool->countFree(), simdRegNumber - 1); + RegistersPool::Reg reg2 = std::move(reg); + ASSERT_ANY_THROW(static_cast(reg)); + ASSERT_NO_THROW(static_cast(reg2)); + ASSERT_EQ(regPool->countFree(), simdRegNumber - 1); +} + + +TEST(RegistersPoolTests, fixed_idx) { + if (!x64::mayiuse(x64::avx2)) { return; } + RegistersPool::Ptr regPool = RegistersPool::create({}); + using Ptr = std::shared_ptr>; + std::vector regs(simdRegNumber); + for (int c = 0; c < simdRegNumber; ++c) { + regs[c] = std::make_shared>(regPool, c); + ASSERT_EQ(regs[c]->getIdx(), c); + } + regs[0]->release(); + ASSERT_ANY_THROW(RegistersPool::Reg(regPool, 1)); + ASSERT_NO_THROW(RegistersPool::Reg(regPool, 0)); +} + +TEST(RegistersPoolTests, exclude) { + if (!x64::mayiuse(x64::avx2)) { return; } + static constexpr int excludedIdx = 0; + RegistersPool::Ptr regPool = RegistersPool::create({ + Xbyak::Xmm(excludedIdx) + }); + using Ptr = std::shared_ptr>; + std::vector regs(simdRegNumber - 1); + std::set idxsInUse; + for (int c = 0; c < simdRegNumber - 1; ++c) { + regs[c] = std::make_shared>(regPool); + idxsInUse.emplace(regs[c]->getIdx()); + } + ASSERT_EQ(regPool->countFree(), 0); + ASSERT_TRUE(idxsInUse.find(excludedIdx) == idxsInUse.end()); +} \ No newline at end of file From aa238d4fc8cdca8d5e3c7b7a05f5a8430b7b7b7f Mon Sep 17 00:00:00 2001 From: Oleksii Zaderykhin Date: Fri, 9 Sep 2022 18:30:51 +0300 Subject: [PATCH 3/8] The virtual function call in the RegistersPool constructor bug was fixed. --- .../src/nodes/kernels/registers_pool.hpp | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp b/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp index 449f94a9da0793..cde9751a54d163 100644 --- a/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp +++ b/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp @@ -177,17 +177,12 @@ class RegistersPool { virtual void returnOpmaskToPool(int idx) { IE_THROW() << "returnOpmaskToPool: The Opmask is not supported in current instruction set"; } virtual size_t countUnusedOpmask() const { IE_THROW() << "countUnusedOpmask: The Opmask is not supported in current instruction set"; } - virtual void excludeOpmask(const Xbyak::Opmask& reg) { - } - RegistersPool(std::initializer_list regsToExclude, int simdRegistersNumber) : simdSet(simdRegistersNumber) { checkUniqueAndUpdate(); for (auto& reg : regsToExclude) { if (reg.isXMM() || reg.isYMM() || reg.isZMM()) { simdSet.exclude(reg); - } else if (reg.isOPMASK()) { - excludeOpmask(Xbyak::Opmask{reg.getIdx()}); } else if (reg.isREG()) { generalSet.exclude(reg); } @@ -251,7 +246,13 @@ template <> class IsaRegistersPool : public RegistersPool { public: IsaRegistersPool(std::initializer_list regsToExclude) - : RegistersPool(regsToExclude, x64::cpu_isa_traits::n_vregs) {} + : RegistersPool(regsToExclude, x64::cpu_isa_traits::n_vregs) { + for (auto& reg : regsToExclude) { + if (reg.isOPMASK()) { + opmaskSet.exclude(reg); + } + } + } int getFreeOpmask(int requestedIdx) override { auto idx = opmaskSet.getUnused(requestedIdx); @@ -263,10 +264,6 @@ class IsaRegistersPool : public RegistersPool { opmaskSet.setAsUnused(idx); } - void excludeOpmask(const Xbyak::Opmask& reg) override { - opmaskSet.exclude(reg); - } - size_t countUnusedOpmask() const override { return opmaskSet.countUnused(); } From b1c8809e91a5b964fa67f5d01f16c436cecfb484 Mon Sep 17 00:00:00 2001 From: Oleksii Zaderykhin Date: Tue, 13 Sep 2022 16:04:54 +0300 Subject: [PATCH 4/8] The Reg8, Reg16 were added to the RegisterPool. The tests were added for different ISA and different Xbyak register types. --- .../src/nodes/kernels/registers_pool.hpp | 35 ++- .../intel_cpu/tests/unit/registers_pool.cpp | 258 +++++++++++++----- 2 files changed, 217 insertions(+), 76 deletions(-) diff --git a/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp b/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp index cde9751a54d163..89f354e612f9b4 100644 --- a/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp +++ b/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp @@ -46,7 +46,9 @@ class RegistersPool { public: Reg() { static_assert(std::is_same::value || std::is_same::value || std::is_same::value || - std::is_same::value || std::is_same::value || std::is_same::value, + std::is_same::value || std::is_same::value || + std::is_same::value || std::is_same::value || + std::is_same::value, "The type is not supported for the RegistersPool::Reg template"); } Reg(const RegistersPool::Ptr& regPool) { initialize(regPool); } @@ -102,7 +104,8 @@ class RegistersPool { size_t countFree() const { if (std::is_same::value || std::is_same::value || std::is_same::value) { return simdSet.countUnused(); - } else if (std::is_same::value || std::is_same::value) { + } else if (std::is_same::value || std::is_same::value || + std::is_same::value || std::is_same::value) { return generalSet.countUnused(); } else if (std::is_same::value) { return countUnusedOpmask(); @@ -138,6 +141,9 @@ class RegistersPool { if (requestedIdx == anyIdx) { return getFirstFreeIndex(); } else { + if (requestedIdx >= isFreeIndexVector.size() || requestedIdx < 0) { + IE_THROW() << "requestedIdx is out of bounds in RegistersPool::PhysicalSet::getUnused()"; + } if (!isFreeIndexVector[requestedIdx]) { IE_THROW() << "The register with index #" << requestedIdx << " already used in the RegistersPool"; } @@ -194,13 +200,17 @@ class RegistersPool { template int getFree(int requestedIdx) { static_assert(std::is_same::value || std::is_same::value || std::is_same::value || - std::is_same::value || std::is_same::value || std::is_same::value, - "Unsupported TReg by RegistersPool. Please, use the following registers either Reg32, Reg64, Xmm, Ymm, Zmm or Opmask"); + std::is_same::value || std::is_same::value || + std::is_same::value || std::is_same::value || + std::is_same::value, + "Unsupported TReg by RegistersPool. Please, use the following Xbyak registers either " + "Reg8, Reg16, Reg32, Reg64, Xmm, Ymm, Zmm or Opmask"); if (std::is_same::value || std::is_same::value || std::is_same::value) { auto idx = simdSet.getUnused(requestedIdx); simdSet.setAsUsed(idx); return idx; - } else if (std::is_same::value || std::is_same::value) { + } else if (std::is_same::value || std::is_same::value || + std::is_same::value || std::is_same::value) { auto idx = generalSet.getUnused(requestedIdx); generalSet.setAsUsed(idx); return idx; @@ -213,7 +223,8 @@ class RegistersPool { void returnToPool(TReg reg) { if (std::is_same::value || std::is_same::value || std::is_same::value) { simdSet.setAsUnused(reg.getIdx()); - } else if (std::is_same::value || std::is_same::value) { + } else if (std::is_same::value || std::is_same::value || + std::is_same::value || std::is_same::value) { generalSet.setAsUnused(reg.getIdx()); } else if (std::is_same::value) { returnOpmaskToPool(reg.getIdx()); @@ -277,5 +288,17 @@ RegistersPool::Ptr RegistersPool::create(std::initializer_list regsT return std::make_shared>(regsToExclude); } +template <> +class IsaRegistersPool : public IsaRegistersPool { +public: + IsaRegistersPool(std::initializer_list regsToExclude) : IsaRegistersPool(regsToExclude) {} +}; + +template <> +class IsaRegistersPool : public IsaRegistersPool { +public: + IsaRegistersPool(std::initializer_list regsToExclude) : IsaRegistersPool(regsToExclude) {} +}; + } // namespace intel_cpu } // namespace ov diff --git a/src/plugins/intel_cpu/tests/unit/registers_pool.cpp b/src/plugins/intel_cpu/tests/unit/registers_pool.cpp index 3b5e1dfcba8977..8848d2a8e4983f 100644 --- a/src/plugins/intel_cpu/tests/unit/registers_pool.cpp +++ b/src/plugins/intel_cpu/tests/unit/registers_pool.cpp @@ -6,11 +6,196 @@ #include "nodes/kernels/registers_pool.hpp" using namespace ov::intel_cpu; + +template +class RegPoolTest : public ::testing::Test { +protected: + void SetUp() override { + if (typename T::RegT(0).isREG()) { // for general purpose registers Reg8, Reg16, Reg32, Reg64 + regNumber = 15; // the RSP register excluded by default + } else if (typename T::RegT(0).isOPMASK()) { + regNumber = 8; + } else { // SIMD registers + regNumber = x64::cpu_isa_traits::n_vregs; + } + } + + int regNumber; +}; + +TYPED_TEST_SUITE_P(RegPoolTest); + +TYPED_TEST_P(RegPoolTest, get_return_by_scope) { + using XbyakRegT = typename TypeParam::RegT; + RegistersPool::Ptr regPool = RegistersPool::create({}); + ASSERT_EQ(regPool->countFree(), this->regNumber); + { + RegistersPool::Reg reg{regPool}; + ASSERT_NO_THROW(static_cast(reg)); + ASSERT_EQ(regPool->countFree(), this->regNumber - 1); + } + ASSERT_EQ(regPool->countFree(), this->regNumber); +} + +TYPED_TEST_P(RegPoolTest, get_return_by_method) { + using XbyakRegT = typename TypeParam::RegT; + RegistersPool::Ptr regPool = RegistersPool::create({}); + ASSERT_EQ(regPool->countFree(), this->regNumber); + RegistersPool::Reg reg{regPool}; + ASSERT_NO_THROW(static_cast(reg)); + ASSERT_EQ(regPool->countFree(), this->regNumber - 1); + reg.release(); + ASSERT_ANY_THROW(static_cast(reg)); + ASSERT_EQ(regPool->countFree(), this->regNumber); + reg = RegistersPool::Reg{regPool}; + ASSERT_NO_THROW(static_cast(reg)); + ASSERT_EQ(regPool->countFree(), this->regNumber - 1); +} + +TYPED_TEST_P(RegPoolTest, default_ctor) { + using XbyakRegT = typename TypeParam::RegT; + RegistersPool::Ptr regPool = RegistersPool::create({}); + RegistersPool::Reg reg; + ASSERT_EQ(regPool->countFree(), this->regNumber); + ASSERT_ANY_THROW(static_cast(reg)); +} + +TYPED_TEST_P(RegPoolTest, get_all) { + using XbyakRegT = typename TypeParam::RegT; + RegistersPool::Ptr regPool = RegistersPool::create({}); + using Ptr = std::shared_ptr>; + std::vector regs(this->regNumber); + for (int c = 0; c < this->regNumber; ++c) { + regs[c] = std::make_shared>(regPool); + } + ASSERT_EQ(regPool->countFree(), 0); + ASSERT_ANY_THROW(RegistersPool::Reg{regPool}); + regs.clear(); + ASSERT_EQ(regPool->countFree(), this->regNumber); +} + +TYPED_TEST_P(RegPoolTest, move) { + using XbyakRegT = typename TypeParam::RegT; + RegistersPool::Ptr regPool = RegistersPool::create({}); + RegistersPool::Reg reg{regPool}; + ASSERT_NO_THROW(static_cast(reg)); + ASSERT_EQ(regPool->countFree(), this->regNumber - 1); + RegistersPool::Reg reg2 = std::move(reg); + ASSERT_ANY_THROW(static_cast(reg)); + ASSERT_NO_THROW(static_cast(reg2)); + ASSERT_EQ(regPool->countFree(), this->regNumber - 1); +} + + +TYPED_TEST_P(RegPoolTest, fixed_idx) { + using XbyakRegT = typename TypeParam::RegT; + RegistersPool::Ptr regPool = RegistersPool::create({}); + using Ptr = std::shared_ptr>; + std::vector regs(this->regNumber); + for (int c = 0; c < this->regNumber; ++c) { + if (c == Xbyak::Operand::RSP) continue; + regs[c] = std::make_shared>(regPool, c); + ASSERT_EQ(regs[c]->getIdx(), c); + } + regs[0]->release(); + ASSERT_ANY_THROW(RegistersPool::Reg(regPool, 1)); + ASSERT_NO_THROW(RegistersPool::Reg(regPool, 0)); +} + +TYPED_TEST_P(RegPoolTest, exclude) { + using XbyakRegT = typename TypeParam::RegT; + static constexpr int excludedIdx = 0; + RegistersPool::Ptr regPool = RegistersPool::create({ + XbyakRegT(excludedIdx) + }); + using Ptr = std::shared_ptr>; + std::vector regs(this->regNumber - 1); + std::set idxsInUse; + for (int c = 0; c < this->regNumber - 1; ++c) { + regs[c] = std::make_shared>(regPool); + idxsInUse.emplace(regs[c]->getIdx()); + } + ASSERT_EQ(regPool->countFree(), 0); + ASSERT_TRUE(idxsInUse.find(excludedIdx) == idxsInUse.end()); +} + +namespace combiner { + +template +struct Case { + using RegT = Reg; + using IsaParam = Isa; +}; + +template +struct make_case { + static constexpr std::size_t N = std::tuple_size::value; + + using type = Case::type, + typename std::tuple_element::type>; +}; + +template +struct make_combinations; + +template +struct make_combinations> { + using tuples = std::tuple::type...>; +}; + +template +using Combinations_t = typename make_combinations + , + tbb::internal::make_index_sequence<(std::tuple_size::value) *(sizeof...(Params))>>::tuples; + +template +struct TestTypesCombiner; + +template +struct TestTypesCombiner> { + using Types = ::testing::Types; +}; + +} // namespace combiner + +template +struct IsaParam { static constexpr x64::cpu_isa_t isa = Isa; }; + +using TestTypes = combiner::TestTypesCombiner, + IsaParam, + IsaParam, + IsaParam, + IsaParam >>::Types; + +using TestTypesAvx512 = combiner::TestTypesCombiner, + IsaParam, + IsaParam, + IsaParam >>::Types; + +REGISTER_TYPED_TEST_SUITE_P(RegPoolTest, + get_return_by_scope, + get_return_by_method, + default_ctor, + get_all, + move, + fixed_idx, + exclude); + +INSTANTIATE_TYPED_TEST_SUITE_P(testIsaAndRegTypes, RegPoolTest, TestTypes); +INSTANTIATE_TYPED_TEST_SUITE_P(testIsaAndRegTypesAvx512, RegPoolTest, TestTypesAvx512); + + const int simdRegNumber = x64::cpu_isa_traits::n_vregs; const int freeGeneralRegNumber = 15; -TEST(RegistersPoolTests, get_return_by_scope) { - if (!x64::mayiuse(x64::avx2)) { return; } +TEST(RegistersPoolTests, simd_and_general) { RegistersPool::Ptr regPool = RegistersPool::create({}); ASSERT_EQ(regPool->countFree(), simdRegNumber); ASSERT_EQ(regPool->countFree(), freeGeneralRegNumber); @@ -22,39 +207,15 @@ TEST(RegistersPoolTests, get_return_by_scope) { ASSERT_EQ(regPool->countFree(), freeGeneralRegNumber); } ASSERT_EQ(regPool->countFree(), simdRegNumber); -} - -TEST(RegistersPoolTests, get_return_by_method) { - if (!x64::mayiuse(x64::avx2)) { return; } - RegistersPool::Ptr regPool = RegistersPool::create({}); - ASSERT_EQ(regPool->countFree(), simdRegNumber); - RegistersPool::Reg reg{regPool}; - ASSERT_NO_THROW(static_cast(reg)); - ASSERT_EQ(regPool->countFree(), simdRegNumber - 1); - reg.release(); - ASSERT_ANY_THROW(static_cast(reg)); - ASSERT_EQ(regPool->countFree(), simdRegNumber); - reg = RegistersPool::Reg{regPool}; - ASSERT_NO_THROW(static_cast(reg)); - ASSERT_EQ(regPool->countFree(), simdRegNumber - 1); + ASSERT_EQ(regPool->countFree(), freeGeneralRegNumber); } TEST(RegistersPoolTests, second_pool_exception) { - if (!x64::mayiuse(x64::avx2)) { return; } RegistersPool::Ptr regPool = RegistersPool::create({}); ASSERT_ANY_THROW(RegistersPool::create({})); } -TEST(RegistersPoolTests, default_ctor) { - if (!x64::mayiuse(x64::avx2)) { return; } - RegistersPool::Ptr regPool = RegistersPool::create({}); - RegistersPool::Reg reg; - ASSERT_EQ(regPool->countFree(), simdRegNumber); - ASSERT_ANY_THROW(static_cast(reg)); -} - TEST(RegistersPoolTests, get_all) { - if (!x64::mayiuse(x64::avx2)) { return; } RegistersPool::Ptr regPool = RegistersPool::create({}); using Ptr = std::shared_ptr>; std::vector regs(simdRegNumber); @@ -73,46 +234,3 @@ TEST(RegistersPoolTests, get_all) { ASSERT_EQ(regPool->countFree(), simdRegNumber); } -TEST(RegistersPoolTests, move) { - if (!x64::mayiuse(x64::avx2)) { return; } - RegistersPool::Ptr regPool = RegistersPool::create({}); - RegistersPool::Reg reg{regPool}; - ASSERT_NO_THROW(static_cast(reg)); - ASSERT_EQ(regPool->countFree(), simdRegNumber - 1); - RegistersPool::Reg reg2 = std::move(reg); - ASSERT_ANY_THROW(static_cast(reg)); - ASSERT_NO_THROW(static_cast(reg2)); - ASSERT_EQ(regPool->countFree(), simdRegNumber - 1); -} - - -TEST(RegistersPoolTests, fixed_idx) { - if (!x64::mayiuse(x64::avx2)) { return; } - RegistersPool::Ptr regPool = RegistersPool::create({}); - using Ptr = std::shared_ptr>; - std::vector regs(simdRegNumber); - for (int c = 0; c < simdRegNumber; ++c) { - regs[c] = std::make_shared>(regPool, c); - ASSERT_EQ(regs[c]->getIdx(), c); - } - regs[0]->release(); - ASSERT_ANY_THROW(RegistersPool::Reg(regPool, 1)); - ASSERT_NO_THROW(RegistersPool::Reg(regPool, 0)); -} - -TEST(RegistersPoolTests, exclude) { - if (!x64::mayiuse(x64::avx2)) { return; } - static constexpr int excludedIdx = 0; - RegistersPool::Ptr regPool = RegistersPool::create({ - Xbyak::Xmm(excludedIdx) - }); - using Ptr = std::shared_ptr>; - std::vector regs(simdRegNumber - 1); - std::set idxsInUse; - for (int c = 0; c < simdRegNumber - 1; ++c) { - regs[c] = std::make_shared>(regPool); - idxsInUse.emplace(regs[c]->getIdx()); - } - ASSERT_EQ(regPool->countFree(), 0); - ASSERT_TRUE(idxsInUse.find(excludedIdx) == idxsInUse.end()); -} \ No newline at end of file From db819b7a13a71cd24cd0daccca1f4372c466a6ff Mon Sep 17 00:00:00 2001 From: Oleksii Zaderykhin Date: Wed, 14 Sep 2022 15:12:47 +0300 Subject: [PATCH 5/8] The release register on the move leakage was fixed. --- src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp | 1 + src/plugins/intel_cpu/tests/unit/registers_pool.cpp | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp b/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp index 89f354e612f9b4..c940cb0717eba6 100644 --- a/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp +++ b/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp @@ -55,6 +55,7 @@ class RegistersPool { Reg(const RegistersPool::Ptr& regPool, int requestedIdx) { initialize(regPool, requestedIdx); } ~Reg() { release(); } Reg& operator=(Reg&& other) noexcept { + release(); reg = other.reg; regPool = std::move(other.regPool); return *this; diff --git a/src/plugins/intel_cpu/tests/unit/registers_pool.cpp b/src/plugins/intel_cpu/tests/unit/registers_pool.cpp index 8848d2a8e4983f..692b1e155d513a 100644 --- a/src/plugins/intel_cpu/tests/unit/registers_pool.cpp +++ b/src/plugins/intel_cpu/tests/unit/registers_pool.cpp @@ -80,7 +80,12 @@ TYPED_TEST_P(RegPoolTest, move) { RegistersPool::Reg reg{regPool}; ASSERT_NO_THROW(static_cast(reg)); ASSERT_EQ(regPool->countFree(), this->regNumber - 1); - RegistersPool::Reg reg2 = std::move(reg); + RegistersPool::Reg reg2{regPool}; + ASSERT_EQ(regPool->countFree(), this->regNumber - 2); + auto regIdx = reg.getIdx(); + reg2 = std::move(reg); + ASSERT_EQ(reg2.getIdx(), regIdx); + ASSERT_EQ(regPool->countFree(), this->regNumber - 1); ASSERT_ANY_THROW(static_cast(reg)); ASSERT_NO_THROW(static_cast(reg2)); ASSERT_EQ(regPool->countFree(), this->regNumber - 1); From 8e893c45ca8a2b6139bdd10829f2b3e4a7ccc371 Mon Sep 17 00:00:00 2001 From: Oleksii Zaderykhin Date: Fri, 16 Sep 2022 12:48:22 +0300 Subject: [PATCH 6/8] The default constructor was added for the IsaRegistersPool class, which exclude some critical registers by default. --- .../src/nodes/kernels/registers_pool.hpp | 21 +++++++++++++++++-- .../intel_cpu/tests/unit/registers_pool.cpp | 10 +++++++-- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp b/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp index c940cb0717eba6..b94b30e5845a18 100644 --- a/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp +++ b/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp @@ -75,10 +75,11 @@ class RegistersPool { regPool.reset(); } } + bool isInitialized() const { return static_cast(regPool); } private: void ensureValid() const { - if (!regPool) { + if (!isInitialized()) { IE_THROW() << "RegistersPool::Reg is either not initialized or released"; } } @@ -184,6 +185,16 @@ class RegistersPool { virtual void returnOpmaskToPool(int idx) { IE_THROW() << "returnOpmaskToPool: The Opmask is not supported in current instruction set"; } virtual size_t countUnusedOpmask() const { IE_THROW() << "countUnusedOpmask: The Opmask is not supported in current instruction set"; } + RegistersPool(int simdRegistersNumber) + : simdSet(simdRegistersNumber) { + checkUniqueAndUpdate(); + generalSet.exclude(Xbyak::Reg64(Xbyak::Operand::RSP)); + generalSet.exclude(Xbyak::Reg64(Xbyak::Operand::RAX)); + generalSet.exclude(Xbyak::Reg64(Xbyak::Operand::RCX)); + generalSet.exclude(Xbyak::Reg64(Xbyak::Operand::RDI)); + generalSet.exclude(Xbyak::Reg64(Xbyak::Operand::RBP)); + } + RegistersPool(std::initializer_list regsToExclude, int simdRegistersNumber) : simdSet(simdRegistersNumber) { checkUniqueAndUpdate(); @@ -221,7 +232,7 @@ class RegistersPool { } template - void returnToPool(TReg reg) { + void returnToPool(const TReg& reg) { if (std::is_same::value || std::is_same::value || std::is_same::value) { simdSet.setAsUnused(reg.getIdx()); } else if (std::is_same::value || std::is_same::value || @@ -257,6 +268,10 @@ class IsaRegistersPool : public RegistersPool { template <> class IsaRegistersPool : public RegistersPool { public: + IsaRegistersPool() : RegistersPool(x64::cpu_isa_traits::n_vregs) { + opmaskSet.exclude(Xbyak::Opmask(0)); // the Opmask(0) has special meaning for some instructions, like gather instruction + } + IsaRegistersPool(std::initializer_list regsToExclude) : RegistersPool(regsToExclude, x64::cpu_isa_traits::n_vregs) { for (auto& reg : regsToExclude) { @@ -293,12 +308,14 @@ template <> class IsaRegistersPool : public IsaRegistersPool { public: IsaRegistersPool(std::initializer_list regsToExclude) : IsaRegistersPool(regsToExclude) {} + IsaRegistersPool() : IsaRegistersPool() {} }; template <> class IsaRegistersPool : public IsaRegistersPool { public: IsaRegistersPool(std::initializer_list regsToExclude) : IsaRegistersPool(regsToExclude) {} + IsaRegistersPool() : IsaRegistersPool() {} }; } // namespace intel_cpu diff --git a/src/plugins/intel_cpu/tests/unit/registers_pool.cpp b/src/plugins/intel_cpu/tests/unit/registers_pool.cpp index 692b1e155d513a..0215dfde370c19 100644 --- a/src/plugins/intel_cpu/tests/unit/registers_pool.cpp +++ b/src/plugins/intel_cpu/tests/unit/registers_pool.cpp @@ -4,6 +4,7 @@ #include #include "nodes/kernels/registers_pool.hpp" +#include "common/nstl.hpp" using namespace ov::intel_cpu; @@ -143,8 +144,13 @@ struct make_case { template struct make_combinations; +template struct index_sequence { }; +template struct make_index_sequence_impl : make_index_sequence_impl { }; +template struct make_index_sequence_impl <0, S...> { using type = index_sequence; }; +template using make_index_sequence = typename make_index_sequence_impl::type; + template -struct make_combinations> { +struct make_combinations> { using tuples = std::tuple::type...>; }; @@ -152,7 +158,7 @@ template using Combinations_t = typename make_combinations , - tbb::internal::make_index_sequence<(std::tuple_size::value) *(sizeof...(Params))>>::tuples; + make_index_sequence<(std::tuple_size::value) *(sizeof...(Params))>>::tuples; template struct TestTypesCombiner; From ade1c35a05d082ed7e973c47f2c651a624b008c4 Mon Sep 17 00:00:00 2001 From: Oleksii Zaderykhin Date: Wed, 21 Sep 2022 14:06:31 +0300 Subject: [PATCH 7/8] The implementation for is_any_of was added and used in the static_assert. The std::is_base_of and Xbyak::Mmx was used to identify vector registers. The factory method was added for RegistersPool with isa non template parameter. --- .../src/nodes/kernels/registers_pool.hpp | 57 ++++++++++++------- src/plugins/intel_cpu/src/utils/cpu_utils.hpp | 13 +++++ 2 files changed, 49 insertions(+), 21 deletions(-) diff --git a/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp b/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp index b94b30e5845a18..8b51ec3c252171 100644 --- a/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp +++ b/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp @@ -7,6 +7,7 @@ #include "cpu/x64/jit_generator.hpp" #include #include "ie_common.h" +#include "utils/cpu_utils.hpp" #include namespace ov { @@ -44,13 +45,7 @@ class RegistersPool { class Reg { friend class RegistersPool; public: - Reg() { - static_assert(std::is_same::value || std::is_same::value || std::is_same::value || - std::is_same::value || std::is_same::value || - std::is_same::value || std::is_same::value || - std::is_same::value, - "The type is not supported for the RegistersPool::Reg template"); - } + Reg() {} Reg(const RegistersPool::Ptr& regPool) { initialize(regPool); } Reg(const RegistersPool::Ptr& regPool, int requestedIdx) { initialize(regPool, requestedIdx); } ~Reg() { release(); } @@ -85,6 +80,10 @@ class RegistersPool { } void initialize(const RegistersPool::Ptr& pool, int requestedIdx = anyIdx) { + static_assert(is_any_of::value, + "Unsupported TReg by RegistersPool::Reg. Please, use the following Xbyak registers either " + "Reg8, Reg16, Reg32, Reg64, Xmm, Ymm, Zmm or Opmask"); release(); reg = TReg(pool->template getFree(requestedIdx)); regPool = pool; @@ -102,9 +101,15 @@ class RegistersPool { template static Ptr create(std::initializer_list regsToExclude); + static Ptr create(x64::cpu_isa_t isa, std::initializer_list regsToExclude); + template size_t countFree() const { - if (std::is_same::value || std::is_same::value || std::is_same::value) { + static_assert(is_any_of::value, + "Unsupported TReg by RegistersPool::Reg. Please, use the following Xbyak registers either " + "Reg8, Reg16, Reg32, Reg64, Xmm, Ymm, Zmm or Opmask"); + if (std::is_base_of::value) { return simdSet.countUnused(); } else if (std::is_same::value || std::is_same::value || std::is_same::value || std::is_same::value) { @@ -211,13 +216,7 @@ class RegistersPool { private: template int getFree(int requestedIdx) { - static_assert(std::is_same::value || std::is_same::value || std::is_same::value || - std::is_same::value || std::is_same::value || - std::is_same::value || std::is_same::value || - std::is_same::value, - "Unsupported TReg by RegistersPool. Please, use the following Xbyak registers either " - "Reg8, Reg16, Reg32, Reg64, Xmm, Ymm, Zmm or Opmask"); - if (std::is_same::value || std::is_same::value || std::is_same::value) { + if (std::is_base_of::value) { auto idx = simdSet.getUnused(requestedIdx); simdSet.setAsUsed(idx); return idx; @@ -233,7 +232,7 @@ class RegistersPool { template void returnToPool(const TReg& reg) { - if (std::is_same::value || std::is_same::value || std::is_same::value) { + if (std::is_base_of::value) { simdSet.setAsUnused(reg.getIdx()); } else if (std::is_same::value || std::is_same::value || std::is_same::value || std::is_same::value) { @@ -299,11 +298,6 @@ class IsaRegistersPool : public RegistersPool { PhysicalSet opmaskSet {8}; }; -template -RegistersPool::Ptr RegistersPool::create(std::initializer_list regsToExclude) { - return std::make_shared>(regsToExclude); -} - template <> class IsaRegistersPool : public IsaRegistersPool { public: @@ -318,5 +312,26 @@ class IsaRegistersPool : public IsaRegistersPool() {} }; +template +RegistersPool::Ptr RegistersPool::create(std::initializer_list regsToExclude) { + return std::make_shared>(regsToExclude); +} + +inline +RegistersPool::Ptr RegistersPool::create(x64::cpu_isa_t isa, std::initializer_list regsToExclude) { +#define ISA_SWITCH_CASE(isa) case isa: return std::make_shared>(regsToExclude); + switch (isa) { + ISA_SWITCH_CASE(x64::sse41) + ISA_SWITCH_CASE(x64::avx) + ISA_SWITCH_CASE(x64::avx2) + ISA_SWITCH_CASE(x64::avx2_vnni) + ISA_SWITCH_CASE(x64::avx512_core) + ISA_SWITCH_CASE(x64::avx512_core_vnni) + ISA_SWITCH_CASE(x64::avx512_core_bf16) + } + IE_THROW() << "Invalid isa argument in RegistersPool::create()"; +#undef ISA_SWITCH_CASE +} + } // namespace intel_cpu } // namespace ov diff --git a/src/plugins/intel_cpu/src/utils/cpu_utils.hpp b/src/plugins/intel_cpu/src/utils/cpu_utils.hpp index 0bea750573b3d6..25963b1521a153 100644 --- a/src/plugins/intel_cpu/src/utils/cpu_utils.hpp +++ b/src/plugins/intel_cpu/src/utils/cpu_utils.hpp @@ -10,10 +10,23 @@ #include "ie_common.h" #include "ie_layouts.h" +#include "general_utils.h" namespace ov { namespace intel_cpu { +// helper struct to tell wheter type T is any of given types U... +// termination case when U... is empty -> return std::false_type +template +struct is_any_of : public std::false_type {}; + +// helper struct to tell whether type is any of given types (U, Rest...) +// recurrence case when at least one type U is present -> returns std::true_type if std::same::value is true, +// otherwise call is_any_of recurrently +template +struct is_any_of + : public std::conditional::value, std::true_type, is_any_of>::type {}; + /** * @brief Returns normalized by size dims where missing dimensions are filled with units from the beginning * Example: dims = {2, 3, 5}; ndims = 5; result = {1, 1, 2, 3, 5} From b242cb5b4aface2f42535b917ff39a2f6da56844 Mon Sep 17 00:00:00 2001 From: Oleksii Zaderykhin Date: Thu, 29 Sep 2022 11:26:07 +0300 Subject: [PATCH 8/8] The -Wswitch and -Wunused-value errors was fixed for macos build. --- .../src/nodes/kernels/registers_pool.hpp | 12 ++++++++++++ .../intel_cpu/tests/unit/registers_pool.cpp | 18 +++++++++--------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp b/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp index 8b51ec3c252171..35fc795b8ee05a 100644 --- a/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp +++ b/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp @@ -328,6 +328,18 @@ RegistersPool::Ptr RegistersPool::create(x64::cpu_isa_t isa, std::initializer_li ISA_SWITCH_CASE(x64::avx512_core) ISA_SWITCH_CASE(x64::avx512_core_vnni) ISA_SWITCH_CASE(x64::avx512_core_bf16) + case x64::avx_vnni: return std::make_shared>(regsToExclude); + case x64::avx512_core_bf16_ymm: return std::make_shared>(regsToExclude); + case x64::avx512_core_bf16_amx_int8: return std::make_shared>(regsToExclude); + case x64::avx512_core_bf16_amx_bf16: return std::make_shared>(regsToExclude); + case x64::avx512_core_amx: return std::make_shared>(regsToExclude); + case x64::avx512_vpopcnt: return std::make_shared>(regsToExclude); + case x64::isa_any: + case x64::amx_tile: + case x64::amx_int8: + case x64::amx_bf16: + case x64::isa_all: + IE_THROW() << "Invalid isa argument in RegistersPool::create()"; } IE_THROW() << "Invalid isa argument in RegistersPool::create()"; #undef ISA_SWITCH_CASE diff --git a/src/plugins/intel_cpu/tests/unit/registers_pool.cpp b/src/plugins/intel_cpu/tests/unit/registers_pool.cpp index 0215dfde370c19..fa6292c2773441 100644 --- a/src/plugins/intel_cpu/tests/unit/registers_pool.cpp +++ b/src/plugins/intel_cpu/tests/unit/registers_pool.cpp @@ -32,7 +32,7 @@ TYPED_TEST_P(RegPoolTest, get_return_by_scope) { ASSERT_EQ(regPool->countFree(), this->regNumber); { RegistersPool::Reg reg{regPool}; - ASSERT_NO_THROW(static_cast(reg)); + ASSERT_NO_THROW([[maybe_unused]] auto val = static_cast(reg)); ASSERT_EQ(regPool->countFree(), this->regNumber - 1); } ASSERT_EQ(regPool->countFree(), this->regNumber); @@ -43,13 +43,13 @@ TYPED_TEST_P(RegPoolTest, get_return_by_method) { RegistersPool::Ptr regPool = RegistersPool::create({}); ASSERT_EQ(regPool->countFree(), this->regNumber); RegistersPool::Reg reg{regPool}; - ASSERT_NO_THROW(static_cast(reg)); + ASSERT_NO_THROW([[maybe_unused]] auto val = static_cast(reg)); ASSERT_EQ(regPool->countFree(), this->regNumber - 1); reg.release(); - ASSERT_ANY_THROW(static_cast(reg)); + ASSERT_ANY_THROW([[maybe_unused]] auto val = static_cast(reg)); ASSERT_EQ(regPool->countFree(), this->regNumber); reg = RegistersPool::Reg{regPool}; - ASSERT_NO_THROW(static_cast(reg)); + ASSERT_NO_THROW([[maybe_unused]] auto val = static_cast(reg)); ASSERT_EQ(regPool->countFree(), this->regNumber - 1); } @@ -58,7 +58,7 @@ TYPED_TEST_P(RegPoolTest, default_ctor) { RegistersPool::Ptr regPool = RegistersPool::create({}); RegistersPool::Reg reg; ASSERT_EQ(regPool->countFree(), this->regNumber); - ASSERT_ANY_THROW(static_cast(reg)); + ASSERT_ANY_THROW([[maybe_unused]] auto val = static_cast(reg)); } TYPED_TEST_P(RegPoolTest, get_all) { @@ -79,7 +79,7 @@ TYPED_TEST_P(RegPoolTest, move) { using XbyakRegT = typename TypeParam::RegT; RegistersPool::Ptr regPool = RegistersPool::create({}); RegistersPool::Reg reg{regPool}; - ASSERT_NO_THROW(static_cast(reg)); + ASSERT_NO_THROW([[maybe_unused]] auto val = static_cast(reg)); ASSERT_EQ(regPool->countFree(), this->regNumber - 1); RegistersPool::Reg reg2{regPool}; ASSERT_EQ(regPool->countFree(), this->regNumber - 2); @@ -87,8 +87,8 @@ TYPED_TEST_P(RegPoolTest, move) { reg2 = std::move(reg); ASSERT_EQ(reg2.getIdx(), regIdx); ASSERT_EQ(regPool->countFree(), this->regNumber - 1); - ASSERT_ANY_THROW(static_cast(reg)); - ASSERT_NO_THROW(static_cast(reg2)); + ASSERT_ANY_THROW([[maybe_unused]] auto val = static_cast(reg)); + ASSERT_NO_THROW([[maybe_unused]] auto val = static_cast(reg2)); ASSERT_EQ(regPool->countFree(), this->regNumber - 1); } @@ -212,7 +212,7 @@ TEST(RegistersPoolTests, simd_and_general) { ASSERT_EQ(regPool->countFree(), freeGeneralRegNumber); { RegistersPool::Reg reg{regPool}; - ASSERT_NO_THROW(static_cast(reg)); + ASSERT_NO_THROW([[maybe_unused]] auto val = static_cast(reg)); ASSERT_EQ(regPool->countFree(), simdRegNumber - 1); ASSERT_EQ(regPool->countFree(), simdRegNumber - 1); ASSERT_EQ(regPool->countFree(), freeGeneralRegNumber);