diff --git a/cpp/csp/python/Conversions.h b/cpp/csp/python/Conversions.h index 8ac7b3e73..a5113fb48 100644 --- a/cpp/csp/python/Conversions.h +++ b/cpp/csp/python/Conversions.h @@ -446,10 +446,7 @@ inline PyObject * toPython( const CspEnum & e, const CspType & type ) auto & enumType = static_cast( type ); const auto * emeta = static_cast( enumType.meta().get() ); - PyObject * obj = emeta -> pyMeta() -> toPyEnum( e ); - if( !obj ) [[unlikely]] - CSP_THROW( ValueError, e.value() << " is not a valid value on csp.enum type " << emeta -> name() ); - return obj; + return emeta -> toPyEnum( e.value() ); } template<> @@ -457,11 +454,13 @@ inline CspEnum fromPython( PyObject * o, const CspType & type ) { assert( type.type() == CspType::Type::ENUM ); - if( !PyType_IsSubtype( Py_TYPE( o ), &PyCspEnum::PyType ) || - static_cast( o ) -> meta() != static_cast( type ).meta().get() ) - CSP_THROW( TypeError, "Invalid enum type, expected enum type " << static_cast( type ).meta() -> name() << " got " << Py_TYPE( o ) -> tp_name ); - - return static_cast( o ) -> enum_; + auto & enumType = static_cast( type ); + const auto * emeta = static_cast( enumType.meta().get() ); + + if( !PyObject_IsInstance( o, ( PyObject * ) emeta -> pyType().get() ) ) + CSP_THROW( TypeError, "Invalid enum type, expected enum type " << emeta -> pyType() -> tp_name << " got " << Py_TYPE( o ) -> tp_name ); + + return static_cast( type ).meta() -> create( PyLong_AsLong( o ) ); } //TimeDelta diff --git a/cpp/csp/python/CspTypeFactory.cpp b/cpp/csp/python/CspTypeFactory.cpp index a1bfd0273..27e6e7bd2 100644 --- a/cpp/csp/python/CspTypeFactory.cpp +++ b/cpp/csp/python/CspTypeFactory.cpp @@ -7,10 +7,23 @@ namespace csp::python { +CspTypeFactory::CspTypeFactory() +{ + PyObject *enum_mod = PyImport_ImportModule( "enum" ); + m_intEnumPyType = ( PyTypeObject * ) PyObject_GetAttrString( enum_mod, "IntEnum" ); +} + CspTypeFactory & CspTypeFactory::instance() { - static CspTypeFactory s_instance; - return s_instance; + //We let this leak since some csp types ( ie CspEnum ) can hold a ref to DialectCspEnumMeta which holds Ptrs + //to python objects, which cant be destroyed statically after python interpreter is shutdown + static CspTypeFactory * s_instance = new CspTypeFactory(); + return *s_instance; +} + +bool CspTypeFactory::isCspEnumPyType( PyTypeObject * pyType ) +{ + return PyType_IsSubtype( pyType, m_intEnumPyType ); } CspTypePtr & CspTypeFactory::typeFromPyType( PyObject * pyTypeObj ) @@ -59,9 +72,9 @@ CspTypePtr & CspTypeFactory::typeFromPyType( PyObject * pyTypeObj ) auto meta = ( ( PyStructMeta * ) pyType ) -> structMeta; rv.first -> second = std::make_shared( meta ); } - else if( PyType_IsSubtype( pyType, &PyCspEnum::PyType ) ) + else if( isCspEnumPyType( pyType ) ) { - auto meta = ( ( PyCspEnumMeta * ) pyType ) -> enumMeta; + auto meta = createCspEnumMetaFromIntEnum( PyTypeObjectPtr::incref( pyType ) ); rv.first -> second = std::make_shared( meta ); } else if( pyType == PyDateTimeAPI -> DateTimeType ) @@ -88,4 +101,30 @@ void CspTypeFactory::removeCachedType( PyTypeObject * pyType ) m_cache.erase( pyType ); } +std::shared_ptr CspTypeFactory::createCspEnumMetaFromIntEnum( PyTypeObjectPtr pyIntEnumType ) +{ + CspEnumMeta::ValueDef metadef; + + PyObjectPtr iter = PyObjectPtr::check( PyObject_GetIter( ( PyObject * ) pyIntEnumType.get() ) ); + PyObject * member; + while( ( member = PyIter_Next( iter.get() ) ) != NULL ) + { + PyObjectPtr name = PyObjectPtr::check( PyObject_GetAttrString( member, "name" ) ); + + const char * namestr = PyUnicode_AsUTF8( name.get() ); + if( !namestr ) + CSP_THROW( PythonPassthrough, "" ); + + if( !PyLong_Check( member ) ) + CSP_THROW( TypeError, "enum key " << namestr << " expected an integer got " << PyObjectPtr::incref( member ) ); + + int64_t value = fromPython( member ); + metadef[ namestr ] = value; + + Py_DECREF( member ); + } + + return std::make_shared( pyIntEnumType, pyIntEnumType -> tp_name, metadef ); +} + } diff --git a/cpp/csp/python/CspTypeFactory.h b/cpp/csp/python/CspTypeFactory.h index dd44d06b7..c80fc067a 100644 --- a/cpp/csp/python/CspTypeFactory.h +++ b/cpp/csp/python/CspTypeFactory.h @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -17,9 +18,17 @@ class CSPTYPESIMPL_EXPORT CspTypeFactory CspTypePtr & typeFromPyType( PyObject * ); void removeCachedType( PyTypeObject * ); + bool isCspEnumPyType( PyTypeObject * pyType ); + private: using Cache = std::unordered_map; + + std::shared_ptr createCspEnumMetaFromIntEnum( PyTypeObjectPtr pyIntEnumType ); + + CspTypeFactory(); Cache m_cache; + + PyTypeObject * m_intEnumPyType; }; } diff --git a/cpp/csp/python/PyCspEnum.cpp b/cpp/csp/python/PyCspEnum.cpp index 286598b1a..dea634568 100644 --- a/cpp/csp/python/PyCspEnum.cpp +++ b/cpp/csp/python/PyCspEnum.cpp @@ -12,269 +12,24 @@ DialectCspEnumMeta::DialectCspEnumMeta( PyTypeObjectPtr pyType, const std::strin CspEnumMeta( name, def ), m_pyType( pyType ) { -} - -/* -MetaClass Madness NOTES!!! -- see PyStruct.cpp for note, same idea -*/ - -static PyObject * PyCspEnumMeta_new( PyTypeObject *subtype, PyObject *args, PyObject *kwds ) -{ - CSP_BEGIN_METHOD; - - PyObject * pyname; - PyObject * bases; - PyObject * dict; - if( !PyArg_ParseTuple( args, "UO!O!", - &pyname, - &PyTuple_Type, &bases, - &PyDict_Type, &dict ) ) - CSP_THROW( PythonPassthrough, "" ); - - //subtype is python defined CspEnumMeta class - PyCspEnumMeta * pymeta = ( PyCspEnumMeta * ) PyType_Type.tp_new( subtype, args, kwds ); - - //Note that we call ctor without parents so as not to 0-init the base POD PyTypeObject class after its been initialized - new ( pymeta ) PyCspEnumMeta; - - //this would be the CspEnum class on python side, it doesnt create any metadata for itself - if( pymeta -> ht_type.tp_base == &PyCspEnum::PyType ) - return ( PyObject * ) pymeta; - - std::string name = PyUnicode_AsUTF8( pyname ); - - PyObject * metadata = PyDict_GetItemString( dict, "__metadata__" ); - if( !metadata ) - CSP_THROW( KeyError, "CspEnumMeta missing __metadata__" ); - - CspEnumMeta::ValueDef def; - - { - PyObject *key, *value; - Py_ssize_t pos = 0; - while( PyDict_Next( metadata, &pos, &key, &value ) ) - { - const char * keystr = PyUnicode_AsUTF8( key ); - if( !keystr ) - CSP_THROW( PythonPassthrough, "" ); - - if( !PyLong_Check( value ) ) - CSP_THROW( TypeError, "csp.Enum key " << keystr << " expected an integer got " << PyObjectPtr::incref( value ) ); - - def[ keystr ] = fromPython( value ); - } - } - - //back reference to the csp enum type that will be accessible on the csp enum -> meta() - //intentionally dont incref here to break the circular dep of type -> shared_ptr on CspEnumMeta - PyTypeObjectPtr typePtr = PyTypeObjectPtr::own( ( PyTypeObject * ) pymeta ); - auto enumMeta = std::make_shared( typePtr, name, def ); - - pymeta -> enumMeta = enumMeta; - //pre-create instances - pymeta -> enumsByName = PyObjectPtr::own( PyDict_New() ); - pymeta -> enumsByValue = PyObjectPtr::own( PyDict_New() ); + m_enumsByName = PyObjectPtr::own( PyDict_New() ); - for( auto & [ key, value ] : def ) + PyObjectPtr iter = PyObjectPtr::check( PyObject_GetIter( ( PyObject * ) pyType.get() ) ); + PyObject * member; + while( ( member = PyIter_Next( iter.get() ) ) != NULL ) { - PyCspEnum * enum_ = ( PyCspEnum * ) ( (PyTypeObject * ) pymeta ) -> tp_alloc( (PyTypeObject * ) pymeta, 0 ); + PyObjectPtr name = PyObjectPtr::check( PyObject_GetAttrString( member, "name" ) ); - new( enum_ ) PyCspEnum( enumMeta -> create( value ) ); - enum_ -> enumName = PyObjectPtr::own( toPython( key ) ); - enum_ -> enumValue = PyObjectPtr::own( toPython( value ) ); + if( !PyLong_Check( member ) ) + CSP_THROW( TypeError, "enum key " << name << " expected an integer got " << PyObjectPtr::incref( member ) ); + + int64_t value = fromPython( member ); + m_enumsByCValue[ value ] = PyObjectPtr::incref( member ); - pymeta -> enumsByCValue[ value ] = PyObjectPtr::incref( enum_ ); - - if( PyDict_SetItem( pymeta -> enumsByName.get(), enum_ -> enumName.get(), enum_ ) < 0 ) - CSP_THROW( PythonPassthrough, "" ); - - if( PyDict_SetItem( pymeta -> enumsByValue.get(), enum_ -> enumValue.get(), enum_ ) < 0 ) - CSP_THROW( PythonPassthrough, "" ); - - //We also have to update the items in the actual type's dict so FooEnum.A is a PyCspEnum! - if( PyDict_SetItem( ( ( PyTypeObject * ) pymeta ) -> tp_dict, enum_ -> enumName.get(), enum_ ) < 0 ) + if( PyDict_SetItem( m_enumsByName.get(), name.get(), member ) < 0 ) CSP_THROW( PythonPassthrough, "" ); } - - return ( PyObject * ) pymeta; - CSP_RETURN_NULL; -} - -PyObject * PyCspEnumMeta::toPyEnum( CspEnum e ) const -{ - auto it = enumsByCValue.find( e.value() ); - if( it == enumsByCValue.end() ) - return nullptr; - - PyObject * rv = it -> second.get(); - Py_INCREF( rv ); - return rv; -} - -void PyCspEnumMeta_dealloc( PyCspEnumMeta * m ) -{ - CspTypeFactory::instance().removeCachedType( reinterpret_cast( m ) ); - m -> ~PyCspEnumMeta(); - PyCspEnumMeta::PyType.tp_free( m ); -} - -PyObject * PyCspEnumMeta_subscript( PyCspEnumMeta * self, PyObject * key ) -{ - CSP_BEGIN_METHOD; - - PyObject * obj = PyDict_GetItem( self -> enumsByName.get(), key ); - - if( !obj ) - CSP_THROW( ValueError, PyObjectPtr::incref( key ) << " is not a valid name on csp.enum type " << ( ( PyTypeObject * ) self ) -> tp_name ); - - Py_INCREF( obj ); - return obj; - CSP_RETURN_NULL; -} - - -static PyMappingMethods PyCspEnumMeta_MappingMethods = { - 0, /*mp_length */ - (binaryfunc) PyCspEnumMeta_subscript, /*mp_subscript */ -}; - -PyTypeObject PyCspEnumMeta::PyType = { - PyVarObject_HEAD_INIT(nullptr, 0) - "_cspimpl.PyCspEnumMeta", /* tp_name */ - sizeof(PyCspEnumMeta), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor) PyCspEnumMeta_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - &PyCspEnumMeta_MappingMethods, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | - Py_TPFLAGS_BASETYPE | Py_TPFLAGS_TYPE_SUBCLASS, /* tp_flags */ - "csp enum metaclass", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - &PyType_Type, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /*tp_init*/ - 0, /* tp_alloc */ - (newfunc) PyCspEnumMeta_new,/* tp_new */ - PyObject_GC_Del, /* tp_free */ -}; - - -//PyCspEnum -void PyCspEnum_dealloc( PyCspEnum * self ) -{ - self -> ~PyCspEnum(); - PyCspEnum::PyType.tp_free( self ); -} - -PyObject * PyCspEnum_new( PyTypeObject * type, PyObject *args, PyObject *kwds ) -{ - CSP_BEGIN_METHOD; - - PyObject * pyvalue; - if( !PyArg_ParseTuple( args, "O", &pyvalue ) ) - CSP_THROW( PythonPassthrough, "" ); - - auto pymeta = (PyCspEnumMeta * ) type; - PyObject * obj = nullptr; - if( PyLong_Check( pyvalue ) ) - obj = PyDict_GetItem( pymeta -> enumsByValue.get(), pyvalue ); - else if( PyUnicode_Check( pyvalue ) ) - obj = PyDict_GetItem( pymeta -> enumsByName.get(), pyvalue ); - - if( !obj ) - CSP_THROW( ValueError, PyObjectPtr::incref( pyvalue ) << " is not a valid value on csp.enum type " << type -> tp_name ); - - Py_INCREF( obj ); - return obj; - CSP_RETURN_NULL; } -PyObject * PyCspEnum_name( PyCspEnum * self, void * ) -{ - Py_INCREF( self -> enumName.get() ); - return self -> enumName.get(); -} - -PyObject * PyCspEnum_value( PyCspEnum * self, void * ) -{ - Py_INCREF( self -> enumValue.get() ); - return self -> enumValue.get(); -} - -static PyGetSetDef PyCspEnum_getset[] = { - { ( char * ) "name", (getter) PyCspEnum_name, 0, ( char * ) "string name of the enum instance", 0 }, - { ( char * ) "value", (getter) PyCspEnum_value, 0, ( char * ) "long value of the enum instance", 0 }, - { NULL } -}; - -PyTypeObject PyCspEnum::PyType = { - PyVarObject_HEAD_INIT(nullptr, 0) - "_cspimpl.PyCspEnum", /* tp_name */ - sizeof(PyCspEnum), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor) PyCspEnum_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | - Py_TPFLAGS_BASETYPE, /* tp_flags */ - "csp enum", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - PyCspEnum_getset, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - (newfunc) PyCspEnum_new, /* tp_new */ - 0, /* tp_free */ -}; - -REGISTER_TYPE_INIT( &PyCspEnumMeta::PyType, "PyCspEnumMeta" ) -REGISTER_TYPE_INIT( &PyCspEnum::PyType, "PyCspEnum" ) - } diff --git a/cpp/csp/python/PyCspEnum.h b/cpp/csp/python/PyCspEnum.h index 0da9e9dbc..051f65ff5 100644 --- a/cpp/csp/python/PyCspEnum.h +++ b/cpp/csp/python/PyCspEnum.h @@ -10,23 +10,6 @@ namespace csp::python { -//This is the base class of csp.Enum -struct CSPTYPESIMPL_EXPORT PyCspEnumMeta : public PyHeapTypeObject -{ - //convert to PyObject ( new ref ) - PyObject * toPyEnum( CspEnum e ) const; - - std::shared_ptr enumMeta; - - PyObjectPtr enumsByName; - PyObjectPtr enumsByValue; - - //for fast toPython calls - std::unordered_map enumsByCValue; - - static PyTypeObject PyType; -}; - //TODO Windows - need to figure out why adding DLL_PUBLIC to this class leads to weird compilation errors on CspEnumMeta's unordered_map... //This is an extension of csp::CspEnumMeta for python dialect, we need it in order to @@ -40,26 +23,25 @@ class CSPTYPESIMPL_EXPORT DialectCspEnumMeta : public CspEnumMeta const PyTypeObjectPtr & pyType() const { return m_pyType; } - const PyCspEnumMeta * pyMeta() const { return ( const PyCspEnumMeta * ) m_pyType.get(); } - + //returns new ref + PyObject * toPyEnum( int64_t value ) const + { + auto it = m_enumsByCValue.find( value ); + if( it == m_enumsByCValue.end() ) + return nullptr; + + PyObject * rv = it -> second.get(); + Py_INCREF( rv ); + return rv; + } + private: PyTypeObjectPtr m_pyType; -}; - -struct CSPTYPESIMPL_EXPORT PyCspEnum : public PyObject -{ - PyCspEnum( const CspEnum & e ) : enum_( e ) {} - ~PyCspEnum() {} + PyObjectPtr m_enumsByName; - CspEnum enum_; - PyObjectPtr enumName; - PyObjectPtr enumValue; - - PyCspEnumMeta * pyMeta() { return ( PyCspEnumMeta * ) ob_type; }; - const DialectCspEnumMeta * meta() { return static_cast( pyMeta() -> enumMeta.get() ); } - - static PyTypeObject PyType; + //for fast toPython calls + std::unordered_map m_enumsByCValue; }; } diff --git a/cpp/csp/python/PyInputProxy.cpp b/cpp/csp/python/PyInputProxy.cpp index cb936962c..f89666275 100644 --- a/cpp/csp/python/PyInputProxy.cpp +++ b/cpp/csp/python/PyInputProxy.cpp @@ -239,8 +239,8 @@ PyObject *PyInputProxy::valuesAt( ValueType valueType, PyObject *startIndexArg, PyObject *endIndexPolicyArg ) const { int32_t startIndex, endIndex; - auto startPolicy = static_cast( static_cast( startIndexPolicyArg ) -> enum_ ); - auto endPolicy = static_cast( static_cast( endIndexPolicyArg ) -> enum_ ); + auto startPolicy = csp::autogen::TimeIndexPolicy::create( startIndexPolicyArg ); + auto endPolicy = csp::autogen::TimeIndexPolicy::create( endIndexPolicyArg ); if( startIndexArg == Py_None ) startIndex = 1 - ts() -> numTicks(); @@ -513,9 +513,9 @@ static inline PyObject * PyInputProxy_values_at_impl( ValueType valueType, PyInp PyObject * endIndexArg; PyObject * startExclusiveArg; PyObject * endExclusiveArg; - if( !PyArg_ParseTuple( args, "OOO!O!", &startIndexArg, &endIndexArg, - &PyCspEnum::PyType, &startExclusiveArg, - &PyCspEnum::PyType, &endExclusiveArg ) ) + if( !PyArg_ParseTuple( args, "OOOO", &startIndexArg, &endIndexArg, + &startExclusiveArg, + &endExclusiveArg ) ) CSP_THROW( RuntimeException, "Invalid arguments passed to values_at" ); return proxy -> valuesAt( valueType, startIndexArg, endIndexArg, startExclusiveArg, endExclusiveArg ); diff --git a/cpp/csp/python/PyStructToDict.cpp b/cpp/csp/python/PyStructToDict.cpp index faf354312..aa7c02f4a 100644 --- a/cpp/csp/python/PyStructToDict.cpp +++ b/cpp/csp/python/PyStructToDict.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -99,7 +100,6 @@ inline PyObjectPtr StructToDictHelper::parseCspToPython( const T& val, const Csp return PyObjectPtr::own( toPython( val ) ); } -// Helper function to convert Enums into python object recursively template<> inline PyObjectPtr StructToDictHelper::parseCspToPython( const CspEnum& val, const CspType& typ ) { @@ -225,11 +225,21 @@ PyObjectPtr StructToDictHelper::parsePyObject( PyObject * value, bool is_recursi INIT_PYDATETIME; if( ( value == Py_None ) || // None check - ( PyBool_Check( value ) || PyLong_Check( value ) || PyFloat_Check( value ) ) || // Primitives check + ( PyBool_Check( value ) || PyFloat_Check( value ) ) || // Primitives check ( PyUnicode_Check( value ) || PyBytes_Check( value ) ) || // Unicode/bytes check ( PyTime_CheckExact( value ) || PyDate_CheckExact( value ) || PyDateTime_CheckExact( value ) || PyDelta_CheckExact( value ) ) ) // Datetime check return PyObjectPtr::incref( value ); + else if( PyLong_Check( value ) ) //Sepearate LongCheck to account for enums + { + if( CspTypeFactory::instance().isCspEnumPyType( Py_TYPE( value ) ) ) + { + if( m_preserveEnums ) + return PyObjectPtr::incref( value ); + return PyObjectPtr::check( PyObject_GetAttrString( value, "name" ) ); + } + return PyObjectPtr::incref( value ); + } else if( PyTuple_Check( value ) || PyList_Check( value ) || PySet_Check( value ) ) return parsePySequence( value ); else if( PyDict_Check( value ) ) @@ -239,12 +249,6 @@ PyObjectPtr StructToDictHelper::parsePyObject( PyObject * value, bool is_recursi auto struct_ptr = static_cast( value ) -> struct_; return parseStructToDictRecursive( struct_ptr ); } - else if( PyType_IsSubtype( Py_TYPE( value ), &PyCspEnum::PyType ) ) - { - auto enum_ptr = static_cast( value ) -> enum_; - PyObject* py_type = ( PyObject* ) ( Py_TYPE( value ) ); - return parseCspToPython( enum_ptr, *pyTypeAsCspType( py_type ) ); - } else { if( ( m_callable == nullptr ) || is_recursing ) diff --git a/cpp/csp/python/PyStructToJson.cpp b/cpp/csp/python/PyStructToJson.cpp index 65cfac308..da572b2b7 100644 --- a/cpp/csp/python/PyStructToJson.cpp +++ b/cpp/csp/python/PyStructToJson.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -209,7 +210,7 @@ rapidjson::Value pyDictKeyToName( PyObject * py_key, rapidjson::Document& doc, P // JSON encoding requires all names to be strings so convert them to strings static thread_local PyTypeObjectPtr s_tl_enum_type; - // Get the enum type on the first call and save it for future use + // Get the enum type on the first call and save it for future use. Note this is for regular enum, not IntEnum which would be a CspEnum if( s_tl_enum_type.get() == nullptr ) [[unlikely]] { // Import enum module to extract the Enum type @@ -223,7 +224,7 @@ rapidjson::Value pyDictKeyToName( PyObject * py_key, rapidjson::Document& doc, P CSP_THROW( RuntimeException, "Unable to import enum module from the python standard library" ); } } - + rapidjson::Value val; if( py_key == Py_None ) { @@ -244,6 +245,9 @@ rapidjson::Value pyDictKeyToName( PyObject * py_key, rapidjson::Document& doc, P } else if( PyLong_Check( py_key ) ) { + if( CspTypeFactory::instance().isCspEnumPyType( Py_TYPE( py_key ) ) ) + return pyDictKeyToName( PyObjectPtr::own( PyObject_GetAttrString( py_key, "name" ) ).get(), doc, callable ); + auto key = PyLong_AsLong( py_key ); val.SetString( std::to_string( key ), doc.GetAllocator() ); } @@ -280,11 +284,6 @@ rapidjson::Value pyDictKeyToName( PyObject * py_key, rapidjson::Document& doc, P auto v = fromPython( py_key ); val = toJson( v, CspType( CspType::Type::DATETIME ), doc, callable ); } - else if( PyType_IsSubtype( Py_TYPE( py_key ), &PyCspEnum::PyType ) ) - { - auto enum_ptr = static_cast( py_key ) -> enum_; - val = toJson( enum_ptr, CspType( CspType::Type::ENUM ), doc, callable ); - } else if( PyType_IsSubtype( Py_TYPE( py_key ), s_tl_enum_type.get() ) ) { // Use the `name` attribute of the enum for the string representation @@ -365,6 +364,8 @@ rapidjson::Value pyObjectToJson( PyObject * value, rapidjson::Document& doc, PyO } else if( PyLong_Check( value ) ) { + if( CspTypeFactory::instance().isCspEnumPyType( Py_TYPE( value ) ) ) + return pyObjectToJson( PyObjectPtr::own( PyObject_GetAttrString( value, "name" ) ).get(), doc, callable, true ); return rapidjson::Value( fromPython( value ) ); } else if( PyFloat_Check( value ) ) @@ -427,11 +428,6 @@ rapidjson::Value pyObjectToJson( PyObject * value, rapidjson::Document& doc, PyO auto struct_ptr = static_cast( value ) -> struct_; return toJsonRecursive( struct_ptr, doc, callable ); } - else if( PyType_IsSubtype( Py_TYPE( value ), &PyCspEnum::PyType ) ) - { - auto enum_ptr = static_cast( value ) -> enum_; - return toJson( enum_ptr, CspType( CspType::Type::ENUM ), doc, callable ); - } else { if( is_recursing ) diff --git a/csp/build/csp_autogen.py b/csp/build/csp_autogen.py index 0ea9157bf..ed467b8c4 100644 --- a/csp/build/csp_autogen.py +++ b/csp/build/csp_autogen.py @@ -5,6 +5,7 @@ import os.path import sys import types +from enum import IntEnum # We need to patch mock modules into sys.modules to avoid pulling in csp/__init__.py and all of its baggage, which include imports of # _cspimpl, which would be a circular dep @@ -85,7 +86,7 @@ def __init__(self, module_name: str, output_filename: str, namespace: str, gener if issubclass(v, Struct) and v is not Struct: self._struct_types.append(v) - elif issubclass(v, Enum) and v is not Enum: + elif issubclass(v, Enum) and v is not Enum or issubclass(v, IntEnum) and v is not IntEnum: self._enum_types.append(v) def _get_dependent_headers(self): @@ -119,11 +120,9 @@ def cpp_filename(self): return self._cpp_filename def generate_header_code(self): - include_guard = "_IN_CSP_AUTOGEN_" + self._module_name.replace(".", "_").upper() - out = f""" -#ifndef {include_guard} -#define {include_guard} - + out = f"""//// Generated from {self._module_name} +#pragma once + """ out += self._generate_headers() @@ -139,11 +138,11 @@ def generate_header_code(self): for struct_type in self._struct_types: out += self._generate_struct_class(struct_type) - out += "\n}\n#endif" + out += "\n}" return out def _generate_headers(self): - common_headers = ["csp/core/Exception.h", "csp/engine/Struct.h", "cstddef"] + common_headers = ["csp/core/Exception.h", "csp/engine/Struct.h", "csp/python/Conversions.h", "cstddef"] common_headers.extend(self._get_dependent_headers()) return "\n".join(f"#include <{h}>" for h in common_headers) @@ -173,7 +172,11 @@ class {enum_name} : public csp::CspEnum static {enum_name} create( enum_ v ) {{ return s_meta -> create( ( int64_t ) v ); }} static {enum_name} create( const char * name) {{ return s_meta -> fromString( name ); }} static {enum_name} create( const std::string & s ) {{ return create( s.c_str() ); }} - + static {enum_name} create( PyObject * e ) + {{ + return {enum_name}( csp::python::fromPython( e, *s_cspEnumType ) ); + }} + enum_ enum_value() const {{ return ( enum_ ) value(); }} static constexpr uint32_t num_types() {{ return {len([x for x in enum_type])}; }} @@ -183,7 +186,7 @@ class {enum_name} : public csp::CspEnum {enum_name}( const csp::CspEnum & v ) : csp::CspEnum( v ) {{ CSP_TRUE_OR_THROW( v.meta() == s_meta.get(), AssertionError, "Mismatched enum meta" ); }} private: - + static std::shared_ptr s_cspEnumType; static std::shared_ptr s_meta; }}; """ @@ -439,14 +442,18 @@ def generate_cpp_code(self): assert_or_die( enumType != nullptr, "failed to find num type {enum_name} in module {self._module_name}" ); // should add some assertion here.. - csp::python::PyCspEnumMeta * pymeta = ( csp::python::PyCspEnumMeta * ) enumType; - s_meta = pymeta -> enumMeta; + //csp::python::PyCspEnumMeta * pymeta = ( csp::python::PyCspEnumMeta * ) enumType; + //s_meta = pymeta -> enumMeta; + auto type = csp::python::CspTypeFactory::instance().typeFromPyType( enumType ); + s_cspEnumType = std::static_pointer_cast( type ); + s_meta = s_cspEnumType -> meta(); }} return true; }} bool static_init_{enum_name} = {enum_name}::static_init(); +std::shared_ptr {enum_name}::s_cspEnumType; std::shared_ptr {enum_name}::s_meta; {static_decls} """ @@ -460,6 +467,7 @@ def generate_cpp_code(self): #include #include #include +#include #include #include #include diff --git a/csp/impl/enum.py b/csp/impl/enum.py index 8235c6ece..8eaa74de1 100644 --- a/csp/impl/enum.py +++ b/csp/impl/enum.py @@ -1,76 +1,39 @@ -import builtins -import enum -import inspect -import types +import sys import typing +from enum import IntEnum, auto -from csp.impl.__csptypesimpl import _csptypesimpl +class Enum(IntEnum): + auto = staticmethod(auto) -class EnumMeta(_csptypesimpl.PyCspEnumMeta): - def __new__(cls, name, bases, dct): - metadata = {} - last_value = -1 - - # Disallow subclassing an enum - EnumMeta._check_for_existing_members(bases) - - for k, v in dct.items(): - # Allow methods and properties and hidden methods - if ( - (k[0] == "_" and k[-1] == "_") - or v is enum.auto - or isinstance(v, (types.FunctionType, builtins.property, builtins.classmethod, builtins.staticmethod)) - ): - continue - - if isinstance(v, enum.auto): - v = last_value + 1 - - if not isinstance(v, int): - raise TypeError(f"csp.Enum expected int enum value, got {type(v).__name__} for field {k}") - - metadata[k] = v - last_value = v - - dct["__metadata__"] = metadata - return super().__new__(cls, name, bases, dct) - - def __iter__(self): - for k, v in self.__metadata__.items(): - yield self(k) - - @property - def __members__(self): - # For compatibility with python Enum - return types.MappingProxyType(self.__metadata__) + @classmethod + def _missing_(cls, value): + # Allow construction by name, ie MyEnum("FOO") as well as MyEnum(1) + if isinstance(value, str): + try: + return cls._member_map_[value] + except KeyError: + pass + return None @staticmethod - def _check_for_existing_members(bases): - for base in bases: - if isinstance(base, _csptypesimpl.PyCspEnumMeta) and base.__members__: - raise TypeError("Cannot extend csp.Enum %r: inheriting from an Enum is prohibited" % base.__name__) + def _generate_next_value_(name, start, count, last_values): + """keep old csp.Enum behavior of auto starting at 0""" + return last_values[-1] + 1 if last_values else 0 + __hash__ = int.__hash__ -class Enum(_csptypesimpl.PyCspEnum, metaclass=EnumMeta): - auto = enum.auto + def __eq__(self, other): + return type(self) is type(other) and self.value == other.value - def __reduce__(self): - return type(self), (self.value,) - - def __repr__(self): - return f"<{type(self).__name__}.{self.name}: {self.value}>" - - def __str__(self): - return f"{type(self).__name__}.{self.name}" + def __ne__(self, other): + return not self.__eq__(other) @classmethod def _validate(cls, v) -> "Enum": if isinstance(v, cls): return v - elif isinstance(v, str): - return cls[v] - elif isinstance(v, int): + elif isinstance(v, (str, int)): return cls(v) raise ValueError(f"Cannot convert value to enum: {v}") @@ -86,7 +49,7 @@ def __get_pydantic_json_schema__(cls, _core_schema, handler): field_schema.update( type="string", title=cls.__name__, - description=cls.__doc__ or "An enumeration of {}".format(cls.__name__), + description="An enumeration of {}".format(cls.__name__), enum=list(cls.__members__.keys()), ) return field_schema @@ -114,14 +77,10 @@ def DynamicEnum(name: str, values: typing.Union[dict, list], start=0, module_nam :param values: either a dictionary of key : values or a list of enum names :param start: when providing a list of values, will start enumerating from start """ - if isinstance(values, list): values = {k: v + start for v, k in enumerate(values)} - else: - values = values.copy() if module_name is None: - module_name = inspect.currentframe().f_back.f_globals["__name__"] - values["__module__"] = module_name + module_name = sys._getframe(1).f_globals["__name__"] - return EnumMeta(name, (Enum,), values) + return Enum(name, values, module=module_name) diff --git a/csp/tests/impl/test_enum.py b/csp/tests/impl/test_enum.py index 636eb2c85..bfdf91eb6 100644 --- a/csp/tests/impl/test_enum.py +++ b/csp/tests/impl/test_enum.py @@ -78,13 +78,13 @@ def test_basic(self): self.assertEqual(MyEnum2.c1(), 30) self.assertEqual(MyEnum2.s1(), 123) - with self.assertRaisesRegex(ValueError, "123 is not a valid value on csp.enum type MyEnum"): + with self.assertRaisesRegex(ValueError, "123 is not a valid MyEnum"): MyEnum(123) - with self.assertRaisesRegex(ValueError, "ABC is not a valid value on csp.enum type MyEnum"): + with self.assertRaisesRegex(ValueError, "'ABC' is not a valid MyEnum"): MyEnum("ABC") - with self.assertRaisesRegex(TypeError, "csp.Enum expected int enum value, got str for field B"): + with self.assertRaisesRegex(ValueError, "invalid literal for int\\(\\) with base 10: 'hey'"): class FOO(csp.Enum): A = 1 @@ -104,9 +104,6 @@ class MyEnum3(csp.Enum): self.assertEqual(MyEnum3.B.value, 10) self.assertEqual(MyEnum3.C.value, 11) - def test_python_enum_compatibility(self): - self.assertEqual(dict(MyEnum.__members__), dict(MyEnum.__metadata__)) - def test_node(self): """test ability of node to convert to/from enum types properly""" @@ -167,13 +164,11 @@ def test_subclassing(self): class A(csp.Enum): RED = 1 - with self.assertRaises(TypeError) as cm: + with self.assertRaisesRegex(TypeError, "cannot extend"): class B(A): GREEN = 2 - self.assertEqual("Cannot extend csp.Enum 'A': inheriting from an Enum is prohibited", str(cm.exception)) - def test_pydantic_validation(self): assert MyModel(enum="FIELD2").enum == MyEnum3.FIELD2 assert MyModel(enum=0).enum == MyEnum3.FIELD1 diff --git a/csp/tests/impl/test_struct.py b/csp/tests/impl/test_struct.py index b0c5da658..8bd0dc1ec 100644 --- a/csp/tests/impl/test_struct.py +++ b/csp/tests/impl/test_struct.py @@ -2521,11 +2521,6 @@ class B(csp.Struct): a: List[MyEnum] s = B(a=[MyEnum.A, MyEnum.FOO]) - - with self.assertRaises(TypeError) as e: - s.a.sort() - with self.assertRaises(TypeError) as e: - s.a.sort(reverse=True) s.a.sort(reverse=True, key=str) self.assertEqual(s.a, [MyEnum.FOO, MyEnum.A]) @@ -2963,14 +2958,6 @@ class B(csp.Struct): s = B(a=[MyEnum.A, MyEnum.FOO]) t = B(a=[MyEnum.FOO, MyEnum.FOO]) - with self.assertRaises(TypeError) as e: - s.a < t.a - with self.assertRaises(TypeError) as e: - s.a <= t.a - with self.assertRaises(TypeError) as e: - s.a > t.a - with self.assertRaises(TypeError) as e: - s.a >= t.a self.assertEqual(s.a == t.a, False) self.assertEqual(s.a != t.a, True) diff --git a/csp/tests/test_history.py b/csp/tests/test_history.py index 0989ec1ec..f91edb054 100644 --- a/csp/tests/test_history.py +++ b/csp/tests/test_history.py @@ -161,7 +161,7 @@ def collect_index( with self.assertRaises(TypeError): csp.values_at(x, 0, 0, csp.TimeIndexPolicy.EXCLUSIVE, csp.TimeIndexPolicy.EXCLUSIVE) - with self.assertRaises(RuntimeError): + with self.assertRaises(TypeError): csp.items_at(x, -10, 0, False, False) @csp.node @@ -200,7 +200,7 @@ def collect_timedelta( csp.output(values_default, csp.values_at(x)) - with self.assertRaises(RuntimeError): + with self.assertRaises(TypeError): csp.times_at(x, startIndex, endIndex, "abc", False) @csp.node @@ -464,8 +464,8 @@ def graph(): ) enum_values = [result[1] for result in results["values_enum"]] - self.assertTrue(np.array_equal(enum_values[-1], np.array([MyEnum.first] * 3))) - self.assertTrue(np.array_equal(enum_values[-2], np.array([MyEnum.first] * 3))) + self.assertTrue(np.array_equal(enum_values[-1], np.array([MyEnum.first] * 3, dtype=object))) + self.assertTrue(np.array_equal(enum_values[-2], np.array([MyEnum.first] * 3, dtype=object))) list_values = [result[1] for result in results["values_list"]] self.assertTrue(list_values[-1][0] == [1, 2, 3, 4]) @@ -587,16 +587,27 @@ def graph(): ) extrapolate_enum_values = [result[1] for result in results["values_extrapolate_enum"]] - self.assertTrue(np.array_equal(extrapolate_enum_values[-1], np.array([MyEnum.second, MyEnum.first]))) - self.assertTrue(np.array_equal(extrapolate_enum_values[-2], np.array([MyEnum.second, MyEnum.second]))) - self.assertTrue(np.array_equal(extrapolate_enum_values[-3], np.array([MyEnum.first, MyEnum.first]))) - self.assertTrue(np.array_equal(extrapolate_enum_values[-4], np.array([MyEnum.first, MyEnum.first]))) + self.assertTrue( + np.array_equal(extrapolate_enum_values[-1], np.array([MyEnum.second, MyEnum.first], dtype=object)) + ) + self.assertTrue( + np.array_equal(extrapolate_enum_values[-2], np.array([MyEnum.second, MyEnum.second], dtype=object)) + ) + self.assertTrue( + np.array_equal(extrapolate_enum_values[-3], np.array([MyEnum.first, MyEnum.first], dtype=object)) + ) + self.assertTrue( + np.array_equal(extrapolate_enum_values[-4], np.array([MyEnum.first, MyEnum.first], dtype=object)) + ) enum_boundary_values = [result[1] for result in results["values_enum_boundary"]] self.assertTrue( np.array_equal( enum_boundary_values[-1], - np.array([MyEnum.first, MyEnum.second, MyEnum.second, MyEnum.second, MyEnum.first, MyEnum.first]), + np.array( + [MyEnum.first, MyEnum.second, MyEnum.second, MyEnum.second, MyEnum.first, MyEnum.first], + dtype=object, + ), ) )