Make optim interface work cross-platform

.github/workflows/qa.yaml

@@ -14,7 +14,7 @@ jobs:
         strategy:
             matrix:
                 python-version: ['3.10', '3.11', '3.12', '3.13', '3.14']
-                julia-version: ['1.11', '1.12']
+                julia-version: ['1.11']  # Excluding Julia 1.12 due to memory crashes , '1.12']
         defaults:
             run:
                 shell: bash -l {0}

Makefile

@@ -32,7 +32,7 @@ help:
 .PHONY : test
 test :
 	@echo "=== C tests ==="
-	cd build && ctest --output-on-failure
+	cd build && ctest --verbose --output-on-failure
 	@echo "=== Julia tests ==="
 	julia tests/lang_julia/runtests.jl
 	@echo "=== Python tests ==="

Project.toml

@@ -1,4 +1,5 @@
 [deps]
+DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
 Ipopt = "b6b21f68-93f8-5de0-b562-5493be1d77c9"
 JuMP = "4076af6c-e467-56ae-b986-b466b2749572"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
@@ -16,3 +17,9 @@ SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
 [sources]
 OpenInterfaces = {path = "lang_julia/OpenInterfaces"}
 OpenInterfacesImpl = {path = "lang_julia/OpenInterfacesImpl"}
+
+[compat]
+MsgPack = "1.2.1"
+Optim = "2.0.1"
+OrdinaryDiffEq = "6.102.1"
+SciMLBase = "2.120.0"

lang_julia/OpenInterfaces/src/OpenInterfaces.jl

@@ -10,6 +10,7 @@ export OIF_TYPE_INT,
     OIF_TYPE_STRING,
     OIF_TYPE_CALLBACK,
     OIF_USER_DATA,
+    OIF_TYPE_USER_DATA,
     OIF_TYPE_CONFIG_DICT
 
 # Language ids
@@ -100,21 +101,26 @@ mutable struct OIFArrayF64
     flags::Int32
 
     function OIFArrayF64(arr::AbstractArray{Float64})
+        nd = ndims(arr)
         dims = collect(Cssize_t, size(arr))
         flags = OIF_ARRAY_F_CONTIGUOUS
 
         if length(dims) == 1
             flags = OIF_ARRAY_C_CONTIGUOUS | OIF_ARRAY_F_CONTIGUOUS
         end
 
-        self = new(
-            ndims(arr),
-            Base.unsafe_convert(Ptr{Cssize_t}, dims),
-            Base.unsafe_convert(Ptr{Float64}, arr),
-            flags,
-        )
+        dimensions = Ptr{Cssize_t}(Libc.malloc(nd * sizeof(Cssize_t)))
+        unsafe_copyto!(dimensions, pointer(dims), nd)
+
+        self = new(nd, dimensions, Base.unsafe_convert(Ptr{Float64}, arr), flags)
+
+        finalizer(finalizing, self)
 
-        return GC.@preserve dims self
+        return self
+    end
+
+    function finalizing(self)
+        Libc.free(self.dimensions)
     end
 end
 
@@ -558,7 +564,7 @@ end
 # otherwise, the will be crashes or something like that
 # as Julia's garbage collector is pretty fast to remove reference
 # that it thinks are unused.
-function make_oif_user_data(data::Ref{Any})::OIFUserData
+function make_oif_user_data(data)::OIFUserData
     data_ptr = Base.unsafe_convert(Ptr{Cvoid}, data)
     OIFUserData(OIF_LANG_JULIA, C_NULL, data_ptr, C_NULL)
 end

lang_julia/OpenInterfacesImpl/Project.toml

@@ -1,7 +1,7 @@
 name = "OpenInterfacesImpl"
 uuid = "daff4927-7858-4806-96da-915047fb441c"
-authors = ["Dmitry Kabanov <dmitry.kabanov@uni-muenster.de>", "Stephan Rave <stephan.rave@uni-muenster.de", "Mario Ohlberger <mario.ohlberger@uni-muenster.de"]
 version = "0.6.3"
+authors = ["Dmitry Kabanov <dmitry.kabanov@uni-muenster.de>", "Stephan Rave <stephan.rave@uni-muenster.de", "Mario Ohlberger <mario.ohlberger@uni-muenster.de"]
 
 [deps]
 MsgPack = "99f44e22-a591-53d1-9472-aa23ef4bd671"
@@ -11,7 +11,7 @@ OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed"
 SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
 
 [sources]
-OpenInterfaces = {path = "../../lang_julia/OpenInterfaces"}
+OpenInterfaces = {path = "../OpenInterfaces"}
 
 [compat]
 MsgPack = "1.2.1"

lang_julia/OpenInterfacesImpl/src/callback.jl

@@ -1,29 +1,89 @@
 module CallbackWrapper
 export make_wrapper_over_c_callback
 
-import SciMLBase
+using OpenInterfaces
 
-using OpenInterfaces: OIFArrayF64, OIF_ARRAY_C_CONTIGUOUS, OIF_ARRAY_F_CONTIGUOUS
+function make_wrapper_over_c_callback(fn_c::Ptr{Cvoid}, oif_argtypes, oif_restype)::Function
+    c_argtypes = Tuple(
+        map(argtype -> begin
+                if argtype == OIF_TYPE_I32
+                    Cint
+                elseif argtype == OIF_TYPE_F64
+                    Cdouble
+                elseif argtype == OIF_TYPE_ARRAY_F64
+                    Ref{OIFArrayF64}
+                elseif argtype == OIF_TYPE_USER_DATA
+                    Ptr{Cvoid}
+                else
+                    error("Unsupported argument type: $argtype")
+                end
+            end, oif_argtypes),
+    )
 
-function make_wrapper_over_c_callback(fn_c::Ptr{Cvoid})::Function
-    function wrapper(t::Float64, y::Vector{Float64}, ydot::Vector{Float64}, user_data)::Int
-        if typeof(user_data) == SciMLBase.NullParameters
-            user_data = C_NULL
-        end
+    # Build a concrete Tuple type e.g. Tuple{Cdouble,Cdouble}.
+    # This IS a valid type parameter, unlike a plain tuple value (Cdouble, Cdouble).
+    c_argtypes_type = Core.apply_type(Tuple, c_argtypes...)
 
-        oif_y = Ref(OIFArrayF64(y))
-        oif_ydot = Ref(OIFArrayF64(ydot))
+    c_restype = if oif_restype == OIF_TYPE_I32 || oif_restype == OIF_TYPE_INT
+        # The second condition is just for explicitness, as `INT` is an alias to `I32`.
+        Cint
+    elseif oif_restype == OIF_TYPE_F64
+        Cdouble
+    else
+        error(
+            "Unsupported return type: $oif_restype. Only supported types are Cint and Cdouble",
+        )
+    end
 
-        @ccall $fn_c(
-            t::Float64,
-            oif_y::Ptr{OIFArrayF64},
-            oif_ydot::Ptr{OIFArrayF64},
-            user_data::Ptr{Cvoid},
-        )::Cint
+    function wrapper(args...)
+        c_args_from_jl_args = Tuple(
+            map(
+                (argtype, argvalue) -> begin
+                    if argtype == OIF_TYPE_I32
+                        argvalue
+                    elseif argtype == OIF_TYPE_F64
+                        argvalue
+                    elseif argtype == OIF_TYPE_ARRAY_F64
+                        OIFArrayF64(argvalue)
+                    elseif argtype == OIF_TYPE_USER_DATA
+                        argvalue === nothing ? C_NULL : argvalue
+                    else
+                        error("Unsupported argument type: $argtype")
+                    end
+                end,
+                oif_argtypes,
+                args,
+            ),
+        )
 
-        return 0
+        return GC.@preserve args c_args_from_jl_args call_c(
+            fn_c,
+            c_argtypes_type,
+            c_restype,
+            c_args_from_jl_args,
+        )
     end
+
     return wrapper
 end
 
+@generated function call_c(
+    fn_p,
+    ::Type{argtypes},
+    ::Type{restype},
+    args,
+) where {argtypes,restype}
+
+    n = length(argtypes.parameters)
+
+    argtypes_expr = Expr(:tuple, map(t -> :($t), argtypes.parameters)...)
+    # Generate :(args[1]), :(args[2]), … at code-generation time so that ccall
+    # sees a fixed, statically-known argument list (runtime splatting is not allowed).
+    args_expr = [:(args[$i]) for i = 1:n]
+
+    quote
+        ccall(fn_p, $restype, $argtypes_expr, $(args_expr...))
+    end
+end
+
 end

lang_julia/_impl/ivp/jl_diffeq/jl_diffeq.jl

@@ -3,6 +3,7 @@ export Self,
     set_initial_value, set_rhs_fn, set_tolerances, integrate, set_user_data, set_integrator
 
 using OrdinaryDiffEq
+import SciMLBase
 
 mutable struct Self
     t0::Float64
@@ -79,6 +80,10 @@ function _rhs_wrapper(rhs)
     # Note that the `p` argument will always be in the function signature
     # regardless of if the problem is defined with parameters!
     function wrapper(du, u, p, t)
+        if typeof(p) == SciMLBase.NullParameters
+            return rhs(t, u, du, nothing)
+        end
+
         return rhs(t, u, du, p)
     end
     return wrapper

lang_julia/_impl/optim/optim_jl/optim_jl.jl

@@ -47,6 +47,7 @@ end
 
 function set_objective_fn(self::Self, objective_fn)
     self.objective_fn = objective_fn
+    self.objective_fn(self.x0, self.user_data)
 end
 
 function set_method(self, method_name, method_params)

lang_julia/bridge_julia.c

@@ -203,10 +203,85 @@ build_julia_tuple_from_size_t_array(intptr_t *dimensions, size_t ndims)
     return tuple;
 }
 
+/**
+ * Build a Julia tuple from an `int32_t *` array.
+ *
+ * @param vals Array with elements of type int (32-bit integers)
+ * @param n Number of the elements in the array
+ * @return Julia tuple on success, `NULL` otherwise
+ */
+jl_value_t *
+build_julia_tuple_from_oifargtype_array(OIFArgType *vals, size_t n)
+{
+    char *tuple_string = malloc(sizeof(char) * 1024);
+    char *cursor = tuple_string;
+    char *right_bound = &tuple_string[1024 - 1];
+    intptr_t diff = right_bound - cursor;
+    snprintf(tuple_string, diff, "%s", "(");
+    cursor++;
+    char buffer[BUFFER_SIZE_];
+    int chars_written;
+    jl_value_t *tuple = NULL;
+
+    for (size_t i = 0; i < n; ++i) {
+        chars_written = snprintf(buffer, BUFFER_SIZE_, "%d", vals[i]);
+        assert(chars_written >= 1);
+        if (chars_written >= BUFFER_SIZE_) {
+            goto report_too_long;
+        }
+        diff = right_bound - cursor;
+        chars_written = snprintf(cursor, diff, "%s", buffer);
+        assert(chars_written >= 1);
+        if (chars_written >= diff) {
+            goto report_too_long;
+        }
+        cursor += chars_written;
+        diff = right_bound - cursor;
+        if (i < n - 1 || n == 1) {
+            chars_written = snprintf(cursor, diff, "%s", ",");
+            assert(chars_written >= 1);
+            if (chars_written >= diff) {
+                goto report_too_long;
+            }
+            cursor++;
+        }
+    }
+    diff = right_bound - cursor;
+    chars_written = snprintf(cursor, diff, "%s", ")");
+    if (chars_written >= diff) {
+        fprintf(stderr, "ERROR: the string to copy is too long\n");
+        exit(1);
+    }
+
+    tuple = jl_eval_string(tuple_string);
+    if (jl_exception_occurred()) {
+        const char *p = jl_string_ptr(
+            jl_eval_string("sprint(showerror, ccall(:jl_exception_occurred, Any, ()))"));
+        fprintf(stderr, "%s\n", p);
+    }
+    goto cleanup;
+
+report_too_long:
+    fprintf(stderr,
+            "[build_julia_tuple_from_int32_t_array] The string representation of the julia "
+            "tuple does not fit in the buffer\n");
+
+cleanup:
+    free(tuple_string);
+
+    return tuple;
+}
+
 static jl_value_t *
 make_wrapper_over_c_callback(OIFCallback *p)
 {
     jl_value_t *wrapper = NULL;
+
+    jl_value_t *fn_p_c_wrapped = NULL;
+    jl_value_t *oif_argtypes = NULL;
+    jl_value_t *oif_restype = NULL;
+    JL_GC_PUSH3(&fn_p_c_wrapped, &oif_argtypes, &oif_restype);  // NOLINT
+
     if (CALLBACK_MODULE_ == NULL) {
         jl_eval_string("using OpenInterfacesImpl.CallbackWrapper");
         if (jl_exception_occurred()) {
@@ -224,10 +299,14 @@ make_wrapper_over_c_callback(OIFCallback *p)
         jl_get_function(CALLBACK_MODULE_, "make_wrapper_over_c_callback");
     assert(fn_callback != NULL);
     assert(p->fn_p_c != NULL);
-    jl_value_t *fn_p_c_wrapped = jl_box_voidpointer(p->fn_p_c);
-    wrapper = jl_call1(fn_callback, fn_p_c_wrapped);
+    fn_p_c_wrapped = jl_box_voidpointer(p->fn_p_c);
+    oif_argtypes = build_julia_tuple_from_oifargtype_array(p->arg_types, p->nargs);
+    oif_restype = jl_box_int32(p->restype);
+
+    wrapper = jl_call3(fn_callback, fn_p_c_wrapped, oif_argtypes, oif_restype);
 
 cleanup:
+    JL_GC_POP();
     return wrapper;
 }
 

lang_python/oif/openinterfaces/core.py

@@ -139,6 +139,10 @@ class OIFCallback(ctypes.Structure):
         ("fn_p_c", ctypes.c_void_p),
         ("fn_p_jl", ctypes.c_void_p),
         ("fn_p_py", ctypes.c_void_p),
+        ("nargs", ctypes.c_uint32),
+        ("arg_types", ctypes.POINTER(OIFArgType)),
+        ("restype", OIFArgType),
+        ("hidden", ctypes.c_void_p),
     ]
 
 
@@ -197,7 +201,15 @@ def make_oif_callback(
 
     fn_p_c = ctypes.cast(c_wrapper_fn, ctypes.c_void_p)
     assert fn_p_c.value is not None
-    oifcallback = OIFCallback(OIF_LANG_PYTHON, fn_p_c, None, fn_p_py)
+
+    nargs = len(argtypes)
+    oif_arg_types = ctypes.cast(
+        (ctypes.c_int * nargs)(*argtypes), ctypes.POINTER(OIFArgType)
+    )
+
+    oifcallback = OIFCallback(
+        OIF_LANG_PYTHON, fn_p_c, None, fn_p_py, nargs, oif_arg_types, restype, None
+    )
     return oifcallback
 
 

lang_python/oif_impl/openinterfaces/_impl/optim/scipy_optimize/scipy_optimize.py

@@ -84,6 +84,7 @@ def minimize(self, out_x):
         )
 
         out_x[:] = result.x
+        print(result)
         return (result.status, result.message)
 
     # def print_stats(self):

tests/CMakeLists.txt

@@ -25,3 +25,4 @@ endif()
 include(GoogleTest)
 
 add_subdirectory(lang_c)
+add_subdirectory(lang_julia)

tests/lang_julia/CMakeLists.txt

@@ -0,0 +1,2 @@
+add_library(oif_aux_funcs SHARED aux_funcs.c)
+target_include_directories(oif_aux_funcs PRIVATE ${CMAKE_SOURCE_DIR}/include)