diff --git a/deploy/packaging/debian/devel.noarch b/deploy/packaging/debian/devel.noarch index 0c0c5e041b..3e3f68587e 100644 --- a/deploy/packaging/debian/devel.noarch +++ b/deploy/packaging/debian/devel.noarch @@ -66,6 +66,7 @@ ./usr/local/mdsplus/include/tcl_messages.h ./usr/local/mdsplus/include/tdishr.h ./usr/local/mdsplus/include/tdishr_messages.h +./usr/local/mdsplus/include/treefileio.h ./usr/local/mdsplus/include/treeioperf.h ./usr/local/mdsplus/include/treeshr.h ./usr/local/mdsplus/include/treeshr_hooks.h diff --git a/deploy/packaging/redhat/devel.noarch b/deploy/packaging/redhat/devel.noarch index 936e4605b7..69052a61ef 100644 --- a/deploy/packaging/redhat/devel.noarch +++ b/deploy/packaging/redhat/devel.noarch @@ -69,6 +69,7 @@ ./usr/local/mdsplus/include/tcl_messages.h ./usr/local/mdsplus/include/tdishr.h ./usr/local/mdsplus/include/tdishr_messages.h +./usr/local/mdsplus/include/treefileio.h ./usr/local/mdsplus/include/treeioperf.h ./usr/local/mdsplus/include/treeshr.h ./usr/local/mdsplus/include/treeshr_hooks.h diff --git a/treeshr/CMakeLists.txt b/treeshr/CMakeLists.txt index 9805c37c0a..da0c256a09 100644 --- a/treeshr/CMakeLists.txt +++ b/treeshr/CMakeLists.txt @@ -68,3 +68,8 @@ target_compile_definitions( mdsplus_add_static_copy(TreeShr _static_target) install(TARGETS TreeShr ${_static_target}) + +# Public contract header for out-of-tree tree-file backend plugins (the vtable +# and "MdsTreeFile" / "FileIo" loader convention). Self-contained +# (only includes ), so it installs standalone. +install(FILES treefileio.h DESTINATION include) diff --git a/treeshr/RemoteAccess.c b/treeshr/RemoteAccess.c index df4df9d77c..91fb486cf8 100644 --- a/treeshr/RemoteAccess.c +++ b/treeshr/RemoteAccess.c @@ -82,6 +82,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // #define DEBUG #include <_mdsshr.h> +#include "treefileio.h" static inline char *replaceBackslashes(char *filename) { @@ -1017,6 +1018,9 @@ typedef struct int conid; int fd; int enhanced; + const TreeFileIo *io; /* backend for a local fd (NULL for remote/conid>=0). + Carries the open/read/write/lseek/lock vtable used + to service this descriptor. */ } fdinfo_t; static struct fd_info_struct @@ -1065,11 +1069,129 @@ char *ParseFile(char *filename, char **hostpart, char **filepart) return tmp; } +/* ---- Pluggable tree-file IO backend (see treefileio.h) ---------------- */ + +static int default_open(const char *filename, int options, mode_t mode) +{ + return open(filename, options, mode); +} +static int default_close(int fd) { return close(fd); } +static ssize_t default_read(int fd, void *buff, size_t count) +{ + return read(fd, buff, count); +} +static ssize_t default_write(int fd, const void *buff, size_t count) +{ + return write(fd, buff, count); +} +static off_t default_lseek(int fd, off_t offset, int whence) +{ + return lseek(fd, offset, whence); +} +/* default_lock: the fcntl-based local lock. Defined below io_lock_local to keep + * the lock code together; forward-declared here because default_tree_file_io is + * initialized just below. */ +static int default_lock(int fd, off_t offset, size_t size, int mode_in, + int *deleted); + +static const TreeFileIo default_tree_file_io = { + default_open, default_close, default_read, + default_write, default_lseek, default_lock, + /* .mmap_capable = */ 1}; + +/* Per-scheme cache. Negative results (NULL io) are cached too, so an unknown + * scheme is only probed once. */ +static struct backend_cache_entry +{ + char *scheme; /* uppercased, no "://" */ + const TreeFileIo *io; +} *g_backends = NULL; +static int g_nbackends = 0; +static pthread_mutex_t g_backends_lock = PTHREAD_MUTEX_INITIALIZER; + +static const TreeFileIo *load_backend(const char *scheme_upper) +{ + char *image = malloc(strlen(scheme_upper) + 12); /* "MdsTreeFile" (11) + NUL */ + strcpy(image, "MdsTreeFile"); + strcat(image, scheme_upper); + const TreeFileIo *(*rtn)(void) = NULL; + int status = LibFindImageSymbol_C(image, "FileIo", (void **)&rtn); + free(image); + if (STATUS_OK && rtn) + return rtn(); + return NULL; +} + +const TreeFileIo *GetTreeFileIo(const char *filename) +{ + const char *sep = filename ? strstr(filename, "://") : NULL; + if (!sep || sep == filename) + return &default_tree_file_io; /* no scheme -> default path */ + size_t len = (size_t)(sep - filename); + char *scheme = malloc(len + 1); + size_t i; + for (i = 0; i < len; i++) + scheme[i] = (char)toupper((unsigned char)filename[i]); + scheme[len] = '\0'; + + const TreeFileIo *io = NULL; + int found = 0; + pthread_mutex_lock(&g_backends_lock); + for (i = 0; i < (size_t)g_nbackends; i++) + { + if (strcmp(g_backends[i].scheme, scheme) == 0) + { + io = g_backends[i].io; + found = 1; + break; + } + } + pthread_mutex_unlock(&g_backends_lock); + + if (!found) + { + /* Resolve OUTSIDE the lock: load_backend() calls dlopen, whose library + * constructors could re-enter GetTreeFileIo and deadlock the non-recursive + * mutex. */ + const TreeFileIo *loaded = load_backend(scheme); /* may be NULL */ + pthread_mutex_lock(&g_backends_lock); + /* Re-scan: another thread may have inserted this scheme while we loaded. */ + for (i = 0; i < (size_t)g_nbackends; i++) + { + if (strcmp(g_backends[i].scheme, scheme) == 0) + { + io = g_backends[i].io; + found = 1; + break; + } + } + if (!found) + { + /* Cache the result (negative results too). On realloc failure, skip + * caching rather than clobber the table; the caller still gets a usable + * result this call. */ + struct backend_cache_entry *tmp = realloc( + g_backends, sizeof(*g_backends) * (size_t)(g_nbackends + 1)); + if (tmp) + { + g_backends = tmp; + g_backends[g_nbackends].scheme = strdup(scheme); + g_backends[g_nbackends].io = loaded; + g_nbackends++; + } + io = loaded; + } + pthread_mutex_unlock(&g_backends_lock); + } + free(scheme); + return io; /* NULL for a URL scheme => fail-closed */ +} + static pthread_mutex_t fds_lock = PTHREAD_MUTEX_INITIALIZER; #define FDS_LOCK MUTEX_LOCK_PUSH(&fds_lock) #define FDS_UNLOCK MUTEX_LOCK_POP(&fds_lock) -int ADD_FD(int fd, int conid, int enhanced) +int ADD_FD(int fd, int conid, int enhanced, const TreeFileIo *io) { int idx; FDS_LOCK; @@ -1081,6 +1203,7 @@ int ADD_FD(int fd, int conid, int enhanced) FDS[idx].i.conid = conid; FDS[idx].i.fd = fd; FDS[idx].i.enhanced = enhanced; + FDS[idx].i.io = io; FDS_UNLOCK; return idx + 1; } @@ -1095,7 +1218,7 @@ inline static fdinfo_t RM_FD(int idx) FDS[idx - 1].in_use = B_FALSE; } else - fdinfo = (fdinfo_t){-1, -1, -1}; + fdinfo = (fdinfo_t){-1, -1, -1, NULL}; FDS_UNLOCK; return fdinfo; } @@ -1107,7 +1230,7 @@ inline static fdinfo_t GET_FD(int idx) if (idx > 0 && idx <= ALLOCATED_FDS && FDS[idx - 1].in_use) fdinfo = FDS[idx - 1].i; else - fdinfo = (fdinfo_t){-1, -1, -1}; + fdinfo = (fdinfo_t){-1, -1, -1, NULL}; FDS_UNLOCK; return fdinfo; } @@ -1134,6 +1257,18 @@ EXPORT int MDS_IO_FD(int idx) return fd; } +EXPORT int MDS_IO_MMAP_CAPABLE(int idx) +{ + int cap; + FDS_LOCK; + if (idx > 0 && idx <= ALLOCATED_FDS && FDS[idx - 1].in_use && FDS[idx - 1].i.io) + cap = FDS[idx - 1].i.io->mmap_capable; + else + cap = 0; /* remote (conid>=0, io==NULL) or invalid: not mmap-able */ + FDS_UNLOCK; + return cap; +} + static int (*SendArg)() = NULL; static int (*GetAnswerInfoTS)() = NULL; static inline int mds_io_request(int conid, mds_io_mode idx, size_t size, @@ -1263,6 +1398,7 @@ EXPORT int MDS_IO_OPEN(char *filename_in, int options, mode_t mode) INIT_AND_FREE_ON_EXIT(char *, filename); INIT_AND_FREE_ON_EXIT(char *, tmp); int conid = -1, fd = -1, enhanced = 0; + const TreeFileIo *io = NULL; filename = replaceBackslashes(strdup(filename_in)); char *hostpart, *filepart; tmp = ParseFile(filename, &hostpart, &filepart); @@ -1270,15 +1406,15 @@ EXPORT int MDS_IO_OPEN(char *filename_in, int options, mode_t mode) fd = io_open_remote(hostpart, filepart, options, mode, &conid, &enhanced); else { - - fd = open(filename, options | O_BINARY | O_RANDOM, mode); + io = GetTreeFileIo(filename); + fd = io ? io->open(filename, options | O_BINARY | O_RANDOM, mode) : -1; MDSDBG("fd=%d, filename='%s'", fd, filename); #ifndef _WIN32 if ((fd >= 0) && ((options & O_CREAT) != 0)) set_mdsplus_file_protection(filename); #endif } - idx = fd < 0 ? fd : ADD_FD(fd, conid, enhanced); + idx = fd < 0 ? fd : ADD_FD(fd, conid, enhanced, io); FREE_NOW(tmp); FREE_NOW(filename); return idx; @@ -1313,7 +1449,7 @@ EXPORT int MDS_IO_CLOSE(int idx) if (i.conid >= 0) return io_close_remote(i.conid, i.fd); MDSDBG("I fd=%d", i.fd); - return close(i.fd); + return (i.io ? i.io : &default_tree_file_io)->close(i.fd); } inline static off_t io_lseek_remote(int conid, int fd, off_t offset, @@ -1351,7 +1487,7 @@ EXPORT off_t MDS_IO_LSEEK(int idx, off_t offset, int whence) return -1; if (i.conid >= 0) return io_lseek_remote(i.conid, i.fd, offset, whence); - return lseek(i.fd, offset, whence); + return (i.io ? i.io : &default_tree_file_io)->lseek(i.fd, offset, whence); } inline static ssize_t io_write_remote(int conid, int fd, void *buff, @@ -1391,7 +1527,7 @@ EXPORT ssize_t MDS_IO_WRITE(int idx, void *buff, size_t count) #ifdef USE_TREE_PERF TreePerfWrite(count); #endif - return write(i.fd, buff, (uint32_t)count); + return (i.io ? i.io : &default_tree_file_io)->write(i.fd, buff, count); } inline static ssize_t io_read_remote(int conid, int fd, void *buff, @@ -1427,7 +1563,7 @@ EXPORT ssize_t MDS_IO_READ(int idx, void *buff, size_t count) #ifdef USE_TREE_PERF TreePerfRead(count); #endif - return read(i.fd, buff, count); + return (i.io ? i.io : &default_tree_file_io)->read(i.fd, buff, count); } inline static ssize_t io_read_x_remote(int conid, int fd, off_t offset, @@ -1480,12 +1616,13 @@ EXPORT ssize_t MDS_IO_READ_X(int idx, off_t offset, void *buff, size_t count, return ans; } ssize_t ans; + const TreeFileIo *io = i.io ? i.io : &default_tree_file_io; IO_RDLOCK_FILE(io_lock_local, i, offset, count, deleted); - lseek(i.fd, offset, SEEK_SET); + io->lseek(i.fd, offset, SEEK_SET); #ifdef USE_TREE_PERF TreePerfRead(count); #endif - ans = read(i.fd, buff, (unsigned int)count); + ans = io->read(i.fd, buff, (unsigned int)count); IO_UNLOCK_FILE(); return ans; } @@ -1514,13 +1651,11 @@ inline static int io_lock_remote(fdinfo_t fdinfo, off_t offset, size_t size, return ret; } -static int io_lock_local(fdinfo_t fdinfo, off_t offset, size_t size, - int mode_in, int *deleted) +static int default_lock(int fd, off_t offset, size_t size, int mode_in, + int *deleted) { - MDSDBG("I fd=%d, offset=%" PRIu64 ", size=%" PRIu64 ", mode=%d", - fdinfo.fd, offset, size, mode_in); - int fd = fdinfo.fd; + fd, offset, size, mode_in); int err; int mode = mode_in & MDS_IO_LOCK_MASK; int nowait = mode_in & MDS_IO_LOCK_NOWAIT; @@ -1539,7 +1674,6 @@ static int io_lock_local(fdinfo_t fdinfo, off_t offset, size_t size, : LOCKFILE_EXCLUSIVE_LOCK; if (nowait) flags |= LOCKFILE_FAIL_IMMEDIATELY; - // UnlockFileEx(h, 0, (DWORD) size, 0, &overlapped); err = !LockFileEx(h, flags, 0, (DWORD)size, 0, &overlapped); } else @@ -1560,7 +1694,7 @@ static int io_lock_local(fdinfo_t fdinfo, off_t offset, size_t size, (mode == 0) ? SEEK_SET : ((offset >= 0) ? SEEK_SET : SEEK_END); flock.l_start = (mode == 0) ? 0 : ((offset >= 0) ? offset : 0); flock.l_len = (mode == 0) ? 0 : size; - static int use_ofd_locks = 1; // atomic? + static int use_ofd_locks = 1; if (use_ofd_locks == 1) { flock.l_pid = 0; @@ -1586,10 +1720,17 @@ static int io_lock_local(fdinfo_t fdinfo, off_t offset, size_t size, *deleted = stat.st_nlink <= 0; #endif MDSDBG("O fd=%d, offset=%" PRIu64 ", size=%" PRIu64 ", mode=%d, err=%d", - fdinfo.fd, (uint64_t)offset, (uint64_t)size, mode_in, err); + fd, (uint64_t)offset, (uint64_t)size, mode_in, err); return err ? TreeLOCK_FAILURE : TreeSUCCESS; } +static int io_lock_local(fdinfo_t fdinfo, off_t offset, size_t size, + int mode_in, int *deleted) +{ + const TreeFileIo *io = fdinfo.io ? fdinfo.io : &default_tree_file_io; + return io->lock(fdinfo.fd, offset, size, mode_in, deleted); +} + EXPORT int MDS_IO_LOCK(int idx, off_t offset, size_t size, int mode_in, int *deleted) { @@ -1857,7 +1998,7 @@ inline static int io_open_one_remote(char *host, char *filepath, status = *fd == -1 ? TreeFAILURE : TreeSUCCESS; if ((*fd >= 0) && edit && (type == TREE_TREEFILE_TYPE)) { - if (IS_NOT_OK(io_lock_remote((fdinfo_t){*conid, *fd, *enhanced}, 1, 1, + if (IS_NOT_OK(io_lock_remote((fdinfo_t){*conid, *fd, *enhanced, NULL}, 1, 1, MDS_IO_LOCK_RD | MDS_IO_LOCK_NOWAIT, 0))) { @@ -1917,6 +2058,7 @@ EXPORT int MDS_IO_OPEN_ONE(char *filepath_in, char const *treename_in, int shot, int enhanced = 0; int conid = -1; int fd = -1; + const TreeFileIo *io = NULL; char treename[13]; char *hostpart, *filepart; size_t i; @@ -1955,6 +2097,7 @@ EXPORT int MDS_IO_OPEN_ONE(char *filepath_in, char const *treename_in, int shot, free(fullpath); if (hostpart) { + io = NULL; fullpath = NULL; status = io_open_one_remote(hostpart, filepart, treename, shot, type, new, @@ -1972,12 +2115,13 @@ EXPORT int MDS_IO_OPEN_ONE(char *filepath_in, char const *treename_in, int shot, fullpath = generate_fullpath(filepart, treename, shot, type); int options, mode; getOptionsMode(new, edit, &options, &mode); - fd = open(fullpath, options | O_BINARY | O_RANDOM, mode); + io = GetTreeFileIo(fullpath); + fd = io ? io->open(fullpath, options | O_BINARY | O_RANDOM, mode) : -1; if (type == TREE_DIRECTORY) { if (fd != -1) { - close(fd); + io->close(fd); fd = -3; } } @@ -1989,8 +2133,12 @@ EXPORT int MDS_IO_OPEN_ONE(char *filepath_in, char const *treename_in, int shot, #endif if ((fd != -1) && edit && (type == TREE_TREEFILE_TYPE)) { - if (IS_NOT_OK(io_lock_local((fdinfo_t){conid, fd, enhanced}, 1, 1, - MDS_IO_LOCK_RD | MDS_IO_LOCK_NOWAIT, 0))) + /* Probe editability with the DEFAULT (real fcntl) lock, not the + * fd's backend lock: a URL backend's pseudo-fd makes fcntl fail + * with EBADF, correctly surfacing TreeEDITING for read-only + * sources, while a real local file locks normally. */ + if (IS_NOT_OK(default_tree_file_io.lock(fd, 1, 1, + MDS_IO_LOCK_RD | MDS_IO_LOCK_NOWAIT, 0))) { status = TreeEDITING; fd = -2; @@ -2014,7 +2162,7 @@ EXPORT int MDS_IO_OPEN_ONE(char *filepath_in, char const *treename_in, int shot, } free(filepath); } - *idx = fd < 0 ? fd : ADD_FD(fd, conid, enhanced); + *idx = fd < 0 ? fd : ADD_FD(fd, conid, enhanced, io); FREE_NOW(fullpath); return status; } diff --git a/treeshr/TreeOpen.c b/treeshr/TreeOpen.c index 07ae3bea5f..46170dcd36 100644 --- a/treeshr/TreeOpen.c +++ b/treeshr/TreeOpen.c @@ -279,7 +279,6 @@ int _TreeClose(void **dbid, char const *tree, int shot) PINO_DATABASE *prev_db; status = TreeNOT_OPEN; - // printf("TREE CLOSE\n"); if (dblist && *dblist) { if (tree) @@ -424,6 +423,8 @@ static int close_top_tree(PINO_DATABASE *dblist, int call_hook) (size_t)local_info->alq * 512); #endif free(local_info->vm_addr); + local_info->vm_addr = NULL; + local_info->section_addr[0] = NULL; } TreeWait(local_info); if (local_info->data_file) @@ -999,7 +1000,7 @@ int OpenOne(TREE_INFO *info, TREE_INFO *root, tree_type_t type, int new, else { #ifndef _WIN32 - info->mapped = (MDS_IO_ID(fd) == -1); + info->mapped = (MDS_IO_ID(fd) == -1) && MDS_IO_MMAP_CAPABLE(fd); #ifdef __APPLE__ /* from python-mmap Issue #11277: fsync(2) is not enough on OS X - a special, OS X specific fcntl(2) is necessary to force DISKSYNC and @@ -1049,6 +1050,7 @@ static int MapTree(TREE_INFO *info, TREE_INFO *root, int edit_flag) nomap = !info->mapped; } } + if (status == TreeSUCCESS) status = MapFile(fd, info, nomap); return status; @@ -1089,7 +1091,6 @@ static int MapFile(int fd, TREE_INFO *info, int nomap) First we get virtual memory for the tree to be mapped into. ********************************************/ - status = GetVmForTree(info, nomap); if (status == TreeSUCCESS) { @@ -1149,7 +1150,11 @@ static int MapFile(int fd, TREE_INFO *info, int nomap) status = TreeSUCCESS; } else + { free(info->vm_addr); + info->vm_addr = NULL; + info->section_addr[0] = NULL; + } } return status; } diff --git a/treeshr/testing/CMakeLists.txt b/treeshr/testing/CMakeLists.txt index 863ccb57db..5a34f8c4ae 100644 --- a/treeshr/testing/CMakeLists.txt +++ b/treeshr/testing/CMakeLists.txt @@ -2,6 +2,7 @@ set(_test_source_list TreeDeleteNodeTest.c TreeSegmentTest.c + TreeFileIoTest.c ) foreach(_test_source ${_test_source_list}) @@ -31,4 +32,26 @@ foreach(_test_source ${_test_source_list}) ${_no_valgrind} ) -endforeach() \ No newline at end of file +endforeach() + +### +### Reference TreeFileIo backend for TreeFileIoTest (scheme "testmem://") +### +# A loadable backend the seam test dlopens by image name "MdsTreeFileTESTMEM" +# (-> libMdsTreeFileTESTMEM.so), exercising GetTreeFileIo() dispatch through a +# non-default backend. It builds into CMAKE_LIBRARY_OUTPUT_DIRECTORY, which +# mdsplus_add_test() already puts on MDSPLUS_LIBRARY_PATH / LD_LIBRARY_PATH, so +# the loader finds it at test time with no extra environment wiring. +add_library( + MdsTreeFileTESTMEM + MdsTreeFileTESTMEM.c +) + +target_include_directories( + MdsTreeFileTESTMEM + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/.. +) + +# Ensure the backend is built before the test that loads it runs. +add_dependencies(TreeFileIoTest MdsTreeFileTESTMEM) \ No newline at end of file diff --git a/treeshr/testing/MdsTreeFileTESTMEM.c b/treeshr/testing/MdsTreeFileTESTMEM.c new file mode 100644 index 0000000000..51c0ab1a7b --- /dev/null +++ b/treeshr/testing/MdsTreeFileTESTMEM.c @@ -0,0 +1,51 @@ +/* Copyright (c) 2026 MIT/GA. Apache-2.0. + * In-tree reference TreeFileIo backend for tests, scheme "testmem://". + * Backed by a real temp file (so read/write/lseek round-trip), but advertises + * mmap_capable=0 and a no-op read lock to emulate a remote/URL backend. */ +#include +#include +#include +#include +#include +#include + +#include "../treefileio.h" + +/* Value matches treeshr's TreeSUCCESS; kept local to avoid private headers. */ +#ifndef TreeSUCCESS +#define TreeSUCCESS 1 +#endif + +/* Map "testmem://" to a temp file path /tmp/mdstreefiletest-. */ +static int tm_open(const char *filename, int options, mode_t mode) +{ + const char *sep = strstr(filename, "://"); + const char *name = sep ? sep + 3 : filename; + char path[512]; + snprintf(path, sizeof(path), "/tmp/mdstreefiletest-%s", name); + return open(path, options, mode ? mode : 0644); +} +static int tm_close(int fd) { return close(fd); } +static ssize_t tm_read(int fd, void *b, size_t n) { return read(fd, b, n); } +static ssize_t tm_write(int fd, const void *b, size_t n) +{ + return write(fd, b, n); +} +static off_t tm_lseek(int fd, off_t off, int whence) +{ + return lseek(fd, off, whence); +} +/* No-op read lock (emulating an immutable remote source). */ +static int tm_lock(int fd, off_t offset, size_t size, int mode_in, int *deleted) +{ + (void)fd; (void)offset; (void)size; (void)mode_in; + if (deleted) *deleted = 0; + return TreeSUCCESS; +} + +static const TreeFileIo tm_io = { + tm_open, tm_close, tm_read, tm_write, tm_lseek, tm_lock, + /* .mmap_capable = */ 0}; + +__attribute__((visibility("default"))) +const TreeFileIo *FileIo(void) { return &tm_io; } diff --git a/treeshr/testing/TreeFileIoTest.c b/treeshr/testing/TreeFileIoTest.c new file mode 100644 index 0000000000..1330ce1373 --- /dev/null +++ b/treeshr/testing/TreeFileIoTest.c @@ -0,0 +1,66 @@ +/* Copyright (c) 2026 MIT/GA. Apache-2.0. + * Dependency-free test of the TreeFileIo backend seam. Requires the + * libMdsTreeFileTESTMEM backend (built alongside) on the image search path. */ +#include +#include +#include +#include +#include +#include + +#include "testing.h" + +extern int MDS_IO_OPEN(char *, int, mode_t); +extern int MDS_IO_CLOSE(int); +extern ssize_t MDS_IO_READ(int, void *, size_t); +extern ssize_t MDS_IO_WRITE(int, void *, size_t); +extern off_t MDS_IO_LSEEK(int, off_t, int); +extern int MDS_IO_MMAP_CAPABLE(int); + +int main(void) +{ + BEGIN_TESTING(TreeFileIo backend seam); + + char hello[] = "hello"; /* writable buffers: MDS_IO_WRITE takes void* */ + char abcd[] = "abcdABCD"; + char buf[16] = {0}; + + /* 1. Default path: a plain (no-scheme) path round-trips via default backend. */ + char plain[] = "/tmp/mdstreefiletest-plain"; + unlink(plain); + int fd = MDS_IO_OPEN(plain, O_RDWR | O_CREAT, 0644); + TEST1(fd >= 0); + TEST1(MDS_IO_WRITE(fd, hello, 5) == 5); + TEST1(MDS_IO_LSEEK(fd, 0, SEEK_SET) == 0); + TEST1(MDS_IO_READ(fd, buf, 5) == 5); + TEST1(memcmp(buf, "hello", 5) == 0); + TEST1(MDS_IO_MMAP_CAPABLE(fd) == 1); + TEST1(MDS_IO_CLOSE(fd) == 0); + + /* 2. Scheme dispatch: testmem:// routes to the loaded backend and round-trips. */ + unlink("/tmp/mdstreefiletest-A"); + int a = MDS_IO_OPEN("testmem://A", O_RDWR | O_CREAT, 0644); + TEST1(a >= 0); + TEST1(MDS_IO_MMAP_CAPABLE(a) == 0); + TEST1(MDS_IO_WRITE(a, abcd, 8) == 8); + TEST1(MDS_IO_LSEEK(a, 0, SEEK_SET) == 0); + memset(buf, 0, sizeof(buf)); + /* 3. Short read then EOF=0. */ + TEST1(MDS_IO_READ(a, buf, 4) == 4); + TEST1(MDS_IO_READ(a, buf, 4) == 4); + TEST1(MDS_IO_READ(a, buf, 4) == 0); + TEST1(MDS_IO_READ(a, buf, 4) == 0); + TEST1(MDS_IO_CLOSE(a) == 0); + + /* 2b. Cache: a second testmem open succeeds (backend loaded once, reused). */ + int a2 = MDS_IO_OPEN("testmem://A", O_RDONLY, 0); + TEST1(a2 >= 0); + TEST1(MDS_IO_CLOSE(a2) == 0); + + /* 4. Fail-closed: an unregistered scheme fails the open. */ + int bad = MDS_IO_OPEN("nosuch://whatever", O_RDONLY, 0); + TEST1(bad < 0); + + END_TESTING; + return 0; +} diff --git a/treeshr/treefileio.h b/treeshr/treefileio.h new file mode 100644 index 0000000000..1f0dbefd4c --- /dev/null +++ b/treeshr/treefileio.h @@ -0,0 +1,78 @@ +/* +Copyright (c) 2026, Massachusetts Institute of Technology and General Atomics. +All rights reserved. Licensed under the Apache License, Version 2.0. +*/ + +/* Pluggable backend for local MDSplus tree-file IO. + * + * Mirrors the mdsip transport-plugin pattern (IoRoutines / LoadIo in + * mdstcpip/): a tree-file path with a URL scheme (e.g. "https://...") is + * served by a backend loaded by image name "MdsTreeFile" exporting a + * function symbol "FileIo" of type FileIoFn. A path with no scheme uses the + * built-in default backend (plain open/read/write/lseek/close). */ +#ifndef TREEFILEIO_H +#define TREEFILEIO_H + +#include /* off_t, ssize_t, mode_t */ + +/* Backend plugin contract + * ----------------------- + * To serve tree-file paths with URL scheme "x://", provide a shared library + * named "libMdsTreeFileX" (scheme uppercased; the loader adds the platform + * "lib" prefix and ".so"/".dll" suffix) that exports a function: + * + * const TreeFileIo *FileIo(void); + * + * The exported FileIo symbol MUST be visible to the dynamic loader: annotate it + * with __attribute__((visibility("default"))) on Unix when the library is built + * with -fvisibility=hidden, or __declspec(dllexport) on Windows; otherwise the + * loader's symbol lookup (dlsym / GetProcAddress) will not find it. + * + * The backend is discovered lazily on the first MDS_IO_OPEN of an "x://" path + * via LibFindImageSymbol_C and cached per scheme, mirroring how mdsip loads + * transport backends ("MdsIp" exporting "Io"; see mdstcpip/). + * + * Backend semantics: + * - Set mmap_capable = 0 for sources whose fds are not real mmap-able kernel + * descriptors (e.g. HTTP/remote object stores); MapFile() then uses the + * calloc+read path instead of mmap(). Set 1 only for real local files. + * - lock() should succeed (no-op) for read locks on an immutable/read-only + * source and fail for write/edit locks, so editing such a tree is refused. + * - An unknown or unloadable scheme makes GetTreeFileIo() return NULL and the + * open fail (fail-closed) rather than treating "x://..." as a local path. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct tree_file_io +{ + int (*open)(const char *filename, int options, mode_t mode); + int (*close)(int fd); + ssize_t (*read)(int fd, void *buff, size_t count); + ssize_t (*write)(int fd, const void *buff, size_t count); + off_t (*lseek)(int fd, off_t offset, int whence); + /* lock: same contract as the legacy io_lock_local fcntl helper. + * mode_in uses MDS_IO_LOCK_* flags; *deleted (if non-NULL) is set to + * whether the file has been unlinked. Returns TreeSUCCESS / TreeLOCK_FAILURE. */ + int (*lock)(int fd, off_t offset, size_t size, int mode_in, int *deleted); + /* mmap_capable: 1 if fds from open() are real kernel descriptors that + * MapFile() may mmap(); 0 if MapFile() must use the calloc+read path. */ + int mmap_capable; +} TreeFileIo; + +/* Exported-symbol signature a backend plugin must provide under name "FileIo". */ +typedef const TreeFileIo *(*FileIoFn)(void); + +/* Returns the backend for `filename`: + * - no "scheme://" prefix -> built-in default backend (never NULL) + * - known/loadable scheme -> plugin backend + * - unknown/unloadable -> NULL (caller must fail the open: fail-closed) */ +const TreeFileIo *GetTreeFileIo(const char *filename); + +#ifdef __cplusplus +} +#endif + +#endif /* TREEFILEIO_H */ diff --git a/treeshr/treeshrp.h b/treeshr/treeshrp.h index 44e278ac23..750d2c57de 100644 --- a/treeshr/treeshrp.h +++ b/treeshr/treeshrp.h @@ -943,6 +943,7 @@ extern int tree_put_dsc(PINO_DATABASE *dbid, TREE_INFO *info, int nid, extern int MDS_IO_ID(int fd); extern int MDS_IO_FD(int fd); +extern int MDS_IO_MMAP_CAPABLE(int fd); #ifdef _WIN32 #ifndef HAVE_PTHREAD_H #define ssize_t int64_t