svdhoog · svdhoog · May 30, 2026 · May 30, 2026 · May 30, 2026 · May 30, 2026
diff --git a/Doxyfile b/Doxyfile
diff --git a/Doxyfile.tmpl b/Doxyfile.tmpl
diff --git a/Doxyfile.txt b/Doxyfile.txt
diff --git a/Makefile b/Makefile
diff --git a/Makefile.tmpl b/Makefile.tmpl
@@ -1,82 +1,98 @@
-# Makefile template file for FLAME
-#
-# FLAME is the FLAME source files
-# SOURCE is the user source files
-#
-# The assumption is that the user only provides C files
-# and a model files <file>.xml
-#
-#===================================================
-# Change this variable for the location of the 
-# Message Board Library installation
-#===================================================
-LIBMBOARD_DIR = /usr/local
+# ==========================================================================
+# FLAME AUTOMATED GENERATION MAKEFILE TEMPLATE WITH PARQUET SERIALIZATION
+# ==========================================================================
+
+# Message Board Library paths
+LIBMBOARD_DIR = /home/sander/build/libmboard
 LIBMBOARD_INC = $(LIBMBOARD_DIR)/include
 LIBMBOARD_LIB = $(LIBMBOARD_DIR)/lib
 
-#====================================================
-# Change these for your compliler/loaded
-#====================================================
-SERIAL_CCOMP	= gcc
-PARALLEL_CCOMP	= mpicc
+# Compilers and Loaders (Using g++ for linking mixed C/C++ objects)
+SERIAL_CCOMP    = gcc
+PARALLEL_CCOMP  = mpicc
+SERIAL_LOADER   = g++
+PARALLEL_LOADER = mpicxx
 
-SERIAL_LOADER	= gcc
-PARALLEL_LOADER	= mpif77
+# Feature Flags
+DEFINES         =
+<?if gsl_lib?>DEFINES        += -DGSL_LIB<?end if?>
 
-DEFINES		=
-<?if gsl_lib?>DEFINES		+= -DGSL_LIB<?end if?>
-#DEFINES	+= -DSTART_END 
-#DEFINES	+= -DGRAPHICS
+# Toolchain Assignment
+CC       = <?if serial?>$(SERIAL_CCOMP)<?end if?><?if parallel?>$(PARALLEL_CCOMP)<?end if?>
+CXX      = g++
+LD       = <?if serial?>$(SERIAL_LOADER)<?end if?><?if parallel?>$(PARALLEL_LOADER)<?end if?>
 
-# C Compiler
-CC      = <?if serial?>$(SERIAL_CCOMP)<?end if?><?if parallel?>$(PARALLEL_CCOMP)<?end if?>
-CFLAGS  =  -I$(LIBMBOARD_INC) ${DEFINES}
-<?if debug?>CFLAGS += -std=c99 -Wall -D_DEBUG_MODE -g<?end if?>
-<?if final?>CFLAGS += -O3<?end if?>
+# Compilation Flags (Included -I. so generated source files can find parquet_engine.h)
+CFLAGS   = -I$(LIBMBOARD_INC) -I. $(DEFINES) -fcommon -Wall
+CXXFLAGS = -I$(LIBMBOARD_INC) -I. $(DEFINES) -Wall -std=c++20
 
-LD       = <?if serial?>$(SERIAL_LOADER)<?end if?><?if parallel?>$(PARALLEL_LOADER)<?end if?>
+<?if debug?>
+CFLAGS   += -std=c11 -D_DEBUG_MODE -g
+CXXFLAGS += -D_DEBUG_MODE -g
+<?end if?>
 
-LDFLAGS  = -L$(LIBMBOARD_LIB)
-<?if gsl_lib?>
-LIBS	+= -lgsl -lgslcblas
+<?if final?>
+CFLAGS   += -O3
+CXXFLAGS += -O3
 <?end if?>
-LIBS    += <?if serial?>-lmboard_s<?if debug?>d<?end if?> <?end if?><?if parallel?>-lmboard_p<?if debug?>d<?end if?> <?end if?> -lm
 
-#FLAME source files
-FLAME   = main.c memory.c xml.c messageboards.c partitioning.c rules.c timing.c
+LDFLAGS  = -L$(LIBMBOARD_LIB)
+
+# Library Dependencies (Standard FLAME + Apache Arrow ecosystem)
+LIBS     = <?if serial?>-lmboard_s<?if debug?>d<?end if?> <?end if?><?if parallel?>-lmboard_p<?if debug?>d<?end if?> <?end if?> -lm
+LIBS    += -larrow -lparquet -lstdc++
+<?if gsl_lib?>LIBS    += -lgsl -lgslcblas<?end if?>
+
+# FLAME Core Source manifests
+FLAME    = main.c memory.c xml.c messageboards.c partitioning.c rules.c timing.c
+AUX      = stategraph.dot stategraph_colour.dot process_order_graph.dot Doxyfile latex.tex
 
-#FLAME auxilary files
-AUX 	= stategraph.dot stategraph_colour.dot process_order_graph.dot Doxyfile latex.tex
+# Dynamic Model Manifests (Parsed dynamically by xparser)
+SOURCES  = <?foreach functionfiles?> $filepath <?end foreach?>
+HEADERS  = header.h low_primes.h mboard.h parquet_engine.h <?foreach xagent?> $name_agent_header.h <?end foreach?>
 
-# FLAME generated model files
-SOURCES = <?foreach functionfiles?> $filepath <?end foreach?>
-HEADERS = header.h low_primes.h mboard.h <?foreach xagent?> $name_agent_header.h <?end foreach?>
+# External Parquet Tracking System Engine Code
+PARQUET_SRC = parquet.cpp parquet_engine.cpp
 
-DEPS    = Makefile header.h low_primes.h
+# Consolidated dependencies (Dynamic generation drops parquet_engine.h straight into place)
+DEPS     = Makefile header.h low_primes.h parquet_engine.h
 
-OBJECTS = $(SOURCES:.c=.o) $(FLAME:.c=.o)
+# Unified Object target arrays derived dynamically from source collections
+OBJECTS  = $(SOURCES:.c=.o) $(FLAME:.c=.o) $(PARQUET_SRC:.cpp=.o)
 EXECUTABLE = main
-RM = rm -f
+RM       = rm -f
 
+# Pipeline Execution Targets
 all: $(SOURCES) $(FLAME) $(EXECUTABLE)
 
 $(EXECUTABLE): $(OBJECTS)
 	$(LD) $(LDFLAGS) $(OBJECTS) -o $@ $(LIBS)
 
+# Core objects depend on native framework headers and generated parquet engine header
 $(OBJECTS): $(DEPS)
 
+# Structural Suffix Processing Patterns
 .c.o:
 	$(CC) -c $(CFLAGS) $< -o $@
 
+.cpp.o:
+	$(CXX) -c $(CXXFLAGS) $< -o $@
+
+# Cleanup Framework Operations (Preserving your *.parquet data files)
 clean:
-	$(RM) $(OBJECTS) $(EXECUTABLE) $(EXECUTABLE).exe 
+	$(RM) $(OBJECTS) $(EXECUTABLE) $(EXECUTABLE).exe
+
 vclean:
-	$(RM) main $(EXECUTABLE) $(EXECUTABLE).exe $(OBJECTS)  $(FLAME) $(HEADERS)  $(AUX) Makefile
+	$(RM) main $(EXECUTABLE) $(EXECUTABLE).exe $(OBJECTS) $(FLAME) $(HEADERS) $(AUX) Makefile
+
 format:
 	astyle -A3 $(SOURCES)
-	for file in $(patsubst %.c,%.h, $(SOURCES)) ; do \
-	echo $$file ; \
-	astyle -A3 $$file ; \
+	for file in $(patsubst %.c,%.h, $(SOURCES)) ; \
+	do \
+		echo $$file ; \
+		astyle -A3 $$file ; \
 	done 
+
 print:
 	a2ps -R -f10 --column=1 $(SOURCES)
+
diff --git a/Makefile.tmpl.custom b/Makefile.tmpl.custom
diff --git a/README.txt b/README.txt
diff --git a/Readme2.txt b/Readme2.txt
diff --git a/dependencygraph.c b/dependencygraph.c
diff --git a/docs/developer_manual/Makefile b/docs/developer_manual/Makefile
diff --git a/docs/developer_manual/communication.tex b/docs/developer_manual/communication.tex
diff --git a/docs/developer_manual/developer_manual.tex b/docs/developer_manual/developer_manual.tex
diff --git a/docs/developer_manual/execution.tex b/docs/developer_manual/execution.tex
diff --git a/docs/developer_manual/memory.tex b/docs/developer_manual/memory.tex
diff --git a/docs/user_manual/Makefile b/docs/user_manual/Makefile
diff --git a/docs/user_manual/XMML.dtd b/docs/user_manual/XMML.dtd
diff --git a/docs/user_manual/iteration_1.dot b/docs/user_manual/iteration_1.dot
diff --git a/docs/user_manual/model_description.tex b/docs/user_manual/model_description.tex
diff --git a/docs/user_manual/model_design.tex b/docs/user_manual/model_design.tex
diff --git a/docs/user_manual/model_execution.tex b/docs/user_manual/model_execution.tex
diff --git a/docs/user_manual/model_implementation.tex b/docs/user_manual/model_implementation.tex
diff --git a/docs/user_manual/overview.tex b/docs/user_manual/overview.tex
diff --git a/docs/user_manual/swarm_1.dot b/docs/user_manual/swarm_1.dot
diff --git a/docs/user_manual/swarm_2.dot b/docs/user_manual/swarm_2.dot
diff --git a/docs/user_manual/swarm_3.dot b/docs/user_manual/swarm_3.dot
diff --git a/docs/user_manual/user_manual.tex b/docs/user_manual/user_manual.tex
diff --git a/docs/user_manual/xmachine.dia b/docs/user_manual/xmachine.dia
diff --git a/docs/user_manual/xmachine.eps b/docs/user_manual/xmachine.eps
diff --git a/docs/xparser_parquet/manual.md b/docs/xparser_parquet/manual.md
@@ -0,0 +1,41 @@
+# Xparser Parquet
+
+The main focus was not on modifying xparser itself to write Parquet files. Instead, the focus was on enhancing the simulation code templates that xparser reads, allowing the compiled simulation binary to write Parquet files natively at runtime.
+
+Here is how those responsibilities are split:
+
+## 1. What xparser Actually Does
+
+xparser is a pre-compilation code generator, not the simulation engine. Its only job is to read your model's XML specification file (which defines your agents, states, and variables) and unroll code templates (like parquet.cpp.tmpl and parquet_engine.cpp.tmpl) into concrete C++ source files (parquet.cpp and parquet_engine.cpp).
+
+xparser itself doesn't know anything about Apache Arrow, memory allocation, or Parquet compression. It just substitutes template variables like <?foreach xagent?> and $name into plain text.
+
+## 2. Where the Real Focus Was: The Runtime Engine
+
+Our actual focus was on building a high-performance runtime data pipeline inside the template files that executes while the simulation is running.
+
+Instead of changing how xparser parses files, we wrote a standard C++ engine that hooks into FLAME's loop structure to intercept agent data before it gets dumped into the traditional, bulky XML iteration logs.
+The Pipeline Architecture We Built:
+```text
+[FLAME Simulation Loop]
+         │
+         ▼
+[parquet.cpp]  ───► Captures simulation state & iteration paths (outputpath)
+         │
+         ▼
+[parquet_engine.cpp]  
+         │
+         ├──► 1. Buffers raw C++ vectors (iterations, IDs, agent variables)
+         ├──► 2. Standardizes memory layouts & resets states (buf.columns.clear())
+         ├──► 3. Feeds data into Apache Arrow Builders (Int32Builder, DoubleBuilder)
+         ├──► 4. Assembles an in-memory Arrow Table with strict Schema enforcement
+         │
+         ▼
+[Physical Disk] ───► Compresses and writes highly optimized .parquet files
+```
+
+## Summary of the Goal
+
+The goal was to replace FLAME's default XML runtime output mechanism with an Apache Arrow/Parquet serialization engine.
+
+We accomplished this by feeding clean, robust C++20 memory management and Arrow API calls into the .tmpl files, so that whenever you run ./xparser and compile your model, the resulting executable natively records data into Parquet format instead of millions of lines of uncompressed XML text.
diff --git a/header.h b/header.h
diff --git a/header.tmpl b/header.tmpl
@@ -529,7 +529,17 @@ void readprepartitionedinitialstates(char * filename, char * filelocation, int *
 <?end if?>
 void readinitialstates(char * filename, char * filelocation, int * itno, double cloud_data[],
 					   int partition_method, int flag);
+
 void saveiterationdata(int iteration_number);
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void saveiterationdata_parquet(int iteration_number);
+
+#ifdef __cplusplus
+}
+#endif
 <?foreach datatype?>
 void free_$name($name * temp);
 void free_$name_static_array($name * array, int size);

diff --git a/low_primes.tmpl b/low_primes.tmpl